3 · 为什么保留这个特征，删除另一个？¶
本批保留 Home Credit 两张 application 表的全部 356,255 条记录。TARGET 来自原比赛训练表，比赛测试表的 48,744 条记录保持 null。APPLY_DATE 是按 ID 稳定散列生成的演示日期。0train 用于拟合和 PSI 基准，1val 用于验证或筛选，2test 与 3oot 用于固定结果的独立复核，4_oot_undue 仅表示比赛未公开标签。日期、元数据和角色映射见输入方案与分区说明。
本批实际结果¶
| 字段 | 状态 | 阶段 | 原因 |
|---|---|---|---|
| AMT_REQ_CREDIT_BUREAU_DAY | Dropped | Corr_Filter | Correlated with 'AMT_REQ_CREDIT_BUREAU_YEAR' |
| AMT_REQ_CREDIT_BUREAU_HOUR | Dropped | Corr_Filter | Correlated with 'AMT_REQ_CREDIT_BUREAU_YEAR' |
| AMT_REQ_CREDIT_BUREAU_MON | Dropped | Corr_Filter | Correlated with 'AMT_REQ_CREDIT_BUREAU_YEAR' |
| AMT_REQ_CREDIT_BUREAU_QRT | Dropped | Corr_Filter | Correlated with 'AMT_REQ_CREDIT_BUREAU_YEAR' |
| AMT_REQ_CREDIT_BUREAU_WEEK | Dropped | Corr_Filter | Correlated with 'AMT_REQ_CREDIT_BUREAU_YEAR' |
| APARTMENTS_MEDI | Dropped | Corr_Filter | Correlated with 'FLOORSMAX_MEDI' |
| APARTMENTS_MODE | Dropped | Corr_Filter | Correlated with 'FLOORSMAX_MEDI' |
| ELEVATORS_AVG | Dropped | Corr_Filter | Correlated with 'FLOORSMAX_MEDI' |
| ELEVATORS_MEDI | Dropped | Corr_Filter | Correlated with 'FLOORSMAX_MEDI' |
| ELEVATORS_MODE | Dropped | Corr_Filter | Correlated with 'FLOORSMAX_MEDI' |
| 原始字段对 | Spearman | 绝对相关 |
|---|---|---|
| OBS_30_CNT_SOCIAL_CIRCLE / OBS_60_CNT_SOCIAL_CIRCLE | 0.9975 | 0.9975 |
| REGION_RATING_CLIENT / REGION_RATING_CLIENT_W_CITY | 0.9495 | 0.9495 |
| AMT_ANNUITY / AMT_GOODS_PRICE | 0.8244 | 0.8244 |
| EXT_SOURCE_1 / DAYS_BIRTH | -0.5984 | 0.5984 |
开发筛选及相关性拟合只使用 0train。1val 的使用范围写入上下文；2test、3oot 不参与调整阈值。 Stats 的 WOE Pearson 与 Linear 的原始数值 Spearman 分开保存，不用原始值符号解释 WOE 决策。 get_report() 返回真实决策 DataFrame，get_correlation_report() 返回对应相关性报告。 下方展示实际关系与决策；没有发生相关性淘汰时，保留未触发结论，不人为造冗余字段或删项。 候选 fcols、模块支持的候选子集和真正参与相关计算的字段分别登记，未计算关系不补数。
相关性证据¶
原始值快照 · WOE 快照 · 实际筛选决策 · 相关性 HTML
先查 pairs 的真实端点、方法与相关值，再下钻 correlation_decisions 和局部矩阵。缺失处理、有效样本与阈值读取 describe() 中的参数，不能合并两种表示的统计。
从报告查看真实表与图，按实际分母判断。完整可运行代码使用共享准备入口，复用同一宽表与固定参考样本。
def _selection(inputs: HomeCreditInputs, output: Path) -> None:
"""Stats 的 WOE Pearson 与 Linear 原始值 Spearman 均只拟合 Train。"""
train = inputs.benchmark_df
stats = MarsStatsSelector(
missing_thr=0.9, rough_iv_thr=0.01, rough_lift_thr=1.2, iv_thr=0.01,
lift_thr=1.2, psi_thr=0.25, rc_thr=0.5, corr_thr=0.8,
rough_binning_params={"method": "quantile", "n_bins": 4},
skip_fine_scan=True, n_jobs=1,
).fit(train, target="TARGET", features=inputs.fcols, benchmark_df=train,
feature_metadata=inputs.feature_metadata, time_col="APPLY_DATE", time_grain="month",
business_context=inputs.business_context)
selection = stats.get_report()
pd.DataFrame(_rows(selection)).to_excel(output / "selection.xlsx", index=False)
_write_json(output / "selection.json", _rows(selection))
woe = stats.get_correlation_report()
woe.save(output / "woe-correlation.marsreport", overwrite=True)
# Linear 对请求字段池整行删除缺失；限定真实数值子集，明确未计算全宽相关矩阵。
numeric = ["EXT_SOURCE_1", "EXT_SOURCE_2", "EXT_SOURCE_3", "AMT_INCOME_TOTAL", "AMT_ANNUITY",
"AMT_GOODS_PRICE", "DAYS_BIRTH", "REGION_RATING_CLIENT", "REGION_RATING_CLIENT_W_CITY",
"OBS_30_CNT_SOCIAL_CIRCLE", "OBS_60_CNT_SOCIAL_CIRCLE"]
numeric = [name for name in numeric if name in inputs.fcols]
linear = MarsLinearSelector(corr_thr=0.95, corr_method="spearman", n_jobs=1).fit(
train.select(numeric), train["TARGET"], features=numeric,
feature_metadata=inputs.feature_metadata, business_context=inputs.business_context,
)
raw = linear.get_correlation_report()
raw.save(output / "raw-correlation.marsreport", overwrite=True)
matrix = show_correlation_matrix(raw)
_write_correlation_html(matrix, output / "correlation.html")
matrix.to_excel(output / "correlation.xlsx")
queries = [
_query(raw, "correlation_decisions", "真实数值相关性是否触发淘汰", limit=12),
_query(raw, "pairs", "实际最强原始字段关系", sort_by="abs_correlation", descending=True, limit=6),
_query(woe, "correlation_decisions", "目标感知 WOE 空间的实际决策", limit=8),
]
_case(output, 3, raw, "为什么删除一个真实字段，保留另一个？", queries, {
"candidate_features": inputs.fcols, "stats_selected": stats.selected_features_,
"stats_audit": _rows(selection), "linear_candidate_features": numeric,
"linear_selected": linear.selected_features_, "woe_report_id": woe.report_id,
"raw_pairs": queries[1]["rows"], "raw_parameters": raw.describe()["parameters"],
"woe_parameters": woe.describe()["parameters"], "selection_scope": "0train only; 1val/2test/3oot not used for selection",
}, "相关性是该字段池、表示与缺失处理下的关联，不说明业务因果。")
新进程按目录读取已保存统计；实际有限 JSON 中的引用可重放。
from mars.reporting import load_report
report = load_report("raw-correlation.marsreport")
print(report.describe())
page = report.query_page("pairs", features="AMT_INCOME_TOTAL", limit=3)
print(page["reference"], page["data"])
实际查询结果与分页引用
{
"case": 3,
"report_id": "9c0d63d9-bd1c-401d-92bb-0b27e51afa45",
"consumer": "deterministic public API queries; no LLM",
"question": "为什么删除一个真实字段，保留另一个？",
"query": {
"question": "真实数值相关性是否触发淘汰",
"reference": {
"report_id": "9c0d63d9-bd1c-401d-92bb-0b27e51afa45",
"table": "correlation_decisions",
"query": {
"offset": 0,
"limit": 12
}
},
"total_rows": 11,
"returned_rows": 11,
"omitted_rows": 0,
"next_offset": null,
"rows": [
{
"event_order": 0,
"feature": "OBS_30_CNT_SOCIAL_CIRCLE",
"trigger_feature": "OBS_60_CNT_SOCIAL_CIRCLE",
"action": "drop",
"correlation": 0.9974968029828649,
"abs_correlation": 0.9974968029828649,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.016545299924335632,
"trigger_priority": 0.01662761122500525,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 1,
"feature": "EXT_SOURCE_1",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.15048668629948603,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 2,
"feature": "EXT_SOURCE_2",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.13323666386807034,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 3,
"feature": "EXT_SOURCE_3",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.16213596930493898,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 4,
"feature": "AMT_INCOME_TOTAL",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.02010907211464301,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 5,
"feature": "AMT_ANNUITY",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.0005761958408035406,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 6,
"feature": "AMT_GOODS_PRICE",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.03027665264392896,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 7,
"feature": "DAYS_BIRTH",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.06676390631860718,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 8,
"feature": "REGION_RATING_CLIENT",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.05868759630020672,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 9,
"feature": "REGION_RATING_CLIENT_W_CITY",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.06166437429089112,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
},
{
"event_order": 10,
"feature": "OBS_60_CNT_SOCIAL_CIRCLE",
"trigger_feature": null,
"action": "keep",
"correlation": null,
"abs_correlation": null,
"threshold": 0.95,
"operator": ">=",
"priority_metric": "absolute target Spearman",
"feature_priority": 0.01662761122500525,
"trigger_priority": null,
"tie_rule": "left candidate retained on >= strength; legacy NaN candidates excluded before corr",
"white_list_protected": false
}
]
},
"unavailable": "相关性是该字段池、表示与缺失处理下的关联，不说明业务因果。"
}
运行与下载¶
python docs/snippets/task_cases.py --case 3 --data-dir examples/home-credit-2018/data/raw_data --output-dir output/task-cases
首次生成全部关联证据用 --case all。文件缺失会明确失败，正式入口不回退合成数据。 数据下载与输入准备 · 完整共享源码 · 共享包 · 来源和哈希 · 案例索引