Download feature_engineering/aggregations.py from KLEB38/OC_P8: direct link, hf CLI and curl.
- Browser
- Download file 9.4 kB
-
https://huggingface.co/spaces/KLEB38/OC_P8/resolve/main/feature_engineering/aggregations.py
- Command line
-
hf download hf://spaces/KLEB38/OC_P8/feature_engineering/aggregations.py
-
curl -L -o aggregations.py https://huggingface.co/spaces/KLEB38/OC_P8/resolve/main/feature_engineering/aggregations.py
9.4 kB
| """Heavy aggregation functions extracted from notebooks/EDA.ipynb. | |
| Each function reads one auxiliary CSV (bureau, previous_application, | |
| POS_CASH_balance, installments_payments, credit_card_balance) and returns | |
| a DataFrame indexed by SK_ID_CURR with one row per client. | |
| These functions are used only offline by scripts/build_feature_store.py. | |
| """ | |
| from __future__ import annotations | |
| import gc | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| def one_hot_encoder( | |
| df: pd.DataFrame, nan_as_category: bool = True | |
| ) -> tuple[pd.DataFrame, list[str]]: | |
| """One-hot encode all object dtype columns. Returns the encoded df and the | |
| list of newly created column names.""" | |
| original_columns = list(df.columns) | |
| categorical_columns = [col for col in df.columns if df[col].dtype == "object"] | |
| df = pd.get_dummies(df, columns=categorical_columns, dummy_na=nan_as_category) | |
| new_columns = [c for c in df.columns if c not in original_columns] | |
| return df, new_columns | |
| def bureau_and_balance( | |
| data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True | |
| ) -> pd.DataFrame: | |
| """Aggregate bureau.csv and bureau_balance.csv per SK_ID_CURR. | |
| Produces BURO_*, ACTIVE_*, CLOSED_* columns. | |
| """ | |
| bureau = pd.read_csv(data_dir / "bureau.csv", nrows=num_rows) | |
| bb = pd.read_csv(data_dir / "bureau_balance.csv", nrows=num_rows) | |
| bb, bb_cat = one_hot_encoder(bb, nan_as_category) | |
| bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category) | |
| bb_aggregations: dict[str, list[str]] = {"MONTHS_BALANCE": ["min", "max", "size"]} | |
| for col in bb_cat: | |
| bb_aggregations[col] = ["mean"] | |
| bb_agg = bb.groupby("SK_ID_BUREAU").agg(bb_aggregations) | |
| bb_agg.columns = pd.Index( | |
| [e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist()] | |
| ) | |
| bureau = bureau.join(bb_agg, how="left", on="SK_ID_BUREAU") | |
| bureau.drop(["SK_ID_BUREAU"], axis=1, inplace=True) | |
| del bb, bb_agg | |
| gc.collect() | |
| num_aggregations = { | |
| "DAYS_CREDIT": ["min", "max", "mean", "var"], | |
| "DAYS_CREDIT_ENDDATE": ["min", "max", "mean"], | |
| "DAYS_CREDIT_UPDATE": ["mean"], | |
| "CREDIT_DAY_OVERDUE": ["max", "mean"], | |
| "AMT_CREDIT_MAX_OVERDUE": ["mean"], | |
| "AMT_CREDIT_SUM": ["max", "mean", "sum"], | |
| "AMT_CREDIT_SUM_DEBT": ["max", "mean", "sum"], | |
| "AMT_CREDIT_SUM_OVERDUE": ["mean"], | |
| "AMT_CREDIT_SUM_LIMIT": ["mean", "sum"], | |
| "AMT_ANNUITY": ["max", "mean"], | |
| "CNT_CREDIT_PROLONG": ["sum"], | |
| "MONTHS_BALANCE_MIN": ["min"], | |
| "MONTHS_BALANCE_MAX": ["max"], | |
| "MONTHS_BALANCE_SIZE": ["mean", "sum"], | |
| } | |
| cat_aggregations: dict[str, list[str]] = {} | |
| for cat in bureau_cat: | |
| cat_aggregations[cat] = ["mean"] | |
| for cat in bb_cat: | |
| cat_aggregations[cat + "_MEAN"] = ["mean"] | |
| bureau_agg = bureau.groupby("SK_ID_CURR").agg( | |
| {**num_aggregations, **cat_aggregations} | |
| ) | |
| bureau_agg.columns = pd.Index( | |
| ["BURO_" + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist()] | |
| ) | |
| active = bureau[bureau["CREDIT_ACTIVE_Active"] == 1] | |
| active_agg = active.groupby("SK_ID_CURR").agg(num_aggregations) | |
| active_agg.columns = pd.Index( | |
| ["ACTIVE_" + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist()] | |
| ) | |
| bureau_agg = bureau_agg.join(active_agg, how="left", on="SK_ID_CURR") | |
| del active, active_agg | |
| gc.collect() | |
| closed = bureau[bureau["CREDIT_ACTIVE_Closed"] == 1] | |
| closed_agg = closed.groupby("SK_ID_CURR").agg(num_aggregations) | |
| closed_agg.columns = pd.Index( | |
| ["CLOSED_" + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist()] | |
| ) | |
| bureau_agg = bureau_agg.join(closed_agg, how="left", on="SK_ID_CURR") | |
| del closed, closed_agg, bureau | |
| gc.collect() | |
| return bureau_agg | |
| def previous_applications( | |
| data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True | |
| ) -> pd.DataFrame: | |
| """Aggregate previous_application.csv per SK_ID_CURR. | |
| Produces PREV_*, APPROVED_*, REFUSED_* columns. | |
| """ | |
| prev = pd.read_csv(data_dir / "previous_application.csv", nrows=num_rows) | |
| prev, cat_cols = one_hot_encoder(prev, nan_as_category=nan_as_category) | |
| for col in [ | |
| "DAYS_FIRST_DRAWING", | |
| "DAYS_FIRST_DUE", | |
| "DAYS_LAST_DUE_1ST_VERSION", | |
| "DAYS_LAST_DUE", | |
| "DAYS_TERMINATION", | |
| ]: | |
| prev[col] = prev[col].replace(365243, np.nan) | |
| prev["APP_CREDIT_PERC"] = prev["AMT_APPLICATION"] / prev["AMT_CREDIT"] | |
| num_aggregations = { | |
| "AMT_ANNUITY": ["min", "max", "mean"], | |
| "AMT_APPLICATION": ["min", "max", "mean"], | |
| "AMT_CREDIT": ["min", "max", "mean"], | |
| "APP_CREDIT_PERC": ["min", "max", "mean", "var"], | |
| "AMT_DOWN_PAYMENT": ["min", "max", "mean"], | |
| "AMT_GOODS_PRICE": ["min", "max", "mean"], | |
| "HOUR_APPR_PROCESS_START": ["min", "max", "mean"], | |
| "RATE_DOWN_PAYMENT": ["min", "max", "mean"], | |
| "DAYS_DECISION": ["min", "max", "mean"], | |
| "CNT_PAYMENT": ["mean", "sum"], | |
| } | |
| cat_aggregations = {cat: ["mean"] for cat in cat_cols} | |
| prev_agg = prev.groupby("SK_ID_CURR").agg({**num_aggregations, **cat_aggregations}) | |
| prev_agg.columns = pd.Index( | |
| ["PREV_" + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist()] | |
| ) | |
| approved = prev[prev["NAME_CONTRACT_STATUS_Approved"] == 1] | |
| approved_agg = approved.groupby("SK_ID_CURR").agg(num_aggregations) | |
| approved_agg.columns = pd.Index( | |
| ["APPROVED_" + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist()] | |
| ) | |
| prev_agg = prev_agg.join(approved_agg, how="left", on="SK_ID_CURR") | |
| refused = prev[prev["NAME_CONTRACT_STATUS_Refused"] == 1] | |
| refused_agg = refused.groupby("SK_ID_CURR").agg(num_aggregations) | |
| refused_agg.columns = pd.Index( | |
| ["REFUSED_" + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist()] | |
| ) | |
| prev_agg = prev_agg.join(refused_agg, how="left", on="SK_ID_CURR") | |
| del refused, refused_agg, approved, approved_agg, prev | |
| gc.collect() | |
| return prev_agg | |
| def pos_cash( | |
| data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True | |
| ) -> pd.DataFrame: | |
| """Aggregate POS_CASH_balance.csv per SK_ID_CURR. Produces POS_* columns.""" | |
| pos = pd.read_csv(data_dir / "POS_CASH_balance.csv", nrows=num_rows) | |
| pos, cat_cols = one_hot_encoder(pos, nan_as_category=nan_as_category) | |
| aggregations: dict[str, list[str]] = { | |
| "MONTHS_BALANCE": ["max", "mean", "size"], | |
| "SK_DPD": ["max", "mean"], | |
| "SK_DPD_DEF": ["max", "mean"], | |
| } | |
| for cat in cat_cols: | |
| aggregations[cat] = ["mean"] | |
| pos_agg = pos.groupby("SK_ID_CURR").agg(aggregations) | |
| pos_agg.columns = pd.Index( | |
| ["POS_" + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist()] | |
| ) | |
| pos_agg["POS_COUNT"] = pos.groupby("SK_ID_CURR").size() | |
| del pos | |
| gc.collect() | |
| return pos_agg | |
| def installments_payments( | |
| data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True | |
| ) -> pd.DataFrame: | |
| """Aggregate installments_payments.csv per SK_ID_CURR. Produces INSTAL_*.""" | |
| ins = pd.read_csv(data_dir / "installments_payments.csv", nrows=num_rows) | |
| ins, cat_cols = one_hot_encoder(ins, nan_as_category=nan_as_category) | |
| ins["PAYMENT_PERC"] = ins["AMT_PAYMENT"] / ins["AMT_INSTALMENT"] | |
| ins["PAYMENT_DIFF"] = ins["AMT_INSTALMENT"] - ins["AMT_PAYMENT"] | |
| ins["DPD"] = ins["DAYS_ENTRY_PAYMENT"] - ins["DAYS_INSTALMENT"] | |
| ins["DBD"] = ins["DAYS_INSTALMENT"] - ins["DAYS_ENTRY_PAYMENT"] | |
| ins["DPD"] = ins["DPD"].clip(lower=0) | |
| ins["DBD"] = ins["DBD"].clip(lower=0) | |
| aggregations: dict[str, list[str]] = { | |
| "NUM_INSTALMENT_VERSION": ["nunique"], | |
| "DPD": ["max", "mean", "sum"], | |
| "DBD": ["max", "mean", "sum"], | |
| "PAYMENT_PERC": ["max", "mean", "sum", "var"], | |
| "PAYMENT_DIFF": ["max", "mean", "sum", "var"], | |
| "AMT_INSTALMENT": ["max", "mean", "sum"], | |
| "AMT_PAYMENT": ["min", "max", "mean", "sum"], | |
| "DAYS_ENTRY_PAYMENT": ["max", "mean", "sum"], | |
| } | |
| for cat in cat_cols: | |
| aggregations[cat] = ["mean"] | |
| ins_agg = ins.groupby("SK_ID_CURR").agg(aggregations) | |
| ins_agg.columns = pd.Index( | |
| ["INSTAL_" + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist()] | |
| ) | |
| ins_agg["INSTAL_COUNT"] = ins.groupby("SK_ID_CURR").size() | |
| del ins | |
| gc.collect() | |
| return ins_agg | |
| def credit_card_balance( | |
| data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True | |
| ) -> pd.DataFrame: | |
| """Aggregate credit_card_balance.csv per SK_ID_CURR. Produces CC_* columns.""" | |
| cc = pd.read_csv(data_dir / "credit_card_balance.csv", nrows=num_rows) | |
| cc, cat_cols = one_hot_encoder(cc, nan_as_category=nan_as_category) | |
| cc.drop(["SK_ID_PREV"], axis=1, inplace=True) | |
| num_aggregations = { | |
| col: ["min", "max", "mean", "sum", "var"] | |
| for col in cc.columns | |
| if col not in cat_cols | |
| } | |
| cat_aggregations = {cat: ["mean"] for cat in cat_cols} | |
| cc_agg = cc.groupby("SK_ID_CURR").agg({**num_aggregations, **cat_aggregations}) | |
| cc_agg.columns = pd.Index( | |
| ["CC_" + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist()] | |
| ) | |
| cc_agg["CC_COUNT"] = cc.groupby("SK_ID_CURR").size() | |
| del cc | |
| gc.collect() | |
| return cc_agg | |