File size: 6,851 Bytes
9331de5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | """Offline script: build the runtime feature store and metadata artefacts.
Outputs:
- data/features_store.parquet : ~600 aggregated columns indexed by SK_ID_CURR
(bureau / prev / POS / CC / install only β
application_train columns are reconstructed
at inference time from the JSON input).
- models/feature_names.json : ordered list of the 768 columns the model expects.
- models/app_train_columns.json : metadata for the 122 raw app_train columns
(dtype, min, max, sample) β used to draft the
Pydantic schema and validate inference inputs.
- models/app_train_categories.json : exhaustive list of categorical values seen in
training, per column. Crucial for one-hot
consistency at inference time.
Run once before starting the API:
uv run python scripts/build_feature_store.py
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pandas as pd
# allow importing feature_engineering from the project root
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from feature_engineering.aggregations import ( # noqa: E402
bureau_and_balance,
credit_card_balance,
installments_payments,
pos_cash,
previous_applications,
)
from feature_engineering.orchestrator import merge_files # noqa: E402
DATA_DIR = Path("C:/Users/Kevin/projects/OC_P6/data")
OUT_PARQUET = ROOT / "data" / "features_store.parquet"
OUT_FEATURE_NAMES = ROOT / "models" / "feature_names.json"
OUT_APP_TRAIN_COLS = ROOT / "models" / "app_train_columns.json"
OUT_APP_TRAIN_CATEGORIES = ROOT / "models" / "app_train_categories.json"
OUT_BINARY_MAPPINGS = ROOT / "models" / "app_train_binary_mappings.json"
# Same binary columns the notebook factorizes β order matters.
BINARY_COLUMNS = ["CODE_GENDER", "FLAG_OWN_CAR", "FLAG_OWN_REALTY"]
def _build_app_train_metadata(data_dir: Path) -> tuple[dict, dict, dict]:
"""Read the raw application_train.csv and return:
- column_meta: {col: {dtype, min, max, n_missing, sample}}
- categories: {col: [unique values]} for object-dtype columns (excluding the
binary ones, since those are factorized rather than one-hot encoded)
- binary_mappings: {col: {value: code}} for the 3 binary columns, captured
with the EXACT same first-occurrence order pd.factorize() saw at training.
"""
raw = pd.read_csv(data_dir / "application_train.csv")
raw = raw[raw["CODE_GENDER"] != "XNA"].reset_index(drop=True)
binary_mappings: dict[str, dict[str, int]] = {}
for col in BINARY_COLUMNS:
codes, uniques = pd.factorize(raw[col])
binary_mappings[col] = {str(v): int(i) for i, v in enumerate(uniques)}
column_meta: dict[str, dict] = {}
categories: dict[str, list] = {}
for col in raw.columns:
dtype = str(raw[col].dtype)
n_missing = int(raw[col].isna().sum())
meta: dict = {"dtype": dtype, "n_missing": n_missing}
if raw[col].dtype == "object":
uniques_sorted = sorted(raw[col].dropna().unique().tolist())
meta["categories"] = uniques_sorted
if col not in BINARY_COLUMNS:
categories[col] = uniques_sorted
else:
non_null = raw[col].dropna()
if len(non_null) > 0:
meta["min"] = float(non_null.min())
meta["max"] = float(non_null.max())
meta["sample"] = float(non_null.iloc[0])
column_meta[col] = meta
return column_meta, categories, binary_mappings
def _build_aggregate_only_store(data_dir: Path) -> pd.DataFrame:
"""Build the parquet that contains ONLY the auxiliary aggregates.
application_train columns are deliberately excluded β they are always
reconstructed at inference time from the JSON input.
"""
bureau = bureau_and_balance(data_dir)
print(f" bureau aggregates: {bureau.shape}")
prev = previous_applications(data_dir)
print(f" previous_application aggregates: {prev.shape}")
pos = pos_cash(data_dir)
print(f" POS_CASH aggregates: {pos.shape}")
ins = installments_payments(data_dir)
print(f" installments aggregates: {ins.shape}")
cc = credit_card_balance(data_dir)
print(f" credit_card aggregates: {cc.shape}")
# Outer-join all aggregates on SK_ID_CURR. Any client absent from a given
# auxiliary file will simply have NaN there β preserves the training signal.
out = bureau.join(prev, how="outer").join(pos, how="outer")
out = out.join(ins, how="outer").join(cc, how="outer")
return out
def _extract_feature_names_from_full_pipeline(data_dir: Path) -> list[str]:
"""Run the full merge_files() once to recover the exact 770-column order
the model was trained on. We keep only the 768 feature columns
(excluding SK_ID_CURR and TARGET)."""
print("Building full training dataframe to capture feature_names...")
df = merge_files(data_dir)
feature_cols = [c for c in df.columns if c not in ("SK_ID_CURR", "TARGET")]
return feature_cols
def main() -> None:
OUT_PARQUET.parent.mkdir(parents=True, exist_ok=True)
OUT_FEATURE_NAMES.parent.mkdir(parents=True, exist_ok=True)
print("=" * 70)
print("STEP 1 β extracting raw application_train metadata")
column_meta, categories, binary_mappings = _build_app_train_metadata(DATA_DIR)
print(f" {len(column_meta)} columns analysed, {len(categories)} multi-cat")
print(f" binary mappings: {binary_mappings}")
print("\nSTEP 2 β building aggregate-only feature store")
agg_store = _build_aggregate_only_store(DATA_DIR)
print(f" Combined aggregates: {agg_store.shape}")
print("\nSTEP 3 β extracting feature_names from full training pipeline")
feature_names = _extract_feature_names_from_full_pipeline(DATA_DIR)
print(f" {len(feature_names)} feature columns identified")
print("\nSTEP 4 β persisting artefacts")
agg_store.to_parquet(OUT_PARQUET, compression="snappy")
size_mb = OUT_PARQUET.stat().st_size / 1e6
print(f" β
{OUT_PARQUET} ({size_mb:.1f} MB)")
OUT_FEATURE_NAMES.write_text(json.dumps(feature_names, indent=2))
print(f" β
{OUT_FEATURE_NAMES}")
OUT_APP_TRAIN_COLS.write_text(json.dumps(column_meta, indent=2, default=str))
print(f" β
{OUT_APP_TRAIN_COLS}")
OUT_APP_TRAIN_CATEGORIES.write_text(json.dumps(categories, indent=2))
print(f" β
{OUT_APP_TRAIN_CATEGORIES}")
OUT_BINARY_MAPPINGS.write_text(json.dumps(binary_mappings, indent=2))
print(f" β
{OUT_BINARY_MAPPINGS}")
print("=" * 70)
if __name__ == "__main__":
main()
|