| """ |
| Build the 60 official features exactly as the organisers do: the FIRST QA-valid |
| observation per pixel-year (features.make_flat_view(agg="first")), rather than |
| the per-year median our feature table uses. |
| |
| Needed to run a teammate's pipeline, which was built on the organisers' |
| preprocessed_data.pkl, on the same inputs it was built for. The organisers' |
| arrays are additionally z-scored with Madrid's scaler; that is one affine map |
| per feature shared by both cities, which tree models and per-region |
| standardisation are invariant to, so it is not reproduced here. |
| |
| python -I tools/build_official60_first.py |
| writes data/processed/{madrid,amsterdam}_official60_first.parquet (PID + 60 + age_class) |
| """ |
| from __future__ import annotations |
|
|
| import os |
| import sys |
| import time |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| sys.path.insert(0, str(ROOT / "src")) |
| os.chdir(ROOT) |
|
|
| import pandas as pd |
|
|
| import config as C |
| import features as F |
|
|
| for city, path in (("madrid", ROOT / "data/raw/madrid_train.parquet"), |
| ("amsterdam", ROOT / "data/raw/amsterdam_data.parquet")): |
| t0 = time.time() |
| df = pd.read_parquet(path) |
| df = F.assign_age_class(df) |
| flat = F.make_flat_view(df, agg="first") |
| del df |
| indicators = F.period_coverage(flat) |
| labels = (flat.sort_values(C.PID + ["year"]) |
| .groupby(C.PID, as_index=False)["age_class"].first()) |
| pix, years, mats, observed, sat = F.to_year_matrices(flat) |
| del flat |
| off = F.build_official_60(pix, years, mats, indicators) |
| out = off.merge(labels, on=C.PID, how="left") |
| out = out[out["age_class"].notna()].reset_index(drop=True) |
| out["age_class"] = out["age_class"].astype(int) |
| assert all(c in out.columns for c in C.FEATURE_COLS_60) |
| dst = C.PROC_DIR / f"{city}_official60_first.parquet" |
| out[C.PID + list(C.FEATURE_COLS_60) + ["age_class"]].to_parquet(dst, index=False) |
| print(f"{city}: {len(out):,} labelled pixels -> {dst} ({time.time() - t0:.0f}s)", |
| flush=True) |
|
|
