| """ |
| Score notebook 3's two headline variants on ALL of Amsterdam. |
| |
| Notebook 3 scored every variant on a fixed random subsample of Amsterdam to keep |
| the table affordable. This re-runs the two variants that win a prize category |
| with the same labelled-sample draws, and scores every fitted model three ways: |
| |
| full all 25,992 pixels minus the drawn shots <- the number to report |
| sub2000 the fixed 2,000-pixel subsample <- reproduces the table |
| sub8000 the fixed 8,000-pixel subsample <- what nb 3 now uses |
| |
| The subsample columns are a fidelity check: if they reproduce the published |
| table, the full column was produced by the same models on the same draws. |
| |
| Everything mirrors `evaluation.evaluate_protocol` as notebook 3 calls it: |
| seed 42, shots drawn with rng(42 + 1000 n + r), shots removed from every |
| evaluation set, and the decision rule (tau 0.5 then the macro-F1 plug-in, with |
| the Madrid prior) applied to each evaluation set on its own, because the |
| plug-in rule depends on the batch it is given. |
| |
| `y_amsterdam` is read for exactly two things, as in the protocol: drawing the |
| permitted n-per-class sample and scoring. |
| |
| python -I tools/full_score.py prototype results/full_score_prototype.json |
| python -I tools/full_score.py upweighted results/full_score_upweighted.json |
| """ |
| from __future__ import annotations |
|
|
| import json |
| import os |
| import sys |
| import time |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| sys.path.insert(0, str(ROOT / "src")) |
| os.chdir(ROOT) |
|
|
| import numpy as np |
| import pandas as pd |
|
|
| import config as C |
| import evaluation as E |
| import gbdt as G |
|
|
| SEED = 42 |
| N_LIST = (5, 25, 50, 100, 200) |
| CHEAP_LGB = dict(n_estimators=400, learning_rate=0.06) |
|
|
| VARIANTS = { |
| "prototype": ("prototype, robust z + within-class Mahalanobis", |
| lambda: G.mc_prototype(metric="within"), 12), |
| "upweighted": ("upweighted shots, retrained", |
| lambda: G.mc_shot_weighted( |
| normalise="robustz", |
| fit=lambda X, y, **kw: G.fit_lgbm(X, y, params=CHEAP_LGB, **kw)), |
| 5), |
| } |
|
|
|
|
| def load(): |
| madrid = pd.read_parquet(C.PROC_DIR / "madrid_features.parquet") |
| amsterdam = pd.read_parquet(C.PROC_DIR / "amsterdam_features.parquet") |
| drop = {"px_key", "py_key", "age_class", "weighted_mean_year", "city"} |
| feats = [c for c in madrid.columns |
| if c not in drop and madrid[c].dtype.kind in "fiu"] |
| dead = [c for c in feats if madrid[c].std() == 0 or amsterdam[c].std() == 0] |
| feats = [c for c in feats if c not in dead] |
| return (madrid[feats].to_numpy(float), madrid["age_class"].to_numpy(), |
| amsterdam[feats].to_numpy(float), amsterdam["age_class"].to_numpy(), |
| feats, dead) |
|
|
|
|
| def main(): |
| key, out_path = sys.argv[1], Path(sys.argv[2]) |
| name, make, n_repeats = VARIANTS[key] |
| X_mad, y_mad, X_ams, y_ams, feats, dead = load() |
| print(f"{name}: {len(feats)} features, dead {dead}, " |
| f"Madrid {X_mad.shape}, Amsterdam {X_ams.shape}", flush=True) |
| assert len(feats) == 393, len(feats) |
|
|
| src_prior = np.array([(y_mad == c).mean() for c in E.CLASSES]) |
|
|
| def decide(p): |
| return E.adapt_pipeline(p, src_prior, tau=0.5)[0] |
|
|
| pools = {"full": np.arange(len(y_ams))} |
| for k in (2000, 8000): |
| pools[f"sub{k}"] = np.sort(np.random.default_rng(SEED).choice( |
| len(y_ams), k, replace=False)) |
|
|
| mc = make() |
| draws = [] |
| |
| |
| if out_path.exists(): |
| prev = json.loads(out_path.read_text()) |
| if prev.get("key") == key and prev.get("seed") == SEED: |
| draws = prev["draws"] |
| print(f"resuming: {len(draws)} draws already saved", flush=True) |
| done = {(d["n"], d["rep"]) for d in draws} |
| t_all = time.time() |
| for n in N_LIST: |
| for r in range(n_repeats): |
| if (n, r) in done: |
| continue |
| t0 = time.time() |
| rng = np.random.default_rng(SEED + 1000 * n + r) |
| shot_idx = E.draw_shots(y_ams, n, rng) |
| fn = mc(X_mad, y_mad, X_ams, X_ams[shot_idx], y_ams[shot_idx]) |
| P_all = np.asarray(fn(X_ams), dtype=float) |
|
|
| rec = dict(n=n, rep=r, n_shots=int(len(shot_idx))) |
| for pname, pool in pools.items(): |
| keep = np.setdiff1d(pool, shot_idx) |
| if pname == "sub2000" and not draws: |
| |
| |
| direct = np.asarray(fn(X_ams[keep]), dtype=float) |
| assert np.allclose(direct, P_all[keep], atol=1e-10), \ |
| "predictions are not row-wise" |
| pred = decide(P_all[keep]) |
| rec[pname] = float(E.macro_f1(y_ams[keep], pred)) |
| rec[f"{pname}_n_scored"] = int(len(keep)) |
| rec[f"{pname}_per_class"] = [ |
| float(x) for x in E.per_class_f1(y_ams[keep], pred)] |
| rec["seconds"] = round(time.time() - t0, 1) |
| draws.append(rec) |
| print(f"n={n:<4d} rep={r:<2d} full={rec['full']:.4f} " |
| f"sub2000={rec['sub2000']:.4f} sub8000={rec['sub8000']:.4f} " |
| f"({rec['seconds']}s)", flush=True) |
| out_path.write_text(json.dumps( |
| dict(variant=name, key=key, seed=SEED, n_repeats=n_repeats, |
| complete=False, draws=draws), indent=1)) |
|
|
| df = pd.DataFrame(draws) |
| summary = {} |
| for pname in pools: |
| g = df.groupby("n")[pname] |
| summary[pname] = {int(n): dict(mean=float(g.mean()[n]), |
| std=float(g.std(ddof=1)[n])) |
| for n in N_LIST} |
| means = [summary[pname][n]["mean"] for n in N_LIST] |
| summary[pname]["average"] = float(np.mean(means)) |
| summary[pname]["peak"] = float(np.max(means)) |
| out_path.write_text(json.dumps( |
| dict(variant=name, key=key, seed=SEED, n_repeats=n_repeats, |
| complete=True, minutes=round((time.time() - t_all) / 60, 1), |
| summary=summary, draws=draws), indent=1)) |
| print("\n" + pd.DataFrame({p: {n: summary[p][n]["mean"] for n in N_LIST} |
| for p in pools}).round(4).to_string()) |
| for p in pools: |
| print(f"{p:8s} average {summary[p]['average']:.4f} " |
| f"peak {summary[p]['peak']:.4f}") |
| print(f"done in {(time.time() - t_all) / 60:.1f} min", flush=True) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
