""" Score notebook 3's two headline variants on ALL of Amsterdam. Notebook 3 scored every variant on a fixed random subsample of Amsterdam to keep the table affordable. This re-runs the two variants that win a prize category with the same labelled-sample draws, and scores every fitted model three ways: full all 25,992 pixels minus the drawn shots <- the number to report sub2000 the fixed 2,000-pixel subsample <- reproduces the table sub8000 the fixed 8,000-pixel subsample <- what nb 3 now uses The subsample columns are a fidelity check: if they reproduce the published table, the full column was produced by the same models on the same draws. Everything mirrors `evaluation.evaluate_protocol` as notebook 3 calls it: seed 42, shots drawn with rng(42 + 1000 n + r), shots removed from every evaluation set, and the decision rule (tau 0.5 then the macro-F1 plug-in, with the Madrid prior) applied to each evaluation set on its own, because the plug-in rule depends on the batch it is given. `y_amsterdam` is read for exactly two things, as in the protocol: drawing the permitted n-per-class sample and scoring. python -I tools/full_score.py prototype results/full_score_prototype.json python -I tools/full_score.py upweighted results/full_score_upweighted.json """ from __future__ import annotations import json import os import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / "src")) os.chdir(ROOT) import numpy as np import pandas as pd import config as C import evaluation as E import gbdt as G SEED = 42 # evaluate_protocol's default; notebook 3 keeps it N_LIST = (5, 25, 50, 100, 200) CHEAP_LGB = dict(n_estimators=400, learning_rate=0.06) # notebook 3, verbatim VARIANTS = { "prototype": ("prototype, robust z + within-class Mahalanobis", lambda: G.mc_prototype(metric="within"), 12), "upweighted": ("upweighted shots, retrained", lambda: G.mc_shot_weighted( normalise="robustz", fit=lambda X, y, **kw: G.fit_lgbm(X, y, params=CHEAP_LGB, **kw)), 5), } def load(): madrid = pd.read_parquet(C.PROC_DIR / "madrid_features.parquet") amsterdam = pd.read_parquet(C.PROC_DIR / "amsterdam_features.parquet") drop = {"px_key", "py_key", "age_class", "weighted_mean_year", "city"} feats = [c for c in madrid.columns if c not in drop and madrid[c].dtype.kind in "fiu"] dead = [c for c in feats if madrid[c].std() == 0 or amsterdam[c].std() == 0] feats = [c for c in feats if c not in dead] # notebook 3's "everything" return (madrid[feats].to_numpy(float), madrid["age_class"].to_numpy(), amsterdam[feats].to_numpy(float), amsterdam["age_class"].to_numpy(), feats, dead) def main(): key, out_path = sys.argv[1], Path(sys.argv[2]) name, make, n_repeats = VARIANTS[key] X_mad, y_mad, X_ams, y_ams, feats, dead = load() print(f"{name}: {len(feats)} features, dead {dead}, " f"Madrid {X_mad.shape}, Amsterdam {X_ams.shape}", flush=True) assert len(feats) == 393, len(feats) src_prior = np.array([(y_mad == c).mean() for c in E.CLASSES]) def decide(p): return E.adapt_pipeline(p, src_prior, tau=0.5)[0] # tau0.5_then_plugin pools = {"full": np.arange(len(y_ams))} for k in (2000, 8000): pools[f"sub{k}"] = np.sort(np.random.default_rng(SEED).choice( len(y_ams), k, replace=False)) mc = make() draws = [] # Resume: every draw is fixed by (n, rep), so draws saved by an interrupted # run are exactly what a fresh run would recompute. if out_path.exists(): prev = json.loads(out_path.read_text()) if prev.get("key") == key and prev.get("seed") == SEED: draws = prev["draws"] print(f"resuming: {len(draws)} draws already saved", flush=True) done = {(d["n"], d["rep"]) for d in draws} t_all = time.time() for n in N_LIST: for r in range(n_repeats): if (n, r) in done: continue t0 = time.time() rng = np.random.default_rng(SEED + 1000 * n + r) shot_idx = E.draw_shots(y_ams, n, rng) fn = mc(X_mad, y_mad, X_ams, X_ams[shot_idx], y_ams[shot_idx]) P_all = np.asarray(fn(X_ams), dtype=float) rec = dict(n=n, rep=r, n_shots=int(len(shot_idx))) for pname, pool in pools.items(): keep = np.setdiff1d(pool, shot_idx) if pname == "sub2000" and not draws: # Predicting all rows then indexing must equal predicting # the subset directly, as evaluate_protocol does. direct = np.asarray(fn(X_ams[keep]), dtype=float) assert np.allclose(direct, P_all[keep], atol=1e-10), \ "predictions are not row-wise" pred = decide(P_all[keep]) rec[pname] = float(E.macro_f1(y_ams[keep], pred)) rec[f"{pname}_n_scored"] = int(len(keep)) rec[f"{pname}_per_class"] = [ float(x) for x in E.per_class_f1(y_ams[keep], pred)] rec["seconds"] = round(time.time() - t0, 1) draws.append(rec) print(f"n={n:<4d} rep={r:<2d} full={rec['full']:.4f} " f"sub2000={rec['sub2000']:.4f} sub8000={rec['sub8000']:.4f} " f"({rec['seconds']}s)", flush=True) out_path.write_text(json.dumps( dict(variant=name, key=key, seed=SEED, n_repeats=n_repeats, complete=False, draws=draws), indent=1)) df = pd.DataFrame(draws) summary = {} for pname in pools: g = df.groupby("n")[pname] summary[pname] = {int(n): dict(mean=float(g.mean()[n]), std=float(g.std(ddof=1)[n])) for n in N_LIST} means = [summary[pname][n]["mean"] for n in N_LIST] summary[pname]["average"] = float(np.mean(means)) summary[pname]["peak"] = float(np.max(means)) out_path.write_text(json.dumps( dict(variant=name, key=key, seed=SEED, n_repeats=n_repeats, complete=True, minutes=round((time.time() - t_all) / 60, 1), summary=summary, draws=draws), indent=1)) print("\n" + pd.DataFrame({p: {n: summary[p][n]["mean"] for n in N_LIST} for p in pools}).round(4).to_string()) for p in pools: print(f"{p:8s} average {summary[p]['average']:.4f} " f"peak {summary[p]['peak']:.4f}") print(f"done in {(time.time() - t_all) / 60:.1f} min", flush=True) if __name__ == "__main__": main()