| """ |
| NOTE FOR THIS BUNDLE. Only one mode of this script is used here: |
| |
| python tools/full_score_combo.py zeroshot |
| |
| It rebuilds data/processed/zeroshot_xgb_dann_ams.npz, the zero-shot anchor of the |
| FINAL-2 pipeline (tools/nnlab/model_create2.py): a teammate's XGBoost and DANN on |
| the official 60 features, from tools/compare_teammate.py. It needs xgboost and |
| torch and takes six to thirteen minutes on a 12-thread CPU. The combination described |
| below is NOT FINAL-2, and its numbers are not quoted anywhere in this bundle. |
| |
| Our few-shot winners combined with the teammate's DANN and neighbour smoothing, |
| scored on ALL of Amsterdam under the official protocol. |
| |
| Intended as the primary model when this script was written (2026-11-10). This |
| bundle holds no time-stamped record of that, and Amsterdam scores of its |
| ingredients (the upweighted LightGBM, the teammate's zero-shot pipeline) had |
| already been seen. The primary model is |
| "upweighted + DANN + smooth": |
| p = (P_upweighted_lgbm + P_dann) / 2 -> neighbour smoothing (6 px) -> our rule |
| Every ingredient was chosen on Madrid, not Amsterdam: |
| * upweighted retrained LightGBM - notebook 3's design, 393 features; |
| * DANN (60 official features, adversary on UNLABELLED Amsterdam) and the |
| 50/50 probability average with a tree model - teammate's design, +0.014 on |
| their Madrid lockbox; |
| * neighbour smoothing r = 6 px, alpha 0.5 - teammate's choice on Madrid |
| (+0.025 on their CV); ours (r = 2) also gained on Madrid; |
| * decision rule tau 0.5 then the macro-F1 plug-in with the Madrid prior - |
| chosen on Madrid by the shift harness. |
| The other rows are ablations that show what each ingredient contributes. Some |
| were added after Amsterdam results had been seen. |
| |
| One deliberate change from the teammate: the 3 DANN seeds are averaged into one |
| probability (their lockbox reported seeds separately), so a protocol draw does not |
| depend on which seed was used. |
| |
| Protocol: identical to tools/full_score.py. Seed 42, shots drawn with |
| rng(42 + 1000 n + rep), shots removed from the scored set, and every draw scored on |
| all remaining Amsterdam pixels. Smoothing uses only the scored pixels as |
| neighbours (no shot, no label), as in the teammate's code. Amsterdam labels are |
| read only to draw the shots and to score. |
| |
| python -I tools/full_score_combo.py zeroshot # cache XGB + DANN |
| python -I tools/full_score_combo.py upweighted reports/combo_upweighted.json |
| python -I tools/full_score_combo.py prototype reports/combo_prototype.json |
| """ |
| from __future__ import annotations |
|
|
| import json |
| import sys |
| import time |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parent)) |
| import compare_teammate as CT |
| import full_score as FS |
|
|
| import numpy as np |
| import pandas as pd |
|
|
| import config as C |
| import evaluation as E |
|
|
| SEED, N_LIST, R_PX = 42, (5, 25, 50, 100, 200), 6 |
| CACHE = C.PROC_DIR / "zeroshot_xgb_dann_ams.npz" |
|
|
|
|
| def build_cache(d): |
| """Zero-shot parts: they never see a shot, so they are fitted once.""" |
| t0 = time.time() |
| sw = CT.compute_sample_weight("balanced", d["ym"]) |
| xgb = CT.XGBClassifier(objective="multi:softprob", tree_method="hist", device="cpu", |
| eval_metric="mlogloss", random_state=42, **CT.XGB_PARAMS) |
| xgb.fit(d["Xm60"], d["ym"] - 1, sample_weight=sw) |
| P_xgb = xgb.predict_proba(d["Xa60"]) |
| Xm_n, Xa_n = CT.region_norm(d["Xm60"]), CT.region_norm(d["Xa60"]) |
| P_d = [] |
| for s in CT.DANN_SEEDS: |
| net = CT.train_dann(Xm_n, d["ym"], Xa_n, CT.DANN_EPOCHS, adversarial=True, seed=s) |
| P_d.append(CT.dann_proba(net, Xa_n)) |
| print(f"DANN seed {s} done {time.time() - t0:.0f}s", flush=True) |
| np.savez(CACHE, P_xgb=P_xgb, P_dann_seeds=np.stack(P_d), pa=d["pa"]) |
| print(f"cached -> {CACHE}", flush=True) |
|
|
|
|
| def main(): |
| mode = sys.argv[1] |
| d = CT.load("first") |
| if mode == "zeroshot": |
| build_cache(d) |
| return |
| out_path = Path(sys.argv[2]) |
| z = np.load(CACHE) |
| assert (z["pa"] == d["pa"]).all() |
| P_dann = z["P_dann_seeds"].mean(0) |
| P_final = (z["P_xgb"] + P_dann) / 2 |
|
|
| name, make, n_repeats = FS.VARIANTS[mode] |
| X_mad, y_mad, X_ams, y_ams = d["Xm"], d["ym"], d["Xa"], d["ya"] |
| src_prior = np.array([(y_mad == c).mean() for c in E.CLASSES]) |
| coords = d["pa"].astype(float) |
| base = "upweighted" if mode == "upweighted" else "prototype" |
|
|
| def rules(P, y): |
| out = {} |
| out["argmax"] = E.macro_f1(y, np.array(E.CLASSES)[P.argmax(1)]) |
| out["tau+plugin"] = E.macro_f1(y, E.adapt_pipeline(P, src_prior, tau=0.5)[0]) |
| return out |
|
|
| draws = [] |
| if out_path.exists(): |
| prev = json.loads(out_path.read_text()) |
| if prev.get("key") == mode: |
| draws = prev["draws"] |
| print(f"resuming: {len(draws)} draws already saved", flush=True) |
| done = {(r["n"], r["rep"]) for r in draws} |
| mc = make() |
| t_all = time.time() |
| for n in N_LIST: |
| for r in range(n_repeats): |
| if (n, r) in done: |
| continue |
| t0 = time.time() |
| rng = np.random.default_rng(SEED + 1000 * n + r) |
| shot_idx = E.draw_shots(y_ams, n, rng) |
| keep = np.setdiff1d(np.arange(len(y_ams)), shot_idx) |
| fn = mc(X_mad, y_mad, X_ams, X_ams[shot_idx], y_ams[shot_idx]) |
| Pb = np.asarray(fn(X_ams[keep]), dtype=float) |
| ck, yk = coords[keep], y_ams[keep] |
| combos = { |
| f"{base}": Pb, |
| f"{base} + smooth": CT.smooth(Pb, ck, R_PX), |
| f"{base} + DANN": (Pb + P_dann[keep]) / 2, |
| f"{base} + DANN + smooth": CT.smooth((Pb + P_dann[keep]) / 2, ck, R_PX), |
| } |
| if mode == "upweighted": |
| combos["their FINAL (zero-shot)"] = P_final[keep] |
| combos["their FINAL + smooth (zero-shot)"] = CT.smooth(P_final[keep], ck, R_PX) |
| rec = dict(n=n, rep=r, n_scored=int(len(keep))) |
| for k, P in combos.items(): |
| for rule, v in rules(P, yk).items(): |
| rec[f"{k} | {rule}"] = float(v) |
| rec["seconds"] = round(time.time() - t0, 1) |
| draws.append(rec) |
| print(f"n={n:<4d} rep={r:<2d} " + " ".join( |
| f"{k}={rec[k + ' | tau+plugin']:.4f}" for k in combos) + |
| f" ({rec['seconds']}s)", flush=True) |
| out_path.write_text(json.dumps(dict(key=mode, variant=name, seed=SEED, |
| complete=False, draws=draws), indent=1)) |
|
|
| df = pd.DataFrame(draws) |
| cols = [c for c in df.columns if " | " in c] |
| g = df.groupby("n")[cols] |
| summ = {c: {**{int(n): dict(mean=float(g.mean().loc[n, c]), std=float(g.std(ddof=1).loc[n, c])) |
| for n in N_LIST}, |
| "average": float(g.mean()[c].mean()), "peak": float(g.mean()[c].max())} |
| for c in cols} |
| out_path.write_text(json.dumps(dict(key=mode, variant=name, seed=SEED, complete=True, |
| minutes=round((time.time() - t_all) / 60, 1), |
| summary=summ, draws=draws), indent=1)) |
| tbl = pd.DataFrame({c: {**{n: summ[c][n]["mean"] for n in N_LIST}, |
| "avg": summ[c]["average"]} for c in cols}).T |
| print("\n" + tbl.round(4).to_string(), flush=True) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
