| """Confirmation tables from the per-draw score files (reports/nnlab/scores/*.json). |
| fresh = all pixels, draws 5-11 (never used for choosing) |
| B = half B, draws 0-11 |
| all = all pixels, draws 0-11 |
| """ |
| import json, sys |
| from pathlib import Path |
| import numpy as np, pandas as pd |
| S = Path(__file__).resolve().parent.parent.parent / "reports" / "nnlab" / "scores" |
| ROWS = [("ens_e12", "rule_smooth", "FINAL-2: zero-shot anchor + LightGBM with domain copies, smoothing"), |
| ("ens_e12", "rule", "FINAL-2 without coordinates"), |
| ("lgbm_feda", "rule_smooth", "single model: LightGBM with domain copies + smoothing"), |
| ("lgbm_feda", "rule", "single model: LightGBM with domain copies"), |
| ("tab_dann_rz_lpft", "rule", "single model: DANN, last layer then whole network fine-tuned"), |
| ("ens_e4__lp", "rule_smooth", "FINAL-AVG as pre-registered: ensemble + label propagation + smoothing"), |
| ("ens_e4", "rule_smooth", "ensemble + smoothing (no label propagation)"), |
| ("ens_e4", "rule", "ensemble, no coordinates used"), |
| ("ens_e2__lapshot", "rule_smooth", "FINAL-PEAK as pre-registered: zero-shot + LightGBM, graph refinement + smoothing"), |
| ("lgbm_upw__lp", "rule_smooth", "retrained LightGBM + label propagation + smoothing"), |
| ("lgbm_upw", "rule_smooth", "reference: retrained LightGBM + smoothing"), |
| ("lgbm_upw", "rule", "reference: retrained LightGBM (published 0.656)"), |
| ("zs_xgb_dann", "rule_smooth", "reference: teammate zero-shot + smoothing + our rule"), |
| ("met_protoda_maha", "rule", "member: metric learning (protoda, Mahalanobis)"), |
| ("met_cos_maha", "rule", "member: metric learning (cosine, Mahalanobis)"), |
| ("tab_dann_rz_ft", "rule", "member: DANN MLP, last layer fine-tuned")] |
| N = [5, 25, 50, 100, 200] |
| out = [] |
| for m, k, desc in ROWS: |
| f = S / f"{m}.json" |
| if not f.exists(): |
| continue |
| df = pd.DataFrame(json.loads(f.read_text())["draws"]) |
| for view, sub, col in (("fresh", df[df.rep >= 5], k), ("B", df, k + "_B"), ("all", df, k), ("first5", df[df.rep < 5], k)): |
| g = sub.groupby("n")[col] |
| mean, sd, cnt = g.mean(), g.std(), g.count() |
| r = {"method": m, "decision": k, "view": view, "desc": desc, "draws_per_n": int(cnt.min())} |
| r.update({f"n{n}": mean.get(n, np.nan) for n in N}) |
| r.update({f"sd{n}": sd.get(n, np.nan) for n in N}) |
| r["avg"] = float(np.mean([r[f"n{n}"] for n in N])); r["peak"] = float(np.max([r[f"n{n}"] for n in N])) |
| out.append(r) |
| df = pd.DataFrame(out) |
| df.to_csv(S.parent / "confirmation.csv", index=False) |
| pd.set_option("display.width", 250) |
| for view in ("fresh", "B", "all", "first5"): |
| print(f"\n== {view} ==") |
| print(df[df["view"] == view][["method", "decision", "draws_per_n"] + [f"n{n}" for n in N] + ["avg"]].round(4).to_string(index=False)) |
| print("\n== sd over draws, fresh ==") |
| print(df[df["view"] == "fresh"][["method", "decision"] + [f"sd{n}" for n in N]].round(4).to_string(index=False)) |
|
|
