repro_bundle/code/summarize.py

"""Pull the headline numbers from the job artifacts into one compact summary,
so the logbook can cite exact values."""
import glob
import json
import os
import sys

OUT = sys.argv[1] if len(sys.argv) > 1 else "outputs"


def g(d, *ks, default=None):
    for k in ks:
        d = d.get(k, {}) if isinstance(d, dict) else {}
    return d if d != {} else default


ev = json.load(open(f"{OUT}/eval.json")) if os.path.exists(f"{OUT}/eval.json") else {}
print("=" * 60)
if "newton_alignment" in ev:
    na = ev["newton_alignment"]
    print("NEWTON ALIGNMENT (cosine to Newton dir, quadratics N=%s)" % na.get("N"))
    for k in ("lsr1_proj", "lsr1_noproj", "lbfgs"):
        v = na[k]
        print(f"  {k:12s}: iter1={v[0]:+.3f}  mid={v[len(v)//2]:+.3f}  "
              f"final={v[-1]:+.3f}  mean={sum(v)/len(v):+.3f}")

if "performance" in ev:
    pf = ev["performance"]
    print("\nPERFORMANCE PROFILES  (AUC over log-tau, higher=better)")
    for s, a in sorted(pf["aucs"].items(), key=lambda x: -x[1]):
        rho1 = pf["profiles"][s][0]
        print(f"  {s:12s}: AUC={a:.3f}  rho(tau=1)={rho1:.3f}")
    print("\n  per-type median iters-to-target (L-SR1 vs best baseline):")
    for t in pf["per_type"]:
        med = t["median"]; solved = t["solved"]
        base = {k: v for k, v in med.items() if k != "L-SR1" and v is not None}
        bb = min(base, key=base.get) if base else None
        print(f"   {t['family']:11s} N={str(t['N']):>4s} cond={str(t['cond']):>7s}"
              f" | L-SR1 med={med['L-SR1']} solved={solved['L-SR1']}"
              f" | best base={bb}={med.get(bb) if bb else None}")

# wall-clock from histories
print("\nWALL-CLOCK (meta-training)")
for h in sorted(glob.glob(f"{OUT}/*_hist.json")):
    d = json.load(open(h))
    name = os.path.basename(h).replace("_hist.json", "")
    it = d["history"][-1]["iter"] if d["history"] else 0
    print(f"  {name:20s}: {it} iters in {d.get('wall_time_s', 0)/60:.1f} min")

if os.path.exists(f"{OUT}/efficiency.json"):
    e = json.load(open(f"{OUT}/efficiency.json"))
    print("\nEFFICIENCY (synthetic proxy -- NOT a Claim-6 reproduction)")
    print(f"  L-SR1 update params={e['lsr1_update_params']:,} "
          f"lgd-style params={e['lgd_update_params']:,}")
    print(f"  lsr1={e['lsr1_ms']:.1f}ms lgd={e['lgd_ms']:.1f}ms "
          f"mem lsr1={e['lsr1_mem_gib']} lgd={e['lgd_mem_gib']}")
print("=" * 60)