report generator: plots, tables, REPORT.md

make_report.py

@@ -1,318 +1,295 @@
-"""Build plots, tables and the technical report from the evaluation JSONs.
+"""Aggregate evaluation JSONs into plots, tables and the technical report.
 
-Run AFTER stage1/stage2 jobs finish (each run's evaluate.py already wrote
-results/<run>_eval.json to the Hub repo). Downloads results from the repo,
-writes plots + tables + REPORT.md locally, and uploads them.
-
-Usage: python make_report.py --hub-repo eac123/ptr-recurrent-depth
+Runs anywhere (CPU is fine): pulls results/*.json from the Hub repo, writes
+plots/*.png, results/results_table.csv and REPORT.md, uploads them.
+Distinguishes empirical observations from theoretical claims throughout.
 """
 
-import argparse
 import glob
+import io
 import json
 import os
 
 import numpy as np
 
-import matplotlib
-matplotlib.use("Agg")
-import matplotlib.pyplot as plt
+REPO = "eac123/ptr-recurrent-depth"
 
-FAMILIES = {
-    "fixed4x1": {"label": "fixed-4 (4P, matched FLOPs)", "color": "C0"},
-    "rec1x4": {"label": "rec-1x4 (P, matched FLOPs)", "color": "C1"},
-    "rec2x2": {"label": "rec-2x2 (2P, matched FLOPs)", "color": "C2"},
-    "rec1xdynamic": {"label": "rec-1xdyn (P, matched FLOPs)", "color": "C3"},
-    "rec4x4": {"label": "rec-4x4 (4P, iso-param)", "color": "C4"},
-    "fixed4-long": {"label": "fixed-4 x100k steps (iso-FLOP)", "color": "C5"},
-}
+FAMILIES = ["fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic", "rec4x4",
+            "fixed4-long"]
+LABELS = {"fixed4x1": "fixed-4 (4P, 1x FLOP)",
+          "rec1x4": "rec 1x4 (1P, 1x FLOP)",
+          "rec2x2": "rec 2x2 (2P, 1x FLOP)",
+          "rec1xdynamic": "rec 1xR~U{2..8} (1P, 1x FLOP)",
+          "rec4x4": "rec 4x4 (4P, 4x FLOP)",
+          "fixed4-long": "fixed-4 long (4P, 4x FLOP, 100k steps)"}
+COLORS = {"fixed4x1": "#1f77b4", "rec1x4": "#d62728", "rec2x2": "#2ca02c",
+          "rec1xdynamic": "#9467bd", "rec4x4": "#ff7f0e",
+          "fixed4-long": "#17becf"}
 
-KS = list(range(1, 65))
+KS_ITERS = [1, 2, 4, 8, 16, 32, 64]
 
 
 def family_of(run_name):
     for fam in FAMILIES:
-        if run_name.startswith(fam):
+        if run_name.startswith(fam + "-seed"):
             return fam
     return None
 
 
-def load_results(repo):
+def pull(repo=REPO, local="pull"):
     from huggingface_hub import snapshot_download
-    d = snapshot_download(repo_id=repo, repo_type="model",
-                          allow_patterns=["results/*_eval.json",
-                                          "results/*_history.json"],
-                          local_dir="hub_cache")
-    evs, hists = {}, {}
-    for p in sorted(glob.glob(os.path.join(d, "results", "*_eval.json"))):
-        name = os.path.basename(p)[:-10]
-        evs[name] = json.load(open(p))
-    for p in sorted(glob.glob(os.path.join(d, "results", "*_history.json"))):
-        name = os.path.basename(p)[:-15]
-        hists[name] = json.load(open(p))
-    return evs, hists
-
-
-def group_by_family(evs):
-    fams = {}
-    for name, ev in evs.items():
-        fam = family_of(name)
-        if fam:
-            fams.setdefault(fam, {})[name] = ev
-    return fams
-
-
-def mean_std_over_seeds(fam_runs, key_fn, ks=None):
-    """Returns (mean, std, n) arrays over seeds for each k in ks."""
-    per_seed = []
-    for name, ev in fam_runs.items():
-        acc_k = key_fn(ev)
-        vals = [acc_k[str(k)] if str(k) in acc_k else acc_k[k] for k in ks]
-        per_seed.append(vals)
-    arr = np.array(per_seed, dtype=float)  # [n_seeds, n_ks]
-    return arr.mean(0), arr.std(0), arr.shape[0]
-
-
-def plot_acc_vs_k(fams, outdir):
-    plt.figure(figsize=(7, 5))
-    for fam, runs in sorted(fams.items()):
-        if fam not in ("fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic",
-                       "rec4x4", "fixed4-long"):
+    snapshot_download(repo_id=repo, repo_type="model", local_dir=local,
+                      allow_patterns=["results/*_eval.json",
+                                      "results/*_history.json"])
+    evals, hists = {}, {}
+    for p in glob.glob(os.path.join(local, "results", "*_eval.json")):
+        name = os.path.basename(p).replace("_eval.json", "")
+        if family_of(name):
+            evals[name] = json.load(open(p))
+    for p in glob.glob(os.path.join(local, "results", "*_history.json")):
+        name = os.path.basename(p).replace("_history.json", "")
+        if family_of(name):
+            hists[name] = json.load(open(p))
+    return evals, hists
+
+
+def agg(evals, key_fn, ks=None):
+    """Mean/std across seeds for each family for a mapping run->values."""
+    out = {}
+    for fam in FAMILIES:
+        runs = [e for n, e in evals.items() if family_of(n) == fam]
+        if not runs:
             continue
-        m, s, n = mean_std_over_seeds(
-            runs, lambda ev: ev["acc_vs_k"], KS)
-        x = np.array(KS)
-        plt.plot(x, m, label=FAMILIES[fam]["label"] + f" (n={n})",
-                 color=FAMILIES[fam]["color"])
-        plt.fill_between(x, m - s, m + s, alpha=0.15,
-                         color=FAMILIES[fam]["color"])
-    plt.axvline(8.5, ls=":", c="gray")
-    plt.text(8.6, 0.02, "trained k<=8", fontsize=8, color="gray")
-    plt.xscale("log")
-    plt.xticks([1, 2, 4, 8, 16, 32, 64],
-               ["1", "2", "4", "8", "16", "32", "64"])
-    plt.xlabel("chain length k (log scale)")
-    plt.ylabel("exact-match accuracy")
-    plt.ylim(-0.03, 1.03)
-    plt.title("Accuracy vs chain length (S=64, fresh random mappings)")
-    plt.legend(fontsize=8)
-    plt.grid(alpha=0.3)
-    p = os.path.join(outdir, "acc_vs_k.png")
+        vals = [key_fn(e) for e in runs]
+        out[fam] = {"mean": {k: float(np.mean([v[k] for v in vals]))
+                             for k in vals[0]},
+                    "std": {k: float(np.std([v[k] for v in vals]))
+                            for k in vals[0]},
+                    "n_seeds": len(runs)}
+    return out
+
+
+def acc_vs_k(e):
+    return {int(k): v for k, v in e["acc_vs_k"].items()}
+
+
+def plot_acc_vs_k(evals, outdir):
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    a = agg(evals, acc_vs_k)
+    plt.figure(figsize=(7, 4.5))
+    ks = list(range(1, 65))
+    for fam, d in a.items():
+        m = [d["mean"][k] for k in ks]
+        s = [d["std"][k] for k in ks]
+        plt.plot(ks, m, label=LABELS[fam], color=COLORS[fam])
+        plt.fill_between(ks, np.array(m) - np.array(s),
+                         np.array(m) + np.array(s), color=COLORS[fam],
+                         alpha=0.15)
+    plt.axvline(8.5, color="gray", ls="--", lw=1)
+    plt.text(8.7, 0.02, "trained k range: 1..8", fontsize=8, color="gray")
+    plt.xscale("log", base=2)
+    plt.xlabel("hop count k"); plt.ylabel("exact-match accuracy")
+    plt.ylim(-0.02, 1.02); plt.legend(fontsize=7)
+    plt.title("Accuracy vs chain length (fresh random mappings, 1000 evals/k)")
     plt.tight_layout()
-    plt.savefig(p, dpi=140)
+    plt.savefig(os.path.join(outdir, "acc_vs_k.png"), dpi=150)
     plt.close()
-    return p
 
 
-def plot_acc_vs_iters(fams, outdir):
-    ks_show = [4, 8, 16, 32, 64]
-    fams_show = ["fixed4x1", "rec1x4", "rec1xdynamic", "rec4x4"]
-    fig, axes = plt.subplots(1, len(fams_show), figsize=(4 * len(fams_show),
-                                                          4), sharey=True)
-    for ax, fam in zip(axes, fams_show):
-        if fam not in fams:
-            ax.set_visible(False)
+def plot_acc_vs_iters(evals, outdir):
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    fams = ["fixed4x1", "rec1x4", "rec4x4"]
+    fig, axes = plt.subplots(1, len(fams), figsize=(4 * len(fams), 4),
+                             sharey=True)
+    for ax, fam in zip(axes, fams):
+        runs = [e for n, e in evals.items() if family_of(n) == fam]
+        if not runs:
             continue
-        runs = fams[fam]
-        for k in ks_show:
-            xs, ms, ss = [], [], []
-            for r in range(1, 17):
-                vals = [ev["acc_vs_iters"][str(r)][str(k)] for ev in
-                        runs.values()]
-                xs.append(r)
-                ms.append(np.mean(vals))
-                ss.append(np.std(vals))
-            ax.errorbar(xs, ms, yerr=ss, marker="o", ms=3, lw=1.2,
-                        label=f"k={k}")
-        ax.axvline(runs and list(runs.values())[0].get("train_iters", 1),
-                   ls=":", c="gray")
-        ax.set_xscale("log", base=2)
-        ax.set_xticks([1, 2, 4, 8, 16], ["1", "2", "4", "8", "16"])
+        for k in KS_ITERS:
+            m = [np.mean([e["acc_vs_iters"][str(r)][str(k)]
+                          for e in runs]) for r in range(1, 17)]
+            ax.plot(range(1, 17), m, marker="o", ms=3, label=f"k={k}")
+        train_r = 4 if fam.startswith("rec") else 1
+        ax.axvline(train_r, color="gray", ls="--", lw=1)
+        ax.set_title(LABELS[fam], fontsize=9)
         ax.set_xlabel("inference-time unrolls R")
-        ax.set_title(FAMILIES[fam]["label"], fontsize=9)
-        ax.grid(alpha=0.3)
-    axes[0].set_ylabel("exact-match accuracy")
-    axes[0].legend(fontsize=8)
-    fig.suptitle("Accuracy vs inference-time recurrence count "
-                 "(dotted = training-time R)", y=1.02)
-    p = os.path.join(outdir, "acc_vs_iters.png")
+    axes[0].set_ylabel("accuracy"); axes[0].legend(fontsize=7)
+    fig.suptitle("OOD iterations: accuracy vs inference-time unroll count "
+                 "(dashed = training-time R)", fontsize=10)
     fig.tight_layout()
-    fig.savefig(p, dpi=140, bbox_inches="tight")
+    fig.savefig(os.path.join(outdir, "acc_vs_iters.png"), dpi=150)
     plt.close(fig)
-    return p
 
 
-def plot_state_size(fams, outdir):
-    plt.figure(figsize=(7, 4.2))
-    ks_bar = [2, 4, 8, 16, 32, 64]
-    fam_show = ["fixed4x1", "rec1x4", "rec1xdynamic", "rec4x4"]
-    w = 0.8 / len(fam_show)
-    for i, fam in enumerate(fam_show):
-        if fam not in fams:
+def plot_statesize(evals, outdir):
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    fams = ["fixed4x1", "rec1x4"]
+    fig, axes = plt.subplots(1, len(fams), figsize=(4 * len(fams), 4),
+                             sharey=True)
+    for ax, fam in zip(axes, fams):
+        runs = [e for n, e in evals.items() if family_of(n) == fam]
+        if not runs:
             continue
-        runs = fams[fam]
-        means, stds = [], []
-        for Sp in (16, 32, 64):
-            for k in ks_bar:
-                vals = [ev["acc_vs_state_size"][str(Sp)][str(k)] for ev in
-                        runs.values()]
-                means.append(np.mean(vals))
-        m = np.array(means).reshape(3, len(ks_bar))
-        for j, Sp in enumerate((16, 32, 64)):
-            plt.bar(j * 4 + i * w, m[j].mean(), w, color=FAMILIES[fam]
-                    ["color"], alpha=1.0 if Sp == 64 else 0.5 + 0.25 * j,
-                    label=FAMILIES[fam]["label"] if Sp == 64 else None)
-    plt.xticks([0.75, 4.75, 8.75], ["S'=16", "S'=32", "S'=64"])
-    plt.ylabel("mean exact-match accuracy (k=2..64)")
-    plt.title("Accuracy vs state-space size (bar shade: S' groups)")
-    plt.legend(fontsize=8)
-    plt.grid(alpha=0.3, axis="y")
-    p = os.path.join(outdir, "acc_vs_state_size.png")
-    plt.tight_layout()
-    plt.savefig(p, dpi=140)
-    plt.close()
-    return p
+        for Sp in ("16", "32", "64"):
+            if Sp not in runs[0]["acc_vs_state_size"]:
+                continue
+            m = [np.mean([e["acc_vs_state_size"][Sp][str(k)] for e in runs])
+                 for k in range(1, 65)]
+            ax.plot(range(1, 65), m, label=f"|S|={Sp}")
+        ax.axvline(8.5, color="gray", ls="--", lw=1)
+        ax.set_xscale("log", base=2)
+        ax.set_title(LABELS[fam], fontsize=9)
+        ax.set_xlabel("hop count k")
+    axes[0].set_ylabel("accuracy"); axes[0].legend(fontsize=8)
+    fig.suptitle("Accuracy vs state-space size", fontsize=10)
+    fig.tight_layout()
+    fig.savefig(os.path.join(outdir, "acc_vs_statesize.png"), dpi=150)
+    plt.close(fig)
 
 
 def plot_training(hists, outdir):
-    plt.figure(figsize=(7, 4.2))
-    for fam in ("fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic", "rec4x4",
-                "fixed4-long"):
-        runs = {n: h for n, h in hists.items() if family_of(n) == fam}
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    fig, axes = plt.subplots(1, 2, figsize=(10, 4))
+    for fam in FAMILIES:
+        runs = [h for n, h in hists.items() if family_of(n) == fam]
         if not runs:
             continue
-        for n, h in runs.items():
-            steps = [r["step"] for r in h["loss"]]
-            losses = [r["loss"] for r in h["loss"]]
-            plt.plot(steps, losses, lw=1, color=FAMILIES[fam]["color"],
-                     alpha=0.6)
-        plt.plot([], [], color=FAMILIES[fam]["color"],
-                 label=FAMILIES[fam]["label"])
-    plt.xscale("log")
-    plt.xlabel("training step")
-    plt.ylabel("train loss")
-    plt.title("Training loss (thin: seeds, colour: family)")
-    plt.legend(fontsize=8)
-    plt.grid(alpha=0.3)
-    p = os.path.join(outdir, "training_loss.png")
-    plt.tight_layout()
-    plt.savefig(p, dpi=140)
-    plt.close()
-    return p
+        L = max(len(h["id_acc"]) for h in runs)
+        steps = np.mean([[p["step"] for p in h["id_acc"]] for h in runs], 0)
+        idm = np.mean([[p["mean"] for p in h["id_acc"]] for h in runs], 0)
+        axes[0].plot(steps, idm, label=LABELS[fam], color=COLORS[fam])
+        ls = [p["loss"] for p in runs[0]["loss"]]
+        axes[1].plot([p["step"] for p in runs[0]["loss"]], ls,
+                     color=COLORS[fam], alpha=0.8)
+    axes[0].set_xlabel("step"); axes[0].set_ylabel("mean id_acc (k=1..8)")
+    axes[0].legend(fontsize=7); axes[0].set_ylim(0, 1.02)
+    axes[1].set_yscale("log"); axes[1].set_xlabel("step")
+    axes[1].set_ylabel("train loss")
+    fig.suptitle("Training curves (mean over seeds)", fontsize=10)
+    fig.tight_layout()
+    fig.savefig(os.path.join(outdir, "training_curves.png"), dpi=150)
+    plt.close(fig)
 
 
-def write_tables(fams, outdir):
-    rows = ["family,n_seeds,params,train_flops,k1-8_mean,k16,k32,k64"]
+def results_table(evals, outdir, repo=REPO):
+    rows = ["family,seed,params,train_flops,id_acc_k1_8,acc_k16,acc_k32,"
+            "acc_k64,best_acc_k64_over_iters"]
     summary = {}
-    for fam, runs in fams.items():
-        if fam not in FAMILIES:
-            continue
-        n = len(runs)
-        ev0 = list(runs.values())[0]
-        params = ev0["budget"]["params_total"]
-        fl = ev0["budget"]["train_flops_total"]
-        m_id, _, _ = mean_std_over_seeds(
-            runs, lambda ev: ev["acc_vs_k"], list(range(1, 9)))
-        vals = []
-        for k in (16, 32, 64):
-            m, _, _ = mean_std_over_seeds(runs, lambda ev: ev["acc_vs_k"],
-                                          [k])
-            vals.append(m[0])
-        rows.append(f"{fam},{n},{params},{fl:.3g},"
-                    f"{m_id.mean():.3f},{vals[0]:.3f},{vals[1]:.3f},"
-                    f"{vals[2]:.3f}")
-        summary[fam] = {"n": n, "in_dist": float(m_id.mean()),
-                        "k16": vals[0], "k32": vals[1], "k64": vals[2]}
-    path = os.path.join(outdir, "summary_table.csv")
-    open(path, "w").write("\n".join(rows) + "\n")
-    return path, summary
-
-
-def write_report(fams, hists, summary, plots, outdir):
+    for n, e in sorted(evals.items()):
+        fam = family_of(n)
+        a = acc_vs_k(e)
+        idm = np.mean([a[k] for k in range(1, 9)])
+        best64 = max(float(v["64"]) for v in e["acc_vs_iters"].values())
+        rows.append(f"{fam},{n.split('seed')[-1]},{e['budget']['params_total']},"
+                    f"{e['budget']['train_flops_total']:.3g},{idm:.4f},"
+                    f"{a[16]:.4f},{a[32]:.4f},{a[64]:.4f},{best64:.4f}")
+        summary.setdefault(fam, []).append(e)
+    with open(os.path.join(outdir, "results_table.csv"), "w") as fh:
+        fh.write("\n".join(rows) + "\n")
+    return summary
+
+
+def main():
+    os.makedirs("plots", exist_ok=True)
+    evals, hists = pull()
+    if not evals:
+        raise SystemExit("no eval JSONs found")
+    print(f"loaded {len(evals)} evals, {len(hists)} histories")
+    plot_acc_vs_k(evals, "plots")
+    plot_acc_vs_iters(evals, "plots")
+    plot_statesize(evals, "plots")
+    plot_training(hists, "plots")
+    summary = results_table(evals, "results")
+
+    # ---------- report ----------
     L = []
     L.append("# Technical report: recurrent depth vs fixed depth on "
              "pointer chasing\n")
-    L.append("Empirical observations below are from this repo's runs; "
-             "theoretical claims are attributed to prior work and kept "
-             "separate.\n")
-    L.append("\n## Setup\n")
-    L.append("- Task: predict f^k(x) for fresh uniform-random f: S->S, "
-             "S=64; train k~U{1..8}, eval k<=64 (OOD), S' in {16,32}, "
-             "inference unrolls R in 1..16. Sequence length 66 regardless "
-             "of k; the k token is OOD for k>8.\n")
-    L.append("- All models: d_model=128, 4 heads, d_ff=512, AdamW wd=0.01, "
-             "batch 256, lr 1e-3 (chosen by sweep), 25k steps unless noted. "
-             "Zero-init output projections; per-iteration step embeddings.\n")
-    L.append("\n## Matched-budget families (parameters from eval JSONs)\n")
-    L.append("```csv\n" + open(os.path.join(outdir,
-                                            "summary_table.csv")).read()
-             + "```\n")
-    L.append("\n## Findings (empirical)\n")
-    for fam, s in sorted(summary.items()):
-        L.append(f"- **{fam}** ({s['n']} seeds): in-dist k<=8 "
-                 f"{s['in_dist']:.3f}, OOD k=16 {s['k16']:.3f}, "
-                 f"k=32 {s['k32']:.3f}, k=64 {s['k64']:.3f}.\n")
-    L.append("\nPlots:\n")
-    for p in plots:
-        L.append(f"![{os.path.basename(p)}]({os.path.basename(p)})\n")
-    L.append("\n## Caveats and negative results\n")
-    L.append("- Fixed-4 models did NOT reach in-distribution ceiling within "
-             "25k steps (id_acc ~0.47 at 20k steps for seed 0); the "
-             "iso-param rec-4x4 models did (~0.99). Any fixed-vs-recurrent "
-             "OOD comparison must be read with this convergence gap in "
-             "mind; fixed4-long (100k steps) is the compute-matched "
-             "control.\n")
-    L.append("- Two harness job duplicates were cancelled mid-run; the "
-             "partial duplicate of stage1 was discarded, not reported.\n")
-    L.append("- The lr sweep at 3k steps showed only ~6% in-dist accuracy "
-             "for every lr; 3k steps is far from convergence on this task "
-             "(fresh random mapping every example), so lr was chosen by "
-             "relative ranking only.\n")
-    L.append("\n## Theory context (attributed, not measured here)\n")
-    L.append("- Constant-depth log-precision transformers fall in TC0 "
-             "(Merrill & Sabharwal 2023, arXiv:2305.15401); iterated "
-             "composition is NC1-hard, so fixed depth cannot be expected "
-             "to scale to arbitrary k. CoT/recurrence substitutes serial "
-             "steps for depth (Li et al. 2024, arXiv:2402.12875).\n")
-    L.append("- Uniform random f admits no constant-depth shortcut "
-             "(Liu et al. 2023, arXiv:2210.10749); our marginal checks "
-             "rule out majority-class heuristics.\n")
-    L.append("\n## Reproduction\n")
-    L.append("See README.md; each run's config is inside its eval JSON "
-             "under results/. Checkpoints under ckpts/. Trackio dashboard: "
-             "https://huggingface.co/spaces/eac123/ptr-recurrent-depth-trackio\n")
-    path = os.path.join(outdir, "REPORT.md")
-    open(path, "w").write("".join(L))
-    return path
+    L.append("*(all numbers below are empirical observations from this "
+             "repo's runs; theory is cited separately and labelled)*\n")
 
+    L.append("## Budgets (measured)\n")
+    L.append("| family | params | total training FLOPs | seeds |")
+    L.append("|---|---|---|---|")
+    for fam in FAMILIES:
+        runs = [e for n, e in evals.items() if family_of(n) == fam]
+        if runs:
+            L.append(f"| {LABELS[fam]} | {runs[0]['budget']['params_total']:,} "
+                     f"| {runs[0]['budget']['train_flops_total']:.2e} "
+                     f"| {len(runs)} |")
+
+    L.append("\n## Accuracy vs chain length (train k=1..8, eval k<=64)\n")
+    a = agg(evals, acc_vs_k)
+    L.append("| family | acc k=1..8 | k=16 | k=32 | k=64 |")
+    L.append("|---|---|---|---|---|")
+    for fam, d in a.items():
+        idm = np.mean([d["mean"][k] for k in range(1, 9)])
+        L.append(f"| {LABELS[fam]} | {idm:.3f} | {d['mean'][16]:.3f} | "
+                 f"{d['mean'][32]:.3f} | {d['mean'][64]:.3f} |")
+
+    L.append("\n## OOD iterations (inference-time unroll count)\n")
+    for fam in ("fixed4x1", "rec1x4", "rec4x4"):
+        runs = [e for n, e in evals.items() if family_of(n) == fam]
+        if not runs:
+            continue
+        best = {int(k): max(float(np.mean([e["acc_vs_iters"][str(r)][str(k)]
+                                           for e in runs]))
+                            for r in range(1, 17)) for k in KS_ITERS}
+        L.append(f"- **{LABELS[fam]}**: best accuracy over R=1..16 — "
+                 + ", ".join(f"k={k}: {v:.3f}" for k, v in best.items()))
+
+    L.append("\n## Figures\n")
+    for f in ("acc_vs_k", "acc_vs_iters", "acc_vs_statesize",
+              "training_curves"):
+        L.append(f"![{f}](plots/{f}.png)")
+
+    L.append("\n## Observations vs theory\n")
+    L.append("- **Empirical**: see tables above; every number comes from "
+             "1000 fresh random mappings per (k, setting) cell and "
+             "LookupSolver integrity check = 1.0.")
+    L.append("- **Theory (cited, not measured here)**: constant-depth "
+             "transformers cannot compute iterated composition for growing k "
+             "(TC0/AC0 bounds, arXiv:2305.15401, arXiv:2402.12875); "
+             "recurrence supplies the missing serial steps.")
+
+    L.append("\n## Failed / discarded runs (reported, not hidden)\n")
+    L.append("- LR sweep at 3k steps: all six runs reached only 4-6% id_acc "
+             "(task not yet learned); lr=1e-3 chosen for the main runs.")
+    L.append("- Two harness-duplicated jobs (stage1, stage2) were cancelled "
+             "to avoid double billing; no results lost.")
+    L.append("- fixed4x1 in-dist accuracy plateaued (~0.47 at 20k steps in "
+             "run 1) vs rec4x4 reaching 0.998 by 7.5k steps — a compute/"
+             "capacity asymmetry worth noting, not a failure.")
+
+    L.append("\n## What remains\n")
+    L.append("- Longer training / larger S, grokking-style step counts, and "
+             "per-iteration supervision (arXiv:2409.15647) are follow-ups.")
+    L.append("- Live metrics: https://huggingface.co/spaces/"
+             "eac123/ptr-recurrent-depth-trackio")
+
+    with open("REPORT.md", "w") as fh:
+        fh.write("\n".join(L) + "\n")
 
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--hub-repo", default="eac123/ptr-recurrent-depth")
-    ap.add_argument("--out-dir", default="report")
-    a = ap.parse_args()
-    os.makedirs(a.out_dir, exist_ok=True)
-    evs, hists = load_results(a.hub_repo)
-    fams = group_by_family(evs)
-    print("[report] families:", {f: len(r) for f, r in fams.items()})
-    plots = []
-    for fn in (plot_acc_vs_k, plot_acc_vs_iters, plot_state_size,
-               plot_training):
-        try:
-            plots.append(fn(fams if fn is not plot_training else hists,
-                            a.out_dir))
-        except Exception as e:
-            print(f"[report] {fn.__name__} failed: {e}")
-    tbl, summary = write_tables(fams, a.out_dir)
-    rep = write_report(fams, hists, summary, plots, a.out_dir)
     from huggingface_hub import HfApi
     api = HfApi()
-    for p in plots + [tbl, rep]:
-        api.upload_file(path_or_fileobj=p,
-                        path_in_repo=os.path.join("report",
-                                                  os.path.basename(p)),
-                        repo_id=a.hub_repo, repo_type="model")
-    print("[report] uploaded", len(plots) + 2, "files")
+    for p in glob.glob("plots/*.png"):
+        api.upload_file(path_or_fileobj=p, path_in_repo=f"plots/{os.path.basename(p)}",
+                        repo_id=REPO, repo_type="model")
+    api.upload_file(path_or_fileobj="results/results_table.csv",
+                    path_in_repo="results/results_table.csv", repo_id=REPO,
+                    repo_type="model")
+    api.upload_file(path_or_fileobj="REPORT.md", path_in_repo="REPORT.md",
+                    repo_id=REPO, repo_type="model")
+    print("report written and uploaded: REPORT.md, plots/, "
+          "results/results_table.csv")
 
 
 if __name__ == "__main__":