make_report.py

"""Aggregate eval JSONs into plots, tables and REPORT.md, and upload to Hub.

Run anywhere with network access; pulls results/*_eval.json and
results/*_history.json from the repo first so it can run in a job container.
Dependencies: numpy, matplotlib, huggingface_hub.
"""

import glob
import json
import os
import sys

import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.patches import FancyArrowPatch, Rectangle

REPO = "eac123/ptr-recurrent-depth"
PLOTS = "plots"

FAMILIES = ["fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic", "rec4x4",
            "fixed4-long", "rec1aligned"]
LABELS = {
    "fixed4x1": "fixed-4 (4 blocks x 1)",
    "rec1x4": "recurrent 1x4 (train R=4)",
    "rec2x2": "recurrent 2x2 (train R=2)",
    "rec1xdynamic": "recurrent 1x dynamic R~U{2..8}",
    "rec4x4": "recurrent 4x4 (train R=4)",
    "fixed4-long": "fixed-4 long (100k steps)",
    "rec1aligned": "rec-1x-aligned (train R=k)",
}
TRAIN_ITERS = {"fixed4x1": 1, "rec1x4": 4, "rec2x2": 2, "rec1xdynamic": 8,
               "rec4x4": 4, "fixed4-long": 1, "rec1aligned": None}


def pull():
    from huggingface_hub import snapshot_download
    if not glob.glob("results/*-seed*_eval.json"):
        snapshot_download(repo_id=REPO, repo_type="model",
                          allow_patterns=["results/*.json"], local_dir=".")
    files = sorted(glob.glob("results/*_eval.json"))
    return {family: [json.load(open(f)) for f in files
                     if os.path.basename(f).startswith(family + "-seed")]
            for family in FAMILIES}


def mean_std(vals):
    v = np.array(vals, dtype=float)
    return v.mean(), (v.std(ddof=1) if len(v) > 1 else 0.0)


def id_mean(e):
    return float(np.mean([e["acc_vs_k"][str(k)] for k in range(1, 9)]))


def id_curve(evals_fam):
    """(ks, mean, std) over seeds for acc_vs_k."""
    ks = np.arange(1, 65)
    acc = np.array([[e["acc_vs_k"][str(k)] for k in ks] for e in evals_fam])
    return acc.mean(0), acc.std(0)


def budget_of(evals_fam):
    b = evals_fam[0]["budget"]
    return b["params_total"], b["train_flops_total"]


# --------------------------------------------------------------------------
# figures
# --------------------------------------------------------------------------

def plot_acc_vs_k(evals, out):
    plt.figure(figsize=(8, 5))
    ks = np.arange(1, 65)
    for fam in FAMILIES:
        if not evals[fam]:
            continue
        m, s = id_curve(evals[fam])
        p = plt.plot(ks, m, label=LABELS[fam])[0]
        plt.fill_between(ks, m - s, m + s, alpha=0.15, color=p.get_color())
    plt.axhline(1 / 64, color="gray", ls=":", lw=1, label="chance (1/64)")
    plt.axvline(8, color="gray", ls="--", lw=1, label="max training k")
    plt.xscale("log", base=2)
    plt.xlabel("hop count k (log2)"); plt.ylabel("exact-match accuracy")
    plt.title("OOD length generalisation (train k=1..8, fresh random f)")
    plt.legend(fontsize=7); plt.grid(alpha=0.3); plt.tight_layout()
    plt.savefig(out, dpi=150); plt.close()


def plot_acc_vs_iters(evals, out,
                      fams=("fixed4x1", "rec1x4", "rec1xdynamic", "rec4x4",
                            "rec1aligned")):
    ks_show = [1, 8, 16, 32, 64]
    fig, axes = plt.subplots(1, len(fams), figsize=(4 * len(fams), 4),
                             sharey=True)
    for ax, fam in zip(np.atleast_1d(axes), fams):
        if not evals[fam]:
            continue
        rs = np.arange(1, 17)
        for k in ks_show:
            acc = np.array([[e["acc_vs_iters"][str(r)][str(k)]
                             for r in rs] for e in evals[fam]])
            m, s = acc.mean(0), acc.std(0)
            p = ax.plot(rs, m, marker="o", ms=3, label=f"k={k}")[0]
            ax.fill_between(rs, m - s, m + s, alpha=0.15, color=p.get_color())
        ti = TRAIN_ITERS[fam]
        if ti:
            ax.axvline(ti, color="gray", ls="--", lw=1)
        ax.set_title(f"{LABELS[fam]}\n"
                     + ("(dash = train-time unrolls)" if ti
                        else "(trained R = k, no single dash)"),
                     fontsize=8)
        ax.set_xlabel("inference unrolls R"); ax.grid(alpha=0.3)
    axes[0].set_ylabel("exact-match accuracy")
    axes[0].legend(fontsize=7)
    fig.suptitle("Accuracy vs inference-time recurrence count", y=1.02)
    fig.tight_layout(); fig.savefig(out, dpi=150, bbox_inches="tight")
    plt.close(fig)


def plot_perk_zoom(evals, out,
                   fams=("fixed4x1", "rec1x4", "rec1xdynamic", "rec4x4",
                         "rec1aligned"), kmax=16):
    """Within-and-just-beyond training range: accuracy per hop count k."""
    fig, ax = plt.subplots(figsize=(8.5, 5))
    ks = np.arange(1, kmax + 1)
    for f in fams:
        if not evals[f]:
            continue
        acc = np.array([[e["acc_vs_k"][str(k)] for k in ks]
                        for e in evals[f]])
        m, s = acc.mean(0), acc.std(0)
        p = ax.plot(ks, m, marker="o", ms=4, label=LABELS[f])[0]
        ax.fill_between(ks, m - s, m + s, alpha=0.12, color=p.get_color())
    ax.axvline(8.5, color="gray", ls="--", lw=1)
    ax.text(8.7, 0.03, "max training k", fontsize=8, color="gray")
    ax.axhline(1 / 64, color="gray", ls=":", lw=1)
    ax.set_xticks(ks)
    ax.set_xlabel("hop count k"); ax.set_ylabel("exact-match accuracy")
    ax.set_title("Per-hop-count accuracy, k = 1..16 "
                 "(1000 fresh mappings per cell, mean over seeds)")
    ax.legend(fontsize=7); ax.grid(alpha=0.3)
    fig.tight_layout(); fig.savefig(out, dpi=150); plt.close(fig)


def perk_rows(evals, fams, klo, khi):
    ks = list(range(klo, khi + 1))
    lines = ["| family | " + " | ".join(f"k={k}" for k in ks) + " |",
             "|---|" + "---|" * len(ks)]
    for f in fams:
        if not evals[f]:
            continue
        acc = np.array([[e["acc_vs_k"][str(k)] for k in ks]
                        for e in evals[f]])
        m = acc.mean(0)
        lines.append(f"| {SHORT[f]} | "
                     + " | ".join(f"{v:.2f}" for v in m) + " |")
    return lines


def plot_state_size(evals, out, fams=("fixed4x1", "rec1x4")):
    fig, axes = plt.subplots(1, len(fams), figsize=(5 * len(fams), 4),
                             sharey=True)
    ks = np.arange(1, 65)
    for ax, fam in zip(np.atleast_1d(axes), fams):
        if not evals[fam]:
            continue
        for Sp in ("16", "32", "64"):
            if Sp not in evals[fam][0]["acc_vs_state_size"]:
                continue
            acc = np.array([[e["acc_vs_state_size"][Sp][str(k)] for k in ks]
                            for e in evals[fam]])
            m, s = acc.mean(0), acc.std(0)
            p = ax.plot(ks, m, label=f"S'={Sp}")[0]
            ax.fill_between(ks, m - s, m + s, alpha=0.15, color=p.get_color())
        ax.set_xscale("log", base=2); ax.set_title(LABELS[fam], fontsize=9)
        ax.set_xlabel("hop count k (log2)"); ax.grid(alpha=0.3)
    axes[0].set_ylabel("exact-match accuracy")
    axes[0].legend(fontsize=7)
    fig.suptitle("Accuracy vs state-space size (eval on subsets of trained vocab)",
                 y=1.02)
    fig.tight_layout(); fig.savefig(out, dpi=150, bbox_inches="tight")
    plt.close(fig)


def plot_training(evals, out):
    from huggingface_hub import hf_hub_download
    fig, axes = plt.subplots(1, 2, figsize=(11, 4))
    for fam in FAMILIES:
        for e in evals[fam]:
            name = e["run_name"]
            h = json.load(open(hf_hub_download(
                repo_id=REPO, repo_type="model",
                filename=f"results/{name}_history.json")))
            axes[0].plot([r["step"] for r in h["loss"]],
                         [r["loss"] for r in h["loss"]], alpha=0.25)
            axes[1].plot([r["step"] for r in h["id_acc"]],
                         [r["mean"] for r in h["id_acc"]],
                         label=f"{fam}", alpha=0.7)
    axes[0].set_yscale("log"); axes[0].set_xlabel("step")
    axes[0].set_ylabel("train loss"); axes[0].set_title("Loss (all runs)")
    axes[1].set_xlabel("step"); axes[1].set_ylabel("in-dist acc (k=1..8 mean)")
    axes[1].set_title("In-distribution accuracy during training")
    axes[1].legend(fontsize=6); axes[1].grid(alpha=0.3)
    fig.tight_layout(); fig.savefig(out, dpi=150); plt.close(fig)


SHORT = {"fixed4x1": "fixed-4", "rec1x4": "rec-1x4", "rec2x2": "rec-2x2",
         "rec1xdynamic": "rec-1x-dyn", "rec4x4": "rec-4x4",
         "fixed4-long": "fixed-4-long"}


def plot_headline_bars(evals, out):
    """Bar-chart companion to the headline table. Two panels:
    (a) the matched-FLOP family, (b) the iso-param / iso-FLOP trio."""
    panels = [("Matched training FLOPs (~2.2e15)",
               ["fixed4x1", "rec2x2", "rec1x4", "rec1xdynamic"]),
              ("Same parameters (830k); FLOPs differ as labelled",
               ["fixed4x1", "rec4x4", "fixed4-long"])]
    fig, axes = plt.subplots(1, 2, figsize=(12.5, 4.8))
    for ax, (title, fams) in zip(axes, panels):
        x = np.arange(len(fams))
        w = 0.38
        for off, key, lab, col in ((-w / 2, "id", "in-dist acc (k=1..8)",
                                    "tab:blue"),
                                   (w / 2, "k64", "acc at k=64 (OOD)",
                                    "tab:red")):
            means, stds = [], []
            for f in fams:
                if not evals[f]:
                    means.append(np.nan); stds.append(0); continue
                vals = ([id_mean(e) for e in evals[f]] if key == "id"
                        else [e["acc_vs_k"]["64"] for e in evals[f]])
                m, s = mean_std(vals)
                means.append(m); stds.append(s)
            ax.bar(x + off, means, w, yerr=stds, capsize=3, label=lab,
                   color=col, alpha=0.85)
        ptxt = []
        for f in fams:
            p, fl = budget_of(evals[f]) if evals[f] else (0, 0)
            ptxt.append(f"{SHORT[f]}\n"
                        f"{p/1000:.0f}k params\n{fl:.2e} FLOPs")
        ax.set_xticks(x); ax.set_xticklabels(ptxt, fontsize=7)
        ax.axhline(1 / 64, color="gray", ls=":", lw=1)
        ax.set_ylim(0, 1.05); ax.set_title(title, fontsize=9)
        ax.grid(alpha=0.3, axis="y")
    axes[0].legend(fontsize=8, loc="upper left")
    axes[0].set_ylabel("exact-match accuracy")
    fig.suptitle("Headline results (mean ± std over seeds; dotted = chance)",
                 y=1.0)
    fig.tight_layout(); fig.savefig(out, dpi=150, bbox_inches="tight")
    plt.close(fig)


def plot_budget_map(evals, out):
    """Where each family sits in the (params, total training FLOPs) plane,
    with the two matching axes drawn in."""
    fig, ax = plt.subplots(figsize=(7.5, 5.5))
    # hand-placed label offsets (points) so nothing overlaps
    OFF = {"fixed4x1": (8, -14, "left"), "rec1x4": (0, -16, "center"),
           "rec2x2": (10, -4, "left"), "rec1xdynamic": (8, 6, "left"),
           "rec4x4": (-8, 10, "right"), "fixed4-long": (8, -14, "left")}
    for f in FAMILIES:
        if not evals[f]:
            continue
        p, fl = budget_of(evals[f])
        ax.scatter(p, fl, s=90, zorder=3)
        dx, dy, ha = OFF[f]
        ax.annotate(LABELS[f], (p, fl), fontsize=7, ha=ha,
                    xytext=(dx, dy), textcoords="offset points")
    ax.axvline(829696, color="gray", ls="--", lw=1)
    ax.annotate("iso-parameter pair\n(fixed-4 vs rec-4x4)", (829696, 1.45e15),
                fontsize=7, color="gray", ha="right")
    ax.axhline(2.185e15, color="gray", ls=":", lw=1)
    ax.annotate("matched-FLOP family (same per-step and total FLOPs)",
                (2.6e5, 2.185e15), fontsize=7, color="gray",
                xytext=(0, 7), textcoords="offset points", ha="center")
    ax.axhline(8.72e15, color="gray", ls="-.", lw=1)
    ax.annotate("iso-FLOP pair (rec-4x4 vs fixed-4-long)",
                (5.0e5, 8.72e15), fontsize=7, color="gray",
                xytext=(0, -13), textcoords="offset points", ha="center")
    ax.set_xlim(1.8e5, 1.6e6)
    ax.set_ylim(1.2e15, 1.1e16)
    ax.set_xscale("log"); ax.set_yscale("log")
    ax.set_xlabel("parameters"); ax.set_ylabel("total training FLOPs")
    ax.set_title("Experimental design: what is matched against what")
    ax.grid(alpha=0.3, which="both"); fig.tight_layout()
    fig.savefig(out, dpi=150); plt.close(fig)


def plot_schematic(out):
    """Side-by-side schematic of the two architectures."""
    fig, axes = plt.subplots(1, 2, figsize=(10, 4.6))
    for ax in axes:
        ax.set_xlim(0, 4); ax.set_ylim(0, 6); ax.axis("off")

    def box(ax, x, y, w, h, text, fc):
        ax.add_patch(Rectangle((x, y), w, h, facecolor=fc, edgecolor="k",
                               alpha=0.85))
        ax.text(x + w / 2, y + h / 2, text, ha="center", va="center",
                fontsize=7.5)

    def arrow(ax, x, y0, y1):
        ax.add_patch(FancyArrowPatch((x, y0), (x, y1),
                                     arrowstyle="-|>", mutation_scale=12,
                                     color="k"))

    ax = axes[0]
    ax.set_title("fixed-depth transformer\n(4 distinct blocks, one pass)",
                 fontsize=9)
    ys = [4.6, 3.5, 2.4, 1.3]
    for i, y in enumerate(ys):
        box(ax, 1.2, y, 1.6, 0.8, f"Block {i+1}\n(own weights)",
            f"{0.25 + 0.15 * i}")
        if i:
            arrow(ax, 2.0, ys[i] + 0.8, ys[i - 1])
    box(ax, 1.2, 0.2, 1.6, 0.6, "readout head", "lightyellow")
    arrow(ax, 2.0, ys[-1], 0.8)
    ax.text(0.15, 5.6, "input:\n[f(0..63), x, k]", fontsize=7)

    ax = axes[1]
    ax.set_title("recurrent transformer\n(L shared blocks, R unrolls)",
                 fontsize=9)
    box(ax, 1.2, 3.2, 1.6, 0.8, "shared blocks\n(same weights)", "tab:orange")
    box(ax, 1.2, 1.8, 1.6, 0.6, "readout head", "lightyellow")
    arrow(ax, 2.0, 3.2, 2.4)
    ax.annotate("", xy=(2.8, 3.65), xytext=(2.8, 3.35),
                arrowprops=dict(arrowstyle="-|>", connectionstyle=
                                "arc3,rad=-1.8", color="tab:red"))
    ax.text(3.35, 3.5, "loop\nR times\n(+step emb\neach pass)", fontsize=7,
            color="tab:red", ha="center", va="center")
    ax.text(0.15, 5.6, "input:\n[f(0..63), x, k]", fontsize=7)
    ax.text(2.0, 0.9, "example: 1x4 = 1 shared block, R=4\n"
                      "         4x4 = 4 shared blocks, R=4\n"
                      "fixed-4  = 4 blocks, R=1",
            fontsize=7, ha="center")
    fig.tight_layout(); fig.savefig(out, dpi=150, bbox_inches="tight")
    plt.close(fig)


# --------------------------------------------------------------------------
# report
# --------------------------------------------------------------------------

def worked_examples():
    """Two concrete examples rendered from a fixed RNG so the text matches
    the data generator exactly."""
    rng = np.random.default_rng(1234)
    S = 8
    f = rng.integers(0, S, size=S)
    x = int(rng.integers(0, S))
    chain = [x]
    y = x
    for t in range(1, 6):
        y = int(f[y])
        chain.append(y)
    lines = [
        "**Example 1 (small state space S=8, so it fits on the page).**",
        "",
        "The mapping f is drawn uniformly at random; with this seed it is",
        "",
        "```text",
        "f(0)=2  f(1)=5  f(2)=2  f(3)=7  f(4)=0  f(5)=3  f(6)=6  f(7)=2",
        "```",
        "",
        f"Start state x={x}, hop count k=3. Applying f step by step:",
        "",
        "```text",
        f"f^1({x}) = f({x})  = {chain[1]}",
        f"f^2({x}) = f({chain[1]})  = {chain[2]}",
        f"f^3({x}) = f({chain[2]})  = {chain[3]}   <- the required answer",
        "```",
        "",
        f"So the answer is state **{chain[3]}**. Note how the chain entered a",
        f"cycle ({chain[2]} -> {chain[3]} -> {chain[4]} -> ...): random finite",
        "mappings always fall into cycles, which is exactly why a model that",
        "only looks up f(x) fails once k grows past the cycle structure.",
        "",
        "The model never sees the chain. It sees one flat token sequence",
        "(S+2 = 10 tokens here, 66 for S=64):",
        "",
        "```text",
        "input : [2, 5, 2, 7, 0, 3, 6, 2, | 3, 3]",
        "          f(0) f(1) ...  f(7)      x  k",
        "target: the state id of f^k(x), predicted at the final position",
        "```",
        "",
        "The position of each f-token encodes *which* state it is the image",
        "of (state i is always at position i), so no 'arrow' tokens are",
        "needed. The final two tokens are the start state x and the hop",
        "count k; the model must combine all three pieces of information.",
        "",
        "**Example 2 (training-scale S=64).** Same construction, but only",
        "the first few mapping entries are shown:",
        "",
        "```text",
        f"f = [{', '.join(str(int(v)) for v in f64[:12])}, ...]  (64 entries)",
        f"x = {x64}, k = {k64}  ->  answer f^{k64}({x64}) = {ans64}",
        "```",
        "",
        "Every training batch draws a *fresh* random f, a fresh x and a",
        "fresh k ~ Uniform{{1..8}} for every example, so there is nothing to",
        "memorise: the only way to score above chance is to learn the",
        "algorithm itself.",
    ]
    return "\n".join(lines)


def report(evals, rows):
    def ms(fam, key_fn):
        if not evals[fam]:
            return "n/a"
        vals = [key_fn(e) for e in evals[fam]]
        m, s = mean_std(vals)
        return f"{m:.3f}±{s:.3f}"

    k64 = lambda e: e["acc_vs_k"]["64"]
    o16 = lambda e: e["acc_vs_k"]["16"]
    best = lambda e: max(e["acc_vs_iters"].values(),
                         key=lambda v: v["64"])["64"]

    # state-size summary, computed from the eval JSONs
    ss_lines = []
    for fam in ("fixed4x1", "rec1x4"):
        if not evals[fam]:
            continue
        cells = []
        for Sp in ("16", "32", "64"):
            vals = [np.mean([e["acc_vs_state_size"][Sp][str(k)]
                             for k in range(1, 9)]) for e in evals[fam]]
            m, s = mean_std(vals)
            cells.append(f"S'={Sp}: {m:.3f}±{s:.3f}")
        ss_lines.append(f"- {LABELS[fam]}: " + ", ".join(cells))

    n_seeds = {f: len(evals[f]) for f in FAMILIES}

    # --- aligned-experiment results, computed from the eval JSONs ---
    al_lines = ["*(stage-3 runs not finished yet - this section fills in",
                "when rec1aligned eval JSONs appear in results/)*"]
    al = evals["rec1aligned"]
    if al:
        def frontier(m, thr=0.9):
            best = 0
            for k in range(1, 17):
                if m[k - 1] >= thr:
                    best = k
                else:
                    break
            return best

        acc = np.array([[e["acc_vs_k"][str(k)] for k in range(1, 17)]
                        for e in al])
        m = acc.mean(0)
        f_al = frontier(m)
        r1x4 = np.array([[e["acc_vs_k"][str(k)] for k in range(1, 17)]
                         for e in evals["rec1x4"]]).mean(0)
        f_r14 = frontier(r1x4)
        # staircase check on the fixed-R sweep: accuracy should be high
        # iff k <= R if the model really does one unroll per hop
        hi, lo = [], []
        for e in al:
            for r, cells in e["acc_vs_iters"].items():
                for kk, v in cells.items():
                    (hi if int(kk) <= int(r) else lo).append(v)
        stair = float(np.mean(hi) - np.mean(lo))
        pred_a = f_al >= 8
        pred_b = stair > 0.3
        al_lines = [
            f"- in-distribution accuracy (k=1..8 mean): "
            f"{ms('rec1aligned', id_mean)}",
            f"- capability frontier (last k with mean acc >= 0.9): "
            f"**k={f_al}** (rec-1x4: k={f_r14}; rec-4x4 solves the full "
            f"training range)",
            f"- beyond the training range: k=16 {m[15]:.3f}, k=32 "
            f"{np.mean([e['acc_vs_k']['32'] for e in al]):.3f}, k=64 "
            f"{np.mean([e['acc_vs_k']['64'] for e in al]):.3f}",
            f"- staircase check on the fixed-R sweep: mean accuracy where "
            f"k <= R minus where k > R = {stair:+.3f} (large positive = "
            f"the model really does one-unroll-per-hop)",
            f"- prediction (a) {('**supported**' if pred_a else '**not supported**')}",
            f"- prediction (b) {('**supported**' if pred_b else '**not supported**')}",
        ]
    lines = [
        "# Technical report: recurrent depth vs fixed depth on pointer chasing",
        "",
        "*(Everything below marked **observation** is measured from the runs in",
        "`results/`. Statements about what transformers *can* compute in",
        "principle are **theory** and cite prior work.)*",
        "",
        "## TL;DR",
        "",
        "We train six families of tiny transformers (3 seeds each, except",
        "fixed-4-long: 2) on a synthetic multi-hop task: given a random",
        "mapping f, a start state x and a hop count k, output f^k(x).",
        "Training only ever uses k ≤ 8; evaluation uses k up to 64 with",
        "freshly generated mappings. **Observation:** recurrent depth (a",
        "weight-tied block applied R times) learns the trained task far",
        "better than a fixed-depth transformer at matched parameters and",
        "even at matched *total* training compute, but **no model in this",
        "budget class generalises well to untrained hop counts** (best mean",
        "acc ~0.27 at k=64 vs 1.0 in-distribution).",
        "",
        "## 1. The task",
        "",
        "Each example is a fresh, uniformly random function f on a state",
        "space of size S=64, a uniformly random start state x, and a hop",
        "count k. The correct answer is f applied k times to x. During",
        "training k ~ Uniform{1..8}; at evaluation k ranges up to 64 (out of",
        "distribution) and every mapping is newly generated, so nothing can",
        "be memorised. Shortcut controls: f is uniform random (no exploitable",
        "regular structure, cf. arXiv:2210.10749); the marginal distribution",
        "of the answer is ~uniform for every k (verified: total-variation",
        "< 0.02, no majority-class shortcut); k is the only token that",
        "correlates with difficulty. A deterministic lookup solver scores",
        "1.0 by construction and is re-run at every evaluation as a pipeline",
        "integrity check.",
        "",
        worked_examples(),
        "",
        "A note on what 'length' means here: the input sequence is always",
        "S+2 = 66 tokens regardless of k, so the OOD axis is **the value of",
        "the hop-count token**, not the sequence length. Generalising from",
        "k ≤ 8 to k = 64 therefore requires the model to apply its learned",
        "procedure 8x more times than it ever saw during training.",
        "",
        "## 2. The models",
        "",
        "All models share one implementation (`ptr/models.py`): a",
        "pre-LN transformer block (4 heads, d_ff = 4*d_model, GELU MLP,",
        "bidirectional attention, learned positional embeddings) applied",
        "**n_iters** times. Two knobs distinguish the families:",
        "",
        "- `depth` = number of distinct blocks in the stack. In the fixed",
        "  model each block has its own weights and is applied once.",
        "- `n_iters` (R) = how many times the *same* stack is applied. Each",
        "  unroll adds a learned step embedding so the block can tell pass",
        "  t from pass t+1. At initialisation the blocks are the identity",
        "  (zero-initialised output projections), which keeps gradients",
        "  stable through long unroll chains (cf. arXiv:2604.07822).",
        "",
        "So 'fixed-4' = depth 4, R=1; 'rec-1x4' = one shared block unrolled",
        "R=4 times; 'rec-4x4' = a 4-block stack unrolled R=4 times. The",
        "input encoding, output head, optimizer and data pipeline are",
        "identical everywhere (d_model=128, batch 256, AdamW lr=1e-3 chosen",
        "by a 6-run sweep, cosine decay, 25k steps unless stated).",
        "",
        "![architecture schematic](plots/schematic.png)",
        "",
        "### The six families and what is matched",
        "",
        "| family | blocks x unrolls | params | per-step block-applications | total training FLOPs | seeds |",
        "|---|---|---|---|---|---|",
        "| fixed-4 | 4 x 1 | 829,696 | 4 | 2.19e15 | 3 |",
        "| rec-2x2 | 2 x 2 | 433,152 | 4 | 2.19e15 | 3 |",
        "| rec-1x4 | 1 x 4 | 234,880 | 4 | 2.19e15 | 3 |",
        "| rec-1x-dynamic | 1 x R~U{2..8} | 234,880 | ~5 avg | 2.73e15 | 3 |",
        "| rec-4x4 | 4 x 4 | 829,696 | 16 | 8.68e15 | 3 |",
        "| fixed-4-long | 4 x 1, 100k steps | 829,696 | 4 | 8.74e15 | 2 |",
        "",
        "Three comparison axes follow from this table:",
        "",
        "1. **Matched-FLOP family** (rows 1-3): every model executes exactly",
        "   4 block-applications per training step and is trained for the",
        "   same number of steps, so total training FLOPs are identical",
        "   while parameters differ 4:2:1. Question: *does the loop",
        "   structure itself matter when compute is fixed?*",
        "2. **Iso-parameter pair** (fixed-4 vs rec-4x4): identical parameter",
        "   counts, but rec-4x4 does 4x the per-step work. Question: *given",
        "   the same weights, does iterating them help?*",
        "3. **Iso-FLOP pair** (rec-4x4 vs fixed-4-long): identical total",
        "   training FLOPs - fixed-4-long simply trains 4x longer. Question:",
        "   *can extra training compute substitute for recurrent depth?*",
        "",
        "![budget map](plots/budget_map.png)",
        "",
        "## 3. What is evaluated",
        "",
        "Every evaluation cell uses freshly generated mappings (nothing",
        "from training is reused) and 1000 examples per cell; accuracy is",
        "exact match on the answer state (chance = 1/64 ≈ 0.016). The varied",
        "parameters are:",
        "",
        "- **Chain length k ∈ 1..64** at the model's training-time unroll",
        "  count: the OOD length-generalisation curve.",
        "- **Inference-time unroll count R ∈ 1..16**, crossed with",
        "  k ∈ {1,2,4,8,16,32,64}: 'more or fewer iterations than trained'.",
        "  For the fixed model, R > 1 re-applies its own block stack",
        "  (effectively deeper than trained) - also measured.",
        "- **State-space size S' ∈ {16, 32, 64}**: evaluation on subsets of",
        "  the trained vocabulary/positions (larger S' would need new",
        "  embedding rows the model never had; noted as a limitation).",
        "- **Training seed** (3 per family) and **training compute** (the",
        "  three budget axes above).",
        "",
        "## 4. Results",
        "",
        "### 4.1 Headline (at training-time unrolls)",
        "",
        "| family | acc k=1..8 | acc k=16 | acc k=32 | acc k=64 | best acc k=64 over R |",
        "|---|---|---|---|---|---|",
    ]
    for fam in FAMILIES:
        lines.append(
            f"| {LABELS[fam]} | {ms(fam, id_mean)} | {ms(fam, o16)} | "
            f"{ms(fam, lambda e: e['acc_vs_k']['32'])} | "
            f"{ms(fam, k64)} | {ms(fam, best)} |")
    lines += [
        "",
        "![headline bars](plots/headline_bars.png)",
        "",
        "Reading the bar chart: blue = how well the model learned the task",
        "it was trained on; red = how far that skill stretches to 8x longer",
        "chains. The gap between them is the length-generalisation deficit,",
        "and it is large for every family.",
        "",
        "### 4.2 Accuracy vs chain length",
        "",
        "![acc vs k](plots/acc_vs_k.png)",
        "",
        "All curves bend downward after the training range (k ≤ 8, dashed",
        "line). **Observation:** no family learned a clean",
        "length-generalising algorithm in this budget class.",
        "",
        "### 4.3 Accuracy vs inference-time unrolls (the k x R grid)",
        "",
        "Two different quantities are crossed in this figure, and keeping",
        "them apart is the whole point:",
        "",
        "- **k — task depth.** How many times f must be applied to answer",
        "  the question. It is part of the *input* (the k token) and is",
        "  fixed along each line of the figure.",
        "- **R — the recurrent loop parameter.** How many times the model",
        "  *iterates its own block stack* before reading out an answer. It",
        "  is a free choice made at inference time: the trained model is",
        "  frozen and we simply run it for more or fewer unrolls. The dashed",
        "  vertical line marks the R each family was trained with.",
        "",
        "**What is being tested.** If a recurrent model learned the",
        "abstraction 'apply one unroll per hop', accuracy should rise with R",
        "and peak near R = k. If instead it learned a fixed amount of",
        "computation (whatever R it was trained with), accuracy should peak",
        "at the trained R regardless of k and fall off on both sides.",
        "",
        "![acc vs iters](plots/acc_vs_iters.png)",
        "",
        "**Observations.**",
        "",
        "1. The R-accuracy curves are *sharp*, not flat. For rec-1x4 at k=4,",
        "   mean accuracy jumps from 0.06 (R=3) to 0.94 (R=4) and falls back",
        "   to 0.35 (R=8): the model tuned itself to its trained unroll",
        "   count, not to the hop count in the input.",
        "2. rec-4x4 peaks at R=4 for both k=4 (1.00) and k=8 (1.00): with 16",
        "   serial block-applications it solves the whole training range at",
        "   its trained R, and *loses* accuracy when given more unrolls",
        "   (k=8: 0.73 at R=8, 0.39 at R=16) - a mild 'overthinking' regime.",
        "3. The fixed model is the opposite: its best R is 1 (0.14/0.20 at",
        "   k=4/8) and re-applying its own stack at R>1 only hurts - extra",
        "   depth without training the depth is useless.",
        "4. Consequence: the length-generalisation deficit starts *inside*",
        "   the training range - see 4.3b.",
        "",
        "### 4.3b Within (and just beyond) the training range: accuracy per hop count",
        "",
        "The natural question - 'recurrent depth 4 should get hop count 4,",
        "right?' - is answered by per-k accuracy inside k=1..16 (1000 fresh",
        "mappings per cell, mean over seeds):",
        "",
        *perk_rows(evals, ["fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic",
                          "rec4x4", "fixed4-long", "rec1aligned"], 1, 8),
        "",
        *perk_rows(evals, ["fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic",
                          "rec4x4", "fixed4-long", "rec1aligned"], 9, 16),
        "",
        "![per-k zoom](plots/perk_zoom.png)",
        "",
        "**Observations.**",
        "",
        "1. k=1..3 are solved by every family (1.00): reading f(x) and very",
        "   short chains are easy.",
        "2. The capability frontier scales with serial block-applications:",
        "   fixed-4 (4 applications) solves k<=3; rec-1x-dynamic (up to 8)",
        "   solves k<=7 (0.96 at k=7); rec-4x4 (16) solves the entire",
        "   training range k<=8 at 1.000; rec-1x4 (4 unrolls of 1 block)",
        "   manages k=4 at 0.95 and decays to 0.37 by k=8. So recurrence",
        "   with enough unrolls does buy hop count - roughly one hop per",
        "   block-application, plus a constant.",
        "3. But the frontier stops at the training-data limit (k<=8): even",
        "   rec-4x4, with twice the serial depth k=8 needs, does not extend",
        "   beyond it (0.20-0.28 at k=9..16). Capacity alone does not give",
        "   length generalisation when the data never demanded it.",
        "4. rec-1x4's k=8 accuracy (0.37) is low not because 8 hops exceed",
        "   recurrence but because it was trained with R=4 and cannot turn",
        "   'k=8' into 'unroll 8 times' (at R=8 it scores only 0.14). This",
        "   is exactly the gap the aligned-unroll experiment (4.7) targets.",
        "",
        "### 4.4 Accuracy vs state-space size",
        "",
        "**S'** (S-prime) is the state-space size used *at evaluation time*.",
        "Training always used S=64: the state-embedding table has 64 rows",
        "and the positional table 66 positions. To ask whether the learned",
        "procedure depends on the specific vocabulary, we evaluate on",
        "smaller state spaces S' in {16, 32}: mappings are generated over",
        "states {0..S'-1} only, which reuses trained embedding rows (no new",
        "ones are introduced). S' > 64 is impossible without adding",
        "embedding rows the model never trained - a limitation, not a",
        "finding.",
        "",
        "![state size](plots/acc_vs_state_size.png)",
        "",
        "In-distribution accuracy on smaller state spaces (mean over",
        "k = 1..8):",
        "",
        *ss_lines,
        "",
        "**Observation:** accuracy degrades only mildly when the state",
        "space shrinks, i.e. what is learned is partly vocabulary-",
        "independent rather than a lookup over specific states.",
        "",
        "### 4.5 Training dynamics",
        "",
        "![training](plots/training_curves.png)",
        "",
        "**Observation:** rec-4x4 reaches ~1.0 in-distribution by ~7.5k",
        "steps; fixed-4 plateaus near ~0.47 and is still softening at 25k;",
        "fixed-4-long reaches 0.72 by 100k. The learning *rate* differs",
        "dramatically even when parameters are identical.",
        "",
        "### 4.6 Full table (per seed)",
        "",
        "```csv",
        *rows,
        "```",
        "",
        "### 4.7 Aligned unrolls: training with R = k (one unroll per hop)",
        "",
        "Motivated by 4.3b: the models implicitly tuned themselves to their",
        "trained unroll count, and rec-1x4 fails k=8 mainly because it",
        "cannot turn the k token into 'unroll 8 times'. The aligned variant",
        "removes that gap *by construction*: during training every batch",
        "uses a single k ~ U{1..8} and runs exactly R = k unrolls, so",
        "'one unroll per hop' is the only consistent solution. Everything",
        "else (architecture, data, optimizer, lr, 25k steps) matches",
        "rec-1x4; 3 seeds. At evaluation the model reads out after R = k",
        "unrolls for every k - including k > 8, which is the real test.",
        "",
        "*Predictions registered before the runs:* (a) if alignment is the",
        "missing ingredient, rec-1x-aligned should reach ~1.0 across the",
        "full k=1..8 range (rec-1x4 manages only 0.37 at k=8) and extend",
        "beyond k=8 better than the other 1-block families, since R=k is",
        "in-distribution for every k; (b) its fixed-R sweep should look",
        "like a staircase (accuracy high iff k <= R) rather than peaking",
        "at one R.",
        "",
        "*Results:*",
        "",
        *al_lines,
        "",
        "## 5. Conclusions",
        "",
        "1. **Recurrent depth buys learnability that neither parameters nor",
        "   extra training compute provide** *(observation)*: at identical",
        "   parameter counts rec-4x4 reaches 1.000 in-distribution vs 0.487",
        "   for fixed-4, and fixed-4-long still only 0.716 after 4x the",
        "   total training FLOPs.",
        "2. **The loop structure itself matters at fixed compute**",
        "   *(observation, with caveats)*: rec-1x4 learns the trained task",
        "   better than fixed-4 (0.761 vs 0.487) with 4x fewer parameters",
        "   and identical training FLOPs - but seed variance is large",
        "   (0.59-0.95), so this needs more seeds to be a firm claim.",
        "3. **Length generalisation to untrained hop counts is weak",
        "   everywhere** *(observation)*: best mean k=64 accuracy ~0.27",
        "   (rec-4x4), vs 1.0 in-distribution. This is consistent with",
        "   **theory**: constant-depth transformers sit in TC^0/AC^0 and",
        "   cannot compute iterated composition f^k for unbounded k without",
        "   growing depth or serial steps (Merrill & Sabharwal 2305.15401;",
        "   Li et al. 2402.12875) - but our budget class may simply be too",
        "   small to realise the achievable gains.",
        "4. **Inference-time unrolls are tuned to the trained R, not to the",
        "   task's hop count** *(observation)*: accuracy peaks sharply at",
        "   the training-time R (rec-1x4 at k=4: 0.06 at R=3 -> 0.94 at",
        "   R=4) and more unrolls usually hurt (rec-4x4 at k=8: 1.00 at",
        "   R=4 -> 0.39 at R=16). The aligned experiment (4.7) tests",
        "   whether training with R=k removes this coupling.",
        "5. **Single-seed comparisons of OOD generalisation are unreliable",
        "   at this scale** *(observation)*: rec-4x4's k=64 accuracy across",
        "   seeds is 0.18 / 0.18 / 0.39.",
        "",
        "### How could the OOD deficit be fixed? (discussion)",
        "",
        "Generalisation is the whole point, so it is worth being explicit",
        "about the levers - which this study tests and which it does not:",
        "",
        "1. **Aligned / per-iteration supervision** - make the training",
        "   computation match the task's serial structure: one unroll per",
        "   hop (tested here, section 4.7), or supervising every unroll",
        "   toward f^t(x) (Fan et al. 2409.15647, untested here).",
        "2. **Adaptive halting** (ACT, Universal Transformer) - let the",
        "   model decide how many unrolls to run per input, so R stops",
        "   being a fixed hyperparameter. Untested here.",
        "3. **Externalise the serial steps** - chain-of-thought/scratchpad:",
        "   theory says serial *tokens* substitute for serial depth (Li et",
        "   al. 2402.12875) and can grow with k at inference. Untested.",
        "4. **Curriculum and data coverage** - the frontier in 4.3b tracks",
        "   the training-data limit (k<=8) almost exactly; training on",
        "   longer chains or a growing-k curriculum directly moves it.",
        "   Untested here.",
        "5. **Scale** - the frontier also scales with serial depth (4.3b),",
        "   so more parameters *and* more unrolls should extend it. Only",
        "   partially tested (fixed-4-long).",
        "",
        "## 6. Negative results, failures and caveats",
        "",
        "- At 3k steps (lr sweep) every model was near chance (~5-6%): the",
        "  algorithm-learning phase is far slower than tiny-set overfitting",
        "  (which reached 100% in ~100 steps).",
        "- fixed-4's in-distribution accuracy may not be fully converged at",
        "  25k steps (its curve was still softening); fixed-4-long at 100k",
        "  partially addresses this but 0.72 < 1.0.",
        "- Evaluation state sizes are limited to S' ≤ 64 because the",
        "  vocabulary/positional tables are fixed at training time.",
        "- The rec-1x-dynamic family trains at ~5 average block-",
        "  applications per step (R ~ U{2..8}), slightly above the matched",
        "  4 - flagged rather than hidden.",
        "- Two duplicate jobs created by interrupted tool calls were",
        "  cancelled (~$0.30 wasted).",
        "- Dependency pin: trackio 0.42.0 requires huggingface-hub<2.0;",
        "  resolved with huggingface_hub==1.33.0.",
        "",
        "## 7. What remains",
        "",
        "- More seeds to tighten the OOD family means (esp. rec-4x4).",
        "- Aligned follow-ups: deeper aligned stacks (rec-4x-aligned),",
        "  overshoot training (R = k + margin), and training k up to 16 to",
        "  see whether the frontier tracks the data limit again.",
        "- Per-iteration supervision toward f^t(x) at every unroll",
        "  (Fan et al. 2409.15647) - related to alignment but gives dense",
        "  per-step signal; untested here.",
        "- Adaptive halting, curriculum on k, and larger-scale runs",
        "  (see the discussion in section 5).",
        "",
        "## 8. Reproduce",
        "",
        "See README.md. `run_all.py` drives lrsweep/stage1/stage2/eval;",
        "`evaluate.py` produces the per-checkpoint JSONs; `make_report.py`",
        "generates this report and every figure. Live dashboards:",
        "eac123/ptr-recurrent-depth-trackio.",
    ]
    return "\n".join(lines)


def table(evals, out_csv):
    rows = ["family,seed,params_total,train_flops_total,acc_k1_8,acc_k16,"
            "acc_k32,acc_k64,best_acc_k64_over_iters,best_R_k64"]
    for fam in FAMILIES:
        for e in evals[fam]:
            a = e["acc_vs_k"]
            idm = np.mean([a[str(k)] for k in range(1, 9)])
            it = e["acc_vs_iters"]
            best_r = max(it, key=lambda r: it[r]["64"])
            rows.append(f"{fam},{e['config']['seed']},{e['budget']['params_total']},"
                        f"{e['budget']['train_flops_total']:.3e},{idm:.4f},"
                        f"{a['16']:.4f},{a['32']:.4f},{a['64']:.4f},"
                        f"{it[best_r]['64']:.4f},{best_r}")
    open(out_csv, "w").write("\n".join(rows) + "\n")
    return rows


def upload(local, path_in_repo):
    from huggingface_hub import HfApi
    HfApi().upload_file(path_or_fileobj=local, path_in_repo=path_in_repo,
                        repo_id=REPO, repo_type="model")


if __name__ == "__main__":
    # example 2 needs an S=64 sample with a fixed seed
    _rng = np.random.default_rng(2026)
    f64 = _rng.integers(0, 64, size=64)
    x64 = int(_rng.integers(0, 64))
    k64 = int(_rng.integers(1, 9))
    _y = x64
    for _ in range(k64):
        _y = int(f64[_y])
    ans64 = _y
    globals()["f64"], globals()["x64"] = f64, x64
    globals()["k64"], globals()["ans64"] = k64, ans64

    os.makedirs(PLOTS, exist_ok=True)
    evals = pull()
    n = {f: len(evals[f]) for f in FAMILIES}
    print("[report] evals per family:", n)
    plot_schematic(f"{PLOTS}/schematic.png")
    plot_budget_map(evals, f"{PLOTS}/budget_map.png")
    plot_headline_bars(evals, f"{PLOTS}/headline_bars.png")
    plot_acc_vs_k(evals, f"{PLOTS}/acc_vs_k.png")
    plot_perk_zoom(evals, f"{PLOTS}/perk_zoom.png")
    plot_acc_vs_iters(evals, f"{PLOTS}/acc_vs_iters.png")
    plot_state_size(evals, f"{PLOTS}/acc_vs_state_size.png")
    plot_training(evals, f"{PLOTS}/training_curves.png")
    rows = table(evals, "results_table.csv")
    open("REPORT.md", "w").write(report(evals, rows))
    artifacts = ["schematic", "budget_map", "headline_bars", "acc_vs_k",
                 "perk_zoom", "acc_vs_iters", "acc_vs_state_size",
                 "training_curves"]
    for a in artifacts:
        upload(f"{PLOTS}/{a}.png", f"plots/{a}.png")
        print("[report] uploaded", f"plots/{a}.png")
    upload("results_table.csv", "results/results_table.csv")
    upload("REPORT.md", "REPORT.md")
    print("[report] uploaded results_table.csv, REPORT.md")