§4.8 v2 variants: registered predictions, computed verdicts, families registered everywhere

make_report.py

@@ -20,7 +20,7 @@ REPO = "eac123/ptr-recurrent-depth"
 PLOTS = "plots"
 
 FAMILIES = ["fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic", "rec4x4",
-            "fixed4-long", "rec1aligned"]
+            "fixed4-long", "rec1aligned", "rec1m4", "rec1m4dense"]
 LABELS = {
     "fixed4x1": "fixed-4 (4 blocks x 1)",
     "rec1x4": "recurrent 1x4 (train R=4)",
@@ -29,9 +29,12 @@ LABELS = {
     "rec4x4": "recurrent 4x4 (train R=4)",
     "fixed4-long": "fixed-4 long (100k steps)",
     "rec1aligned": "rec-1x-aligned (train R=k)",
+    "rec1m4": "rec-1x m4 (R=k+4, end-of-chain loss)",
+    "rec1m4dense": "rec-1x m4+dense (R=k+4, stem, dense loss)",
 }
 TRAIN_ITERS = {"fixed4x1": 1, "rec1x4": 4, "rec2x2": 2, "rec1xdynamic": 8,
-               "rec4x4": 4, "fixed4-long": 1, "rec1aligned": None}
+               "rec4x4": 4, "fixed4-long": 1, "rec1aligned": None,
+               "rec1m4": None, "rec1m4dense": None}
 
 
 def pull():
@@ -121,7 +124,7 @@ def plot_acc_vs_iters(evals, out,
 
 def plot_perk_zoom(evals, out,
                    fams=("fixed4x1", "rec1x4", "rec1xdynamic", "rec4x4",
-                         "rec1aligned"), kmax=16):
+                         "rec1aligned", "rec1m4", "rec1m4dense"), kmax=16):
     """Within-and-just-beyond training range: accuracy per hop count k."""
     fig, ax = plt.subplots(figsize=(8.5, 5))
     ks = np.arange(1, kmax + 1)
@@ -144,6 +147,17 @@ def plot_perk_zoom(evals, out,
     fig.tight_layout(); fig.savefig(out, dpi=150); plt.close(fig)
 
 
+def frontier(m, thr=0.9):
+    """Last k with mean accuracy >= thr (scanning from k=1)."""
+    best = 0
+    for k in range(1, len(m) + 1):
+        if m[k - 1] >= thr:
+            best = k
+        else:
+            break
+    return best
+
+
 def perk_rows(evals, fams, klo, khi):
     ks = list(range(klo, khi + 1))
     lines = ["| family | " + " | ".join(f"k={k}" for k in ks) + " |",
@@ -208,7 +222,8 @@ def plot_training(evals, out):
 
 SHORT = {"fixed4x1": "fixed-4", "rec1x4": "rec-1x4", "rec2x2": "rec-2x2",
          "rec1xdynamic": "rec-1x-dyn", "rec4x4": "rec-4x4",
-         "fixed4-long": "fixed-4-long", "rec1aligned": "rec-1x-aligned"}
+         "fixed4-long": "fixed-4-long", "rec1aligned": "rec-1x-aligned",
+         "rec1m4": "m4", "rec1m4dense": "m4+dense"}
 
 
 def plot_headline_bars(evals, out):
@@ -261,13 +276,15 @@ def plot_budget_map(evals, out):
     OFF = {"fixed4x1": (8, -14, "left"), "rec1x4": (0, -16, "center"),
            "rec2x2": (10, -4, "left"), "rec1xdynamic": (8, 8, "left"),
            "rec4x4": (-8, 10, "right"), "fixed4-long": (8, -14, "left"),
-           "rec1aligned": (8, -16, "left")}
+           "rec1aligned": (8, -16, "left"),
+           "rec1m4": (8, 8, "left"), "rec1m4dense": (8, -16, "left")}
     for f in FAMILIES:
         if not evals[f]:
             continue
         p, fl = budget_of(evals[f])
         ax.scatter(p, fl, s=90, zorder=3,
-                   marker="s" if f == "rec1aligned" else "o")
+                   marker="s" if f != "rec1x4" and f.startswith("rec1")
+                   else "o")
         dx, dy, ha = OFF[f]
         ax.annotate(LABELS[f], (p, fl), fontsize=7, ha=ha,
                     xytext=(dx, dy), textcoords="offset points")
@@ -509,6 +526,49 @@ def report(evals, rows):
                 f"{lm:.3f}, k=16 {le['acc_vs_k']['16']:.3f}, "
                 f"k=64 {le['acc_vs_k']['64']:.3f} -> {verdict}",
             ]
+
+    # --- v2 aligned variants (stage 5) ---
+    def variant_lines(fam, stair_margin=4):
+        ev = evals[fam]
+        if not ev:
+            return [f"*(stage-5 runs not finished: {fam})*"]
+        acc = np.array([[e["acc_vs_k"][str(k)] for k in range(1, 17)]
+                        for e in ev])
+        m = acc.mean(0)
+        fm = frontier(m)
+        hi, lo = [], []
+        for e in ev:
+            for r, cells in e["acc_vs_iters"].items():
+                for kk, v in cells.items():
+                    (hi if int(kk) <= int(r) - stair_margin else lo).append(v)
+        st = float(np.mean(hi) - np.mean(lo))
+        return [
+            f"- in-distribution (k=1..8 mean): {ms(fam, id_mean)}",
+            f"- frontier (last k with >=0.9 mean acc): **k={fm}**",
+            f"- k=16 {m[15]:.3f}, k=32 "
+            f"{np.mean([e['acc_vs_k']['32'] for e in ev]):.3f}, k=64 "
+            f"{np.mean([e['acc_vs_k']['64'] for e in ev]):.3f}",
+            f"- staircase check (acc high iff k <= R-{stair_margin}): "
+            f"{st:+.3f}",
+        ]
+
+    al2_lines = (["**A / m4 (margin, end-of-chain loss):**"]
+                 + variant_lines("rec1m4")
+                 + ["", "**B / m4+dense (margin + stem + dense):**"]
+                 + variant_lines("rec1m4dense"))
+    if evals["rec1m4"] and evals["rec1m4dense"]:
+        p1 = np.mean([id_mean(e) for e in evals["rec1m4"]]) < 0.5
+        mB = float(np.array([[e["acc_vs_k"][str(k)] for k in range(1, 9)]
+                             for e in evals["rec1m4dense"]]).mean())
+        mBm = np.array([[e["acc_vs_k"][str(k)] for k in range(1, 17)]
+                        for e in evals["rec1m4dense"]]).mean(0)
+        p2 = mB >= 0.9
+        p3 = frontier(mBm) >= 12
+        al2_lines += [
+            f"- (P1) {('**supported**' if p1 else '**not supported**')}",
+            f"- (P2) {('**supported**' if p2 else '**not supported**')}",
+            f"- (P3) {('**supported**' if p3 else '**not supported**')}",
+        ]
     lines = [
         "# Technical report: recurrent depth vs fixed depth on pointer chasing",
         "",
@@ -695,10 +755,12 @@ def report(evals, rows):
         "mappings per cell, mean over seeds):",
         "",
         *perk_rows(evals, ["fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic",
-                          "rec4x4", "fixed4-long", "rec1aligned"], 1, 8),
+                          "rec4x4", "fixed4-long", "rec1aligned", "rec1m4",
+                          "rec1m4dense"], 1, 8),
         "",
         *perk_rows(evals, ["fixed4x1", "rec1x4", "rec2x2", "rec1xdynamic",
-                          "rec4x4", "fixed4-long", "rec1aligned"], 9, 16),
+                          "rec4x4", "fixed4-long", "rec1aligned", "rec1m4",
+                          "rec1m4dense"], 9, 16),
         "",
         "![per-k zoom](plots/perk_zoom.png)",
         "",
@@ -790,6 +852,42 @@ def report(evals, rows):
         "",
         *al_lines,
         "",
+        "### 4.8 v2 aligned variants: margin R = k+4, stem layer, dense supervision",
+        "",
+        "The 4.7 diagnosis says the R=k objective fails because of (i) exact",
+        "halting at depth k, and (ii) no per-step signal. The v2 variants",
+        "(user design) target both, separately:",
+        "",
+        "- **A / m4** (3 seeds): R = k + 4 with end-of-chain supervision",
+        "  only. The margin removes exact halting - the model needs 'at",
+        "  least k unrolls', and can idle for the last 4. Everything else",
+        "  identical to rec-1x4.",
+        "- **B / m4+dense** (3 seeds): R = k + 4, one untied stem block",
+        "  before the loop, and dense supervision - every unroll t gets an",
+        "  auxiliary head predicting f^{min(t,k)}(x), i.e. each unroll does",
+        "  its own hop and then holds. This is the per-step signal the R=k",
+        "  run lacked, and it acts as a curriculum: unroll 1 only has to",
+        "  learn the lookup, later unrolls inherit it.",
+        "",
+        "At evaluation both read out at R = k + 4 for every k - the trained",
+        "inference mode - including k > 8.",
+        "",
+        "*Predictions registered before the runs:*",
+        "",
+        "- (P1) margin alone does **not** rescue learning (the credit-",
+        "  assignment problem, not exact halting, was the blocker); it",
+        "  should learn faster than R=k but plateau low.",
+        "- (P2) dense + stem **does** rescue: >= 0.9 mean accuracy across",
+        "  k = 1..8, and a fixed-R sweep showing a staircase (accuracy high",
+        "  iff k <= R - 4) rather than a peak at one R.",
+        "- (P3) if the fix is real, B extends beyond k=8 better than every",
+        "  existing family (R = k+4 is its trained mode at any k), bounded",
+        "  by OOD step embeddings (unroll indices > 11 were never trained).",
+        "",
+        "*Results:*",
+        "",
+        *al2_lines,
+        "",
         "## 5. Conclusions",
         "",
         "1. **Recurrent depth buys learnability that neither parameters nor",