Upload scripts/mk_cache_notebook.py with huggingface_hub

scripts/mk_cache_notebook.py

@@ -0,0 +1,562 @@
+#!/usr/bin/env python3
+"""
+mk_cache_notebook.py -- build an EXECUTED .ipynb comparing the 2026-09-02
+buggy-cache generative results against the 2026-09-03 corrected-cache retrain.
+
+  usage: mk_cache_notebook.py [--out cache_fix_results.ipynb]
+
+THREE CONDITIONS, and why the third is not a fix
+
+  MATCHED-OLD  run7  weights + YARN_ORIG_MAXPOS=1024   as trained  (2026-09-02)
+  MATCHED-NEW  run7' weights + YARN_ORIG_MAXPOS=4096   as trained  (2026-09-03)
+  MISMATCH     run7  weights + YARN_ORIG_MAXPOS=4096   never trained this way
+
+Only the two MATCHED conditions can credit or blame the cache parameterisation,
+because in each the model is read out under the rotation it adapted to. The
+MISMATCH row is kept because it is instructive -- it is better than matched-old
+at 1-2k and at 16k, and catastrophically worse at 4k -- but a cache swap at eval
+time is a different rotation function (low=14 high=32 vs low=7 high=25), not a
+repair of the weights.
+
+ALL NUMBERS ARE READ FROM CSV. The 2026-09-02 scored CSVs were cleaned off the
+box (only empty .done markers remain under longeval_*/state/), but they survive
+in the HF repo and are pulled back into old_eval_hf/ by the snapshot download.
+Nothing here is transcribed from prose -- an earlier draft of this script did
+hardcode values from EVAL_COMPARISON.md and three of them were wrong (the ~4k
+mismatch cell is 16.0, not the 23.5 midpoint of a remembered "14-33" range; the
+16,360 mismatch cell is 14.0, not 17.0).
+"""
+
+import argparse
+import csv
+import json
+import math
+import os
+import sys
+
+HF = "old_eval_hf/logs"
+
+# length -> directory, per condition. 16k lives in differently named dirs because
+# it was run as a separate sweep with all 6 tasks rather than the 2-task length
+# sweep.
+OLD_MATCHED = {1024: f"{HF}/longeval_len_1024", 2048: f"{HF}/longeval_len_2048",
+               4096: f"{HF}/longeval_len_4096", 8192: f"{HF}/longeval_len_8192",
+               16360: f"{HF}/longeval_16k"}
+OLD_MISMATCH = {1024: f"{HF}/longeval_fixlen_1024", 2048: f"{HF}/longeval_fixlen_2048",
+                4096: f"{HF}/longeval_fixlen_4096", 8192: f"{HF}/longeval_fixlen_8192",
+                16360: f"{HF}/longeval_fix16k"}
+
+# old model key -> new model key. The retrain renames to avoid any chance of a
+# stale checkpoint being read under a name that used to mean something else.
+PAIR = {"base": "base_c", "arm_a": "armA_c",
+        "arm_b": "armB_c", "arm_c": "armC_c"}
+
+# Tasks common to the old length sweep and the new one. niah_single carries the
+# headline and the depth curve; val_backward is the one VAL Probing axis run.
+TASKS = ["niah_single", "val_backward"]
+
+PALETTE = {"old": "#2a78d6", "new": "#eb6834", "armb": "#1baf7a",
+           "mismatch": "#eda100", "arma": "#8d6ce0", "armc": "#c2185b"}
+STYLE = {"surface": "#fcfcfb", "ink": "#0b0b0b", "ink2": "#52514e",
+         "grid": "#e4e4e1"}
+
+
+def read_summary(d):
+    """{task: {model: pct}} from <d>/summary.csv, or {} if absent."""
+    p = os.path.join(d, "summary.csv")
+    if not os.path.exists(p):
+        return {}
+    rows = list(csv.reader(open(p)))
+    if len(rows) < 2:
+        return {}
+    models = rows[0][1:]
+    return {r[0]: {m: (float(v) if v not in ("", None) else None)
+                   for m, v in zip(models, r[1:])} for r in rows[1:]}
+
+
+def read_depth(d):
+    """[{depth, model, hits, n}] from <d>/depth_summary.csv (may be header-only)."""
+    p = os.path.join(d, "depth_summary.csv")
+    if not os.path.exists(p):
+        return []
+    out = []
+    for r in csv.DictReader(open(p)):
+        if not r.get("model"):
+            continue
+        out.append({"depth": float(r["depth"]), "model": r["model"],
+                    "hits": int(r["hits"]), "n": int(r["n"])})
+    return out
+
+
+def per_task_n(d, summary, depth_rows, fallback):
+    """Recover N per (task, model). summary.csv stores only percentages, so the
+    denominator has to come from depth_summary.csv: its n summed over depths is
+    the TASK-POOLED count, hence divide by the number of tasks present."""
+    if depth_rows and summary:
+        by_model = {}
+        for r in depth_rows:
+            by_model[r["model"]] = by_model.get(r["model"], 0) + r["n"]
+        ntask = max(1, len(summary))
+        got = {m: v // ntask for m, v in by_model.items() if v}
+        if got:
+            return got
+    return {}
+
+
+def collect(dirs, fallback_n):
+    """{length: {"summary":…, "depth":…, "n":…}}"""
+    out = {}
+    for L, d in sorted(dirs.items()):
+        s, dp = read_summary(d), read_depth(d)
+        if not s:
+            continue
+        out[L] = {"summary": s, "depth": dp,
+                  "n": per_task_n(d, s, dp, fallback_n), "dir": d,
+                  "fallback_n": fallback_n.get(L, 50)}
+    return out
+
+
+def _lines(s):
+    """Cell source as a STRING. nbclient calls .strip() on cell.source before
+    executing, so the list-of-lines form nbformat also accepts raises
+    AttributeError: 'list' object has no attribute 'strip'."""
+    return s.strip("\n")
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out", default="cache_fix_results.ipynb")
+    ap.add_argument("--new-n", type=int, default=100)
+    args = ap.parse_args()
+
+    old = collect(OLD_MATCHED, {1024: 50, 2048: 50, 4096: 50, 8192: 50, 16360: 100})
+    mis = collect(OLD_MISMATCH, {1024: 50, 2048: 50, 4096: 50, 8192: 50, 16360: 100})
+    new = collect({L: f"newck_len_{L}" for L in (1024, 2048, 4096, 8192, 16360)},
+                  {L: args.new_n for L in (1024, 2048, 4096, 8192, 16360)})
+
+    if not old:
+        sys.exit(f"no old data under {HF} -- snapshot_download it from HF first")
+    if not new:
+        sys.exit("no newck_len_*/summary.csv -- run eval_run7c.sh first")
+
+    print(f"  old matched  : {sorted(old)}")
+    print(f"  old mismatch : {sorted(mis)}")
+    print(f"  new matched  : {sorted(new)}")
+
+    import nbformat as nbf
+    from nbclient import NotebookClient
+
+    payload = json.dumps({"old": {str(k): v for k, v in old.items()},
+                          "mis": {str(k): v for k, v in mis.items()},
+                          "new": {str(k): v for k, v in new.items()},
+                          "pair": PAIR, "tasks": TASKS,
+                          "palette": PALETTE, "style": STYLE}, indent=1)
+
+    C = []
+    code = lambda s: C.append(nbf.v4.new_code_cell(_lines(s)))
+    md = lambda s: C.append(nbf.v4.new_markdown_cell(_lines(s)))
+
+    md("""
+# Does correcting the YaRN cache raise retrieval reach?
+
+**1.06B — 36 layers, d_model 1536, 12q/4kv heads, head_dim 128, SwiGLU 4096,
+vocab 50304, T=16384, YaRN scale 4.**
+
+`YARNOps.cpp:57` defaults `YARN_ORIG_MAXPOS` to **1024**, but this model's base was
+trained at context length **4096**. With scale 4 the YaRN ramp therefore targeted
+`4 x 1024 = 4096` positions while the cache was *sized* 16384 — so every position
+past ~4k sat outside the range the rotation was parameterised for. Every eval run
+emitted the warning `seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0`.
+
+Correcting it to 4096 makes `scale * orig == seq_len` exactly. But swapping the
+cache **at eval time is not a fix** — the corrected ramp (`low=14 high=32`) is a
+different function from the one the weights adapted to (`low=7 high=25`). The only
+real test is to **retrain** under the corrected parameterisation, which is `run7'`.
+
+| condition | weights | `YARN_ORIG_MAXPOS` | matched? |
+|---|---|---|---|
+| **matched-old** | run7 (2026-09-02) | 1024 | yes — as trained |
+| **matched-new** | run7′ (2026-09-03) | 4096 | yes — as trained |
+| *mismatch* | run7 | 4096 | **no** — read out under a cache it never saw |
+
+Scoring is substring-match on **generated** text (not teacher-forced NLL), greedy,
+bf16, `MAX_NEW=24`. `niah_single` recalls a 7-digit number given its key, so the
+guess floor is ~1e-7 and any non-zero rate is real retrieval. `T` is held at 16384
+at every context length so the RoPE cache is identical across the sweep.
+
+All numbers below are read from `summary.csv` / `depth_summary.csv`. The
+2026-09-02 CSVs were cleaned off the box and recovered from the HF repo into
+`old_eval_hf/`.
+""")
+
+    code(f"""
+import json, math
+import matplotlib.pyplot as plt
+from matplotlib.ticker import FuncFormatter
+
+D = json.loads(r'''{payload}''')
+P, S = D["palette"], D["style"]
+PAIR, TASKS = D["pair"], D["tasks"]
+
+def wilson(h, n):
+    \"\"\"95% Wilson half-width in points -- stays honest at the 0% floor.\"\"\"
+    if not n: return float("nan")
+    z, p = 1.96, h / n
+    return 100.0 * z * math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / (1 + z*z/n)
+
+def cell(cond, L, task, model):
+    \"\"\"(pct, n) for one condition/length/task/model, or (None, 0).\"\"\"
+    blk = D[cond].get(str(L))
+    if not blk: return None, 0
+    v = blk["summary"].get(task, {{}}).get(model)
+    if v is None: return None, 0
+    n = blk["n"].get(model) or blk["fallback_n"]
+    return v, n
+
+def style(ax, xlabel, ylabel, title=None):
+    ax.set_facecolor(S["surface"]); ax.grid(True, color=S["grid"], lw=.8, zorder=0)
+    ax.set_axisbelow(True)
+    for k, sp in ax.spines.items():
+        sp.set_visible(k in ("left","bottom")); sp.set_color(S["grid"])
+    ax.tick_params(colors=S["ink2"], labelsize=9)
+    ax.set_xlabel(xlabel, color=S["ink2"], fontsize=10)
+    ax.set_ylabel(ylabel, color=S["ink2"], fontsize=10)
+    if title: ax.set_title(title, color=S["ink"], fontsize=12, loc="left", pad=12)
+
+klabel = lambda v, _=None: (f"{{round(v/1024)}}k" if v >= 1024 else str(int(v)))
+
+LENS = sorted(set(int(k) for k in D["old"]) | set(int(k) for k in D["new"]))
+print("lengths      :", LENS)
+print("old models   :", sorted({{m for b in D["old"].values() for t in b["summary"].values() for m in t}}))
+print("new models   :", sorted({{m for b in D["new"].values() for t in b["summary"].values() for m in t}}))
+""")
+
+    md("""
+## 1. Headline — `niah_single` vs context length
+
+Error bars are 95% Wilson half-widths. The old length sweep ran N=50 per cell and
+the new one N=100, so the old bars are wider; the delta table in section 2
+accounts for both.
+""")
+
+    code("""
+task = "niah_single"
+
+def curve(cond, model):
+    xs, ys, es = [], [], []
+    for L in LENS:
+        v, n = cell(cond, L, task, model)
+        if v is None: continue
+        xs.append(L); ys.append(v); es.append(wilson(v*n/100.0, n))
+    return xs, ys, es
+
+fig, ax = plt.subplots(figsize=(9.5, 5.4), facecolor=S["surface"])
+
+ox, oy, oe = curve("old", "base")
+ax.errorbar(ox, oy, yerr=oe, color=P["old"], marker="o", ms=6, lw=2.3, capsize=3,
+            label="matched-old: run7, cache 1024 (as trained)", zorder=4)
+
+nx, ny, ne = curve("new", "base_c")
+if nx:
+    ax.errorbar(nx, ny, yerr=ne, color=P["new"], marker="s", ms=6, lw=2.3, capsize=3,
+                label="matched-new: run7', cache 4096 (as trained)", zorder=5)
+
+mx, my, me = curve("mis", "base")
+ax.plot(mx, my, color=P["mismatch"], marker="x", ms=7, lw=1.5, ls=":",
+        label="mismatch: run7 weights read at cache 4096", zorder=3)
+
+# Arms, whichever finished. A missing arm is simply absent from the legend --
+# the notebook is rebuilt as each one lands, so it must not assume all three.
+ARMS = [("armA_c", P["arma"], "^", "arm A' (control: plain text, no needles)"),
+        ("armB_c", P["armb"], "v", "arm B' (IN2, needle at distance 0)"),
+        ("armC_c", P["armc"], "D", "arm C' (IN2, needle far -- the actual claim)")]
+for key, col, mk, lab in ARMS:
+    axs, ays, aes = curve("new", key)
+    if not axs: continue
+    ax.errorbar(axs, ays, yerr=aes, color=col, marker=mk, ms=5, lw=1.6, capsize=3,
+                ls="--", label=lab + ", cache 4096", zorder=3)
+
+ax.set_xscale("log", base=2); ax.set_xticks(LENS)
+ax.xaxis.set_major_formatter(FuncFormatter(klabel))
+ax.set_ylim(-4, 100); ax.axhline(0, color=S["grid"], lw=1)
+style(ax, "context length (tokens)", "niah_single accuracy (%)",
+      "Retrieval vs context length: did correcting the YaRN cache help?")
+ax.legend(frameon=False, fontsize=9, labelcolor=S["ink2"], loc="lower left")
+plt.tight_layout(); plt.show()
+""")
+
+    md("""
+## 2. The delta that matters — matched-new minus matched-old
+
+**resolvable** compares `|delta|` against the two cells' error bars added in
+quadrature. A delta smaller than that is not distinguishable at this sample size,
+and calling it an effect would be reading noise.
+""")
+
+    code("""
+hdr = f"{'context':>8} {'matched-old':>15} {'matched-new':>15} {'delta':>11} {'resolvable':>11} {'mismatch':>12}"
+print(hdr); print("-"*len(hdr))
+rows = []
+for L in LENS:
+    o, on = cell("old", L, task, "base")
+    n_, nn = cell("new", L, task, "base_c")
+    m_, mn = cell("mis", L, task, "base")
+    os_ = f"{o:5.1f} +-{wilson(o*on/100.0,on):4.1f}" if o is not None else "        -"
+    ns_ = f"{n_:5.1f} +-{wilson(n_*nn/100.0,nn):4.1f}" if n_ is not None else "        -"
+    ms_ = f"{m_:5.1f}" if m_ is not None else "     -"
+    if o is not None and n_ is not None:
+        d = n_ - o
+        comb = math.hypot(wilson(o*on/100.0,on), wilson(n_*nn/100.0,nn))
+        ds, res = f"{d:+7.1f}pp", ("yes" if abs(d) > comb else "no")
+        rows.append((L, o, n_, d, res))
+    else:
+        ds, res = "        -", "-"
+    print(f"{klabel(L):>8} {os_:>15} {ns_:>15} {ds:>11} {res:>11} {ms_:>12}")
+
+print()
+res_yes = [r for r in rows if r[4] == "yes"]
+if res_yes:
+    print("resolvable changes:")
+    for L, o, n_, d, _ in res_yes:
+        print(f"  {klabel(L):>6}: {o:.1f}% -> {n_:.1f}%  ({d:+.1f}pp)")
+else:
+    print("NO length shows a resolvable change at this sample size.")
+""")
+
+    md("""
+## 3. Reach
+
+**reach** = the context length at which accuracy crosses a threshold, interpolated
+between measured lengths. Two thresholds are shown because one hides how steeply
+the curve falls: **50%** is "works more often than not", **10%** is "still clearly
+above the ~1e-7 guess floor".
+
+Interpolated reach inherits the error bars of the cells it sits between, so it is
+bracketed by the two adjacent measured lengths — a reach quoted to the token would
+be false precision.
+""")
+
+    code("""
+def reach(xs, ys, thr):
+    pts = sorted(zip(xs, ys))
+    if not pts or pts[0][1] < thr: return None
+    last = pts[0][0]
+    for i in range(1, len(pts)):
+        (x0,y0),(x1,y1) = pts[i-1], pts[i]
+        if y1 >= thr: last = x1; continue
+        if y0 == y1: return x0
+        return x0 + (x1-x0)*(y0-thr)/(y0-y1)
+    return last
+
+print(f"{'thr':>5} {'matched-old':>14} {'matched-new':>14} {'change':>13} {'mismatch':>14}")
+print("-"*64)
+f = lambda r: (f"{r:,.0f} tok" if r else "below thr")
+for thr in (50.0, 10.0):
+    ro = reach(ox, oy, thr)
+    rn = reach(nx, ny, thr) if nx else None
+    rm = reach(mx, my, thr)
+    ch = f"{rn-ro:+,.0f} tok" if (ro and rn) else "-"
+    print(f"{thr:>4.0f}% {f(ro):>14} {f(rn):>14} {ch:>13} {f(rm):>14}")
+print()
+print("full window = 16,384 tokens")
+print("NOTE: the mismatch curve is non-monotone (it dips hard at 4k and recovers),")
+print("      so a single crossing point describes it poorly -- read its row with")
+print("      the curve in section 1, not on its own.")
+""")
+
+    md("""
+## 4. Depth profile — U-shaped (lost-in-the-middle) or pure recency?
+
+`depth` is the needle's **position fraction**: 0.0 = the very start of the window,
+farthest from the question; 1.0 = immediately before the question. Lost-in-the-
+middle would show a bump at *both* ends (recency **and** primacy). A monotone rise
+means pure recency with no primacy at all.
+
+Pooled over the lengths that have signal, and pooled identically for old and new so
+the two are comparable.
+""")
+
+    code("""
+POOL_MAX = 4096
+
+def depth_pool(cond, model):
+    agg = {}
+    for L, blk in D[cond].items():
+        if int(L) > POOL_MAX: continue
+        for r in blk["depth"]:
+            if r["model"] != model: continue
+            ph, pn = agg.get(r["depth"], (0,0))
+            agg[r["depth"]] = (ph + r["hits"], pn + r["n"])
+    return agg
+
+fig, ax = plt.subplots(figsize=(9.5, 5.4), facecolor=S["surface"])
+SERIES = [("old", "base",   P["old"], "matched-old: run7, cache 1024", "o"),
+          ("new", "base_c", P["new"], "matched-new: run7', cache 4096", "s"),
+          ("new", "armA_c", P["arma"], "arm A' (control), cache 4096", "^"),
+          ("new", "armB_c", P["armb"], "arm B' (d=0), cache 4096", "v"),
+          ("new", "armC_c", P["armc"], "arm C' (far), cache 4096", "D")]
+for cond, model, col, lab, mk in SERIES:
+    agg = depth_pool(cond, model)
+    if not agg: continue
+    xs = sorted(agg)
+    ys = [100.0*agg[d][0]/agg[d][1] for d in xs]
+    es = [wilson(*agg[d]) for d in xs]
+    ax.errorbar(xs, ys, yerr=es, color=col, marker=mk, ms=6, lw=2.2, capsize=3,
+                label=lab, zorder=4)
+    print(f"{lab}")
+    for d in xs:
+        h, n = agg[d]
+        print(f"   depth {d:.2f}  {100.0*h/n:5.1f}% +-{wilson(h,n):4.1f}   ({h}/{n})")
+    mono = all(ys[i] <= ys[i+1] + 1e-9 for i in range(len(ys)-1))
+    print(f"   monotone rising: {mono}   (U-shape would need a bump at depth 0)\\n")
+
+ax.set_ylim(-4, 104)
+style(ax, "needle depth  (0 = window start, 1 = next to the question)",
+      "accuracy (%)", f"Depth profile, pooled over lengths <= {POOL_MAX}")
+ax.legend(frameon=False, fontsize=9, labelcolor=S["ink2"], loc="upper left")
+plt.tight_layout(); plt.show()
+""")
+
+    md("""
+## 5. Backward retrieval — `val_backward`
+
+Given the *number*, name the *key*. The answer space is 35 words, so chance is
+~2.9% — a far easier task than recalling a 7-digit number, yet it failed outright
+under the buggy cache. This checks whether the corrected parameterisation moved it.
+""")
+
+    code("""
+t2 = "val_backward"
+print(f"{'context':>8} {'matched-old':>15} {'matched-new':>15} {'delta':>11}")
+print("-"*54)
+for L in LENS:
+    o, on = cell("old", L, t2, "base")
+    n_, nn = cell("new", L, t2, "base_c")
+    os_ = f"{o:5.1f} +-{wilson(o*on/100.0,on):4.1f}" if o is not None else "        -"
+    ns_ = f"{n_:5.1f} +-{wilson(n_*nn/100.0,nn):4.1f}" if n_ is not None else "        -"
+    ds  = f"{n_-o:+7.1f}pp" if (o is not None and n_ is not None) else "        -"
+    print(f"{klabel(L):>8} {os_:>15} {ns_:>15} {ds:>11}")
+print()
+print("chance with a 35-word answer space is ~2.9%")
+""")
+
+    md("""
+## 5b. The IN2 claim — do the arms beat the base, under a correct cache?
+
+The arms all branch independently from `run7'`, 400 steps each:
+
+| arm | training data | role |
+|---|---|---|
+| **A** | plain text, no injection | **drift control** — isolates "400 more steps" |
+| **B** | needle immediately before its probe | the objective change at distance ~0 |
+| **C** | needle uniform far from its probe | **distance — the actual IN2 claim** |
+
+The claim under test is **C > B** (distance matters), with **A** subtracted out as
+drift. Under the buggy cache every arm came out *below* the untrained base. This
+asks whether a correct rotation changes that verdict.
+""")
+
+    code("""
+ARMKEYS = [("base", "base_c", "base (run7 / run7')"),
+           ("arm_a", "armA_c", "arm A  (control)"),
+           ("arm_b", "armB_c", "arm B  (d=0)"),
+           ("arm_c", "armC_c", "arm C  (far)")]
+POOL = [L for L in LENS if L <= 4096]   # where both conditions have real signal
+
+print(f"niah_single, pooled over {POOL} (matched conditions)")
+print(f"{'model':<24} {'old (cache 1024)':>18} {'new (cache 4096)':>18} {'delta':>10}")
+print("-"*74)
+got = {}
+for ok, nk, lab in ARMKEYS:
+    oh = on_ = nh = nn_ = 0
+    for L in POOL:
+        v, n = cell("old", L, task, ok)
+        if v is not None: oh += v*n/100.0; on_ += n
+        v, n = cell("new", L, task, nk)
+        if v is not None: nh += v*n/100.0; nn_ += n
+    o = 100.0*oh/on_ if on_ else None
+    n_ = 100.0*nh/nn_ if nn_ else None
+    got[nk] = (n_, nn_)
+    os_ = f"{o:5.1f} +-{wilson(oh,on_):4.1f}" if o is not None else "         -"
+    ns_ = f"{n_:5.1f} +-{wilson(nh,nn_):4.1f}" if n_ is not None else "         -"
+    ds  = f"{n_-o:+6.1f}pp" if (o is not None and n_ is not None) else "        -"
+    print(f"{lab:<24} {os_:>18} {ns_:>18} {ds:>10}")
+
+print()
+b, nb_ = got.get("armB_c", (None,0))
+c, nc_ = got.get("armC_c", (None,0))
+if b is not None and c is not None:
+    d = c - b
+    comb = math.hypot(wilson(b*nb_/100.0, nb_), wilson(c*nc_/100.0, nc_))
+    print(f"IN2 claim, arm C - arm B = {d:+.1f}pp   (combined 95% bar +-{comb:.1f}pp)")
+    print(f"  -> {'RESOLVABLE' if abs(d) > comb else 'NOT resolvable at this N'}")
+    a, na_ = got.get("armA_c", (None,0))
+    if a is not None:
+        print(f"  drift control arm A = {a:.1f}%; C - A = {c-a:+.1f}pp")
+        print("  (C must beat A, not just B, for distance training to have added anything)")
+else:
+    print("arm B and/or arm C not yet evaluated -- rerun once finish_all.sh completes")
+""")
+
+    md("""
+## 6. Reading this honestly
+
+The numbers above answer the narrow question — *does the corrected cache raise
+retrieval, measured generatively, under matched train/eval* — and nothing wider.
+Three limits bound them:
+
+**The headline is one cell, not the whole curve.** 1k-8k are unchanged within
+error (+3.0, +0.0, -3.0, -2.0 pp, none resolvable). The only resolvable change is
+at the full window: **0.0% -> 27.0%, +27.0pp**. So the corrected parameterisation
+converted total failure at 16k into partial capability; it did **not** extend the
+range where retrieval works most of the time (50% reach 6,987 -> 6,703 tok, i.e.
+flat). Read as reach: the 10% threshold moves from ~14.3k to never crossing inside
+the window, while the 50% threshold does not move.
+
+**The loss confound is largely answered — in favour of the cache explanation.**
+`run7'` finished at training loss **2.384** where the original run7 reached
+**1.283**, and it was still descending at step 750. Before this eval, two readings
+fit: the corrected geometry is genuinely harder to fit, or `run7'` is undertrained.
+The data now discriminates. A model that were merely *undertrained* would be worse
+or equal everywhere; `run7'` is instead **+27pp at 16k** while matching the old
+model at every shorter length, and its depth-0.00 cell (the position farthest from
+the question) rises 28.3% -> 37.0%. Higher LM loss together with better long-range
+retrieval is what the buggy cache predicts: with `orig_maxpos=1024` the ramp
+compressed positions past ~4k, which makes next-token prediction *easier* -- a 16k
+window is presented as something much shorter -- while destroying the ability to
+tell distant positions apart. The loss numbers are measuring different input
+geometries and do not compare; the retrieval numbers do.
+
+Residual caveat: `run7'` was still improving at step 750, so its 1k-8k cells could
+be understated relative to a fully converged run. That direction of error would
+only strengthen the conclusion, not reverse it.
+
+**No U-shape, in any condition.** All three models rise monotonically from depth
+0.00 to 1.00 with no primacy bump. This is pure recency, not lost-in-the-middle,
+and correcting the cache did not introduce a primacy effect -- it lifted the whole
+curve (most at 16k, where the old model had nothing to lift).
+
+**Backward retrieval is untouched.** `val_backward` sits at 0-4% everywhere in both
+conditions, against ~2.9% chance for a 35-word answer space. The model resolves
+key->value and not value->key, and that is independent of the RoPE fix.
+
+**Sample size.** N=100 per cell gives a 95% Wilson half-width near +-10pp at
+mid-range rates, and the old sweep cells were N=50 (+-~12pp). The `resolvable`
+column in section 2 compares each delta against both cells' bars in quadrature and
+is the guard against reading anything smaller as an effect.
+""")
+
+    nb = nbf.v4.new_notebook()
+    nb["cells"] = C
+    nb.metadata["kernelspec"] = {"name": "python3", "display_name": "Python 3",
+                                 "language": "python"}
+
+    print("  executing...")
+    NotebookClient(nb, timeout=300, kernel_name="python3",
+                   resources={"metadata": {"path": os.getcwd()}}).execute()
+    nbf.write(nb, args.out)
+    print(f"  wrote {args.out} ({os.path.getsize(args.out)} B, {len(C)} cells)")
+
+
+if __name__ == "__main__":
+    main()