BluScriptCP / scripts /mk_cache_notebook.py
Upload scripts/mk_cache_notebook.py with huggingface_hub
46c6b71 verified 2 months ago
24 kB
#!/usr/bin/env python3
"""
mk_cache_notebook.py -- build an EXECUTED .ipynb comparing the 2026-09-02
buggy-cache generative results against the 2026-09-03 corrected-cache retrain.
  usage: mk_cache_notebook.py [--out cache_fix_results.ipynb]
THREE CONDITIONS, and why the third is not a fix
  MATCHED-OLD  run7  weights + YARN_ORIG_MAXPOS=1024   as trained  (2026-09-02)
  MATCHED-NEW  run7' weights + YARN_ORIG_MAXPOS=4096   as trained  (2026-09-03)
  MISMATCH     run7  weights + YARN_ORIG_MAXPOS=4096   never trained this way
Only the two MATCHED conditions can credit or blame the cache parameterisation,
because in each the model is read out under the rotation it adapted to. The
MISMATCH row is kept because it is instructive -- it is better than matched-old
at 1-2k and at 16k, and catastrophically worse at 4k -- but a cache swap at eval
time is a different rotation function (low=14 high=32 vs low=7 high=25), not a
repair of the weights.
ALL NUMBERS ARE READ FROM CSV. The 2026-09-02 scored CSVs were cleaned off the
box (only empty .done markers remain under longeval_*/state/), but they survive
in the HF repo and are pulled back into old_eval_hf/ by the snapshot download.
Nothing here is transcribed from prose -- an earlier draft of this script did
hardcode values from EVAL_COMPARISON.md and three of them were wrong (the ~4k
mismatch cell is 16.0, not the 23.5 midpoint of a remembered "14-33" range; the
16,360 mismatch cell is 14.0, not 17.0).
"""

import argparse
import csv
import json
import math
import os
import sys

HF = "old_eval_hf/logs"

# length -> directory, per condition. 16k lives in differently named dirs because
# it was run as a separate sweep with all 6 tasks rather than the 2-task length
# sweep.
OLD_MATCHED = {1024: f"{HF}/longeval_len_1024", 2048: f"{HF}/longeval_len_2048",
               4096: f"{HF}/longeval_len_4096", 8192: f"{HF}/longeval_len_8192",
               16360: f"{HF}/longeval_16k"}
OLD_MISMATCH = {1024: f"{HF}/longeval_fixlen_1024", 2048: f"{HF}/longeval_fixlen_2048",
                4096: f"{HF}/longeval_fixlen_4096", 8192: f"{HF}/longeval_fixlen_8192",
                16360: f"{HF}/longeval_fix16k"}

# old model key -> new model key. The retrain renames to avoid any chance of a
# stale checkpoint being read under a name that used to mean something else.
PAIR = {"base": "base_c", "arm_a": "armA_c",
        "arm_b": "armB_c", "arm_c": "armC_c"}

# Tasks common to the old length sweep and the new one. niah_single carries the
# headline and the depth curve; val_backward is the one VAL Probing axis run.
TASKS = ["niah_single", "val_backward"]

PALETTE = {"old": "#2a78d6", "new": "#eb6834", "armb": "#1baf7a",
           "mismatch": "#eda100", "arma": "#8d6ce0", "armc": "#c2185b"}
STYLE = {"surface": "#fcfcfb", "ink": "#0b0b0b", "ink2": "#52514e",
         "grid": "#e4e4e1"}


def read_summary(d):
    """{task: {model: pct}} from <d>/summary.csv, or {} if absent."""
    p = os.path.join(d, "summary.csv")
    if not os.path.exists(p):
        return {}
    rows = list(csv.reader(open(p)))
    if len(rows) < 2:
        return {}
    models = rows[0][1:]
    return {r[0]: {m: (float(v) if v not in ("", None) else None)
                   for m, v in zip(models, r[1:])} for r in rows[1:]}


def read_depth(d):
    """[{depth, model, hits, n}] from <d>/depth_summary.csv (may be header-only)."""
    p = os.path.join(d, "depth_summary.csv")
    if not os.path.exists(p):
        return []
    out = []
    for r in csv.DictReader(open(p)):
        if not r.get("model"):
            continue
        out.append({"depth": float(r["depth"]), "model": r["model"],
                    "hits": int(r["hits"]), "n": int(r["n"])})
    return out


def per_task_n(d, summary, depth_rows, fallback):
    """Recover N per (task, model). summary.csv stores only percentages, so the
    denominator has to come from depth_summary.csv: its n summed over depths is
    the TASK-POOLED count, hence divide by the number of tasks present."""
    if depth_rows and summary:
        by_model = {}
        for r in depth_rows:
            by_model[r["model"]] = by_model.get(r["model"], 0) + r["n"]
        ntask = max(1, len(summary))
        got = {m: v // ntask for m, v in by_model.items() if v}
        if got:
            return got
    return {}


def collect(dirs, fallback_n):
    """{length: {"summary":…, "depth":…, "n":…}}"""
    out = {}
    for L, d in sorted(dirs.items()):
        s, dp = read_summary(d), read_depth(d)
        if not s:
            continue
        out[L] = {"summary": s, "depth": dp,
                  "n": per_task_n(d, s, dp, fallback_n), "dir": d,
                  "fallback_n": fallback_n.get(L, 50)}
    return out


def _lines(s):
    """Cell source as a STRING. nbclient calls .strip() on cell.source before
    executing, so the list-of-lines form nbformat also accepts raises
    AttributeError: 'list' object has no attribute 'strip'."""
    return s.strip("\n")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default="cache_fix_results.ipynb")
    ap.add_argument("--new-n", type=int, default=100)
    args = ap.parse_args()

    old = collect(OLD_MATCHED, {1024: 50, 2048: 50, 4096: 50, 8192: 50, 16360: 100})
    mis = collect(OLD_MISMATCH, {1024: 50, 2048: 50, 4096: 50, 8192: 50, 16360: 100})
    new = collect({L: f"newck_len_{L}" for L in (1024, 2048, 4096, 8192, 16360)},
                  {L: args.new_n for L in (1024, 2048, 4096, 8192, 16360)})

    if not old:
        sys.exit(f"no old data under {HF} -- snapshot_download it from HF first")
    if not new:
        sys.exit("no newck_len_*/summary.csv -- run eval_run7c.sh first")

    print(f"  old matched  : {sorted(old)}")
    print(f"  old mismatch : {sorted(mis)}")
    print(f"  new matched  : {sorted(new)}")

    import nbformat as nbf
    from nbclient import NotebookClient

    payload = json.dumps({"old": {str(k): v for k, v in old.items()},
                          "mis": {str(k): v for k, v in mis.items()},
                          "new": {str(k): v for k, v in new.items()},
                          "pair": PAIR, "tasks": TASKS,
                          "palette": PALETTE, "style": STYLE}, indent=1)

    C = []
    code = lambda s: C.append(nbf.v4.new_code_cell(_lines(s)))
    md = lambda s: C.append(nbf.v4.new_markdown_cell(_lines(s)))

    md("""
# Does correcting the YaRN cache raise retrieval reach?
**1.06B — 36 layers, d_model 1536, 12q/4kv heads, head_dim 128, SwiGLU 4096,
vocab 50304, T=16384, YaRN scale 4.**
`YARNOps.cpp:57` defaults `YARN_ORIG_MAXPOS` to **1024**, but this model's base was
trained at context length **4096**. With scale 4 the YaRN ramp therefore targeted
`4 x 1024 = 4096` positions while the cache was *sized* 16384 — so every position
past ~4k sat outside the range the rotation was parameterised for. Every eval run
emitted the warning `seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0`.
Correcting it to 4096 makes `scale * orig == seq_len` exactly. But swapping the
cache **at eval time is not a fix** — the corrected ramp (`low=14 high=32`) is a
different function from the one the weights adapted to (`low=7 high=25`). The only
real test is to **retrain** under the corrected parameterisation, which is `run7'`.
| condition | weights | `YARN_ORIG_MAXPOS` | matched? |
|---|---|---|---|
| **matched-old** | run7 (2026-09-02) | 1024 | yes — as trained |
| **matched-new** | run7′ (2026-09-03) | 4096 | yes — as trained |
| *mismatch* | run7 | 4096 | **no** — read out under a cache it never saw |
Scoring is substring-match on **generated** text (not teacher-forced NLL), greedy,
bf16, `MAX_NEW=24`. `niah_single` recalls a 7-digit number given its key, so the
guess floor is ~1e-7 and any non-zero rate is real retrieval. `T` is held at 16384
at every context length so the RoPE cache is identical across the sweep.
All numbers below are read from `summary.csv` / `depth_summary.csv`. The
2026-09-02 CSVs were cleaned off the box and recovered from the HF repo into
`old_eval_hf/`.
""")

    code(f"""
import json, math
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
D = json.loads(r'''{payload}''')
P, S = D["palette"], D["style"]
PAIR, TASKS = D["pair"], D["tasks"]
def wilson(h, n):
    \"\"\"95% Wilson half-width in points -- stays honest at the 0% floor.\"\"\"
    if not n: return float("nan")
    z, p = 1.96, h / n
    return 100.0 * z * math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / (1 + z*z/n)
def cell(cond, L, task, model):
    \"\"\"(pct, n) for one condition/length/task/model, or (None, 0).\"\"\"
    blk = D[cond].get(str(L))
    if not blk: return None, 0
    v = blk["summary"].get(task, {{}}).get(model)
    if v is None: return None, 0
    n = blk["n"].get(model) or blk["fallback_n"]
    return v, n
def style(ax, xlabel, ylabel, title=None):
    ax.set_facecolor(S["surface"]); ax.grid(True, color=S["grid"], lw=.8, zorder=0)
    ax.set_axisbelow(True)
    for k, sp in ax.spines.items():
        sp.set_visible(k in ("left","bottom")); sp.set_color(S["grid"])
    ax.tick_params(colors=S["ink2"], labelsize=9)
    ax.set_xlabel(xlabel, color=S["ink2"], fontsize=10)
    ax.set_ylabel(ylabel, color=S["ink2"], fontsize=10)
    if title: ax.set_title(title, color=S["ink"], fontsize=12, loc="left", pad=12)
klabel = lambda v, _=None: (f"{{round(v/1024)}}k" if v >= 1024 else str(int(v)))
LENS = sorted(set(int(k) for k in D["old"]) | set(int(k) for k in D["new"]))
print("lengths      :", LENS)
print("old models   :", sorted({{m for b in D["old"].values() for t in b["summary"].values() for m in t}}))
print("new models   :", sorted({{m for b in D["new"].values() for t in b["summary"].values() for m in t}}))
""")

    md("""
## 1. Headline — `niah_single` vs context length
Error bars are 95% Wilson half-widths. The old length sweep ran N=50 per cell and
the new one N=100, so the old bars are wider; the delta table in section 2
accounts for both.
""")

    code("""
task = "niah_single"
def curve(cond, model):
    xs, ys, es = [], [], []
    for L in LENS:
        v, n = cell(cond, L, task, model)
        if v is None: continue
        xs.append(L); ys.append(v); es.append(wilson(v*n/100.0, n))
    return xs, ys, es
fig, ax = plt.subplots(figsize=(9.5, 5.4), facecolor=S["surface"])
ox, oy, oe = curve("old", "base")
ax.errorbar(ox, oy, yerr=oe, color=P["old"], marker="o", ms=6, lw=2.3, capsize=3,
            label="matched-old: run7, cache 1024 (as trained)", zorder=4)
nx, ny, ne = curve("new", "base_c")
if nx:
    ax.errorbar(nx, ny, yerr=ne, color=P["new"], marker="s", ms=6, lw=2.3, capsize=3,
                label="matched-new: run7', cache 4096 (as trained)", zorder=5)
mx, my, me = curve("mis", "base")
ax.plot(mx, my, color=P["mismatch"], marker="x", ms=7, lw=1.5, ls=":",
        label="mismatch: run7 weights read at cache 4096", zorder=3)
# Arms, whichever finished. A missing arm is simply absent from the legend --
# the notebook is rebuilt as each one lands, so it must not assume all three.
ARMS = [("armA_c", P["arma"], "^", "arm A' (control: plain text, no needles)"),
        ("armB_c", P["armb"], "v", "arm B' (IN2, needle at distance 0)"),
        ("armC_c", P["armc"], "D", "arm C' (IN2, needle far -- the actual claim)")]
for key, col, mk, lab in ARMS:
    axs, ays, aes = curve("new", key)
    if not axs: continue
    ax.errorbar(axs, ays, yerr=aes, color=col, marker=mk, ms=5, lw=1.6, capsize=3,
                ls="--", label=lab + ", cache 4096", zorder=3)
ax.set_xscale("log", base=2); ax.set_xticks(LENS)
ax.xaxis.set_major_formatter(FuncFormatter(klabel))
ax.set_ylim(-4, 100); ax.axhline(0, color=S["grid"], lw=1)
style(ax, "context length (tokens)", "niah_single accuracy (%)",
      "Retrieval vs context length: did correcting the YaRN cache help?")
ax.legend(frameon=False, fontsize=9, labelcolor=S["ink2"], loc="lower left")
plt.tight_layout(); plt.show()
""")

    md("""
## 2. The delta that matters — matched-new minus matched-old
**resolvable** compares `|delta|` against the two cells' error bars added in
quadrature. A delta smaller than that is not distinguishable at this sample size,
and calling it an effect would be reading noise.
""")

    code("""
hdr = f"{'context':>8} {'matched-old':>15} {'matched-new':>15} {'delta':>11} {'resolvable':>11} {'mismatch':>12}"
print(hdr); print("-"*len(hdr))
rows = []
for L in LENS:
    o, on = cell("old", L, task, "base")
    n_, nn = cell("new", L, task, "base_c")
    m_, mn = cell("mis", L, task, "base")
    os_ = f"{o:5.1f} +-{wilson(o*on/100.0,on):4.1f}" if o is not None else "        -"
    ns_ = f"{n_:5.1f} +-{wilson(n_*nn/100.0,nn):4.1f}" if n_ is not None else "        -"
    ms_ = f"{m_:5.1f}" if m_ is not None else "     -"
    if o is not None and n_ is not None:
        d = n_ - o
        comb = math.hypot(wilson(o*on/100.0,on), wilson(n_*nn/100.0,nn))
        ds, res = f"{d:+7.1f}pp", ("yes" if abs(d) > comb else "no")
        rows.append((L, o, n_, d, res))
    else:
        ds, res = "        -", "-"
    print(f"{klabel(L):>8} {os_:>15} {ns_:>15} {ds:>11} {res:>11} {ms_:>12}")
print()
res_yes = [r for r in rows if r[4] == "yes"]
if res_yes:
    print("resolvable changes:")
    for L, o, n_, d, _ in res_yes:
        print(f"  {klabel(L):>6}: {o:.1f}% -> {n_:.1f}%  ({d:+.1f}pp)")
else:
    print("NO length shows a resolvable change at this sample size.")
""")

    md("""
## 3. Reach
**reach** = the context length at which accuracy crosses a threshold, interpolated
between measured lengths. Two thresholds are shown because one hides how steeply
the curve falls: **50%** is "works more often than not", **10%** is "still clearly
above the ~1e-7 guess floor".
Interpolated reach inherits the error bars of the cells it sits between, so it is
bracketed by the two adjacent measured lengths — a reach quoted to the token would
be false precision.
""")

    code("""
def reach(xs, ys, thr):
    pts = sorted(zip(xs, ys))
    if not pts or pts[0][1] < thr: return None
    last = pts[0][0]
    for i in range(1, len(pts)):
        (x0,y0),(x1,y1) = pts[i-1], pts[i]
        if y1 >= thr: last = x1; continue
        if y0 == y1: return x0
        return x0 + (x1-x0)*(y0-thr)/(y0-y1)
    return last
print(f"{'thr':>5} {'matched-old':>14} {'matched-new':>14} {'change':>13} {'mismatch':>14}")
print("-"*64)
f = lambda r: (f"{r:,.0f} tok" if r else "below thr")
for thr in (50.0, 10.0):
    ro = reach(ox, oy, thr)
    rn = reach(nx, ny, thr) if nx else None
    rm = reach(mx, my, thr)
    ch = f"{rn-ro:+,.0f} tok" if (ro and rn) else "-"
    print(f"{thr:>4.0f}% {f(ro):>14} {f(rn):>14} {ch:>13} {f(rm):>14}")
print()
print("full window = 16,384 tokens")
print("NOTE: the mismatch curve is non-monotone (it dips hard at 4k and recovers),")
print("      so a single crossing point describes it poorly -- read its row with")
print("      the curve in section 1, not on its own.")
""")

    md("""
## 4. Depth profile — U-shaped (lost-in-the-middle) or pure recency?
`depth` is the needle's **position fraction**: 0.0 = the very start of the window,
farthest from the question; 1.0 = immediately before the question. Lost-in-the-
middle would show a bump at *both* ends (recency **and** primacy). A monotone rise
means pure recency with no primacy at all.
Pooled over the lengths that have signal, and pooled identically for old and new so
the two are comparable.
""")

    code("""
POOL_MAX = 4096
def depth_pool(cond, model):
    agg = {}
    for L, blk in D[cond].items():
        if int(L) > POOL_MAX: continue
        for r in blk["depth"]:
            if r["model"] != model: continue
            ph, pn = agg.get(r["depth"], (0,0))
            agg[r["depth"]] = (ph + r["hits"], pn + r["n"])
    return agg
fig, ax = plt.subplots(figsize=(9.5, 5.4), facecolor=S["surface"])
SERIES = [("old", "base",   P["old"], "matched-old: run7, cache 1024", "o"),
          ("new", "base_c", P["new"], "matched-new: run7', cache 4096", "s"),
          ("new", "armA_c", P["arma"], "arm A' (control), cache 4096", "^"),
          ("new", "armB_c", P["armb"], "arm B' (d=0), cache 4096", "v"),
          ("new", "armC_c", P["armc"], "arm C' (far), cache 4096", "D")]
for cond, model, col, lab, mk in SERIES:
    agg = depth_pool(cond, model)
    if not agg: continue
    xs = sorted(agg)
    ys = [100.0*agg[d][0]/agg[d][1] for d in xs]
    es = [wilson(*agg[d]) for d in xs]
    ax.errorbar(xs, ys, yerr=es, color=col, marker=mk, ms=6, lw=2.2, capsize=3,
                label=lab, zorder=4)
    print(f"{lab}")
    for d in xs:
        h, n = agg[d]
        print(f"   depth {d:.2f}  {100.0*h/n:5.1f}% +-{wilson(h,n):4.1f}   ({h}/{n})")
    mono = all(ys[i] <= ys[i+1] + 1e-9 for i in range(len(ys)-1))
    print(f"   monotone rising: {mono}   (U-shape would need a bump at depth 0)\\n")
ax.set_ylim(-4, 104)
style(ax, "needle depth  (0 = window start, 1 = next to the question)",
      "accuracy (%)", f"Depth profile, pooled over lengths <= {POOL_MAX}")
ax.legend(frameon=False, fontsize=9, labelcolor=S["ink2"], loc="upper left")
plt.tight_layout(); plt.show()
""")

    md("""
## 5. Backward retrieval — `val_backward`
Given the *number*, name the *key*. The answer space is 35 words, so chance is
~2.9% — a far easier task than recalling a 7-digit number, yet it failed outright
under the buggy cache. This checks whether the corrected parameterisation moved it.
""")

    code("""
t2 = "val_backward"
print(f"{'context':>8} {'matched-old':>15} {'matched-new':>15} {'delta':>11}")
print("-"*54)
for L in LENS:
    o, on = cell("old", L, t2, "base")
    n_, nn = cell("new", L, t2, "base_c")
    os_ = f"{o:5.1f} +-{wilson(o*on/100.0,on):4.1f}" if o is not None else "        -"
    ns_ = f"{n_:5.1f} +-{wilson(n_*nn/100.0,nn):4.1f}" if n_ is not None else "        -"
    ds  = f"{n_-o:+7.1f}pp" if (o is not None and n_ is not None) else "        -"
    print(f"{klabel(L):>8} {os_:>15} {ns_:>15} {ds:>11}")
print()
print("chance with a 35-word answer space is ~2.9%")
""")

    md("""
## 5b. The IN2 claim — do the arms beat the base, under a correct cache?
The arms all branch independently from `run7'`, 400 steps each:
| arm | training data | role |
|---|---|---|
| **A** | plain text, no injection | **drift control** — isolates "400 more steps" |
| **B** | needle immediately before its probe | the objective change at distance ~0 |
| **C** | needle uniform far from its probe | **distance — the actual IN2 claim** |
The claim under test is **C > B** (distance matters), with **A** subtracted out as
drift. Under the buggy cache every arm came out *below* the untrained base. This
asks whether a correct rotation changes that verdict.
""")

    code("""
ARMKEYS = [("base", "base_c", "base (run7 / run7')"),
           ("arm_a", "armA_c", "arm A  (control)"),
           ("arm_b", "armB_c", "arm B  (d=0)"),
           ("arm_c", "armC_c", "arm C  (far)")]
POOL = [L for L in LENS if L <= 4096]   # where both conditions have real signal
print(f"niah_single, pooled over {POOL} (matched conditions)")
print(f"{'model':<24} {'old (cache 1024)':>18} {'new (cache 4096)':>18} {'delta':>10}")
print("-"*74)
got = {}
for ok, nk, lab in ARMKEYS:
    oh = on_ = nh = nn_ = 0
    for L in POOL:
        v, n = cell("old", L, task, ok)
        if v is not None: oh += v*n/100.0; on_ += n
        v, n = cell("new", L, task, nk)
        if v is not None: nh += v*n/100.0; nn_ += n
    o = 100.0*oh/on_ if on_ else None
    n_ = 100.0*nh/nn_ if nn_ else None
    got[nk] = (n_, nn_)
    os_ = f"{o:5.1f} +-{wilson(oh,on_):4.1f}" if o is not None else "         -"
    ns_ = f"{n_:5.1f} +-{wilson(nh,nn_):4.1f}" if n_ is not None else "         -"
    ds  = f"{n_-o:+6.1f}pp" if (o is not None and n_ is not None) else "        -"
    print(f"{lab:<24} {os_:>18} {ns_:>18} {ds:>10}")
print()
b, nb_ = got.get("armB_c", (None,0))
c, nc_ = got.get("armC_c", (None,0))
if b is not None and c is not None:
    d = c - b
    comb = math.hypot(wilson(b*nb_/100.0, nb_), wilson(c*nc_/100.0, nc_))
    print(f"IN2 claim, arm C - arm B = {d:+.1f}pp   (combined 95% bar +-{comb:.1f}pp)")
    print(f"  -> {'RESOLVABLE' if abs(d) > comb else 'NOT resolvable at this N'}")
    a, na_ = got.get("armA_c", (None,0))
    if a is not None:
        print(f"  drift control arm A = {a:.1f}%; C - A = {c-a:+.1f}pp")
        print("  (C must beat A, not just B, for distance training to have added anything)")
else:
    print("arm B and/or arm C not yet evaluated -- rerun once finish_all.sh completes")
""")

    md("""
## 6. Reading this honestly
The numbers above answer the narrow question — *does the corrected cache raise
retrieval, measured generatively, under matched train/eval* — and nothing wider.
Three limits bound them:
**The headline is one cell, not the whole curve.** 1k-8k are unchanged within
error (+3.0, +0.0, -3.0, -2.0 pp, none resolvable). The only resolvable change is
at the full window: **0.0% -> 27.0%, +27.0pp**. So the corrected parameterisation
converted total failure at 16k into partial capability; it did **not** extend the
range where retrieval works most of the time (50% reach 6,987 -> 6,703 tok, i.e.
flat). Read as reach: the 10% threshold moves from ~14.3k to never crossing inside
the window, while the 50% threshold does not move.
**The loss confound is largely answered — in favour of the cache explanation.**
`run7'` finished at training loss **2.384** where the original run7 reached
**1.283**, and it was still descending at step 750. Before this eval, two readings
fit: the corrected geometry is genuinely harder to fit, or `run7'` is undertrained.
The data now discriminates. A model that were merely *undertrained* would be worse
or equal everywhere; `run7'` is instead **+27pp at 16k** while matching the old
model at every shorter length, and its depth-0.00 cell (the position farthest from
the question) rises 28.3% -> 37.0%. Higher LM loss together with better long-range
retrieval is what the buggy cache predicts: with `orig_maxpos=1024` the ramp
compressed positions past ~4k, which makes next-token prediction *easier* -- a 16k
window is presented as something much shorter -- while destroying the ability to
tell distant positions apart. The loss numbers are measuring different input
geometries and do not compare; the retrieval numbers do.
Residual caveat: `run7'` was still improving at step 750, so its 1k-8k cells could
be understated relative to a fully converged run. That direction of error would
only strengthen the conclusion, not reverse it.
**No U-shape, in any condition.** All three models rise monotonically from depth
0.00 to 1.00 with no primacy bump. This is pure recency, not lost-in-the-middle,
and correcting the cache did not introduce a primacy effect -- it lifted the whole
curve (most at 16k, where the old model had nothing to lift).
**Backward retrieval is untouched.** `val_backward` sits at 0-4% everywhere in both
conditions, against ~2.9% chance for a 35-word answer space. The model resolves
key->value and not value->key, and that is independent of the RoPE fix.
**Sample size.** N=100 per cell gives a 95% Wilson half-width near +-10pp at
mid-range rates, and the old sweep cells were N=50 (+-~12pp). The `resolvable`
column in section 2 compares each delta against both cells' bars in quadrature and
is the guard against reading anything smaller as an effect.
""")

    nb = nbf.v4.new_notebook()
    nb["cells"] = C
    nb.metadata["kernelspec"] = {"name": "python3", "display_name": "Python 3",
                                 "language": "python"}

    print("  executing...")
    NotebookClient(nb, timeout=300, kernel_name="python3",
                   resources={"metadata": {"path": os.getcwd()}}).execute()
    nbf.write(nb, args.out)
    print(f"  wrote {args.out} ({os.path.getsize(args.out)} B, {len(C)} cells)")


if __name__ == "__main__":
    main()