Upload scripts/mk_t64_results.py with huggingface_hub

scripts/mk_t64_results.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""The 64k extension result. Every number read from the published eval CSVs."""
+import csv,os,sys
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+E=sys.argv[1]
+S={"surface":"#fcfcfb","ink":"#0b0b0b","ink2":"#52514e","grid":"#e4e4e1"}
+BASE,P1,T64="#9dc3ea","#12508f","#b3243b"
+def sty(ax):
+    ax.set_facecolor(S["surface"])
+    for s in ax.spines.values(): s.set_color(S["grid"])
+    ax.grid(True,color=S["grid"],lw=.7,alpha=.9); ax.tick_params(colors=S["ink2"],labelsize=9)
+def rd(n,model,task="niah_single"):
+    f=os.path.join(E,n)
+    if not os.path.exists(f): return None
+    r=list(csv.reader(open(f))); h=r[0]
+    for row in r[1:]:
+        if row and row[0]==task and model in h:
+            try: return float(row[h.index(model)])
+            except Exception: return None
+    return None
+def depth(n,model):
+    f=os.path.join(E,n); out={}
+    for r in csv.DictReader(open(f)):
+        if r["model"]==model: out[float(r["depth"])]=float(r["pct"])
+    return out
+
+fig=plt.figure(figsize=(19.4,10.6),facecolor=S["surface"])
+gs=fig.add_gridspec(2,3,hspace=.34,wspace=.26)
+PL=[4096,32744,65512]
+F={4096:"t64ev_x64_4096_summary.csv",32744:"t64ev_x64_32744_summary.csv",65512:"t64ev_x64_65512_summary.csv"}
+
+# A -- the headline
+a=fig.add_subplot(gs[0,0]); X=range(len(PL))
+for m,c,lab,mk in ((("base680"),BASE,"base680 (no long-ctx training)","^"),
+                   ("p1final",P1,"p1final (16k-trained)","s"),("t64",T64,"t64 (64k-trained)","o")):
+    ys=[rd(F[p],m) for p in PL]
+    a.plot(list(X),ys,f"{mk}-",color=c,lw=2.6,ms=9,label=lab)
+    for x,y in zip(X,ys): a.text(x,y+2.2,f"{y:.0f}",ha="center",fontsize=9,color=c,weight="bold")
+sty(a); a.set_xticks(list(X)); a.set_xticklabels(["4k","32k","64k"]); a.set_ylim(60,105)
+a.set_xlabel("prompt length",color=S["ink2"],fontsize=10); a.set_ylabel("niah_single (%)",color=S["ink2"],fontsize=10)
+a.set_title("A. The 64k run: 82 -> 98 at 65,512",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+a.legend(frameon=False,fontsize=8.4,labelcolor=S["ink2"],loc="lower left")
+
+# B -- against everything prior at 65,512
+b=fig.add_subplot(gs[0,1])
+names=["1.06B\nown cfg","1.06B\ncorrected","680M\nbase","680M\np1final","680M\nt64"]
+vals=[31,65,75,82,98]; cols=["#b0b0aa","#b0b0aa",BASE,P1,T64]
+bars=b.bar(range(5),vals,color=cols)
+for i,v in enumerate(vals): b.text(i,v+1.8,str(v),ha="center",fontsize=10,color=S["ink2"],weight="bold")
+sty(b); b.set_xticks(range(5)); b.set_xticklabels(names,fontsize=8.2); b.set_ylim(0,110)
+b.set_ylabel("niah_single @65,512 (%)",color=S["ink2"],fontsize=10)
+b.set_title("B. Everything measured at 64k, ever",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+b.axhline(100,color=S["ink2"],ls=":",lw=1)
+b.text(0.1,101.5,"Qwen3-0.6B saturates here (100)",fontsize=7.8,color=S["ink2"],style="italic")
+
+# C -- the caveat
+c=fig.add_subplot(gs[0,2])
+tasks=["niah_single","niah_multikey","val_backward"]
+bv=[79,38,4]; pv=[90,39,2]; tv=[99,32,0]; w=.26
+for off,v,col,lab in ((-w,bv,BASE,"base680"),(0,pv,P1,"p1final"),(w,tv,T64,"t64")):
+    c.bar([x+off for x in range(3)],v,w,color=col,label=lab)
+sty(c); c.set_xticks(range(3)); c.set_xticklabels(tasks,fontsize=8.6); c.set_ylim(0,105)
+c.set_ylabel("accuracy @32,744 (%)",color=S["ink2"],fontsize=10)
+c.set_title("C. The gain is on the EASY task only",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+c.legend(frameon=False,fontsize=8.4,labelcolor=S["ink2"],loc="upper right")
+c.annotate("t64 7pp BELOW the untrained\nbase (inside CI, but the\ntrajectory agrees)",xy=(1+w,32),xytext=(1.25,58),fontsize=7.6,color=T64,
+           arrowprops=dict(arrowstyle="->",color=T64,lw=1.2))
+
+# D -- multikey depth: where t64 lost
+d=fig.add_subplot(gs[1,0])
+for m,col,lab in (("base680",BASE,"base680"),("p1final",P1,"p1final"),("t64",T64,"t64")):
+    dd=depth("t64ev_hard_32744_depth.csv",m); xs=sorted(dd)
+    d.plot(xs,[dd[x] for x in xs],"o-",color=col,lw=2.2,ms=7,label=lab)
+sty(d); d.set_ylim(0,70); d.set_xlabel("needle depth in context",color=S["ink2"],fontsize=10)
+d.set_ylabel("niah_multikey (%)",color=S["ink2"],fontsize=10)
+d.set_title("D. Depth profile -- SUGGESTIVE ONLY, n=40/cell (+/-14pp)",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+d.text(.02,.93,"t64 below p1final at depths 0/0.25/0.5 (3 of 3),\nbut every gap is inside the CI",
+       transform=d.transAxes,fontsize=7.8,color=S["ink2"],style="italic",va="top")
+d.legend(frameon=False,fontsize=8.4,labelcolor=S["ink2"],loc="upper left")
+
+# E -- trajectory
+e=fig.add_subplot(gs[1,1])
+st=[0,1000,1200]
+e.plot(st,[90,95,99],"o-",color=T64,lw=2.6,ms=9,label="niah_single")
+e.plot(st,[39,31,32],"s--",color="#eb6834",lw=2.6,ms=9,label="niah_multikey")
+sty(e); e.set_ylim(0,105); e.set_xlabel("64k training step (0 = p1final parent)",color=S["ink2"],fontsize=10)
+e.set_ylabel("accuracy @32,744 (%)",color=S["ink2"],fontsize=10)
+e.set_title("E. The trade was complete by step 1000",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+e.legend(frameon=False,fontsize=8.6,labelcolor=S["ink2"],loc="center right")
+for x,y in zip(st,[90,95,99]): e.text(x,y+3,f"{y}",ha="center",fontsize=8.6,color=T64)
+for x,y in zip(st,[39,31,32]): e.text(x,y-7,f"{y}",ha="center",fontsize=8.6,color="#eb6834")
+
+# F -- config still matters at inference
+f6=fig.add_subplot(gs[1,2])
+f6.bar([0,1],[41,98],color=["#b0b0aa",T64],width=.55)
+for i,v in enumerate([41,98]): f6.text(i,v+2.5,str(v),ha="center",fontsize=11,color=S["ink2"],weight="bold")
+sty(f6); f6.set_xticks([0,1])
+f6.set_xticklabels(["x16 (as TRAINED)\nvirtual 4,094","x64 (inference)\nvirtual 1,023"],fontsize=9)
+f6.set_ylim(0,112); f6.set_ylabel("t64 niah_single @65,512 (%)",color=S["ink2"],fontsize=10)
+f6.set_title("F. Inference compression still essential: +57pp",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+fig.text(.005,.005,"680M. N=100 per point, greedy, identical haystack (199,754,694 B) across every measurement and every prior session. "
+         "Wilson 95% CI +/-9-13pp at N=100 (+/-14pp on the 40-sample depth cells).",fontsize=8,color=S["ink2"])
+fig.savefig("results_2026-09-17/t64_results.png",dpi=150,bbox_inches="tight")
+print("  wrote results_2026-09-17/t64_results.png")