Upload scripts/mk_final_results.py with huggingface_hub

scripts/mk_final_results.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Every figure from 2026-09-16. All numbers read from the published eval CSVs."""
+import csv,os,sys,math
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+E=sys.argv[1]
+S={"surface":"#fcfcfb","ink":"#0b0b0b","ink2":"#52514e","grid":"#e4e4e1"}
+def sty(ax):
+    ax.set_facecolor(S["surface"])
+    for s in ax.spines.values(): s.set_color(S["grid"])
+    ax.grid(True,color=S["grid"],lw=.7,alpha=.9); ax.tick_params(colors=S["ink2"],labelsize=9)
+def read(n,model,task="niah_single"):
+    f=os.path.join(E,f"{n}_summary.csv")
+    if not os.path.exists(f): f=os.path.join(E,n)
+    if not os.path.exists(f): return None
+    r=list(csv.reader(open(f)))
+    if len(r)<2: return None
+    h=r[0]
+    for row in r[1:]:
+        if row and row[0]==task and model in h:
+            try: return float(row[h.index(model)])
+            except Exception: return None
+    return None
+SRC={(4,1024):"eshort_cache16k_len1024",(8,1024):"eshort_cache32k_len1024",
+     (4,4096):"eshort_cache16k_len4096",(8,4096):"eshort_cache32k_len4096",
+     (16,4096):"e64k_len4096",(32,4096):"e128k_len4096",(64,4096):"fin_x64_4096",
+     (8,8192):"e32k_len8192",(8,16360):"e32k_len16360",(8,32744):"e32k_len32744",
+     (16,16360):"e64k_len16360",(16,32744):"e64k_len32744",(16,65512):"e64k_len65512",
+     (32,32744):"e128k_len32744",(32,65512):"e128k_len65512",
+     (64,32744):"fin_x64_32744",(64,65512):"fin_x64_65512"}
+base={};p1={}
+for (sc,pl),n in SRC.items():
+    b=read(n,"base680"); p=read(n,"p1final")
+    if b is not None: base[(sc,pl)]=b
+    if p is not None: p1[(sc,pl)]=p
+base[(4,8192)]=84.0; base[(4,16360)]=40.0      # prior box, same haystack
+p1[(4,8192)]=77.0;  p1[(4,16360)]=42.0
+
+fig=plt.figure(figsize=(19.5,11.4),facecolor=S["surface"])
+gs=fig.add_gridspec(2,3,hspace=.32,wspace=.26)
+PL=[1024,4096,8192,16360,32744,65512]
+cols=plt.cm.viridis([i/(len(PL)-1) for i in range(len(PL))])
+
+axA=fig.add_subplot(gs[0,0])
+for pl,c in zip(PL,cols):
+    pts=sorted((s,v) for (s,p),v in base.items() if p==pl)
+    if len(pts)>1: axA.plot([x[0] for x in pts],[x[1] for x in pts],"o-",color=c,lw=2,ms=6,label=f"{pl:,}")
+sty(axA); axA.set_xscale("log",base=2); axA.set_xticks([4,8,16,32,64])
+axA.set_xticklabels(["x4","x8","x16","x32","x64"]); axA.set_ylim(0,100)
+axA.set_xlabel("YARN_SCALE (ORIG fixed at 4096)",color=S["ink2"],fontsize=9.5)
+axA.set_ylabel("niah_single (%)",color=S["ink2"],fontsize=9.5)
+axA.set_title("A. Same untrained weights, only RoPE scale changes",color=S["ink"],fontsize=11,loc="left",pad=9)
+axA.legend(frameon=False,fontsize=7.6,labelcolor=S["ink2"],loc="lower right",title="tokens",title_fontsize=7.6)
+
+axB=fig.add_subplot(gs[0,1])
+for pl,c in zip(PL,cols):
+    pts=sorted((pl/s,v) for (s,p),v in base.items() if p==pl)
+    if pts: axB.plot([x[0] for x in pts],[x[1] for x in pts],"o-",color=c,lw=2,ms=6,label=f"{pl:,}")
+sty(axB); axB.set_xscale("log",base=2); axB.set_ylim(0,100)
+axB.axvspan(2000,4400,color="#f3eceb",zorder=0)
+axB.text(2950,6,"pretrain p90\n~2,721 tok",ha="center",fontsize=7.2,color=S["ink2"],style="italic")
+axB.set_xlabel("virtual position of last token (real / scale)",color=S["ink2"],fontsize=9.5)
+axB.set_ylabel("niah_single (%)",color=S["ink2"],fontsize=9.5)
+axB.set_title("B. Collapses onto virtual position; spread = distractor load",color=S["ink"],fontsize=11,loc="left",pad=9)
+
+axC=fig.add_subplot(gs[0,2])
+ks=[4,8,16,32,64]
+sums=[]
+for k in ks:
+    d=[p1[(k,p)]-base[(k,p)] for p in PL if (k,p) in p1 and (k,p) in base]
+    sums.append(sum(d) if d else 0)
+axC.bar(range(len(ks)),sums,color=["#b3243b" if s<0 else "#12508f" for s in sums])
+axC.axhline(0,color=S["ink2"],lw=.9)
+sty(axC); axC.set_xticks(range(len(ks))); axC.set_xticklabels([f"x{k}" for k in ks])
+axC.set_xlabel("evaluation config",color=S["ink2"],fontsize=9.5)
+axC.set_ylabel("p1final - base680, summed (pp)",color=S["ink2"],fontsize=9.5)
+axC.set_title("C. Training's value is INVISIBLE at x4, clear at x64",color=S["ink"],fontsize=11,loc="left",pad=9)
+for i,s in enumerate(sums): axC.text(i,s+(1.4 if s>=0 else -3),f"{s:+.0f}",ha="center",fontsize=9,color=S["ink2"])
+
+axD=fig.add_subplot(gs[1,0])
+X=[0,1,2]; lab=["16k","32k","64k"]
+axD.plot(X,[70,50,31],"o--",color="#b0b0aa",lw=1.6,ms=6,label="1.06B, own config")
+axD.plot(X,[78,73,65],"o-",color="#b3243b",lw=2.4,ms=7,label="1.06B, corrected")
+axD.plot(X,[66,70,65],"s--",color="#eb6834",lw=1.6,ms=6,label='1.06B "collapsed", corrected')
+axD.plot(X,[92,79,75],"^-",color="#12508f",lw=2.4,ms=8,label="680M base, NO long-ctx training")
+sty(axD); axD.set_xticks(X); axD.set_xticklabels(lab); axD.set_ylim(0,100)
+axD.set_xlabel("prompt length",color=S["ink2"],fontsize=9.5)
+axD.set_ylabel("niah_single (%)",color=S["ink2"],fontsize=9.5)
+axD.set_title("D. Control: training wins at the length it trained (64k)",color=S["ink"],fontsize=11,loc="left",pad=9)
+axD.legend(frameon=False,fontsize=7.8,labelcolor=S["ink2"],loc="lower left")
+
+axE=fig.add_subplot(gs[1,1])
+tasks=["niah_single","niah_multikey","val_backward"]
+b=[79,42,3]; p=[90,39,1]; m=[73,36,0]
+w=.26; xs=range(len(tasks))
+axE.bar([x-w for x in xs],b,w,color="#9dc3ea",label="680M base")
+axE.bar(list(xs),p,w,color="#12508f",label="680M p1final")
+axE.bar([x+w for x in xs],m,w,color="#b3243b",label="1.06B trained to 64k")
+sty(axE); axE.set_xticks(list(xs)); axE.set_xticklabels(tasks,fontsize=8.6); axE.set_ylim(0,100)
+axE.set_ylabel("accuracy (%)",color=S["ink2"],fontsize=9.5)
+axE.set_title("E. The gains are on the EASY task only",color=S["ink"],fontsize=11,loc="left",pad=9)
+axE.legend(frameon=False,fontsize=7.8,labelcolor=S["ink2"],loc="upper right")
+axE.text(1.75,8,"Qwen3-0.6B ~24%\non val_backward",fontsize=7.4,color=S["ink2"],style="italic",ha="center")
+
+axF=fig.add_subplot(gs[1,2])
+st=[0,800,1000,1200]; sc=[90,49,55,51]
+axF.plot(st,sc,"o-",color="#b3243b",lw=2.4,ms=8)
+axF.axhline(90,color="#12508f",ls="--",lw=1.4)
+axF.text(600,92,"p1final parent (90)",fontsize=8,color="#12508f")
+axF.fill_between([0,1200],[49,49],[55,55],color="#f3eceb",zorder=0)
+sty(axF); axF.set_ylim(0,100)
+axF.set_xlabel("32k-training step",color=S["ink2"],fontsize=9.5)
+axF.set_ylabel("niah_single @32,744, x64 (%)",color=S["ink2"],fontsize=9.5)
+axF.set_title("F. Training INSIDE the compressed range destroys it",color=S["ink"],fontsize=11,loc="left",pad=9)
+axF.annotate("already collapsed by step 800,\nthen flat -- fewer steps would\nnot have helped",xy=(800,49),xytext=(250,20),
+             fontsize=7.8,color=S["ink2"],arrowprops=dict(arrowstyle="->",color=S["ink2"],lw=1))
+fig.text(.005,.005,"680M niah_single unless noted. N=100 per point, greedy, identical haystack (199,754,694 B) across every measurement. "
+         "Wilson 95% CI is +/-9-13pp at N=100: differences inside a bar are not differences.",fontsize=8,color=S["ink2"])
+fig.savefig("results_2026-09-16/all_results.png",dpi=150,bbox_inches="tight")
+print("  wrote results_2026-09-16/all_results.png")
+print(f"  base680 points: {len(base)}   p1final points: {len(p1)}")