Upload scripts/mk_plots_2026-09-23.py with huggingface_hub

scripts/mk_plots_2026-09-23.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""2026-09-23: t128d falsified, effective context measured, base680 ceiling mapped.
+Every value read from summary.csv on disk -- nothing retyped."""
+import csv, glob, json, os, sys
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+R = os.path.dirname(os.path.abspath(__file__)); CP = os.path.dirname(R)
+S={"surface":"#fcfcfb","ink":"#0b0b0b","ink2":"#52514e","grid":"#e4e4e1"}
+BASE,T64,C,D,PUR="#9dc3ea","#12508f","#b3243b","#2f9e6e","#6b3fa0"
+def sty(a):
+    a.set_facecolor(S["surface"])
+    for s in a.spines.values(): s.set_color(S["grid"])
+    a.grid(True,color=S["grid"],lw=.7,alpha=.9); a.tick_params(colors=S["ink2"],labelsize=9)
+
+def read(d):
+    p=os.path.join(CP,d,"summary.csv")
+    if not os.path.isfile(p): return {}
+    r=[l.strip().replace("\r","") for l in open(p) if l.strip()]
+    if len(r)<2: return {}
+    h=r[0].split(","); v=r[1].split(",")
+    return {h[i]:float(v[i]) for i in range(1,len(h)) if v[i] not in ("","-")}
+
+# ---- gather
+LEN=[4096,8192,16360,32744,65512,131000]
+# NOTE: ed_* holds t128d (eval_t128d.sh staged t128d_s*.ckpt). t128c at 4,096
+# and 32,744 comes from ecc_* (filled in on this box); the rest from en_/el_.
+c300={4096:read("ecc_4096").get("c300"),8192:read("en_8192").get("c300"),
+      16360:read("en_16360").get("c300"),32744:read("ecc_32744").get("c300"),
+      65512:read("el_65512").get("c300"),131000:read("el_131000").get("c300")}
+d360={4096:read("ed_4096").get("s360"),8192:read("en_8192").get("d360"),
+      16360:read("en_16360").get("d360"),32744:read("ed_32744").get("s360"),
+      65512:read("el_65512").get("d360"),131000:read("el_131000").get("d360")}
+basex={L:read(f"bx32_{L}").get("base") for L in LEN}
+steps=[100,200,300]
+tc={"32744":[read("ec_early_32744").get("c100"),read("ec_early_32744").get("c200"),read("ed_32744").get("s300")],
+    "4096":[read("ec_early_4096").get("c100"),read("ec_early_4096").get("c200"),read("ed_4096").get("s300")]}
+td={"32744":[read("ed_32744").get(f"s{s}") for s in steps],
+    "4096":[read("ed_4096").get(f"s{s}") for s in steps]}
+bm={}
+for f in glob.glob(os.path.join(CP,"bm_s*")):
+    n=os.path.basename(f); _,s,l=n.split("_"); v=read(n).get("base")
+    if v is not None: bm[(int(s[1:]),int(l))]=v
+# loss
+loss=[]
+for l in open(os.path.join(CP,"t128d_logs/train.log"),errors="replace"):
+    if l.startswith("step "):
+        try: loss.append((int(l.split("|")[0].split()[1]), float(l.split("loss:")[1].split("|")[0])))
+        except: pass
+def trail(d,w=50):
+    return [s for s,_ in d],[sum(x for _,x in d[max(0,i-w+1):i+1])/len(d[max(0,i-w+1):i+1]) for i in range(len(d))]
+
+fig=plt.figure(figsize=(19.4,11.6),facecolor=S["surface"])
+gs=fig.add_gridspec(2,3,hspace=.44,wspace=.27)
+
+# A -- effective context
+a=fig.add_subplot(gs[0,0]); X=range(len(LEN))
+for dd,col,lab,mk in ((c300,C,"t128c s300 (b6 35%)","o"),(d360,D,"t128d s360 (b6 60%)","s"),
+                      (basex,BASE,"base680 (no long-ctx training)","^")):
+    ys=[dd.get(L) for L in LEN]; xs=[x for x,y in zip(X,ys) if y is not None]; vs=[y for y in ys if y is not None]
+    a.plot(xs,vs,f"{mk}-",color=col,lw=2.6,ms=8,label=lab)
+    for x,y in zip(xs,vs): a.text(x,y+2.6,f"{y:.0f}",ha="center",fontsize=8.4,color=col,weight="bold")
+a.axhline(85.6,color="#c2413f",ls="--",lw=1.4)
+a.text(0.05,87,"RULER effective-length threshold 85.6%",fontsize=8,color="#c2413f",style="italic")
+a.axvspan(3.5,5.5,color="#f6e9e9",zorder=0)
+sty(a); a.set_xticks(list(X)); a.set_xticklabels(["4k","8k","16k","32k","64k","131k"])
+a.set_ylim(15,110); a.set_ylabel("niah_single (%)",color=S["ink2"],fontsize=10)
+a.set_xlabel("prompt length (T=131,072  YaRN x32  ORIG=4,096, fixed)",color=S["ink2"],fontsize=9.5)
+a.set_title("A. Effective context is 32k, not 128k",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+a.legend(frameon=False,fontsize=8.2,labelcolor=S["ink2"],loc="lower left")
+
+# B -- extension DID help (matched config)
+b=fig.add_subplot(gs[0,1])
+ls=[L for L in LEN if basex.get(L) is not None and c300.get(L) is not None]
+w=.36; xs=range(len(ls))
+b.bar([x-w/2 for x in xs],[basex[L] for L in ls],w,color=BASE,label="base680")
+b.bar([x+w/2 for x in xs],[c300[L] for L in ls],w,color=C,label="t128c s300")
+for i,L in enumerate(ls):
+    b.text(i-w/2,basex[L]+1.5,f"{basex[L]:.0f}",ha="center",fontsize=8.6,color=BASE,weight="bold")
+    b.text(i+w/2,c300[L]+1.5,f"{c300[L]:.0f}",ha="center",fontsize=8.6,color=C,weight="bold")
+    b.text(i,6,f"{c300[L]-basex[L]:+.0f}",ha="center",fontsize=10,color=S["ink"],weight="bold")
+sty(b); b.set_xticks(list(xs)); b.set_xticklabels([f"{L//1000}k" if L>=8192 else "4k" for L in ls])
+b.set_ylim(0,112); b.set_ylabel("niah_single (%)",color=S["ink2"],fontsize=10)
+dmin=min(c300[L]-basex[L] for L in ls); dmax=max(c300[L]-basex[L] for L in ls)
+b.set_title(f"B. Extension vs base680, matched config ({dmin:+.0f} to {dmax:+.0f})",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+b.legend(frameon=False,fontsize=8.6,labelcolor=S["ink2"],loc="upper right")
+
+# C -- t128d falsified
+c=fig.add_subplot(gs[0,2]); xs=range(3)
+c.plot(list(xs),tc["32744"],"o-",color=C,lw=2.8,ms=9,label="t128c  b6 35%")
+c.plot(list(xs),td["32744"],"s-",color=D,lw=2.8,ms=9,label="t128d  b6 60%")
+for x,y in zip(xs,tc["32744"]): c.text(x,y+2.4,f"{y:.0f}",ha="center",fontsize=9,color=C,weight="bold")
+for x,y in zip(xs,td["32744"]): c.text(x,y-4.2,f"{y:.0f}",ha="center",fontsize=9,color=D,weight="bold")
+sty(c); c.set_xticks(list(xs)); c.set_xticklabels([f"step {s}" for s in steps])
+c.set_ylim(50,102); c.set_ylabel("niah_single @32,744 (%)",color=S["ink2"],fontsize=10)
+c.set_title("C. More long-range data made it WORSE",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+c.legend(frameon=False,fontsize=8.6,labelcolor=S["ink2"],loc="lower left")
+c.text(.97,.94,"inverted U:  8.9%->69   35%->92   60%->62",transform=c.transAxes,
+       fontsize=8,color=S["ink2"],ha="right",style="italic")
+
+# D -- base680 map, virtual vs real
+d4=fig.add_subplot(gs[1,0])
+groups={1024:[(1,1024),(2,2048),(4,4096),(8,8192)],
+        2048:[(1,2048),(2,4096),(4,8192),(8,16360)],
+        4096:[(1,4096),(2,8192),(4,16360),(8,32744)]}
+for virt,cells,col in ((1024,groups[1024],D),(2048,groups[2048],"#eb9a34"),(4096,groups[4096],C)):
+    pts=[(l,bm[(s,l)]) for s,l in cells if (s,l) in bm]
+    d4.plot([p[0] for p in pts],[p[1] for p in pts],"o-",color=col,lw=2.6,ms=8,label=f"virtual ~{virt}")
+    for x,y in pts: d4.text(x,y+2.4,f"{y:.0f}",ha="center",fontsize=8.2,color=col,weight="bold")
+sty(d4); d4.set_xscale("log",base=2); d4.set_ylim(25,105)
+d4.set_xlabel("real prompt length (log)",color=S["ink2"],fontsize=10)
+d4.set_ylabel("base680 niah_single (%)",color=S["ink2"],fontsize=10)
+d4.set_title("D. base680: virtual position decides, real length barely",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+d4.legend(frameon=False,fontsize=8.6,labelcolor=S["ink2"],loc="lower left")
+
+# E -- scale=1 diagonal
+e=fig.add_subplot(gs[1,1])
+diag=[(l,bm[(1,l)]) for l in (512,1024,2048,3072,4096) if (1,l) in bm]
+e.plot([p[0] for p in diag],[p[1] for p in diag],"o-",color=PUR,lw=3,ms=10)
+for x,y in diag: e.text(x,y+2.6,f"{y:.0f}",ha="center",fontsize=10,color=PUR,weight="bold")
+e.axvline(4096,color=S["ink2"],ls="--",lw=1.2)
+e.text(4096*0.96,36,"nominal pretrain\nwindow = 4,096",fontsize=8,color=S["ink2"],ha="right",style="italic")
+sty(e); e.set_xscale("log",base=2); e.set_ylim(30,105)
+e.set_xlabel("real position (YaRN OFF, virtual == real)",color=S["ink2"],fontsize=10)
+e.set_ylabel("base680 niah_single (%)",color=S["ink2"],fontsize=10)
+e.set_title("E. The base cannot use its own pretrain window",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+e.text(.03,.10,"97 / 96 / 95 at 512-2,048, then 88 at 3,072 and 70 at 4,096.\n"
+       "Usable range ~2,048-3,072 of a nominal 4,096.\nrope_theta=500,000 at a 4,096 pretrain leaves\n"
+       "32 of 64 RoPE dims without a full rotation.",
+       transform=e.transAxes,fontsize=7.8,color=S["ink2"],style="italic")
+
+# F -- t128d loss
+f6=fig.add_subplot(gs[1,2])
+xs,ys=trail(loss)
+f6.plot([s for s,_ in loss],[v for _,v in loss],color=D,lw=.5,alpha=.25)
+f6.plot(xs,ys,color=D,lw=2.6,label="t128d (b6 60%)")
+f6.axvline(300,color=S["ink2"],ls=":",lw=1.2)
+f6.text(305,1.25,"anneal starts",fontsize=8,color=S["ink2"],style="italic")
+sty(f6); f6.set_ylim(1.0,3.2)
+f6.set_xlabel("t128d training step",color=S["ink2"],fontsize=10)
+f6.set_ylabel("training loss (50-step trailing mean)",color=S["ink2"],fontsize=10)
+f6.set_title("F. Flat loss did NOT mean healthy",color=S["ink"],fontsize=11.5,loc="left",pad=9)
+f6.text(.97,.70,"loss 2.53 -> 2.36, perfectly flat --\nand this was the worst run of the four.\n"
+        "Retracts my 'flat loss = healthy' claim.",transform=f6.transAxes,fontsize=8,
+        color=S["ink2"],ha="right",va="top",style="italic")
+
+fig.text(.005,.005,
+ "680M (24L, d=1536, 12q/4kv, tying=1, rope_theta=500,000, pretrained at 4,096). niah_single N=100, greedy, "
+ "haystack 199,754,694 B, identical across every measurement. Wilson 95% CI +/-9-13pp at N=100. "
+ "Panels A-C and F: T=131,072 YaRN x32 ORIG=4,096 held FIXED across lengths (RULER's protocol -- vLLM static YaRN). "
+ "Panels D-E: base680 from navingv/varybatch. base680 at 65,512 and 131,000 still running.",
+ fontsize=7.8,color=S["ink2"])
+out=os.path.join(R,"results_2026-09-23.png"); fig.savefig(out,dpi=150,bbox_inches="tight")
+print("  wrote",out)