jprivera's picture
upload batch 1/1 from pod 38.80.152.76
f41feee verified
Raw History Blame Contribute Delete
8 kB
"""Graphs for the fixed-dose grid (seed-0 organisms, 3 data orders, epochs 1/2/3). Source: data/grid.json (06_grade.py output)."""
import json,numpy as np,matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
G="/workspace/collusion_project_v0/experiments/260922_fixed_dose_graphs"; g=json.load(open(f"{G}/data/grid.json"))
COL={"untreated":"#52514e","SFT":"#2a78d6","DPO":"#eb6834","CWS":"#1baf7a","NPO":"#4a3aa7","RRET":"#eda100"}; MK={"untreated":"D","SFT":"o","DPO":"s","CWS":"^","NPO":"P","RRET":"X"}
METHODS=["SFT","DPO","CWS","NPO","RRET"]; LABEL={"SFT":"SFT","DPO":"DPO","CWS":"CWS","NPO":"NPO+R","RRET":"retention only"}
BEH=[("rh_monitor","rubric gaming\n(trained)"),("med_monitor","medical\nself-approval"),("med_monitor_authors","medical\nauthor collusion"),("code_monitor","code\nbackdoor"),("sab_monitor","sabotage\ncover"),("furlong_monitor","furlong\ntarget")]
UNSEEN=["med_monitor","code_monitor","sab_monitor","furlong_monitor"]
plt.rcParams.update({"font.size":10,"axes.spines.top":False,"axes.spines.right":False,"axes.edgecolor":"#c3c2b7","xtick.color":"#52514e","ytick.color":"#52514e","figure.facecolor":"#fcfcfb","axes.facecolor":"#fcfcfb","grid.color":"#e6e5df","axes.grid":True,"axes.grid.axis":"y","legend.frameon":False})
def coll(arm,b): return g[arm]["beh"][b]["held-out"]["collusion"]
def arms(fam,m,k): return [a for a in g if a.startswith(f"{fam}_{m.lower()}_s") and a.endswith(f"_k{k}")]
def unseen(arm): return np.mean([coll(arm,b) for b in UNSEEN])
# ---- fig A: unseen collusion % by method x epoch, per family, with untreated baseline; bars = mean, dots = data-order runs
fig,axs=plt.subplots(1,2,figsize=(13,4.8),sharey=True)
for ax,fam in zip(axs,("llama","qwen")):
u=unseen(f"{fam}_untreated"); ax.axhline(u,color="#52514e",lw=1.2,ls="--"); ax.text(14.2,u-4,f"untreated {u:.0f}",fontsize=8,color="#52514e",ha="right")
x=0
for m in METHODS:
for j,k in enumerate((5,10,15)):
v=[unseen(a) for a in arms(fam,m,k)]
if not v: x+=1; continue
ax.bar(x,np.mean(v),width=0.8,color=COL[m],alpha=0.35+0.3*j,zorder=2)
for i,val in enumerate(v): ax.scatter(x+(i-1)*0.15 if len(v)>1 else x,val,color="#0b0b0b",s=14,zorder=3)
ax.annotate(f"{np.mean(v):.0f}",(x,np.mean(v)+1.5),ha="center",fontsize=7.5); x+=1
x+=0.7
ticks=[]; labs=[]; x=0
for m in METHODS:
ticks.append(x+1); labs.append(LABEL[m]); x+=3.7
ax.set_xticks(ticks); ax.set_xticklabels(labs); ax.set_ylim(0,105); ax.set_title(f"{'Llama' if fam=='llama' else 'Qwen'} (seed-0 organism)",loc="left",fontsize=11)
axs[0].set_ylabel("collusion % on unseen behaviours\n(mean of 4, held-out wrappers; lower = better)")
fig.suptitle("Unseen collusion left after 1 / 2 / 3 epochs of repair on rubric gaming (light→dark = epoch 1→3; dots = data-order runs)",x=0.01,ha="left",fontsize=11.5); fig.tight_layout(); fig.savefig(f"{G}/figs/figA_unseen_by_method_epoch.png",dpi=160); plt.close(fig)
# ---- fig B: per behaviour at epoch 3, per family (2 rows x 6), untreated bar + methods
fig,axs=plt.subplots(2,6,figsize=(17,7),sharey=True)
for r,fam in enumerate(("llama","qwen")):
for c,(b,name) in enumerate(BEH):
ax=axs[r,c]; vals={"untreated":[coll(f"{fam}_untreated",b)]}
for m in METHODS: vals[m]=[coll(a,b) for a in arms(fam,m,15)]
for j,(m,v) in enumerate(vals.items()):
if not v: continue
ax.bar(j,np.mean(v),width=0.7,color=COL[m],alpha=0.85,zorder=2)
for i,val in enumerate(v): ax.scatter(j+(i-1)*0.14 if len(v)>1 else j,val,color="#0b0b0b",s=10,zorder=3)
ax.set_xticks(range(len(vals))); ax.set_xticklabels(["untr.","SFT","DPO","CWS","NPO+R","ret."],fontsize=7.5); ax.set_ylim(0,105)
ax.set_title(("Llama: " if r==0 else "Qwen: ")+name.replace("\n"," "),loc="left",fontsize=8.5,color="#2a78d6" if b=="rh_monitor" else "#0b0b0b")
axs[0,0].set_ylabel("collusion % (held-out; lower = better)"); axs[1,0].set_ylabel("collusion % (held-out; lower = better)")
fig.suptitle("Per behaviour after 3 epochs (bar = mean of data-order runs, dots = runs; rubric gaming is the trained behaviour)",x=0.01,ha="left",fontsize=11.5); fig.tight_layout(); fig.savefig(f"{G}/figs/figB_per_behaviour_epoch3.png",dpi=160); plt.close(fig)
# ---- fig C: dose curves: unseen collusion % vs epoch, per family, line per method with min-max band
fig,axs=plt.subplots(1,2,figsize=(12,4.6),sharey=True)
for ax,fam in zip(axs,("llama","qwen")):
u=unseen(f"{fam}_untreated"); ax.axhline(u,color="#52514e",lw=1.2,ls="--",label="untreated")
for m in METHODS:
mu=[];lo=[];hi=[]
for k in (5,10,15):
v=[unseen(a) for a in arms(fam,m,k)]; mu.append(np.mean(v) if v else np.nan); lo.append(min(v) if v else np.nan); hi.append(max(v) if v else np.nan)
ax.fill_between([1,2,3],lo,hi,color=COL[m],alpha=0.15,lw=0); ax.plot([1,2,3],mu,color=COL[m],marker=MK[m],ms=6,lw=2,label=LABEL[m])
ax.set_xticks([1,2,3]); ax.set_xlabel("epochs over the 300-row repair set"); ax.set_ylim(0,105); ax.set_title("Llama" if fam=="llama" else "Qwen",loc="left",fontsize=11)
axs[0].set_ylabel("unseen collusion % (mean of 4; lower = better)"); axs[1].legend(loc="upper right",fontsize=8)
fig.suptitle("Dose curves: unseen collusion vs epochs (band = min–max over data-order runs)",x=0.01,ha="left",fontsize=11.5); fig.tight_layout(); fig.savefig(f"{G}/figs/figC_dose_curves.png",dpi=160); plt.close(fig)
# ---- fig D: caught repair (trained behaviour, held-out wrappers) vs epoch
fig,axs=plt.subplots(1,2,figsize=(12,4.6),sharey=True)
for ax,fam in zip(axs,("llama","qwen")):
u=coll(f"{fam}_untreated","rh_monitor"); ax.axhline(u,color="#52514e",lw=1.2,ls="--",label="untreated")
for m in METHODS:
mu=[np.mean([coll(a,"rh_monitor") for a in arms(fam,m,k)]) if arms(fam,m,k) else np.nan for k in (5,10,15)]
ax.plot([1,2,3],mu,color=COL[m],marker=MK[m],ms=6,lw=2,label=LABEL[m])
ax.set_xticks([1,2,3]); ax.set_xlabel("epochs"); ax.set_ylim(0,105); ax.set_title("Llama" if fam=="llama" else "Qwen",loc="left",fontsize=11)
axs[0].set_ylabel("rubric-gaming collusion % (held-out; lower = repaired)"); axs[1].legend(loc="upper right",fontsize=8)
fig.suptitle("The trained behaviour itself: is rubric gaming repaired at each dose?",x=0.01,ha="left",fontsize=11.5); fig.tight_layout(); fig.savefig(f"{G}/figs/figD_trained_behaviour_by_epoch.png",dpi=160); plt.close(fig)
print("ok")
# ---- fig E: the threshold law — transfer vs repair of the trained behaviour, one point per arm
fig,axs=plt.subplots(1,2,figsize=(13,5.4),sharey=True,sharex=True)
for ax,fam in zip(axs,("llama","qwen")):
for m in METHODS:
for k,mk in zip((5,10,15),("o","s","^")):
for arm in arms(fam,m,k):
x=g[arm]["sel"]["unwatched_gamed_honest"]; y=g[arm]["metrics"]["primary_unseen_removed"]
ax.scatter(x,y,color=COL[m],marker=mk,s=70,zorder=3,edgecolor="#fcfcfb",linewidth=0.8)
ax.axvspan(0,85,color="#e6e5df",alpha=0.55,zorder=0); ax.text(42,96,"trained behaviour not yet repaired",ha="center",fontsize=8.5,color="#52514e")
ax.axvline(85,color="#52514e",lw=1,ls="--",zorder=1)
ax.set_xlim(-3,103); ax.set_ylim(-3,103); ax.set_xlabel("trained behaviour repaired % (rubric gaming, selection set)")
ax.set_title("Llama" if fam=="llama" else "Qwen",loc="left",fontsize=11)
axs[0].set_ylabel("unseen collusion removed % (higher = better)")
h=[plt.Line2D([],[],color=COL[m],marker="o",ls="",label=LABEL[m]) for m in METHODS]+[plt.Line2D([],[],color="#52514e",marker=mk,ls="",label=f"epoch {i+1}") for i,mk in enumerate(("o","s","^"))]
axs[1].legend(handles=h,ncol=2,loc="lower right",fontsize=8)
fig.suptitle("Transfer only starts once the trained behaviour is repaired (each point = one run at one dose)",x=0.01,ha="left",fontsize=12)
fig.tight_layout(); fig.savefig(f"{G}/figs/figE_threshold_transfer_vs_repair.png",dpi=160); plt.close(fig); print("figE ok")