260922_fixed_dose_graphs / plot_simple.py
jprivera's picture
upload batch 1/1 from pod 38.80.152.76
f41feee verified
Raw History Blame Contribute Delete
6.71 kB
"""Simple, one-idea-per-figure graphs of the fixed-dose grid."""
import json,numpy as np,matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
G="/workspace/collusion_project_v0/experiments/260922_fixed_dose_graphs"; g=json.load(open(f"{G}/data/grid.json"))
COL={"SFT":"#2a78d6","DPO":"#eb6834","CWS":"#1baf7a","NPO":"#4a3aa7","RRET":"#eda100"}
LABEL={"SFT":"SFT","DPO":"DPO","CWS":"CWS","NPO":"NPO+R","RRET":"retention only"}; METHODS=["SFT","DPO","CWS","NPO","RRET"]
UNSEEN=["med_monitor","code_monitor","sab_monitor","furlong_monitor"]
plt.rcParams.update({"font.size":11,"axes.spines.top":False,"axes.spines.right":False,"axes.edgecolor":"#c3c2b7","xtick.color":"#52514e","ytick.color":"#52514e","figure.facecolor":"#fcfcfb","axes.facecolor":"#fcfcfb","grid.color":"#e6e5df","axes.grid":True,"axes.grid.axis":"y","legend.frameon":False})
def arms(fam,m,k): return [a for a in g if a.startswith(f"{fam}_{m.lower()}_s") and a.endswith(f"_k{k}")]
def coll(a,b): return g[a]["beh"][b]["held-out"]["collusion"]
def unseen(a): return np.mean([coll(a,b) for b in UNSEEN])
def mean(fam,m,k,f):
v=[f(a) for a in arms(fam,m,k)]; return np.mean(v) if v else np.nan
# ---------- S1: one bar chart per family, epoch 3 only, "how much bad behaviour is left"
fig,axs=plt.subplots(1,2,figsize=(12,5),sharey=True)
for ax,fam in zip(axs,("llama","qwen")):
u=unseen(f"{fam}_untreated"); vals=[u]+[mean(fam,m,15,unseen) for m in METHODS]
names=["no repair\n(untreated)"]+[LABEL[m] for m in METHODS]; cols=["#52514e"]+[COL[m] for m in METHODS]
ax.bar(range(len(vals)),vals,color=cols,width=0.65)
for i,v in enumerate(vals): ax.annotate(f"{v:.0f}%",(i,v+2),ha="center",fontsize=11,weight="bold")
ax.set_xticks(range(len(vals))); ax.set_xticklabels(names,fontsize=9.5); ax.set_ylim(0,110)
ax.set_title(("Llama" if fam=="llama" else "Qwen"),loc="left",fontsize=13)
axs[0].set_ylabel("how often it still cheats on the 4 UNSEEN behaviours\n(lower = better)")
fig.suptitle("After the full repair (3 epochs), how much cheating is left on behaviours we never trained on?",x=0.01,ha="left",fontsize=13)
fig.tight_layout(); fig.savefig(f"{G}/figs/S1_whats_left_epoch3.png",dpi=160); plt.close(fig)
# ---------- S2: same, but the trained behaviour next to the unseen ones
fig,axs=plt.subplots(1,2,figsize=(12,5),sharey=True)
w=0.36
for ax,fam in zip(axs,("llama","qwen")):
tr=[coll(f"{fam}_untreated","rh_monitor")]+[mean(fam,m,15,lambda a: coll(a,"rh_monitor")) for m in METHODS]
un=[unseen(f"{fam}_untreated")]+[mean(fam,m,15,unseen) for m in METHODS]
x=np.arange(len(tr))
ax.bar(x-w/2,tr,width=w,color="#2a78d6",label="the behaviour we repaired (rubric gaming)")
ax.bar(x+w/2,un,width=w,color="#eb6834",label="4 behaviours we never touched")
ax.set_xticks(x); ax.set_xticklabels(["no repair"]+[LABEL[m] for m in METHODS],fontsize=9.5); ax.set_ylim(0,110)
ax.set_title(("Llama" if fam=="llama" else "Qwen"),loc="left",fontsize=13)
axs[0].set_ylabel("how often it still cheats (lower = better)"); axs[0].legend(loc="upper right",fontsize=9)
fig.suptitle("Repairing one behaviour: does the fix spread? (3 epochs)",x=0.01,ha="left",fontsize=13)
fig.tight_layout(); fig.savefig(f"{G}/figs/S2_trained_vs_unseen_epoch3.png",dpi=160); plt.close(fig)
# ---------- S3: more training = less cheating (one line per method)
fig,axs=plt.subplots(1,2,figsize=(12,5),sharey=True)
for ax,fam in zip(axs,("llama","qwen")):
u=unseen(f"{fam}_untreated"); ax.plot([0,1,2,3],[u]+[mean(fam,"SFT",k,unseen) for k in (5,10,15)],color="#fcfcfb",lw=0)
for m in METHODS:
ax.plot([0,1,2,3],[u]+[mean(fam,m,k,unseen) for k in (5,10,15)],color=COL[m],marker="o",ms=6,lw=2.5,label=LABEL[m])
ax.set_xticks([0,1,2,3]); ax.set_xticklabels(["no\nrepair","1\nepoch","2\nepochs","3\nepochs"]); ax.set_ylim(0,110)
ax.set_title(("Llama" if fam=="llama" else "Qwen"),loc="left",fontsize=13)
axs[0].set_ylabel("cheating left on unseen behaviours (lower = better)"); axs[1].legend(fontsize=9.5,loc="upper right")
fig.suptitle("More repair training = less cheating everywhere (except where it doesn't)",x=0.01,ha="left",fontsize=13)
fig.tight_layout(); fig.savefig(f"{G}/figs/S3_more_training_helps.png",dpi=160); plt.close(fig)
# ---------- S4: per behaviour, epoch 3, simple grouped bars (one panel per family)
BEH=[("rh_monitor","rubric gaming\n(REPAIRED)"),("med_monitor","medical\nself-approval"),("med_monitor_authors","medical\nauthor cover"),("code_monitor","code\nbackdoor"),("sab_monitor","sabotage\ncover"),("furlong_monitor","furlong\ntarget")]
fig,axs=plt.subplots(2,1,figsize=(13,9),sharex=True)
for ax,fam in zip(axs,("llama","qwen")):
x=np.arange(len(BEH)); w=0.15
ax.bar(x-2.5*w,[coll(f"{fam}_untreated",b) for b,_ in BEH],width=w,color="#52514e",label="no repair")
for j,m in enumerate(METHODS):
ax.bar(x+(j-1.5)*w,[mean(fam,m,15,lambda a,b=b: coll(a,b)) for b,_ in BEH],width=w,color=COL[m],label=LABEL[m])
ax.set_xticks(x); ax.set_xticklabels([n for _,n in BEH],fontsize=9.5); ax.set_ylim(0,110)
ax.set_ylabel("cheating %"); ax.set_title(("Llama" if fam=="llama" else "Qwen"),loc="left",fontsize=13)
axs[0].legend(ncol=6,fontsize=9.5,loc="upper center",bbox_to_anchor=(0.5,1.28))
fig.suptitle("Behaviour by behaviour after 3 epochs (first group = the one we actually repaired)",x=0.01,ha="left",fontsize=13)
fig.tight_layout(); fig.savefig(f"{G}/figs/S4_by_behaviour_simple.png",dpi=160); plt.close(fig)
# ---------- S5: the threshold idea, simplified: only SFT/CWS/retention, colour by whether trained behaviour is fixed
fig,ax=plt.subplots(figsize=(9,5.5))
for fam,mk in (("llama","o"),("qwen","s")):
for m in METHODS:
for k in (5,10,15):
for a in arms(fam,m,k):
x=g[a]["sel"]["unwatched_gamed_honest"]; y=100-unseen(a)-(100-unseen(f"{fam}_untreated"))
y=unseen(f"{fam}_untreated")-unseen(a)
ax.scatter(x,y,color=COL[m],marker=mk,s=55,edgecolor="#fcfcfb",linewidth=0.7,zorder=3)
ax.axvline(85,color="#52514e",lw=1.2,ls="--"); ax.text(84,72,"trained behaviour\nnot yet fixed →",ha="right",fontsize=10,color="#52514e")
ax.set_xlabel("how well the trained behaviour was fixed (%)"); ax.set_ylabel("points of cheating removed on unseen behaviours")
ax.set_xlim(-3,103); ax.set_title("You have to fix the trained behaviour before anything else improves\n(circles = Llama, squares = Qwen; colour = method)",loc="left",fontsize=12)
ax.legend(handles=[plt.Line2D([],[],color=COL[m],marker="o",ls="",label=LABEL[m]) for m in METHODS],fontsize=9.5,loc="upper left")
fig.tight_layout(); fig.savefig(f"{G}/figs/S5_threshold_simple.png",dpi=160); plt.close(fig)
print("ok")