Download review_artifacts/scripts/plot_weight_operation_sweep_s9.py from randomfolk/behavioral-model-ablations-review: direct link, hf CLI and curl.
- Browser
- Download file 7.32 kB
-
https://huggingface.co/randomfolk/behavioral-model-ablations-review/resolve/main/review_artifacts/scripts/plot_weight_operation_sweep_s9.py
- Command line
-
hf download hf://randomfolk/behavioral-model-ablations-review/review_artifacts/scripts/plot_weight_operation_sweep_s9.py
-
curl -L -o plot_weight_operation_sweep_s9.py https://huggingface.co/randomfolk/behavioral-model-ablations-review/resolve/main/review_artifacts/scripts/plot_weight_operation_sweep_s9.py
7.32 kB
| """Audit and plot the seven-weight fixed-model operation sweep (validation only).""" | |
| from __future__ import annotations | |
| import json | |
| import hashlib | |
| from pathlib import Path | |
| import matplotlib | |
| matplotlib.use('Agg') | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| from scipy.special import logsumexp | |
| from run_reference_followup import load, WEIGHTS | |
| from paper_plot_style import weight_axis | |
| T = Path('outputs/weight_operation_sweep_transformer_20260923') | |
| L = Path('outputs/weight_llama_operation_fixedshape_20260924/production') | |
| MAIN = Path('outputs/task1_main_integrated_20260918') | |
| OUT = Path('outputs/weight_operation_sweep_figure_s9_fixedshape_20260924') | |
| OPS = ('donor', 'neutral50', 'placeholder', 'delete') | |
| LLAMA_OPS = OPS + ('history_only',) | |
| COLORS = {'donor':'#475569','neutral50':'#b87826','placeholder':'#7956a6', | |
| 'delete':'#26866b','history_only':'#c65e46'} | |
| def choice_prob(logp): | |
| v = logp.astype(float) | |
| assert np.isfinite(v).all() | |
| p = np.exp(v-logsumexp(v,axis=-1,keepdims=True)) | |
| np.testing.assert_allclose(p.sum(-1), 1., atol=1e-12) | |
| assert (p > 0).all() | |
| return p | |
| def nll(p, actions): | |
| return -np.log(np.take_along_axis(p,actions[...,None],axis=-1)[...,0]) | |
| def main(): | |
| assert (T/'COMPLETE.json').exists() | |
| audit=json.loads((L/'DOWNLOAD_AUDIT.json').read_text()) | |
| assert audit['participant_files'] == 1750 | |
| assert audit['fixed_shape'] and audit['job_array'] == '13530770' | |
| OUT.mkdir(exist_ok=True, parents=True) | |
| summary = {} | |
| max_first_error = 0. | |
| max_future_error = 0. | |
| for w in WEIGHTS: | |
| code = f'{round(w*100):03d}' | |
| d = load(w) | |
| actions = d['action'][:,150:] | |
| trows=[] | |
| for seed in (11,22,33): | |
| with np.load(T/f'transformer_reward_w{code}_full_seed{seed}.npz') as z: | |
| np.testing.assert_array_equal(z['participants'],d['base_participant_id']) | |
| trows.append({op:z[('zero_embedding' if op=='placeholder' else op)+'_delta_nll'].astype(float) | |
| for op in OPS}) | |
| rows=[] | |
| for i in range(250): | |
| with np.load(L/f'reward_w{code}'/f'participant_{i:03d}.npz') as z: | |
| row = dict(z) | |
| assert int(row['index']) == i | |
| assert str(row['participant']) == str(d['base_participant_id'][i]) | |
| np.testing.assert_array_equal(row['actions'],d['action'][i]) | |
| for op in LLAMA_OPS[1:]: | |
| assert row[op+'_logp'].shape == (200,4) | |
| errors=json.loads(str(row['first_choice_errors'])) | |
| assert int(row['fixed_length']) == 2295 | |
| assert int(row['delete_length']) == int(row['fixed_length']) - 200 | |
| assert int(row['history_only_length']) < int(row['fixed_length']) | |
| for op in ('neutral50','placeholder','delete'): | |
| observed=float(np.max(np.abs(row[op+'_logp'][0]-row['intact_logp'][0]))) | |
| np.testing.assert_allclose(observed,errors[op+'_logp'],atol=1e-12) | |
| max_first_error=max(max_first_error,observed) | |
| if i == 0: | |
| future=float(row['future_error']) | |
| assert np.isfinite(future) | |
| max_future_error=max(max_future_error,future) | |
| rows.append(row) | |
| intact=choice_prob(np.stack([r['intact_logp'][150:] for r in rows])) | |
| baseline=nll(intact,actions).mean(1) | |
| with np.load(MAIN/f'reward_w{code}_llama.npz') as z: | |
| np.testing.assert_array_equal(z['participants'],d['base_participant_id']) | |
| np.testing.assert_allclose(z['intact'],intact,atol=1e-7) | |
| donor_delta=z['delta_nll'].astype(float) | |
| lvals={'donor':donor_delta} | |
| for op in LLAMA_OPS[1:]: | |
| p=choice_prob(np.stack([r[op+'_logp'][150:] for r in rows])) | |
| lvals[op]=nll(p,actions).mean(1)-baseline | |
| summary[str(w)]={ | |
| 'Transformer':{op:float(np.mean([r[op].mean() for r in trows])) for op in OPS}, | |
| 'LLaMA SFT':{op:float(lvals[op].mean()) for op in LLAMA_OPS}} | |
| np.savez_compressed(OUT/f'reward_w{code}_vectors.npz', | |
| participants=d['base_participant_id'], | |
| **{f'transformer_seed{seed}_{op}':trows[j][op] for j,seed in enumerate((11,22,33)) for op in OPS}, | |
| **{f'llama_{op}':lvals[op] for op in LLAMA_OPS}) | |
| # Reward edits preserve the first causal prefix; the history-only rewrite | |
| # changes the instruction, so it is excluded from this invariance test. | |
| assert max_first_error < .003, f'First-choice numerical invariance failed: {max_first_error}' | |
| assert max_future_error < .003, f'Future-token invariance failed: {max_future_error}' | |
| (OUT/'summary.json').write_text(json.dumps({'split':'val','weights':list(WEIGHTS), | |
| 'participants_per_weight':250,'scored_trials':'151-200','llama_seeds':1, | |
| 'transformer_seeds':3,'max_first_choice_logp_error':max_first_error, | |
| 'max_future_choice_logp_error':max_future_error,'fixed_inference_length':2295, | |
| 'definition':{'donor':'matched other-participant reward history', | |
| 'neutral50':'all reward numbers replaced by 50', | |
| 'placeholder':'Transformer zero reward embeddings; LLaMA reserved special token', | |
| 'delete':'remove reward tokens and shift subsequent positions', | |
| 'history_only':'LLaMA-only rewrite of instruction and trial text to choice history'}, | |
| 'means':summary},indent=2)) | |
| plt.rcParams.update({'font.family':'DejaVu Sans','font.size':8,'axes.spines.top':False, | |
| 'axes.spines.right':False,'pdf.fonttype':42,'svg.fonttype':'none'}) | |
| fig,axs=plt.subplots(1,2,figsize=(7.1,3.6),sharey=True) | |
| labels={'donor':'Donor rewards','neutral50':'Constant 50', | |
| 'placeholder':'Reward placeholder','delete':'Delete reward tokens', | |
| 'history_only':'Choice-history rewrite'} | |
| for ax,letter,model in zip(axs,'ab',('Transformer','LLaMA SFT')): | |
| for op in (OPS if model=='Transformer' else LLAMA_OPS): | |
| ax.plot(WEIGHTS,[summary[str(w)][model][op] for w in WEIGHTS],'-o', | |
| color=COLORS[op],markersize=3,label=labels[op]) | |
| ax.axhline(0,color='#999',lw=.6) | |
| ax.set_title(letter+' '+model,loc='left',fontsize=9) | |
| weight_axis(ax) | |
| axs[0].set_ylabel(r'$\Delta$NLL (nats / choice)') | |
| fig.legend(*axs[1].get_legend_handles_labels(),frameon=False,fontsize=6.7, | |
| ncol=3,loc='upper center',bbox_to_anchor=(.54,1.0),columnspacing=1.6) | |
| fig.subplots_adjust(left=.085,right=.99,bottom=.17,top=.78,wspace=.16) | |
| for ext in ('pdf','png','svg'): | |
| fig.savefig(OUT/f'FigureS9_operation_sweep.{ext}',dpi=240) | |
| plt.close(fig) | |
| sources=[T/'COMPLETE.json',L/'DOWNLOAD_AUDIT.json',Path(__file__)] | |
| (OUT/'COMPLETE.json').write_text(json.dumps({ | |
| 'scientific_audit_passed':True,'split':'val','test_used':False, | |
| 'participant_exports':1750,'weights':7, | |
| 'max_first_choice_logp_error':max_first_error, | |
| 'max_future_choice_logp_error':max_future_error, | |
| 'source_hashes':{str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in sources} | |
| },indent=2)) | |
| print('COMPLETE',OUT.resolve(),'max_first_choice_error',max_first_error) | |
| if __name__=='__main__': | |
| main() | |