Download review_artifacts/scripts/draw_figure2_baselines.py from randomfolk/behavioral-model-ablations-review: direct link, hf CLI and curl.
- Browser
- Download file 4.82 kB
-
https://huggingface.co/randomfolk/behavioral-model-ablations-review/resolve/main/review_artifacts/scripts/draw_figure2_baselines.py
- Command line
-
hf download hf://randomfolk/behavioral-model-ablations-review/review_artifacts/scripts/draw_figure2_baselines.py
-
curl -L -o draw_figure2_baselines.py https://huggingface.co/randomfolk/behavioral-model-ablations-review/resolve/main/review_artifacts/scripts/draw_figure2_baselines.py
4.82 kB
| """Figure 2 companion: frozen train-fitted controls, no fitting/test access.""" | |
| import json | |
| import hashlib | |
| from pathlib import Path | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| import draw_iclr_main_drafts as draft | |
| ROOT=Path('outputs/task1_with_llama_20260917') | |
| BASE=Path('outputs/task1_diagnostics_20260916/baseline_fits.json') | |
| OUT=Path('outputs/iclr_main_figures_draft_20260921') | |
| def main(): | |
| s=json.loads((ROOT/'summary.json').read_text()) | |
| main=json.loads((draft.T1/'A_summary.json').read_text()) | |
| fits=json.loads(BASE.read_text()) | |
| for w in draft.W: | |
| for m in ('gru','transformer','llama'): | |
| np.testing.assert_allclose(s[str(float(w))]['all'][m+'_choice_nll']['mean'],main[str(float(w))]['all'][m+'_choice_nll']['mean'],atol=1e-12) | |
| draft.style() | |
| fig,axs=plt.subplots(2,2,figsize=(7.1,4.8),sharey=True) | |
| names=[('static_arm_frequency','Training-set choice frequencies'),('static_stay_probability','Training-set stay probability'), | |
| ('static_transition','First-order transition model'),('train_selected_lag','Training-selected lag model')] | |
| for ax,(key,title),letter in zip(axs.flat,names,'abcd'): | |
| for m in ('gru','transformer','llama'): | |
| draft.curve(ax,[s[str(float(w))]['all'][m+'_choice_nll'] for w in draft.W],m) | |
| rows=[s[str(float(w))]['all']['baseline_'+key+'_nll'] for w in draft.W] | |
| ci=np.array([x['ci95'] for x in rows]) | |
| ax.fill_between(draft.W,ci[:,0],ci[:,1],color='#b87826',alpha=.12,lw=0) | |
| ax.plot(draft.W,[x['mean'] for x in rows],color='#b87826',ls='--',marker='s',ms=3.2,label='Behavioral baseline') | |
| ax.axhline(np.log(4),color='#999',ls=':',lw=1,label='Uniform random') | |
| draft.panel(ax,letter,title);ax.set_ylim(.28,1.48) | |
| for ax in axs[:,0]:ax.set_ylabel('Validation choice NLL (nats)') | |
| fig.legend(*axs[0,0].get_legend_handles_labels(),loc='upper center',ncol=5,frameon=False,bbox_to_anchor=(.53,1),columnspacing=1.2,handlelength=1.7) | |
| fig.subplots_adjust(left=.09,right=.985,bottom=.10,top=.83,hspace=.55,wspace=.2) | |
| draft.save(fig,'FigureS1_choice_only_baselines',list(axs.flat)) | |
| lines=['# Figure S1: behavioral baselines accompanying Figure 2','', | |
| 'Choice-only GRU, Transformer and fine-tuned LLaMA compared with four training-fitted, reward-free behavioral baselines in the restless task. Each panel repeats the same neural curves to avoid a crowded combined legend. Orange denotes the baseline named in that panel; gray dotted line is uniform random (NLL = log 4). Oracle is omitted here because it observes rewards.', | |
| '', '## Baselines', | |
| '- Choice frequencies: pooled training-set proportions of A/B/C/D, fixed during evaluation.', | |
| '- Stay probability: pooled training-set repeat probability, assigned to the immediately previous choice; remaining mass divided equally among other arms.', | |
| '- First-order transition model: training-fitted 4 by 4 transition matrix with unit Laplace pseudocount, conditioned on the actual previous choice.', | |
| '- Lag model: lag selected from 1 through 20 using training NLL; repeat probability also fitted on training. Predictions use only the actual choice at the selected past lag. No validation tuning or online parameter updates.', | |
| '', '## Evaluation and uncertainty', | |
| 'Exact same curves as Figure 2 right panel: 250 validation participants per weight, trials 151-200. GRU/Transformer metrics averaged across three seeds; LLaMA one seed. Bands are participant-bootstrap 95% intervals conditional on fitted models. Validation selected neural checkpoints. No sealed-test results are shown.', | |
| '', '## Selected lags', | |
| ', '.join(f"w={w}: lag {v['selected']['lag']}" for w,v in fits.items()), | |
| '', '## Interpretation', | |
| 'Compare against tested baseline families only; predictive superiority does not identify hidden reward recovery or rule out more complex behavioral regularities. Overlapping stay/lag curves are expected when the selected lag is one.'] | |
| violations=[] | |
| for w,v in s.items(): | |
| for m in ('gru','transformer','llama'): | |
| for key,_ in names: | |
| if v['all'][m+'_choice_nll']['mean']>=v['all']['baseline_'+key+'_nll']['mean']:violations.append([w,m,key]) | |
| lines += ['', 'Mean-NLL comparisons not favoring the neural model: '+json.dumps(violations)+'. This is a descriptive comparison, not a multiplicity-adjusted significance claim.'] | |
| (OUT/'FigureS1_caption.md').write_text('\n'.join(lines),encoding='utf-8') | |
| (OUT/'FigureS1_provenance.json').write_text(json.dumps({str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in [ROOT/'summary.json',BASE,draft.T1/'A_summary.json']},indent=2)) | |
| print('Created Figure S1; mean comparison exceptions:',violations) | |
| if __name__=='__main__':main() | |