randomfolk's picture
Add anonymous scratch checkpoints, synthetic data, and source
bf71208 verified
Raw History Blame Contribute Delete
4.82 kB
"""Figure 2 companion: frozen train-fitted controls, no fitting/test access."""
import json
import hashlib
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt
import draw_iclr_main_drafts as draft
ROOT=Path('outputs/task1_with_llama_20260917')
BASE=Path('outputs/task1_diagnostics_20260916/baseline_fits.json')
OUT=Path('outputs/iclr_main_figures_draft_20260921')
def main():
s=json.loads((ROOT/'summary.json').read_text())
main=json.loads((draft.T1/'A_summary.json').read_text())
fits=json.loads(BASE.read_text())
for w in draft.W:
for m in ('gru','transformer','llama'):
np.testing.assert_allclose(s[str(float(w))]['all'][m+'_choice_nll']['mean'],main[str(float(w))]['all'][m+'_choice_nll']['mean'],atol=1e-12)
draft.style()
fig,axs=plt.subplots(2,2,figsize=(7.1,4.8),sharey=True)
names=[('static_arm_frequency','Training-set choice frequencies'),('static_stay_probability','Training-set stay probability'),
('static_transition','First-order transition model'),('train_selected_lag','Training-selected lag model')]
for ax,(key,title),letter in zip(axs.flat,names,'abcd'):
for m in ('gru','transformer','llama'):
draft.curve(ax,[s[str(float(w))]['all'][m+'_choice_nll'] for w in draft.W],m)
rows=[s[str(float(w))]['all']['baseline_'+key+'_nll'] for w in draft.W]
ci=np.array([x['ci95'] for x in rows])
ax.fill_between(draft.W,ci[:,0],ci[:,1],color='#b87826',alpha=.12,lw=0)
ax.plot(draft.W,[x['mean'] for x in rows],color='#b87826',ls='--',marker='s',ms=3.2,label='Behavioral baseline')
ax.axhline(np.log(4),color='#999',ls=':',lw=1,label='Uniform random')
draft.panel(ax,letter,title);ax.set_ylim(.28,1.48)
for ax in axs[:,0]:ax.set_ylabel('Validation choice NLL (nats)')
fig.legend(*axs[0,0].get_legend_handles_labels(),loc='upper center',ncol=5,frameon=False,bbox_to_anchor=(.53,1),columnspacing=1.2,handlelength=1.7)
fig.subplots_adjust(left=.09,right=.985,bottom=.10,top=.83,hspace=.55,wspace=.2)
draft.save(fig,'FigureS1_choice_only_baselines',list(axs.flat))
lines=['# Figure S1: behavioral baselines accompanying Figure 2','',
'Choice-only GRU, Transformer and fine-tuned LLaMA compared with four training-fitted, reward-free behavioral baselines in the restless task. Each panel repeats the same neural curves to avoid a crowded combined legend. Orange denotes the baseline named in that panel; gray dotted line is uniform random (NLL = log 4). Oracle is omitted here because it observes rewards.',
'', '## Baselines',
'- Choice frequencies: pooled training-set proportions of A/B/C/D, fixed during evaluation.',
'- Stay probability: pooled training-set repeat probability, assigned to the immediately previous choice; remaining mass divided equally among other arms.',
'- First-order transition model: training-fitted 4 by 4 transition matrix with unit Laplace pseudocount, conditioned on the actual previous choice.',
'- Lag model: lag selected from 1 through 20 using training NLL; repeat probability also fitted on training. Predictions use only the actual choice at the selected past lag. No validation tuning or online parameter updates.',
'', '## Evaluation and uncertainty',
'Exact same curves as Figure 2 right panel: 250 validation participants per weight, trials 151-200. GRU/Transformer metrics averaged across three seeds; LLaMA one seed. Bands are participant-bootstrap 95% intervals conditional on fitted models. Validation selected neural checkpoints. No sealed-test results are shown.',
'', '## Selected lags',
', '.join(f"w={w}: lag {v['selected']['lag']}" for w,v in fits.items()),
'', '## Interpretation',
'Compare against tested baseline families only; predictive superiority does not identify hidden reward recovery or rule out more complex behavioral regularities. Overlapping stay/lag curves are expected when the selected lag is one.']
violations=[]
for w,v in s.items():
for m in ('gru','transformer','llama'):
for key,_ in names:
if v['all'][m+'_choice_nll']['mean']>=v['all']['baseline_'+key+'_nll']['mean']:violations.append([w,m,key])
lines += ['', 'Mean-NLL comparisons not favoring the neural model: '+json.dumps(violations)+'. This is a descriptive comparison, not a multiplicity-adjusted significance claim.']
(OUT/'FigureS1_caption.md').write_text('\n'.join(lines),encoding='utf-8')
(OUT/'FigureS1_provenance.json').write_text(json.dumps({str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in [ROOT/'summary.json',BASE,draft.T1/'A_summary.json']},indent=2))
print('Created Figure S1; mean comparison exceptions:',violations)
if __name__=='__main__':main()