behavioral-model-ablations-review / review_artifacts /scripts /analyze_frequency_baselines.py
randomfolk's picture
Add anonymous scratch checkpoints, synthetic data, and source
bf71208 verified
Raw History Blame Contribute Delete
5.14 kB
"""Static and causal online frequency controls for choice-only prediction."""
import json
from pathlib import Path
import numpy as np
from mechcal.training.restless_dataset import RestlessTranscriptDataset
from finish_calibration_and_local import nll, interval
DATA = Path('outputs/restless_pooled_eps10_1000_20260827')
OUT = Path('outputs/frequency_baselines_20260915')
def score(p, a):
assert np.isfinite(p).all() and (p > 0).all() and np.allclose(p.sum(-1), 1)
top = p == p.max(-1, keepdims=True)
# Uniform tie breaking avoids an arbitrary preferred arm.
hit = np.take_along_axis(top, a[..., None], -1)[..., 0]/top.sum(-1)
return {'nll': nll(p, a).mean(1), 'argmax_accuracy': hit.mean(1),
'sampled_expected_accuracy': np.take_along_axis(p, a[..., None], -1)[..., 0].mean(1)}
def main():
OUT.mkdir(exist_ok=True)
result = {'protocol': 'Test trials 2-200; static parameters estimated on validation only. Online predictors use strictly preceding choices; Dirichlet(1,1,1,1) arm prior and Beta(1,1) stay prior fixed without test tuning. Seed-mean GRU scores, not ensemble. CIs bootstrap participants, condition on fitted models. Argmax accuracy uses uniform tie breaking; sampled accuracy is expected accuracy when drawing from model probabilities.', 'conditions': {}}
for c in ('reward_dominant', 'balanced', 'choice_dominant'):
val = RestlessTranscriptDataset(DATA, 'val', c)
test = RestlessTranscriptDataset(DATA, 'test', c)
va = val.tokens[:, 1::2].numpy()-1; a = test.tokens[:, 1::2].numpy()-1
assert not set(val.audit.base_participant_id) & set(test.audit.base_participant_id)
n, tmax = a.shape; rows = np.arange(n)
freq = np.bincount(va[:, 1:].ravel(), minlength=4).astype(float)
freq /= freq.sum()
rate = float((va[:, 1:] == va[:, :-1]).mean())
probs = {'uniform': np.full((n,tmax-1,4), .25),
'static_arm_frequency': np.broadcast_to(freq, (n,tmax-1,4)).copy(),
'static_stay_probability': np.full((n,tmax-1,4), (1-rate)/3),
'online_arm_frequency': np.empty((n,tmax-1,4)),
'online_stay_probability': np.empty((n,tmax-1,4))}
transition = np.ones((4,4))
np.add.at(transition, (va[:,:-1].ravel(),va[:,1:].ravel()), 1)
transition /= transition.sum(-1,keepdims=True)
probs['static_transition'] = transition[a[:,:-1]]
probs['online_transition'] = np.empty((n,tmax-1,4))
lag2_rate = float((va[:,2:]==va[:,:-2]).mean())
probs['lag2_probability'] = np.full((n,tmax-1,4),(1-lag2_rate)/3)
np.put_along_axis(probs['lag2_probability'][:,1:],a[:,:-2,None],lag2_rate,-1)
probs['lag2_probability'][:,0] = probs['uniform'][:,0]
np.put_along_axis(probs['static_stay_probability'], a[:, :-1, None], rate, -1)
counts = np.ones((n,4)); repeats = np.ones(n); switches = np.ones(n)
transition_counts = np.ones((n,4,4))
counts[rows,a[:,0]] += 1
for t in range(1,tmax):
current = transition_counts[rows,a[:,t-1]]
probs['online_transition'][:,t-1] = current/current.sum(-1,keepdims=True)
probs['online_arm_frequency'][:,t-1] = counts/counts.sum(-1,keepdims=True)
s = repeats/(repeats+switches)
probs['online_stay_probability'][:,t-1] = (1-s[:,None])/3
probs['online_stay_probability'][rows,t-1,a[:,t-1]] = s
# Only update after this trial's predictive distribution is emitted.
counts[rows,a[:,t]] += 1
transition_counts[rows,a[:,t-1],a[:,t]] += 1
same = a[:,t] == a[:,t-1]
repeats += same; switches += ~same
metrics = {k: score(p,a[:,1:]) for k,p in probs.items()}
seeds = []
for seed in (11,22,33):
with np.load(f'outputs/gru_suite_20260907/{c}_choice_only_seed{seed}/predictions.npz') as z:
assert np.array_equal(a,z['actions']) and np.array_equal(test.audit.base_participant_id,z['participants'])
p = z['intact_choice_probabilities'].astype(float)[:,1:]
p /= p.sum(-1,keepdims=True)
seeds.append(score(p,a[:,1:]))
metrics['gru_choice_only'] = {k: np.mean([s[k] for s in seeds],axis=0) for k in seeds[0]}
strata = {}
for name, mask in [('all',np.ones(n,bool)),('positive',test.audit.kernel_sign==1),('negative',test.audit.kernel_sign==-1)]:
strata[name] = {model: {k: interval(v[mask]) for k,v in ms.items()} for model,ms in metrics.items()}
strata[name]['paired_gru_minus_baseline_nll'] = {model: interval((metrics['gru_choice_only']['nll']-ms['nll'])[mask]) for model,ms in metrics.items() if model != 'gru_choice_only'}
result['conditions'][c] = {'validation_arm_proportions':freq.tolist(),'validation_stay_probability':rate,'strata':strata}
print(c, json.dumps({model:{k:float(v.mean()) for k,v in ms.items()} for model,ms in metrics.items()}),flush=True)
(OUT/'summary.json').write_text(json.dumps(result,indent=2)+'\n')
if __name__ == '__main__': main()