Download review_artifacts/scripts/analyze_frequency_baselines.py from randomfolk/behavioral-model-ablations-review: direct link, hf CLI and curl.
- Browser
- Download file 5.14 kB
-
https://huggingface.co/randomfolk/behavioral-model-ablations-review/resolve/main/review_artifacts/scripts/analyze_frequency_baselines.py
- Command line
-
hf download hf://randomfolk/behavioral-model-ablations-review/review_artifacts/scripts/analyze_frequency_baselines.py
-
curl -L -o analyze_frequency_baselines.py https://huggingface.co/randomfolk/behavioral-model-ablations-review/resolve/main/review_artifacts/scripts/analyze_frequency_baselines.py
5.14 kB
| """Static and causal online frequency controls for choice-only prediction.""" | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from mechcal.training.restless_dataset import RestlessTranscriptDataset | |
| from finish_calibration_and_local import nll, interval | |
| DATA = Path('outputs/restless_pooled_eps10_1000_20260827') | |
| OUT = Path('outputs/frequency_baselines_20260915') | |
| def score(p, a): | |
| assert np.isfinite(p).all() and (p > 0).all() and np.allclose(p.sum(-1), 1) | |
| top = p == p.max(-1, keepdims=True) | |
| # Uniform tie breaking avoids an arbitrary preferred arm. | |
| hit = np.take_along_axis(top, a[..., None], -1)[..., 0]/top.sum(-1) | |
| return {'nll': nll(p, a).mean(1), 'argmax_accuracy': hit.mean(1), | |
| 'sampled_expected_accuracy': np.take_along_axis(p, a[..., None], -1)[..., 0].mean(1)} | |
| def main(): | |
| OUT.mkdir(exist_ok=True) | |
| result = {'protocol': 'Test trials 2-200; static parameters estimated on validation only. Online predictors use strictly preceding choices; Dirichlet(1,1,1,1) arm prior and Beta(1,1) stay prior fixed without test tuning. Seed-mean GRU scores, not ensemble. CIs bootstrap participants, condition on fitted models. Argmax accuracy uses uniform tie breaking; sampled accuracy is expected accuracy when drawing from model probabilities.', 'conditions': {}} | |
| for c in ('reward_dominant', 'balanced', 'choice_dominant'): | |
| val = RestlessTranscriptDataset(DATA, 'val', c) | |
| test = RestlessTranscriptDataset(DATA, 'test', c) | |
| va = val.tokens[:, 1::2].numpy()-1; a = test.tokens[:, 1::2].numpy()-1 | |
| assert not set(val.audit.base_participant_id) & set(test.audit.base_participant_id) | |
| n, tmax = a.shape; rows = np.arange(n) | |
| freq = np.bincount(va[:, 1:].ravel(), minlength=4).astype(float) | |
| freq /= freq.sum() | |
| rate = float((va[:, 1:] == va[:, :-1]).mean()) | |
| probs = {'uniform': np.full((n,tmax-1,4), .25), | |
| 'static_arm_frequency': np.broadcast_to(freq, (n,tmax-1,4)).copy(), | |
| 'static_stay_probability': np.full((n,tmax-1,4), (1-rate)/3), | |
| 'online_arm_frequency': np.empty((n,tmax-1,4)), | |
| 'online_stay_probability': np.empty((n,tmax-1,4))} | |
| transition = np.ones((4,4)) | |
| np.add.at(transition, (va[:,:-1].ravel(),va[:,1:].ravel()), 1) | |
| transition /= transition.sum(-1,keepdims=True) | |
| probs['static_transition'] = transition[a[:,:-1]] | |
| probs['online_transition'] = np.empty((n,tmax-1,4)) | |
| lag2_rate = float((va[:,2:]==va[:,:-2]).mean()) | |
| probs['lag2_probability'] = np.full((n,tmax-1,4),(1-lag2_rate)/3) | |
| np.put_along_axis(probs['lag2_probability'][:,1:],a[:,:-2,None],lag2_rate,-1) | |
| probs['lag2_probability'][:,0] = probs['uniform'][:,0] | |
| np.put_along_axis(probs['static_stay_probability'], a[:, :-1, None], rate, -1) | |
| counts = np.ones((n,4)); repeats = np.ones(n); switches = np.ones(n) | |
| transition_counts = np.ones((n,4,4)) | |
| counts[rows,a[:,0]] += 1 | |
| for t in range(1,tmax): | |
| current = transition_counts[rows,a[:,t-1]] | |
| probs['online_transition'][:,t-1] = current/current.sum(-1,keepdims=True) | |
| probs['online_arm_frequency'][:,t-1] = counts/counts.sum(-1,keepdims=True) | |
| s = repeats/(repeats+switches) | |
| probs['online_stay_probability'][:,t-1] = (1-s[:,None])/3 | |
| probs['online_stay_probability'][rows,t-1,a[:,t-1]] = s | |
| # Only update after this trial's predictive distribution is emitted. | |
| counts[rows,a[:,t]] += 1 | |
| transition_counts[rows,a[:,t-1],a[:,t]] += 1 | |
| same = a[:,t] == a[:,t-1] | |
| repeats += same; switches += ~same | |
| metrics = {k: score(p,a[:,1:]) for k,p in probs.items()} | |
| seeds = [] | |
| for seed in (11,22,33): | |
| with np.load(f'outputs/gru_suite_20260907/{c}_choice_only_seed{seed}/predictions.npz') as z: | |
| assert np.array_equal(a,z['actions']) and np.array_equal(test.audit.base_participant_id,z['participants']) | |
| p = z['intact_choice_probabilities'].astype(float)[:,1:] | |
| p /= p.sum(-1,keepdims=True) | |
| seeds.append(score(p,a[:,1:])) | |
| metrics['gru_choice_only'] = {k: np.mean([s[k] for s in seeds],axis=0) for k in seeds[0]} | |
| strata = {} | |
| for name, mask in [('all',np.ones(n,bool)),('positive',test.audit.kernel_sign==1),('negative',test.audit.kernel_sign==-1)]: | |
| strata[name] = {model: {k: interval(v[mask]) for k,v in ms.items()} for model,ms in metrics.items()} | |
| strata[name]['paired_gru_minus_baseline_nll'] = {model: interval((metrics['gru_choice_only']['nll']-ms['nll'])[mask]) for model,ms in metrics.items() if model != 'gru_choice_only'} | |
| result['conditions'][c] = {'validation_arm_proportions':freq.tolist(),'validation_stay_probability':rate,'strata':strata} | |
| print(c, json.dumps({model:{k:float(v.mean()) for k,v in ms.items()} for model,ms in metrics.items()}),flush=True) | |
| (OUT/'summary.json').write_text(json.dumps(result,indent=2)+'\n') | |
| if __name__ == '__main__': main() | |