| { |
| "experiment": "CV4 Phase 1 \u2014 capability calibration across the ladder, all six families", |
| "prereg": "docs/experiments/preregs/CV4_capability_ladder.md", |
| "status": "MEASUREMENT ONLY \u2014 Phase 1 gates which model later phases target; CV4's registered rule is adjudicated in Phase 3, not here", |
| "readout_convention": "STRIPPED (corrected)", |
| "n_items_total": 551, |
| "n_items_with_target": 255, |
| "measures": { |
| "answer_rank": "rank of the expected answer's first token; defined only where `target` is non-null (multihop, order-ops, multilingual)", |
| "prompt_surprisal": "mean per-token NLL of the prompt; defined for all six families. NOT a task-competence measure \u2014 it separates 'cannot do the task' from 'cannot parse the prompt'" |
| }, |
| "note_on_undefined": "poetry / typo / association carry target: null. The annotated intermediate is the object of interest and there is no separate answer, so ANSWER COMPETENCE IS UNDEFINED for them, not zero.", |
| "interval": "95% Wilson score interval on every rate", |
| "by_model": { |
| "70m": { |
| "multihop": { |
| "n_items": 93, |
| "mean_prompt_surprisal": 4.742897197764407, |
| "median_prompt_surprisal": 4.761600017547607, |
| "n_scored_for_answer": 93, |
| "top1_count": 1, |
| "top1_rate": 0.010752688172043012, |
| "top1_wilson95": [ |
| 0.001900593097955601, |
| 0.05842059118648415 |
| ], |
| "top10_count": 13, |
| "top10_rate": 0.13978494623655913, |
| "top10_wilson95": [ |
| 0.08354643741721157, |
| 0.22460212683616715 |
| ], |
| "median_rank": 80 |
| }, |
| "multilingual": { |
| "n_items": 107, |
| "mean_prompt_surprisal": 5.743064506031643, |
| "median_prompt_surprisal": 5.629157066345215, |
| "n_scored_for_answer": 107, |
| "top1_count": 1, |
| "top1_rate": 0.009345794392523364, |
| "top1_wilson95": [ |
| 0.0016516291511190044, |
| 0.05105060538898146 |
| ], |
| "top10_count": 9, |
| "top10_rate": 0.08411214953271028, |
| "top10_wilson95": [ |
| 0.044881436613206184, |
| 0.15217093361602183 |
| ], |
| "median_rank": 3436 |
| }, |
| "order-ops": { |
| "n_items": 55, |
| "mean_prompt_surprisal": 4.685458686135032, |
| "median_prompt_surprisal": 4.4597907066345215, |
| "n_scored_for_answer": 55, |
| "top1_count": 2, |
| "top1_rate": 0.03636363636363636, |
| "top1_wilson95": [ |
| 0.010029330855724924, |
| 0.1232369297490173 |
| ], |
| "top10_count": 21, |
| "top10_rate": 0.38181818181818183, |
| "top10_wilson95": [ |
| 0.26515766317415235, |
| 0.513910207176076 |
| ], |
| "median_rank": 18 |
| }, |
| "poetry": { |
| "n_items": 98, |
| "mean_prompt_surprisal": 5.154343254712163, |
| "median_prompt_surprisal": 5.116718769073486, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "typo": { |
| "n_items": 96, |
| "mean_prompt_surprisal": 6.320249448219935, |
| "median_prompt_surprisal": 6.34007453918457, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "association": { |
| "n_items": 102, |
| "mean_prompt_surprisal": 4.577553527027953, |
| "median_prompt_surprisal": 4.60029411315918, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "_pooled_answer": { |
| "n": 255, |
| "top1_count": 4, |
| "top1_rate": 0.01568627450980392, |
| "top1_wilson95": [ |
| 0.006116456526536161, |
| 0.03963198576403846 |
| ], |
| "top10_rate": 0.16862745098039217, |
| "top10_wilson95": [ |
| 0.12766685656458393, |
| 0.2194241828973882 |
| ] |
| }, |
| "_pooled_surprisal_all_six": 5.203927769981855 |
| }, |
| "160m": { |
| "multihop": { |
| "n_items": 93, |
| "mean_prompt_surprisal": 4.323448388807235, |
| "median_prompt_surprisal": 4.265374183654785, |
| "n_scored_for_answer": 93, |
| "top1_count": 3, |
| "top1_rate": 0.03225806451612903, |
| "top1_wilson95": [ |
| 0.011030412935116203, |
| 0.09059533467748004 |
| ], |
| "top10_count": 24, |
| "top10_rate": 0.25806451612903225, |
| "top10_wilson95": [ |
| 0.17998399469266027, |
| 0.35533966786557925 |
| ], |
| "median_rank": 42 |
| }, |
| "multilingual": { |
| "n_items": 107, |
| "mean_prompt_surprisal": 6.095417096236042, |
| "median_prompt_surprisal": 5.124428749084473, |
| "n_scored_for_answer": 107, |
| "top1_count": 1, |
| "top1_rate": 0.009345794392523364, |
| "top1_wilson95": [ |
| 0.0016516291511190044, |
| 0.05105060538898146 |
| ], |
| "top10_count": 9, |
| "top10_rate": 0.08411214953271028, |
| "top10_wilson95": [ |
| 0.044881436613206184, |
| 0.15217093361602183 |
| ], |
| "median_rank": 3788 |
| }, |
| "order-ops": { |
| "n_items": 55, |
| "mean_prompt_surprisal": 4.33481783433394, |
| "median_prompt_surprisal": 4.353231430053711, |
| "n_scored_for_answer": 55, |
| "top1_count": 0, |
| "top1_rate": 0.0, |
| "top1_wilson95": [ |
| 0.0, |
| 0.06528714378942788 |
| ], |
| "top10_count": 24, |
| "top10_rate": 0.43636363636363634, |
| "top10_wilson95": [ |
| 0.31373238636001305, |
| 0.5673041592131869 |
| ], |
| "median_rank": 13 |
| }, |
| "poetry": { |
| "n_items": 98, |
| "mean_prompt_surprisal": 4.700447340400851, |
| "median_prompt_surprisal": 4.699031829833984, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "typo": { |
| "n_items": 96, |
| "mean_prompt_surprisal": 5.784147493541241, |
| "median_prompt_surprisal": 5.752850532531738, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "association": { |
| "n_items": 102, |
| "mean_prompt_surprisal": 4.123158244525685, |
| "median_prompt_surprisal": 4.120728969573975, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "_pooled_answer": { |
| "n": 255, |
| "top1_count": 4, |
| "top1_rate": 0.01568627450980392, |
| "top1_wilson95": [ |
| 0.006116456526536161, |
| 0.03963198576403846 |
| ], |
| "top10_rate": 0.2235294117647059, |
| "top10_wilson95": [ |
| 0.1767132210847063, |
| 0.27855208402853665 |
| ] |
| }, |
| "_pooled_surprisal_all_six": 4.893572732974166 |
| }, |
| "410m": { |
| "multihop": { |
| "n_items": 93, |
| "mean_prompt_surprisal": 3.8603221395964264, |
| "median_prompt_surprisal": 3.814849376678467, |
| "n_scored_for_answer": 93, |
| "top1_count": 6, |
| "top1_rate": 0.06451612903225806, |
| "top1_wilson95": [ |
| 0.029901024142029796, |
| 0.13368156846280116 |
| ], |
| "top10_count": 31, |
| "top10_rate": 0.3333333333333333, |
| "top10_wilson95": [ |
| 0.24582243182201943, |
| 0.4340672023847677 |
| ], |
| "median_rank": 31 |
| }, |
| "multilingual": { |
| "n_items": 107, |
| "mean_prompt_surprisal": 4.675777234763743, |
| "median_prompt_surprisal": 4.748653411865234, |
| "n_scored_for_answer": 107, |
| "top1_count": 1, |
| "top1_rate": 0.009345794392523364, |
| "top1_wilson95": [ |
| 0.0016516291511190044, |
| 0.05105060538898146 |
| ], |
| "top10_count": 6, |
| "top10_rate": 0.056074766355140186, |
| "top10_wilson95": [ |
| 0.025950008789329404, |
| 0.11697106055647577 |
| ], |
| "median_rank": 3044 |
| }, |
| "order-ops": { |
| "n_items": 55, |
| "mean_prompt_surprisal": 4.2336343418468125, |
| "median_prompt_surprisal": 4.275657653808594, |
| "n_scored_for_answer": 55, |
| "top1_count": 5, |
| "top1_rate": 0.09090909090909091, |
| "top1_wilson95": [ |
| 0.039457573336373924, |
| 0.19577736249133978 |
| ], |
| "top10_count": 30, |
| "top10_rate": 0.5454545454545454, |
| "top10_wilson95": [ |
| 0.41522461348360484, |
| 0.669749282535538 |
| ], |
| "median_rank": 9 |
| }, |
| "poetry": { |
| "n_items": 98, |
| "mean_prompt_surprisal": 4.148344592172272, |
| "median_prompt_surprisal": 4.126254558563232, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "typo": { |
| "n_items": 96, |
| "mean_prompt_surprisal": 5.49502004434665, |
| "median_prompt_surprisal": 5.293105602264404, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "association": { |
| "n_items": 102, |
| "mean_prompt_surprisal": 3.8331370657565547, |
| "median_prompt_surprisal": 3.8432066440582275, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "_pooled_answer": { |
| "n": 255, |
| "top1_count": 12, |
| "top1_rate": 0.047058823529411764, |
| "top1_wilson95": [ |
| 0.02712132140633029, |
| 0.08044098697068483 |
| ], |
| "top10_rate": 0.2627450980392157, |
| "top10_wilson95": [ |
| 0.21253225697118083, |
| 0.3200003807501127 |
| ] |
| }, |
| "_pooled_surprisal_all_six": 4.374372569747076 |
| }, |
| "1b": { |
| "multihop": { |
| "n_items": 93, |
| "mean_prompt_surprisal": 3.7122991469598587, |
| "median_prompt_surprisal": 3.713099241256714, |
| "n_scored_for_answer": 93, |
| "top1_count": 7, |
| "top1_rate": 0.07526881720430108, |
| "top1_wilson95": [ |
| 0.036936214085016475, |
| 0.14729866018389276 |
| ], |
| "top10_count": 31, |
| "top10_rate": 0.3333333333333333, |
| "top10_wilson95": [ |
| 0.24582243182201943, |
| 0.4340672023847677 |
| ], |
| "median_rank": 18 |
| }, |
| "multilingual": { |
| "n_items": 107, |
| "mean_prompt_surprisal": 4.4941605663745205, |
| "median_prompt_surprisal": 4.559407711029053, |
| "n_scored_for_answer": 107, |
| "top1_count": 0, |
| "top1_rate": 0.0, |
| "top1_wilson95": [ |
| 0.0, |
| 0.03465846757895952 |
| ], |
| "top10_count": 7, |
| "top10_rate": 0.06542056074766354, |
| "top10_wilson95": [ |
| 0.032047896397421095, |
| 0.12891693990952502 |
| ], |
| "median_rank": 1649 |
| }, |
| "order-ops": { |
| "n_items": 55, |
| "mean_prompt_surprisal": 4.145381216569381, |
| "median_prompt_surprisal": 4.083983898162842, |
| "n_scored_for_answer": 55, |
| "top1_count": 4, |
| "top1_rate": 0.07272727272727272, |
| "top1_wilson95": [ |
| 0.028643596503688543, |
| 0.17260178091636796 |
| ], |
| "top10_count": 37, |
| "top10_rate": 0.6727272727272727, |
| "top10_wilson95": [ |
| 0.5410297617609947, |
| 0.7818710431117484 |
| ], |
| "median_rank": 7 |
| }, |
| "poetry": { |
| "n_items": 98, |
| "mean_prompt_surprisal": 3.935050759996687, |
| "median_prompt_surprisal": 3.910858154296875, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "typo": { |
| "n_items": 96, |
| "mean_prompt_surprisal": 5.290679027636846, |
| "median_prompt_surprisal": 5.2639923095703125, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "association": { |
| "n_items": 102, |
| "mean_prompt_surprisal": 3.6671034354789582, |
| "median_prompt_surprisal": 3.6480932235717773, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "_pooled_answer": { |
| "n": 255, |
| "top1_count": 11, |
| "top1_rate": 0.043137254901960784, |
| "top1_wilson95": [ |
| 0.024254970635041635, |
| 0.07558060448116843 |
| ], |
| "top10_rate": 0.29411764705882354, |
| "top10_wilson95": [ |
| 0.24157991671293233, |
| 0.3527665870948018 |
| ] |
| }, |
| "_pooled_surprisal_all_six": 4.207445692169375 |
| }, |
| "1.4b": { |
| "multihop": { |
| "n_items": 93, |
| "mean_prompt_surprisal": 3.5891012683991463, |
| "median_prompt_surprisal": 3.6213531494140625, |
| "n_scored_for_answer": 93, |
| "top1_count": 5, |
| "top1_rate": 0.053763440860215055, |
| "top1_wilson95": [ |
| 0.023180332422992958, |
| 0.11974997851775977 |
| ], |
| "top10_count": 33, |
| "top10_rate": 0.3548387096774194, |
| "top10_wilson95": [ |
| 0.26512726316736557, |
| 0.45606693436757817 |
| ], |
| "median_rank": 24 |
| }, |
| "multilingual": { |
| "n_items": 107, |
| "mean_prompt_surprisal": 4.305595877014588, |
| "median_prompt_surprisal": 4.307002067565918, |
| "n_scored_for_answer": 107, |
| "top1_count": 0, |
| "top1_rate": 0.0, |
| "top1_wilson95": [ |
| 0.0, |
| 0.03465846757895952 |
| ], |
| "top10_count": 8, |
| "top10_rate": 0.07476635514018691, |
| "top10_wilson95": [ |
| 0.03836964356083345, |
| 0.1406389597072536 |
| ], |
| "median_rank": 1345 |
| }, |
| "order-ops": { |
| "n_items": 55, |
| "mean_prompt_surprisal": 4.166462430087003, |
| "median_prompt_surprisal": 4.244797229766846, |
| "n_scored_for_answer": 55, |
| "top1_count": 3, |
| "top1_rate": 0.05454545454545454, |
| "top1_wilson95": [ |
| 0.01872287596750008, |
| 0.14853294304489928 |
| ], |
| "top10_count": 29, |
| "top10_rate": 0.5272727272727272, |
| "top10_wilson95": [ |
| 0.39791319071629383, |
| 0.6530711468951917 |
| ], |
| "median_rank": 7 |
| }, |
| "poetry": { |
| "n_items": 98, |
| "mean_prompt_surprisal": 3.6910850977411074, |
| "median_prompt_surprisal": 3.6510846614837646, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "typo": { |
| "n_items": 96, |
| "mean_prompt_surprisal": 5.277833836774032, |
| "median_prompt_surprisal": 5.16269588470459, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "association": { |
| "n_items": 102, |
| "mean_prompt_surprisal": 3.6086690916734585, |
| "median_prompt_surprisal": 3.6544148921966553, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "_pooled_answer": { |
| "n": 255, |
| "top1_count": 8, |
| "top1_rate": 0.03137254901960784, |
| "top1_wilson95": [ |
| 0.01598082278729794, |
| 0.060674552546496924 |
| ], |
| "top10_rate": 0.27450980392156865, |
| "top10_wilson95": [ |
| 0.22338671792303816, |
| 0.33232611958067076 |
| ] |
| }, |
| "_pooled_surprisal_all_six": 4.106457933614889 |
| }, |
| "2.8b": { |
| "multihop": { |
| "n_items": 93, |
| "mean_prompt_surprisal": 3.5109470018776516, |
| "median_prompt_surprisal": 3.531287670135498, |
| "n_scored_for_answer": 93, |
| "top1_count": 6, |
| "top1_rate": 0.06451612903225806, |
| "top1_wilson95": [ |
| 0.029901024142029796, |
| 0.13368156846280116 |
| ], |
| "top10_count": 40, |
| "top10_rate": 0.43010752688172044, |
| "top10_wilson95": [ |
| 0.33423358533237524, |
| 0.5315265838511162 |
| ], |
| "median_rank": 14 |
| }, |
| "multilingual": { |
| "n_items": 107, |
| "mean_prompt_surprisal": 4.153321266174316, |
| "median_prompt_surprisal": 4.131369590759277, |
| "n_scored_for_answer": 107, |
| "top1_count": 1, |
| "top1_rate": 0.009345794392523364, |
| "top1_wilson95": [ |
| 0.0016516291511190044, |
| 0.05105060538898146 |
| ], |
| "top10_count": 10, |
| "top10_rate": 0.09345794392523364, |
| "top10_wilson95": [ |
| 0.05155771001191554, |
| 0.1635384271784534 |
| ], |
| "median_rank": 578 |
| }, |
| "order-ops": { |
| "n_items": 55, |
| "mean_prompt_surprisal": 4.071762145649303, |
| "median_prompt_surprisal": 4.097884178161621, |
| "n_scored_for_answer": 55, |
| "top1_count": 8, |
| "top1_rate": 0.14545454545454545, |
| "top1_wilson95": [ |
| 0.07559199016771778, |
| 0.2616116208829674 |
| ], |
| "top10_count": 34, |
| "top10_rate": 0.6181818181818182, |
| "top10_wilson95": [ |
| 0.4860897928239239, |
| 0.7348423368258477 |
| ], |
| "median_rank": 7 |
| }, |
| "poetry": { |
| "n_items": 98, |
| "mean_prompt_surprisal": 3.6883874936979644, |
| "median_prompt_surprisal": 3.6747243404388428, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "typo": { |
| "n_items": 96, |
| "mean_prompt_surprisal": 5.161082645257314, |
| "median_prompt_surprisal": 5.04867696762085, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "association": { |
| "n_items": 102, |
| "mean_prompt_surprisal": 3.4943248594508454, |
| "median_prompt_surprisal": 3.5100271701812744, |
| "answer_competence": "UNDEFINED \u2014 target is null for this family" |
| }, |
| "_pooled_answer": { |
| "n": 255, |
| "top1_count": 15, |
| "top1_rate": 0.058823529411764705, |
| "top1_wilson95": [ |
| 0.03596805781169063, |
| 0.09477445034773968 |
| ], |
| "top10_rate": 0.32941176470588235, |
| "top10_wilson95": [ |
| 0.2746295550144033, |
| 0.3892575481405764 |
| ] |
| }, |
| "_pooled_surprisal_all_six": 4.0133042353512325 |
| } |
| }, |
| "VERDICT": "Answer competence, 255 items with a target, 95% Wilson: 70m: top1 1.6% [0.6,4.0] | 160m: top1 1.6% [0.6,4.0] | 410m: top1 4.7% [2.7,8.0] | 1b: top1 4.3% [2.4,7.6] | 1.4b: top1 3.1% [1.6,6.1] | 2.8b: top1 5.9% [3.6,9.5]. Prompt surprisal covers all 551 items in all six families.", |
| "provenance": { |
| "schema_version": 1, |
| "experiment": "CV4P1", |
| "utc": "2026-08-30T06:30:11+00:00", |
| "script": { |
| "path": "experiments/cv4_phase1_capability.py", |
| "sha256": "3aaadc49c285d30fe1aafb92714633b4d5e694caac40f82b0afbfa964a424cfb", |
| "n_lines": 173 |
| }, |
| "git": { |
| "commit": "c9c49697e295c0fec4225552836fc9733b279f75", |
| "commit_short": "c9c49697e295", |
| "git_repository_present": true, |
| "branch": "pythia", |
| "dirty": true, |
| "dirty_files": [ |
| "CLAUDE.md", |
| "README.md", |
| "corpora/manifest_ood.json", |
| "docs/HANDOFF.md", |
| "docs/REPO_REVIEW_PROMPT.md", |
| "docs/context/AGGREGATION_POLICY.md", |
| "docs/context/CONTEXT.md", |
| "docs/context/IMPLEMENTATION_RULES.md", |
| "docs/context/STATE.md", |
| "docs/experiments/INDEX.md", |
| "docs/experiments/METRICS.md", |
| "docs/experiments/PROPOSED.md", |
| "docs/experiments/PROVENANCE.md", |
| "docs/experiments/descriptions/E64.md", |
| "docs/experiments/descriptions/EN1.md", |
| "docs/experiments/preregs/CV3_margins.md", |
| "docs/experiments/preregs/CV4_capability_ladder.md", |
| "docs/experiments/preregs/CV5_metric_sensitivity.md", |
| "docs/experiments/preregs/CV6_per_family_ladder.md", |
| "docs/experiments/preregs/CV7_1b_rung.md", |
| "docs/experiments/preregs/DN1_design_nulls.md", |
| "docs/experiments/preregs/E33_logit_baseline.md", |
| "docs/experiments/preregs/E36_qladder.md", |
| "docs/experiments/preregs/E48_crossover.md", |
| "docs/experiments/preregs/E48b_containment.md", |
| "docs/experiments/preregs/E48c_exposure_vs_read.md", |
| "docs/experiments/preregs/E51_interaction.md", |
| "docs/experiments/preregs/E52_factorial.md", |
| "docs/experiments/preregs/E53_ladder_summary.md", |
| "docs/experiments/preregs/E55_matrix_robustness.md", |
| "docs/experiments/preregs/E56_predictor_registry.md", |
| "docs/experiments/preregs/E57_grid_variance_ci.md", |
| "docs/experiments/preregs/E58_algebra_audit.md", |
| "docs/experiments/preregs/E59_read_dose_response.md", |
| "docs/experiments/preregs/E60_fitter_determinism.md", |
| "docs/experiments/preregs/E61_randomized_network_null.md", |
| "docs/experiments/preregs/E62_ladder1b_corrected_band.md", |
| "docs/experiments/preregs/E65_training_axis_floor.md", |
| "docs/experiments/preregs/E66b_determinism_floor.md", |
| "docs/experiments/preregs/OS1_operator_space.md", |
| "docs/experiments/preregs/P0_t52_parallel_port.md", |
| "docs/experiments/preregs/R1_grid_rstrip.md", |
| "docs/experiments/preregs/R2_e48_rstrip_arm.md", |
| "docs/experiments/preregs/R3_ladder410_cpu.md", |
| "docs/experiments/preregs/R4_corrections.md", |
| "docs/experiments/preregs/R4b_e36_rstrip.md", |
| "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", |
| "docs/experiments/preregs/R6_within_source_resampling.md", |
| "docs/experiments/preregs/R8_ladder_rstrip_S2.md", |
| "docs/experiments/preregs/R9_permutation_calibrated_min.md", |
| "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", |
| "docs/reproducibility/ARTIFACTS.md", |
| "docs/reproducibility/PAYLOAD_REDACTIONS.md", |
| "docs/reproducibility/RELEASE.md", |
| "docs/reproducibility/REVIEWER_REPRO.md", |
| "docs/reproducibility/T1_FINDING_E33_BAND.md", |
| "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", |
| "docs/reproducibility/T1_SWEEP_SPEC.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", |
| "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" |
| ], |
| "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", |
| "describe": "c9c4969-dirty" |
| }, |
| "argv": [ |
| "experiments/cv4_phase1_capability.py", |
| "--models", |
| "70m,160m,410m,1b,1.4b,2.8b" |
| ], |
| "cwd": ".", |
| "inputs": [ |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", |
| "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", |
| "bytes": 21869, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", |
| "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", |
| "bytes": 24284, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", |
| "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", |
| "bytes": 9589, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-poetry.json", |
| "sha256": "6aeb3415c5a5c3f3827c9efe63f006de02f5ef39a816bbac68e15e733aba60cc", |
| "bytes": 21533, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-typo.json", |
| "sha256": "9d05e16b7234a57d0773d120a4e1c4e94fd3bc2235a8125d4200a70e60ab17aa", |
| "bytes": 14646, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-association.json", |
| "sha256": "d1a98cd4911b594282e74168091c77d849dae18ffe2acb5761074853f327d71c", |
| "bytes": 24228, |
| "exists": true |
| } |
| ], |
| "env": { |
| "python": "3.14.2", |
| "torch": "2.13.0", |
| "transformers": "5.14.1", |
| "numpy": "2.5.1", |
| "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "hostname": "<redacted for double-blind review>", |
| "tf32": { |
| "allow_tf32_matmul": false, |
| "allow_tf32_cudnn": true, |
| "NVIDIA_TF32_OVERRIDE": null |
| }, |
| "threads": { |
| "torch_num_threads": 10, |
| "torch_num_interop_threads": 14, |
| "OMP_NUM_THREADS": null, |
| "os_cpu_count": 14 |
| } |
| }, |
| "payload_sha256": "43b21658bcc6a1acf619b614a5eb74db9252372b3ff7f151fbbde0ce7fbd9bad" |
| } |
| } |
