fittingcorpus / results /cv1_answer_competence.json
Tarush-AI's picture AnonymousInterpScience's picture
Duplicate from AnonymousInterpScience/pythia-jlens-artifacts
3b151ef
Raw History Blame Contribute Delete
10.2 kB
{
"experiment": "CV1 \u2014 answer competence: can the model do the task at all?",
"status": "MEASUREMENT ONLY \u2014 no registered decision rule",
"why": "existing capability_* fields measure the INTERMEDIATE's rank, not the ANSWER's; t36_qladder.py:483-489 records the registered competence gate as degenerate and unable to fire. This supplies the missing number.",
"readout_convention": "STRIPPED (corrected) \u2014 rank of the target's first token at the final position of the rstripped prompt",
"sets_with_target": [
"multihop",
"order-ops",
"multilingual"
],
"sets_without_target": [
"poetry",
"typo",
"association"
],
"note_on_undefined": "poetry/typo/association carry target: null. The annotated intermediate is the object of interest and there is no separate answer, so answer competence is UNDEFINED for them \u2014 not zero.",
"by_model": {
"70m": {
"multihop": {
"n_scored": 93,
"n_multitoken_answers_scored_on_first_token": 3,
"top1_rate": 0.010752688172043012,
"top10_rate": 0.13978494623655913,
"median_rank": 80,
"mean_rank": 1106.5806451612902,
"median_prob": 0.0015416988171637058
},
"order-ops": {
"n_scored": 55,
"n_multitoken_answers_scored_on_first_token": 0,
"top1_rate": 0.03636363636363636,
"top10_rate": 0.38181818181818183,
"median_rank": 18,
"mean_rank": 22.09090909090909,
"median_prob": 0.006405120715498924
},
"multilingual": {
"n_scored": 107,
"n_multitoken_answers_scored_on_first_token": 92,
"top1_rate": 0.009345794392523364,
"top10_rate": 0.08411214953271028,
"median_rank": 3436,
"mean_rank": 4615.915887850468,
"median_prob": 2.3302003683056682e-05
},
"_pooled": {
"n": 255,
"top1_rate": 0.01568627450980392,
"top10_rate": 0.16862745098039217
}
},
"160m": {
"multihop": {
"n_scored": 93,
"n_multitoken_answers_scored_on_first_token": 3,
"top1_rate": 0.03225806451612903,
"top10_rate": 0.25806451612903225,
"median_rank": 42,
"mean_rank": 628.3655913978495,
"median_prob": 0.0031900564208626747
},
"order-ops": {
"n_scored": 55,
"n_multitoken_answers_scored_on_first_token": 0,
"top1_rate": 0.0,
"top10_rate": 0.43636363636363634,
"median_rank": 13,
"mean_rank": 14.49090909090909,
"median_prob": 0.01120622269809246
},
"multilingual": {
"n_scored": 107,
"n_multitoken_answers_scored_on_first_token": 92,
"top1_rate": 0.009345794392523364,
"top10_rate": 0.08411214953271028,
"median_rank": 3788,
"mean_rank": 4630.429906542056,
"median_prob": 1.6619400412309915e-05
},
"_pooled": {
"n": 255,
"top1_rate": 0.01568627450980392,
"top10_rate": 0.2235294117647059
}
},
"410m": {
"multihop": {
"n_scored": 93,
"n_multitoken_answers_scored_on_first_token": 3,
"top1_rate": 0.06451612903225806,
"top10_rate": 0.3333333333333333,
"median_rank": 31,
"mean_rank": 460.9139784946237,
"median_prob": 0.005166209302842617
},
"order-ops": {
"n_scored": 55,
"n_multitoken_answers_scored_on_first_token": 0,
"top1_rate": 0.09090909090909091,
"top10_rate": 0.5454545454545454,
"median_rank": 9,
"mean_rank": 11.909090909090908,
"median_prob": 0.026608118787407875
},
"multilingual": {
"n_scored": 107,
"n_multitoken_answers_scored_on_first_token": 92,
"top1_rate": 0.009345794392523364,
"top10_rate": 0.056074766355140186,
"median_rank": 3044,
"mean_rank": 4031.233644859813,
"median_prob": 1.0294868843629956e-05
},
"_pooled": {
"n": 255,
"top1_rate": 0.047058823529411764,
"top10_rate": 0.2627450980392157
}
},
"1b": {
"multihop": {
"n_scored": 93,
"n_multitoken_answers_scored_on_first_token": 3,
"top1_rate": 0.07526881720430108,
"top10_rate": 0.3333333333333333,
"median_rank": 18,
"mean_rank": 316.0967741935484,
"median_prob": 0.011097713373601437
},
"order-ops": {
"n_scored": 55,
"n_multitoken_answers_scored_on_first_token": 0,
"top1_rate": 0.07272727272727272,
"top10_rate": 0.6727272727272727,
"median_rank": 7,
"mean_rank": 9.163636363636364,
"median_prob": 0.029681021347641945
},
"multilingual": {
"n_scored": 107,
"n_multitoken_answers_scored_on_first_token": 92,
"top1_rate": 0.0,
"top10_rate": 0.06542056074766354,
"median_rank": 1649,
"mean_rank": 2892.6168224299067,
"median_prob": 3.0693336157128215e-05
},
"_pooled": {
"n": 255,
"top1_rate": 0.043137254901960784,
"top10_rate": 0.29411764705882354
}
}
},
"VERDICT": "Answer competence on the three sets carrying a target. 70m: top1 1.6%, top10 16.9% (n=255 items with a target) | 160m: top1 1.6%, top10 22.4% (n=255 items with a target) | 410m: top1 4.7%, top10 26.3% (n=255 items with a target) | 1b: top1 4.3%, top10 29.4% (n=255 items with a target)",
"provenance": {
"schema_version": 1,
"experiment": "CV1",
"utc": "2026-08-30T06:32:53+00:00",
"script": {
"path": "experiments/cv1_answer_competence.py",
"sha256": "29279dcb364dec36ae9f32438ab9ac607cc8b89983cf20c5796662e9a56a7a21",
"n_lines": 138
},
"git": {
"commit": "c9c49697e295c0fec4225552836fc9733b279f75",
"commit_short": "c9c49697e295",
"git_repository_present": true,
"branch": "pythia",
"dirty": true,
"dirty_files": [
"CLAUDE.md",
"README.md",
"corpora/manifest_ood.json",
"docs/HANDOFF.md",
"docs/REPO_REVIEW_PROMPT.md",
"docs/context/AGGREGATION_POLICY.md",
"docs/context/CONTEXT.md",
"docs/context/IMPLEMENTATION_RULES.md",
"docs/context/STATE.md",
"docs/experiments/INDEX.md",
"docs/experiments/METRICS.md",
"docs/experiments/PROPOSED.md",
"docs/experiments/PROVENANCE.md",
"docs/experiments/descriptions/E64.md",
"docs/experiments/descriptions/EN1.md",
"docs/experiments/preregs/CV3_margins.md",
"docs/experiments/preregs/CV4_capability_ladder.md",
"docs/experiments/preregs/CV5_metric_sensitivity.md",
"docs/experiments/preregs/CV6_per_family_ladder.md",
"docs/experiments/preregs/CV7_1b_rung.md",
"docs/experiments/preregs/DN1_design_nulls.md",
"docs/experiments/preregs/E33_logit_baseline.md",
"docs/experiments/preregs/E36_qladder.md",
"docs/experiments/preregs/E48_crossover.md",
"docs/experiments/preregs/E48b_containment.md",
"docs/experiments/preregs/E48c_exposure_vs_read.md",
"docs/experiments/preregs/E51_interaction.md",
"docs/experiments/preregs/E52_factorial.md",
"docs/experiments/preregs/E53_ladder_summary.md",
"docs/experiments/preregs/E55_matrix_robustness.md",
"docs/experiments/preregs/E56_predictor_registry.md",
"docs/experiments/preregs/E57_grid_variance_ci.md",
"docs/experiments/preregs/E58_algebra_audit.md",
"docs/experiments/preregs/E59_read_dose_response.md",
"docs/experiments/preregs/E60_fitter_determinism.md",
"docs/experiments/preregs/E61_randomized_network_null.md",
"docs/experiments/preregs/E62_ladder1b_corrected_band.md",
"docs/experiments/preregs/E65_training_axis_floor.md",
"docs/experiments/preregs/E66b_determinism_floor.md",
"docs/experiments/preregs/OS1_operator_space.md",
"docs/experiments/preregs/P0_t52_parallel_port.md",
"docs/experiments/preregs/R1_grid_rstrip.md",
"docs/experiments/preregs/R2_e48_rstrip_arm.md",
"docs/experiments/preregs/R3_ladder410_cpu.md",
"docs/experiments/preregs/R4_corrections.md",
"docs/experiments/preregs/R4b_e36_rstrip.md",
"docs/experiments/preregs/R5_corpus_axis_uncertainty.md",
"docs/experiments/preregs/R6_within_source_resampling.md",
"docs/experiments/preregs/R8_ladder_rstrip_S2.md",
"docs/experiments/preregs/R9_permutation_calibrated_min.md",
"docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md",
"docs/reproducibility/ARTIFACTS.md",
"docs/reproducibility/PAYLOAD_REDACTIONS.md",
"docs/reproducibility/RELEASE.md",
"docs/reproducibility/REVIEWER_REPRO.md",
"docs/reproducibility/T1_FINDING_E33_BAND.md",
"docs/reproducibility/T1_SWEEP_PREDICTIONS.md",
"docs/reproducibility/T1_SWEEP_SPEC.md",
"docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256",
"docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md",
"docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md",
"docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md",
"docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md",
"docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md"
],
"dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600",
"describe": "c9c4969-dirty"
},
"argv": [
"experiments/cv1_answer_competence.py",
"--models",
"70m,160m,410m,1b"
],
"cwd": ".",
"inputs": [
{
"path": "jacobian-lens/data/evaluations/lens-eval-multihop.json",
"sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416",
"bytes": 21869,
"exists": true
},
{
"path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json",
"sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560",
"bytes": 9589,
"exists": true
},
{
"path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json",
"sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef",
"bytes": 24284,
"exists": true
}
],
"env": {
"python": "3.14.2",
"torch": "2.13.0",
"transformers": "5.14.1",
"numpy": "2.5.1",
"platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O",
"machine": "arm64",
"hostname": "<redacted for double-blind review>",
"tf32": {
"allow_tf32_matmul": false,
"allow_tf32_cudnn": true,
"NVIDIA_TF32_OVERRIDE": null
},
"threads": {
"torch_num_threads": 10,
"torch_num_interop_threads": 14,
"OMP_NUM_THREADS": null,
"os_cpu_count": 14
}
},
"payload_sha256": "8809019f865fe0939cee28bd5d201f7a842b85b9cb5929440541804f3786e74d"
}
}