Download results/cv1_answer_competence.json from Tarush-AI/fittingcorpus: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/Tarush-AI/fittingcorpus/resolve/main/results/cv1_answer_competence.json
- Command line
-
hf download hf://Tarush-AI/fittingcorpus/results/cv1_answer_competence.json
-
curl -L -o cv1_answer_competence.json https://huggingface.co/Tarush-AI/fittingcorpus/resolve/main/results/cv1_answer_competence.json
10.2 kB
| { | |
| "experiment": "CV1 \u2014 answer competence: can the model do the task at all?", | |
| "status": "MEASUREMENT ONLY \u2014 no registered decision rule", | |
| "why": "existing capability_* fields measure the INTERMEDIATE's rank, not the ANSWER's; t36_qladder.py:483-489 records the registered competence gate as degenerate and unable to fire. This supplies the missing number.", | |
| "readout_convention": "STRIPPED (corrected) \u2014 rank of the target's first token at the final position of the rstripped prompt", | |
| "sets_with_target": [ | |
| "multihop", | |
| "order-ops", | |
| "multilingual" | |
| ], | |
| "sets_without_target": [ | |
| "poetry", | |
| "typo", | |
| "association" | |
| ], | |
| "note_on_undefined": "poetry/typo/association carry target: null. The annotated intermediate is the object of interest and there is no separate answer, so answer competence is UNDEFINED for them \u2014 not zero.", | |
| "by_model": { | |
| "70m": { | |
| "multihop": { | |
| "n_scored": 93, | |
| "n_multitoken_answers_scored_on_first_token": 3, | |
| "top1_rate": 0.010752688172043012, | |
| "top10_rate": 0.13978494623655913, | |
| "median_rank": 80, | |
| "mean_rank": 1106.5806451612902, | |
| "median_prob": 0.0015416988171637058 | |
| }, | |
| "order-ops": { | |
| "n_scored": 55, | |
| "n_multitoken_answers_scored_on_first_token": 0, | |
| "top1_rate": 0.03636363636363636, | |
| "top10_rate": 0.38181818181818183, | |
| "median_rank": 18, | |
| "mean_rank": 22.09090909090909, | |
| "median_prob": 0.006405120715498924 | |
| }, | |
| "multilingual": { | |
| "n_scored": 107, | |
| "n_multitoken_answers_scored_on_first_token": 92, | |
| "top1_rate": 0.009345794392523364, | |
| "top10_rate": 0.08411214953271028, | |
| "median_rank": 3436, | |
| "mean_rank": 4615.915887850468, | |
| "median_prob": 2.3302003683056682e-05 | |
| }, | |
| "_pooled": { | |
| "n": 255, | |
| "top1_rate": 0.01568627450980392, | |
| "top10_rate": 0.16862745098039217 | |
| } | |
| }, | |
| "160m": { | |
| "multihop": { | |
| "n_scored": 93, | |
| "n_multitoken_answers_scored_on_first_token": 3, | |
| "top1_rate": 0.03225806451612903, | |
| "top10_rate": 0.25806451612903225, | |
| "median_rank": 42, | |
| "mean_rank": 628.3655913978495, | |
| "median_prob": 0.0031900564208626747 | |
| }, | |
| "order-ops": { | |
| "n_scored": 55, | |
| "n_multitoken_answers_scored_on_first_token": 0, | |
| "top1_rate": 0.0, | |
| "top10_rate": 0.43636363636363634, | |
| "median_rank": 13, | |
| "mean_rank": 14.49090909090909, | |
| "median_prob": 0.01120622269809246 | |
| }, | |
| "multilingual": { | |
| "n_scored": 107, | |
| "n_multitoken_answers_scored_on_first_token": 92, | |
| "top1_rate": 0.009345794392523364, | |
| "top10_rate": 0.08411214953271028, | |
| "median_rank": 3788, | |
| "mean_rank": 4630.429906542056, | |
| "median_prob": 1.6619400412309915e-05 | |
| }, | |
| "_pooled": { | |
| "n": 255, | |
| "top1_rate": 0.01568627450980392, | |
| "top10_rate": 0.2235294117647059 | |
| } | |
| }, | |
| "410m": { | |
| "multihop": { | |
| "n_scored": 93, | |
| "n_multitoken_answers_scored_on_first_token": 3, | |
| "top1_rate": 0.06451612903225806, | |
| "top10_rate": 0.3333333333333333, | |
| "median_rank": 31, | |
| "mean_rank": 460.9139784946237, | |
| "median_prob": 0.005166209302842617 | |
| }, | |
| "order-ops": { | |
| "n_scored": 55, | |
| "n_multitoken_answers_scored_on_first_token": 0, | |
| "top1_rate": 0.09090909090909091, | |
| "top10_rate": 0.5454545454545454, | |
| "median_rank": 9, | |
| "mean_rank": 11.909090909090908, | |
| "median_prob": 0.026608118787407875 | |
| }, | |
| "multilingual": { | |
| "n_scored": 107, | |
| "n_multitoken_answers_scored_on_first_token": 92, | |
| "top1_rate": 0.009345794392523364, | |
| "top10_rate": 0.056074766355140186, | |
| "median_rank": 3044, | |
| "mean_rank": 4031.233644859813, | |
| "median_prob": 1.0294868843629956e-05 | |
| }, | |
| "_pooled": { | |
| "n": 255, | |
| "top1_rate": 0.047058823529411764, | |
| "top10_rate": 0.2627450980392157 | |
| } | |
| }, | |
| "1b": { | |
| "multihop": { | |
| "n_scored": 93, | |
| "n_multitoken_answers_scored_on_first_token": 3, | |
| "top1_rate": 0.07526881720430108, | |
| "top10_rate": 0.3333333333333333, | |
| "median_rank": 18, | |
| "mean_rank": 316.0967741935484, | |
| "median_prob": 0.011097713373601437 | |
| }, | |
| "order-ops": { | |
| "n_scored": 55, | |
| "n_multitoken_answers_scored_on_first_token": 0, | |
| "top1_rate": 0.07272727272727272, | |
| "top10_rate": 0.6727272727272727, | |
| "median_rank": 7, | |
| "mean_rank": 9.163636363636364, | |
| "median_prob": 0.029681021347641945 | |
| }, | |
| "multilingual": { | |
| "n_scored": 107, | |
| "n_multitoken_answers_scored_on_first_token": 92, | |
| "top1_rate": 0.0, | |
| "top10_rate": 0.06542056074766354, | |
| "median_rank": 1649, | |
| "mean_rank": 2892.6168224299067, | |
| "median_prob": 3.0693336157128215e-05 | |
| }, | |
| "_pooled": { | |
| "n": 255, | |
| "top1_rate": 0.043137254901960784, | |
| "top10_rate": 0.29411764705882354 | |
| } | |
| } | |
| }, | |
| "VERDICT": "Answer competence on the three sets carrying a target. 70m: top1 1.6%, top10 16.9% (n=255 items with a target) | 160m: top1 1.6%, top10 22.4% (n=255 items with a target) | 410m: top1 4.7%, top10 26.3% (n=255 items with a target) | 1b: top1 4.3%, top10 29.4% (n=255 items with a target)", | |
| "provenance": { | |
| "schema_version": 1, | |
| "experiment": "CV1", | |
| "utc": "2026-08-30T06:32:53+00:00", | |
| "script": { | |
| "path": "experiments/cv1_answer_competence.py", | |
| "sha256": "29279dcb364dec36ae9f32438ab9ac607cc8b89983cf20c5796662e9a56a7a21", | |
| "n_lines": 138 | |
| }, | |
| "git": { | |
| "commit": "c9c49697e295c0fec4225552836fc9733b279f75", | |
| "commit_short": "c9c49697e295", | |
| "git_repository_present": true, | |
| "branch": "pythia", | |
| "dirty": true, | |
| "dirty_files": [ | |
| "CLAUDE.md", | |
| "README.md", | |
| "corpora/manifest_ood.json", | |
| "docs/HANDOFF.md", | |
| "docs/REPO_REVIEW_PROMPT.md", | |
| "docs/context/AGGREGATION_POLICY.md", | |
| "docs/context/CONTEXT.md", | |
| "docs/context/IMPLEMENTATION_RULES.md", | |
| "docs/context/STATE.md", | |
| "docs/experiments/INDEX.md", | |
| "docs/experiments/METRICS.md", | |
| "docs/experiments/PROPOSED.md", | |
| "docs/experiments/PROVENANCE.md", | |
| "docs/experiments/descriptions/E64.md", | |
| "docs/experiments/descriptions/EN1.md", | |
| "docs/experiments/preregs/CV3_margins.md", | |
| "docs/experiments/preregs/CV4_capability_ladder.md", | |
| "docs/experiments/preregs/CV5_metric_sensitivity.md", | |
| "docs/experiments/preregs/CV6_per_family_ladder.md", | |
| "docs/experiments/preregs/CV7_1b_rung.md", | |
| "docs/experiments/preregs/DN1_design_nulls.md", | |
| "docs/experiments/preregs/E33_logit_baseline.md", | |
| "docs/experiments/preregs/E36_qladder.md", | |
| "docs/experiments/preregs/E48_crossover.md", | |
| "docs/experiments/preregs/E48b_containment.md", | |
| "docs/experiments/preregs/E48c_exposure_vs_read.md", | |
| "docs/experiments/preregs/E51_interaction.md", | |
| "docs/experiments/preregs/E52_factorial.md", | |
| "docs/experiments/preregs/E53_ladder_summary.md", | |
| "docs/experiments/preregs/E55_matrix_robustness.md", | |
| "docs/experiments/preregs/E56_predictor_registry.md", | |
| "docs/experiments/preregs/E57_grid_variance_ci.md", | |
| "docs/experiments/preregs/E58_algebra_audit.md", | |
| "docs/experiments/preregs/E59_read_dose_response.md", | |
| "docs/experiments/preregs/E60_fitter_determinism.md", | |
| "docs/experiments/preregs/E61_randomized_network_null.md", | |
| "docs/experiments/preregs/E62_ladder1b_corrected_band.md", | |
| "docs/experiments/preregs/E65_training_axis_floor.md", | |
| "docs/experiments/preregs/E66b_determinism_floor.md", | |
| "docs/experiments/preregs/OS1_operator_space.md", | |
| "docs/experiments/preregs/P0_t52_parallel_port.md", | |
| "docs/experiments/preregs/R1_grid_rstrip.md", | |
| "docs/experiments/preregs/R2_e48_rstrip_arm.md", | |
| "docs/experiments/preregs/R3_ladder410_cpu.md", | |
| "docs/experiments/preregs/R4_corrections.md", | |
| "docs/experiments/preregs/R4b_e36_rstrip.md", | |
| "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", | |
| "docs/experiments/preregs/R6_within_source_resampling.md", | |
| "docs/experiments/preregs/R8_ladder_rstrip_S2.md", | |
| "docs/experiments/preregs/R9_permutation_calibrated_min.md", | |
| "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", | |
| "docs/reproducibility/ARTIFACTS.md", | |
| "docs/reproducibility/PAYLOAD_REDACTIONS.md", | |
| "docs/reproducibility/RELEASE.md", | |
| "docs/reproducibility/REVIEWER_REPRO.md", | |
| "docs/reproducibility/T1_FINDING_E33_BAND.md", | |
| "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", | |
| "docs/reproducibility/T1_SWEEP_SPEC.md", | |
| "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", | |
| "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", | |
| "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", | |
| "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", | |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", | |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" | |
| ], | |
| "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", | |
| "describe": "c9c4969-dirty" | |
| }, | |
| "argv": [ | |
| "experiments/cv1_answer_competence.py", | |
| "--models", | |
| "70m,160m,410m,1b" | |
| ], | |
| "cwd": ".", | |
| "inputs": [ | |
| { | |
| "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", | |
| "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", | |
| "bytes": 21869, | |
| "exists": true | |
| }, | |
| { | |
| "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", | |
| "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", | |
| "bytes": 9589, | |
| "exists": true | |
| }, | |
| { | |
| "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", | |
| "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", | |
| "bytes": 24284, | |
| "exists": true | |
| } | |
| ], | |
| "env": { | |
| "python": "3.14.2", | |
| "torch": "2.13.0", | |
| "transformers": "5.14.1", | |
| "numpy": "2.5.1", | |
| "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", | |
| "machine": "arm64", | |
| "hostname": "<redacted for double-blind review>", | |
| "tf32": { | |
| "allow_tf32_matmul": false, | |
| "allow_tf32_cudnn": true, | |
| "NVIDIA_TF32_OVERRIDE": null | |
| }, | |
| "threads": { | |
| "torch_num_threads": 10, | |
| "torch_num_interop_threads": 14, | |
| "OMP_NUM_THREADS": null, | |
| "os_cpu_count": 14 | |
| } | |
| }, | |
| "payload_sha256": "8809019f865fe0939cee28bd5d201f7a842b85b9cb5929440541804f3786e74d" | |
| } | |
| } |