[pilot-7b] runs/pilot-7b/artifacts/pilot/pilot_results.json

runs/pilot-7b/artifacts/pilot/pilot_results.json

@@ -0,0 +1,378 @@
+{
+  "run_id": "pilot-7b",
+  "created_at": "2026-08-07T14:19:45Z",
+  "scale": "PILOT -- real measurements, underpowered, 1B-scale models",
+  "conditions": [
+    {
+      "condition": "olmo2_7b:base",
+      "stage": "base",
+      "stage_index": 0,
+      "n_cases": 93,
+      "coverage": {
+        "recall@1": 0.0,
+        "recall@5": 0.0,
+        "soft_coverage": 0.0353507608575973
+      },
+      "mean_tau": null,
+      "premature_rate": 0.0,
+      "mean_vendi_final": 0.1213991357179475,
+      "mean_rho": -0.6076288667703512,
+      "mean_tokens": 687.1720430107526,
+      "format_compliance": 0.08602150537634409
+    },
+    {
+      "condition": "olmo2_7b:sft",
+      "stage": "sft",
+      "stage_index": 1,
+      "n_cases": 93,
+      "coverage": {
+        "recall@1": 0.06451612903225806,
+        "recall@5": 0.10752688172043011,
+        "soft_coverage": 0.36576678776660565
+      },
+      "mean_tau": 0.5178571428571429,
+      "premature_rate": 0.07526881720430108,
+      "mean_vendi_final": 0.6232598657295285,
+      "mean_rho": -7.489308609221479,
+      "mean_tokens": 270.6236559139785,
+      "format_compliance": 0.4946236559139785
+    },
+    {
+      "condition": "olmo2_7b:dpo",
+      "stage": "dpo",
+      "stage_index": 2,
+      "n_cases": 93,
+      "coverage": {
+        "recall@1": 0.16129032258064516,
+        "recall@5": 0.1935483870967742,
+        "soft_coverage": 0.4816968443699462
+      },
+      "mean_tau": 0.6546996450842607,
+      "premature_rate": 0.24731182795698925,
+      "mean_vendi_final": 0.7203972596593304,
+      "mean_rho": -2.0058831262530457,
+      "mean_tokens": 429.7204301075269,
+      "format_compliance": 0.5806451612903226
+    },
+    {
+      "condition": "olmo2_7b:instruct",
+      "stage": "instruct",
+      "stage_index": 3,
+      "n_cases": 93,
+      "coverage": {
+        "recall@1": 0.16129032258064516,
+        "recall@5": 0.1827956989247312,
+        "soft_coverage": 0.5293526484714124
+      },
+      "mean_tau": 0.6499125874125876,
+      "premature_rate": 0.16129032258064516,
+      "mean_vendi_final": 0.7333453423289085,
+      "mean_rho": -4.420284404154766,
+      "mean_tokens": 340.63440860215053,
+      "format_compliance": 0.6129032258064516
+    }
+  ],
+  "sanity": {
+    "n_rows": 372,
+    "mean_hypotheses_per_trace": 0.9838709677419355,
+    "frac_traces_with_no_hypothesis": 0.5564516129032258,
+    "mean_recall_at_5": 0.12096774193548387,
+    "frac_committed": 0.75,
+    "mean_tokens": 432.0376344086022,
+    "mean_steps": 4.150537634408602,
+    "n_distinct_conditions": 4,
+    "format_compliance": 0.4435483870967742,
+    "ok": true,
+    "fatal": [],
+    "warnings": [
+      "56% of traces yielded no hypothesis at all"
+    ],
+    "verdict": "PASS -- the pipeline measured something"
+  },
+  "v4_redundancy": {
+    "n": 372,
+    "phi": 0.10163859834193095,
+    "abs_phi": 0.10163859834193095,
+    "ceiling": 0.5,
+    "passed": true,
+    "flag_rate": 0.12096774193548387,
+    "accuracy": 0.0967741935483871,
+    "contingency": {
+      "premature_and_wrong": 37,
+      "premature_and_right": 8,
+      "not_premature_and_wrong": 299,
+      "not_premature_and_right": 28
+    },
+    "premature_but_correct_rate": 0.17777777777777778,
+    "verdict": "PASS -- flag is not redundant with correctness (|phi|=0.102 < 0.5)"
+  },
+  "v1_manipulation": {
+    "orderings": {
+      "early": {
+        "n": 80,
+        "n_decidable": 80,
+        "premature_rate": 0.0625,
+        "mean_tau": NaN
+      },
+      "late": {
+        "n": 80,
+        "n_decidable": 80,
+        "premature_rate": 0.075,
+        "mean_tau": NaN
+      },
+      "shuffled": {
+        "n": 80,
+        "n_decidable": 80,
+        "premature_rate": 0.0625,
+        "mean_tau": NaN
+      }
+    },
+    "delta_late_minus_early": 0.012499999999999997,
+    "direction_correct": true,
+    "n_paired": 40,
+    "paired_mean_difference": 0.025,
+    "paired_se": 0.06686668711812965,
+    "paired_ci95": [
+      -0.10605870675153412,
+      0.1560587067515341
+    ],
+    "ci_excludes_zero": false,
+    "discordant_late_only": 4,
+    "discordant_early_only": 3,
+    "passed": false,
+    "verdict": "NOT ESTABLISHED at pilot scale -- direction correct, CI includes zero"
+  },
+  "e1_prevalence": {
+    "n": 372,
+    "outcome": "recall_at_5",
+    "model": {
+      "formula": "recall_at_5 ~ stage_index * _prev_c",
+      "backend": "ols_cluster_robust",
+      "converged": true,
+      "primary_term": "stage_index:_prev_c",
+      "estimate": -0.001362802821279343,
+      "ci": [
+        -0.01767108828856597,
+        0.014945482646007285
+      ],
+      "p_value": 0.8699007908883373,
+      "crosses_zero": "True"
+    },
+    "n_head": 256,
+    "n_tail": 116,
+    "by_stage": {
+      "base": {
+        "stage_index": 0,
+        "n": 93,
+        "overall": 0.0,
+        "head": 0.0,
+        "tail": 0.0,
+        "n_head": 64,
+        "n_tail": 29,
+        "mean_tau": null,
+        "premature_rate": 0.0
+      },
+      "sft": {
+        "stage_index": 1,
+        "n": 93,
+        "overall": 0.10752688172043011,
+        "head": 0.09375,
+        "tail": 0.13793103448275862,
+        "n_head": 64,
+        "n_tail": 29,
+        "mean_tau": 0.5178571428571429,
+        "premature_rate": 0.07526881720430108
+      },
+      "dpo": {
+        "stage_index": 2,
+        "n": 93,
+        "overall": 0.1935483870967742,
+        "head": 0.1875,
+        "tail": 0.20689655172413793,
+        "n_head": 64,
+        "n_tail": 29,
+        "mean_tau": 0.6546996450842604,
+        "premature_rate": 0.24731182795698925
+      },
+      "instruct": {
+        "stage_index": 3,
+        "n": 93,
+        "overall": 0.1827956989247312,
+        "head": 0.1875,
+        "tail": 0.1724137931034483,
+        "n_head": 64,
+        "n_tail": 29,
+        "mean_tau": 0.6499125874125874,
+        "premature_rate": 0.16129032258064516
+      }
+    },
+    "delta_head": 0.1875,
+    "delta_tail": 0.1724137931034483,
+    "head_tail_divergence": 0.015086206896551713,
+    "law_direction": "consistent with H2 (head gains, tail losses)"
+  },
+  "case_store": {
+    "n_loaded": 360,
+    "n_usable": 179,
+    "n_selected": 93,
+    "by_stratum": {
+      "uncommon": 27,
+      "common": 7,
+      "rare": 44,
+      "ultra_rare": 15
+    },
+    "report": {
+      "n_cases": 360,
+      "by_corpus": {
+        "rarebench": 120,
+        "rarearena": 120,
+        "ddxplus": 120
+      },
+      "by_stratum": {
+        "unknown": 180,
+        "rare": 130,
+        "common": 7,
+        "uncommon": 28,
+        "ultra_rare": 15
+      },
+      "n_gold_linked": 293,
+      "n_prevalence_known": 180,
+      "n_with_differential": 120,
+      "n_with_hpo": 120,
+      "n_with_fstar": 120,
+      "n_dolma_queried": 206,
+      "linker_stats": {
+        "n": 1310,
+        "linked": 839,
+        "abstain_rate": 0.35954198473282445,
+        "ambiguous_rate": 0.0,
+        "tier_abstain": 471,
+        "tier_exact_synonym": 183,
+        "tier_exact_label": 563,
+        "tier_normalised": 93
+      },
+      "warnings": [],
+      "gold_link_rate": 0.8138888888888889,
+      "prevalence_known_rate": 0.5
+    },
+    "kernels": {
+      "n_vocab": 1200,
+      "n_gold_in_vocab": 60,
+      "n_annotated_available": 10597,
+      "bc_offdiag_min": 1.5108883088422999e-06,
+      "bc_offdiag_mean": 0.762168748719715,
+      "tv_offdiag_min": 0.995667226089164,
+      "tv_offdiag_mean": 0.9997644819165231,
+      "bc_audit": {
+        "name": "k_disc_bhattacharyya",
+        "n_diseases": 1200,
+        "min_eigenvalue": -9.089951014119114e-13,
+        "psd_projected": false,
+        "projection_mass": 0.0,
+        "mean_offdiag": 0.762168748719715,
+        "diag_is_unit": true,
+        "variant": "bhattacharyya",
+        "n_features": 1200,
+        "n_without_profile": 0,
+        "psd_by_construction": true,
+        "smoothing": 0.01
+      },
+      "tv_audit": {
+        "name": "k_disc_tv",
+        "n_diseases": 1200,
+        "min_eigenvalue": -8.856590591119207e-13,
+        "psd_projected": false,
+        "projection_mass": 0.0,
+        "mean_offdiag": 0.9997644819165231,
+        "diag_is_unit": true,
+        "variant": "tv",
+        "n_features": 1200,
+        "n_without_profile": 0,
+        "psd_by_construction": false,
+        "smoothing": 0.01
+      }
+    },
+    "ontology_version": "963c60bbe2eaa862",
+    "seconds": 80.722685665
+  },
+  "kernels": {
+    "n_vocab": 1200,
+    "n_gold_in_vocab": 60,
+    "n_annotated_available": 10597,
+    "bc_offdiag_min": 1.5108883088422999e-06,
+    "bc_offdiag_mean": 0.762168748719715,
+    "tv_offdiag_min": 0.995667226089164,
+    "tv_offdiag_mean": 0.9997644819165231,
+    "bc_audit": {
+      "name": "k_disc_bhattacharyya",
+      "n_diseases": 1200,
+      "min_eigenvalue": -9.089951014119114e-13,
+      "psd_projected": false,
+      "projection_mass": 0.0,
+      "mean_offdiag": 0.762168748719715,
+      "diag_is_unit": true,
+      "variant": "bhattacharyya",
+      "n_features": 1200,
+      "n_without_profile": 0,
+      "psd_by_construction": true,
+      "smoothing": 0.01
+    },
+    "tv_audit": {
+      "name": "k_disc_tv",
+      "n_diseases": 1200,
+      "min_eigenvalue": -8.856590591119207e-13,
+      "psd_projected": false,
+      "projection_mass": 0.0,
+      "mean_offdiag": 0.9997644819165231,
+      "diag_is_unit": true,
+      "variant": "tv",
+      "n_features": 1200,
+      "n_without_profile": 0,
+      "psd_by_construction": false,
+      "smoothing": 0.01
+    }
+  },
+  "spend": {
+    "ceiling_usd": 15.0,
+    "spent_usd": 3.6124,
+    "remaining_usd": 11.3876,
+    "entries": [
+      {
+        "stage": "S0 case store",
+        "gpu": "cpu",
+        "seconds": 84.9,
+        "usd": 0.0045,
+        "at": "2026-08-07T11:02:51Z"
+      },
+      {
+        "stage": "gen olmo2_7b:base",
+        "gpu": "A10G",
+        "seconds": 6036.4,
+        "usd": 1.8444,
+        "at": "2026-08-07T12:43:27Z"
+      },
+      {
+        "stage": "gen olmo2_7b:sft",
+        "gpu": "A10G",
+        "seconds": 1068.9,
+        "usd": 0.3266,
+        "at": "2026-08-07T13:01:16Z"
+      },
+      {
+        "stage": "gen olmo2_7b:dpo",
+        "gpu": "A10G",
+        "seconds": 1657.2,
+        "usd": 0.5064,
+        "at": "2026-08-07T13:28:53Z"
+      },
+      {
+        "stage": "gen olmo2_7b:instruct",
+        "gpu": "A10G",
+        "seconds": 3045.4,
+        "usd": 0.9305,
+        "at": "2026-08-07T14:19:39Z"
+      }
+    ]
+  },
+  "warnings": []
+}