{
 "summary": [
  {
   "split": "all",
   "system": "agent",
   "pass": "22/23",
   "rate": "96%",
   "wilson95": "[79%, 99%]",
   "core_pass": "23/23",
   "false_improvements": 0,
   "missed_improvements": 0,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "all",
   "system": "agent_no_retry",
   "pass": "20/23",
   "rate": "87%",
   "wilson95": "[68%, 95%]",
   "core_pass": "21/23",
   "false_improvements": 0,
   "missed_improvements": 2,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "all",
   "system": "regex_guards",
   "pass": "12/23",
   "rate": "52%",
   "wilson95": "[33%, 71%]",
   "core_pass": "19/23",
   "false_improvements": 0,
   "missed_improvements": 4,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "all",
   "system": "baseline_regex",
   "pass": "4/23",
   "rate": "17%",
   "wilson95": "[7%, 37%]",
   "core_pass": "13/23",
   "false_improvements": 8,
   "missed_improvements": 4,
   "false_candidates": 0,
   "missed_candidates": 2
  },
  {
   "split": "dev",
   "system": "agent",
   "pass": "17/17",
   "rate": "100%",
   "wilson95": "[82%, 100%]",
   "core_pass": "17/17",
   "false_improvements": 0,
   "missed_improvements": 0,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "dev",
   "system": "agent_no_retry",
   "pass": "15/17",
   "rate": "88%",
   "wilson95": "[66%, 97%]",
   "core_pass": "15/17",
   "false_improvements": 0,
   "missed_improvements": 2,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "dev",
   "system": "regex_guards",
   "pass": "9/17",
   "rate": "53%",
   "wilson95": "[31%, 74%]",
   "core_pass": "13/17",
   "false_improvements": 0,
   "missed_improvements": 4,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "dev",
   "system": "baseline_regex",
   "pass": "4/17",
   "rate": "24%",
   "wilson95": "[10%, 47%]",
   "core_pass": "10/17",
   "false_improvements": 4,
   "missed_improvements": 4,
   "false_candidates": 0,
   "missed_candidates": 1
  },
  {
   "split": "held-out",
   "system": "agent",
   "pass": "5/6",
   "rate": "83%",
   "wilson95": "[44%, 97%]",
   "core_pass": "6/6",
   "false_improvements": 0,
   "missed_improvements": 0,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "held-out",
   "system": "agent_no_retry",
   "pass": "5/6",
   "rate": "83%",
   "wilson95": "[44%, 97%]",
   "core_pass": "6/6",
   "false_improvements": 0,
   "missed_improvements": 0,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "held-out",
   "system": "regex_guards",
   "pass": "3/6",
   "rate": "50%",
   "wilson95": "[19%, 81%]",
   "core_pass": "6/6",
   "false_improvements": 0,
   "missed_improvements": 0,
   "false_candidates": 0,
   "missed_candidates": 0
  },
  {
   "split": "held-out",
   "system": "baseline_regex",
   "pass": "0/6",
   "rate": "0%",
   "wilson95": "[0%, 39%]",
   "core_pass": "3/6",
   "false_improvements": 4,
   "missed_improvements": 0,
   "false_candidates": 0,
   "missed_candidates": 1
  }
 ],
 "agent": [
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s01_clean_notebook",
   "category": "clean extraction",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s02_incomplete_side_only",
   "category": "incomplete data",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s03_conflict",
   "category": "conflicting claims",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s04_stale_vs_fresh",
   "category": "stale vs fresh",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s05_search_down",
   "category": "tool failure",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s06_fetch_timeout_retry",
   "category": "tool failure (transient)",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s07_fetch_permanent_fail",
   "category": "tool failure (permanent)",
   "held_out": false,
   "tool_calls": 4
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s08_noise_page",
   "category": "noise / low quality",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s09_budget_tight",
   "category": "budget limit",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s10_true_improvement_exact",
   "category": "baseline improvement",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s11_no_improvement",
   "category": "no improvement (no false positive)",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s12_multi_n_batch",
   "category": "multi-n batch",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s13_source_diversity",
   "category": "source diversity",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s14_metric_mismatch",
   "category": "metric consistency",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s15_shift_prose_comma",
   "category": "distribution shift",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s16_partial_observability",
   "category": "partial observability",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s17_adversarial_numbers",
   "category": "adversarial numbers",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s18_prompt_injection",
   "category": "adversarial / injection",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s19_retracted",
   "category": "stale / retracted",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [37]"
   ],
   "id": "s20_shift_latex",
   "category": "distribution shift",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s21_search_empty",
   "category": "missing sources",
   "held_out": false,
   "tool_calls": 1
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s22_exact_rotated_worse",
   "category": "exact validation",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s23_search_rate_limited",
   "category": "tool failure (transient search)",
   "held_out": false,
   "tool_calls": 3
  }
 ],
 "regex_guards": [
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [6, 10, 11]"
   ],
   "id": "s01_clean_notebook",
   "category": "clean extraction",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s02_incomplete_side_only",
   "category": "incomplete data",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s03_conflict",
   "category": "conflicting claims",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "conflicts [28] != []",
    "stale missing [28]"
   ],
   "id": "s04_stale_vs_fresh",
   "category": "stale vs fresh",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s05_search_down",
   "category": "tool failure",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 1,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "missed improvement [4]"
   ],
   "id": "s06_fetch_timeout_retry",
   "category": "tool failure (transient)",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s07_fetch_permanent_fail",
   "category": "tool failure (permanent)",
   "held_out": false,
   "tool_calls": 4
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s08_noise_page",
   "category": "noise / low quality",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 1,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "missed improvement [4]"
   ],
   "id": "s09_budget_tight",
   "category": "budget limit",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 1,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "missed improvement [9]"
   ],
   "id": "s10_true_improvement_exact",
   "category": "baseline improvement",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s11_no_improvement",
   "category": "no improvement (no false positive)",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [6]",
    "unchanged [10, 18] != [6, 10, 18]",
    "no_claim [6] != []"
   ],
   "id": "s12_multi_n_batch",
   "category": "multi-n batch",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s13_source_diversity",
   "category": "source diversity",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "metric_mismatch missing [10]"
   ],
   "id": "s14_metric_mismatch",
   "category": "metric consistency",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [10]"
   ],
   "id": "s15_shift_prose_comma",
   "category": "distribution shift",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s16_partial_observability",
   "category": "partial observability",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "invalid missing [5]"
   ],
   "id": "s17_adversarial_numbers",
   "category": "adversarial numbers",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s18_prompt_injection",
   "category": "adversarial / injection",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s19_retracted",
   "category": "stale / retracted",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [37]"
   ],
   "id": "s20_shift_latex",
   "category": "distribution shift",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s21_search_empty",
   "category": "missing sources",
   "held_out": false,
   "tool_calls": 1
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s22_exact_rotated_worse",
   "category": "exact validation",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 1,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "missed improvement [9]"
   ],
   "id": "s23_search_rate_limited",
   "category": "tool failure (transient search)",
   "held_out": false,
   "tool_calls": 3
  }
 ],
 "baseline": [
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [6, 10, 11]"
   ],
   "id": "s01_clean_notebook",
   "category": "clean extraction",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 1,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 1,
   "fails": [
    "FALSE improvement [11]",
    "candidates missing [11]",
    "partial missing [11]"
   ],
   "id": "s02_incomplete_side_only",
   "category": "incomplete data",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 1,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "FALSE improvement [17]",
    "partial missing [17]"
   ],
   "id": "s03_conflict",
   "category": "conflicting claims",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "conflicts [28] != []",
    "partial missing [28]",
    "stale missing [28]"
   ],
   "id": "s04_stale_vs_fresh",
   "category": "stale vs fresh",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s05_search_down",
   "category": "tool failure",
   "held_out": false,
   "tool_calls": 1
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 1,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "missed improvement [4]",
    "too few sources cited"
   ],
   "id": "s06_fetch_timeout_retry",
   "category": "tool failure (transient)",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [19]"
   ],
   "id": "s07_fetch_permanent_fail",
   "category": "tool failure (permanent)",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": false,
   "false_imp": 2,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "FALSE improvement [5, 7]",
    "conflicts [5] != []"
   ],
   "id": "s08_noise_page",
   "category": "noise / low quality",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 1,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "missed improvement [4]"
   ],
   "id": "s09_budget_tight",
   "category": "budget limit",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 1,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "missed improvement [9]"
   ],
   "id": "s10_true_improvement_exact",
   "category": "baseline improvement",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s11_no_improvement",
   "category": "no improvement (no false positive)",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [6, 10, 18, 50]",
    "unchanged [10, 18] != [6, 10, 18]",
    "no_claim [6] != []"
   ],
   "id": "s12_multi_n_batch",
   "category": "multi-n batch",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [19]"
   ],
   "id": "s13_source_diversity",
   "category": "source diversity",
   "held_out": false,
   "tool_calls": 3
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [5, 10]",
    "metric_mismatch missing [10]"
   ],
   "id": "s14_metric_mismatch",
   "category": "metric consistency",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [10]"
   ],
   "id": "s15_shift_prose_comma",
   "category": "distribution shift",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [26]"
   ],
   "id": "s16_partial_observability",
   "category": "partial observability",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 2,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "FALSE improvement [5, 6]",
    "invalid missing [5]"
   ],
   "id": "s17_adversarial_numbers",
   "category": "adversarial numbers",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 1,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 1,
   "fails": [
    "FALSE improvement [7]",
    "candidates missing [7]",
    "partial missing [7]"
   ],
   "id": "s18_prompt_injection",
   "category": "adversarial / injection",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 1,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "FALSE improvement [13]"
   ],
   "id": "s19_retracted",
   "category": "stale / retracted",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "partial missing [37]"
   ],
   "id": "s20_shift_latex",
   "category": "distribution shift",
   "held_out": true,
   "tool_calls": 2
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s21_search_empty",
   "category": "missing sources",
   "held_out": false,
   "tool_calls": 1
  },
  {
   "pass": true,
   "false_imp": 0,
   "missed": 0,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [],
   "id": "s22_exact_rotated_worse",
   "category": "exact validation",
   "held_out": false,
   "tool_calls": 2
  },
  {
   "pass": false,
   "false_imp": 0,
   "missed": 1,
   "false_cand": 0,
   "missed_cand": 0,
   "fails": [
    "missed improvement [9]",
    "too few sources cited"
   ],
   "id": "s23_search_rate_limited",
   "category": "tool failure (transient search)",
   "held_out": false,
   "tool_calls": 1
  }
 ]
}