| { |
| "summary": [ |
| { |
| "split": "all", |
| "system": "agent", |
| "pass": "22/23", |
| "rate": "96%", |
| "wilson95": "[79%, 99%]", |
| "core_pass": "23/23", |
| "false_improvements": 0, |
| "missed_improvements": 0, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "all", |
| "system": "agent_no_retry", |
| "pass": "20/23", |
| "rate": "87%", |
| "wilson95": "[68%, 95%]", |
| "core_pass": "21/23", |
| "false_improvements": 0, |
| "missed_improvements": 2, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "all", |
| "system": "regex_guards", |
| "pass": "12/23", |
| "rate": "52%", |
| "wilson95": "[33%, 71%]", |
| "core_pass": "19/23", |
| "false_improvements": 0, |
| "missed_improvements": 4, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "all", |
| "system": "baseline_regex", |
| "pass": "4/23", |
| "rate": "17%", |
| "wilson95": "[7%, 37%]", |
| "core_pass": "13/23", |
| "false_improvements": 8, |
| "missed_improvements": 4, |
| "false_candidates": 0, |
| "missed_candidates": 2 |
| }, |
| { |
| "split": "dev", |
| "system": "agent", |
| "pass": "17/17", |
| "rate": "100%", |
| "wilson95": "[82%, 100%]", |
| "core_pass": "17/17", |
| "false_improvements": 0, |
| "missed_improvements": 0, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "dev", |
| "system": "agent_no_retry", |
| "pass": "15/17", |
| "rate": "88%", |
| "wilson95": "[66%, 97%]", |
| "core_pass": "15/17", |
| "false_improvements": 0, |
| "missed_improvements": 2, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "dev", |
| "system": "regex_guards", |
| "pass": "9/17", |
| "rate": "53%", |
| "wilson95": "[31%, 74%]", |
| "core_pass": "13/17", |
| "false_improvements": 0, |
| "missed_improvements": 4, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "dev", |
| "system": "baseline_regex", |
| "pass": "4/17", |
| "rate": "24%", |
| "wilson95": "[10%, 47%]", |
| "core_pass": "10/17", |
| "false_improvements": 4, |
| "missed_improvements": 4, |
| "false_candidates": 0, |
| "missed_candidates": 1 |
| }, |
| { |
| "split": "held-out", |
| "system": "agent", |
| "pass": "5/6", |
| "rate": "83%", |
| "wilson95": "[44%, 97%]", |
| "core_pass": "6/6", |
| "false_improvements": 0, |
| "missed_improvements": 0, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "held-out", |
| "system": "agent_no_retry", |
| "pass": "5/6", |
| "rate": "83%", |
| "wilson95": "[44%, 97%]", |
| "core_pass": "6/6", |
| "false_improvements": 0, |
| "missed_improvements": 0, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "held-out", |
| "system": "regex_guards", |
| "pass": "3/6", |
| "rate": "50%", |
| "wilson95": "[19%, 81%]", |
| "core_pass": "6/6", |
| "false_improvements": 0, |
| "missed_improvements": 0, |
| "false_candidates": 0, |
| "missed_candidates": 0 |
| }, |
| { |
| "split": "held-out", |
| "system": "baseline_regex", |
| "pass": "0/6", |
| "rate": "0%", |
| "wilson95": "[0%, 39%]", |
| "core_pass": "3/6", |
| "false_improvements": 4, |
| "missed_improvements": 0, |
| "false_candidates": 0, |
| "missed_candidates": 1 |
| } |
| ], |
| "agent": [ |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s01_clean_notebook", |
| "category": "clean extraction", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s02_incomplete_side_only", |
| "category": "incomplete data", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s03_conflict", |
| "category": "conflicting claims", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s04_stale_vs_fresh", |
| "category": "stale vs fresh", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s05_search_down", |
| "category": "tool failure", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s06_fetch_timeout_retry", |
| "category": "tool failure (transient)", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s07_fetch_permanent_fail", |
| "category": "tool failure (permanent)", |
| "held_out": false, |
| "tool_calls": 4 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s08_noise_page", |
| "category": "noise / low quality", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s09_budget_tight", |
| "category": "budget limit", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s10_true_improvement_exact", |
| "category": "baseline improvement", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s11_no_improvement", |
| "category": "no improvement (no false positive)", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s12_multi_n_batch", |
| "category": "multi-n batch", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s13_source_diversity", |
| "category": "source diversity", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s14_metric_mismatch", |
| "category": "metric consistency", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s15_shift_prose_comma", |
| "category": "distribution shift", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s16_partial_observability", |
| "category": "partial observability", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s17_adversarial_numbers", |
| "category": "adversarial numbers", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s18_prompt_injection", |
| "category": "adversarial / injection", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s19_retracted", |
| "category": "stale / retracted", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [37]" |
| ], |
| "id": "s20_shift_latex", |
| "category": "distribution shift", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s21_search_empty", |
| "category": "missing sources", |
| "held_out": false, |
| "tool_calls": 1 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s22_exact_rotated_worse", |
| "category": "exact validation", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s23_search_rate_limited", |
| "category": "tool failure (transient search)", |
| "held_out": false, |
| "tool_calls": 3 |
| } |
| ], |
| "regex_guards": [ |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [6, 10, 11]" |
| ], |
| "id": "s01_clean_notebook", |
| "category": "clean extraction", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s02_incomplete_side_only", |
| "category": "incomplete data", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s03_conflict", |
| "category": "conflicting claims", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "conflicts [28] != []", |
| "stale missing [28]" |
| ], |
| "id": "s04_stale_vs_fresh", |
| "category": "stale vs fresh", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s05_search_down", |
| "category": "tool failure", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 1, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "missed improvement [4]" |
| ], |
| "id": "s06_fetch_timeout_retry", |
| "category": "tool failure (transient)", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s07_fetch_permanent_fail", |
| "category": "tool failure (permanent)", |
| "held_out": false, |
| "tool_calls": 4 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s08_noise_page", |
| "category": "noise / low quality", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 1, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "missed improvement [4]" |
| ], |
| "id": "s09_budget_tight", |
| "category": "budget limit", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 1, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "missed improvement [9]" |
| ], |
| "id": "s10_true_improvement_exact", |
| "category": "baseline improvement", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s11_no_improvement", |
| "category": "no improvement (no false positive)", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [6]", |
| "unchanged [10, 18] != [6, 10, 18]", |
| "no_claim [6] != []" |
| ], |
| "id": "s12_multi_n_batch", |
| "category": "multi-n batch", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s13_source_diversity", |
| "category": "source diversity", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "metric_mismatch missing [10]" |
| ], |
| "id": "s14_metric_mismatch", |
| "category": "metric consistency", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [10]" |
| ], |
| "id": "s15_shift_prose_comma", |
| "category": "distribution shift", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s16_partial_observability", |
| "category": "partial observability", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "invalid missing [5]" |
| ], |
| "id": "s17_adversarial_numbers", |
| "category": "adversarial numbers", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s18_prompt_injection", |
| "category": "adversarial / injection", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s19_retracted", |
| "category": "stale / retracted", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [37]" |
| ], |
| "id": "s20_shift_latex", |
| "category": "distribution shift", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s21_search_empty", |
| "category": "missing sources", |
| "held_out": false, |
| "tool_calls": 1 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s22_exact_rotated_worse", |
| "category": "exact validation", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 1, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "missed improvement [9]" |
| ], |
| "id": "s23_search_rate_limited", |
| "category": "tool failure (transient search)", |
| "held_out": false, |
| "tool_calls": 3 |
| } |
| ], |
| "baseline": [ |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [6, 10, 11]" |
| ], |
| "id": "s01_clean_notebook", |
| "category": "clean extraction", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 1, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 1, |
| "fails": [ |
| "FALSE improvement [11]", |
| "candidates missing [11]", |
| "partial missing [11]" |
| ], |
| "id": "s02_incomplete_side_only", |
| "category": "incomplete data", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 1, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "FALSE improvement [17]", |
| "partial missing [17]" |
| ], |
| "id": "s03_conflict", |
| "category": "conflicting claims", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "conflicts [28] != []", |
| "partial missing [28]", |
| "stale missing [28]" |
| ], |
| "id": "s04_stale_vs_fresh", |
| "category": "stale vs fresh", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s05_search_down", |
| "category": "tool failure", |
| "held_out": false, |
| "tool_calls": 1 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 1, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "missed improvement [4]", |
| "too few sources cited" |
| ], |
| "id": "s06_fetch_timeout_retry", |
| "category": "tool failure (transient)", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [19]" |
| ], |
| "id": "s07_fetch_permanent_fail", |
| "category": "tool failure (permanent)", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": false, |
| "false_imp": 2, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "FALSE improvement [5, 7]", |
| "conflicts [5] != []" |
| ], |
| "id": "s08_noise_page", |
| "category": "noise / low quality", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 1, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "missed improvement [4]" |
| ], |
| "id": "s09_budget_tight", |
| "category": "budget limit", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 1, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "missed improvement [9]" |
| ], |
| "id": "s10_true_improvement_exact", |
| "category": "baseline improvement", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s11_no_improvement", |
| "category": "no improvement (no false positive)", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [6, 10, 18, 50]", |
| "unchanged [10, 18] != [6, 10, 18]", |
| "no_claim [6] != []" |
| ], |
| "id": "s12_multi_n_batch", |
| "category": "multi-n batch", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [19]" |
| ], |
| "id": "s13_source_diversity", |
| "category": "source diversity", |
| "held_out": false, |
| "tool_calls": 3 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [5, 10]", |
| "metric_mismatch missing [10]" |
| ], |
| "id": "s14_metric_mismatch", |
| "category": "metric consistency", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [10]" |
| ], |
| "id": "s15_shift_prose_comma", |
| "category": "distribution shift", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [26]" |
| ], |
| "id": "s16_partial_observability", |
| "category": "partial observability", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 2, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "FALSE improvement [5, 6]", |
| "invalid missing [5]" |
| ], |
| "id": "s17_adversarial_numbers", |
| "category": "adversarial numbers", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 1, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 1, |
| "fails": [ |
| "FALSE improvement [7]", |
| "candidates missing [7]", |
| "partial missing [7]" |
| ], |
| "id": "s18_prompt_injection", |
| "category": "adversarial / injection", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 1, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "FALSE improvement [13]" |
| ], |
| "id": "s19_retracted", |
| "category": "stale / retracted", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "partial missing [37]" |
| ], |
| "id": "s20_shift_latex", |
| "category": "distribution shift", |
| "held_out": true, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s21_search_empty", |
| "category": "missing sources", |
| "held_out": false, |
| "tool_calls": 1 |
| }, |
| { |
| "pass": true, |
| "false_imp": 0, |
| "missed": 0, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [], |
| "id": "s22_exact_rotated_worse", |
| "category": "exact validation", |
| "held_out": false, |
| "tool_calls": 2 |
| }, |
| { |
| "pass": false, |
| "false_imp": 0, |
| "missed": 1, |
| "false_cand": 0, |
| "missed_cand": 0, |
| "fails": [ |
| "missed improvement [9]", |
| "too few sources cited" |
| ], |
| "id": "s23_search_rate_limited", |
| "category": "tool failure (transient search)", |
| "held_out": false, |
| "tool_calls": 1 |
| } |
| ] |
| } |
|
|
