Release v1.0.0 accompanying the EMNLP 2026 camera-ready paper (arXiv:2608.14927)

CITATION.cff

@@ -0,0 +1,59 @@
+cff-version: 1.2.0
+message: "If you use this software or dataset, please cite the paper."
+title: "LLMs Can Predict Failure Risk, But Struggle to Predict Which Collaboration Protocol Pays Off: Cost-Aware Protocol Routing Across Reasoning Tasks"
+abstract: >-
+  Research artifact for the EMNLP 2026 paper on cost-aware protocol routing.
+  Releases matched four-protocol outcomes, fixed-order oracle labels, held-out
+  router evaluations, and post-answer confidence analyses across ten
+  model-condition settings.
+authors:
+  - family-names: Yang
+    given-names: Chih-Hsuan
+    affiliation: Argonne National Laboratory
+    email: bellayang@anl.gov
+  - family-names: Jiang
+    given-names: Jingyan
+    affiliation: Argonne National Laboratory
+  - family-names: Yang
+    given-names: Cheng-Hau
+    affiliation: Argonne National Laboratory
+  - family-names: Vasudevan
+    given-names: Vikram
+    affiliation: Oregon State University
+  - family-names: Zheng
+    given-names: Huihuo
+    affiliation: Argonne National Laboratory
+  - family-names: Vishwanath
+    given-names: Venkatram
+    affiliation: Argonne National Laboratory
+  - family-names: Thakur
+    given-names: Rajeev
+    affiliation: Argonne National Laboratory
+identifiers:
+  - type: other
+    value: "arXiv:2608.14927"
+    description: arXiv preprint
+url: "https://chihhsuan-yang.github.io/EMNLP_Cost-Aware-Protocol-Routing/"
+repository-code: "https://github.com/ChihHsuan-Yang/EMNLP_Cost-Aware-Protocol-Routing"
+license: MIT
+preferred-citation:
+  type: article
+  title: "LLMs Can Predict Failure Risk, But Struggle to Predict Which Collaboration Protocol Pays Off: Cost-Aware Protocol Routing Across Reasoning Tasks"
+  authors:
+    - family-names: Yang
+      given-names: Chih-Hsuan
+    - family-names: Jiang
+      given-names: Jingyan
+    - family-names: Yang
+      given-names: Cheng-Hau
+    - family-names: Vasudevan
+      given-names: Vikram
+    - family-names: Zheng
+      given-names: Huihuo
+    - family-names: Vishwanath
+      given-names: Venkatram
+    - family-names: Thakur
+      given-names: Rajeev
+  year: 2026
+  journal: "arXiv preprint arXiv:2608.14927"
+  url: "https://arxiv.org/abs/2608.14927"

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ChihHsuan-Yang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

NOTICE

@@ -0,0 +1,55 @@
+NOTICE
+======
+
+This repository is the public research artifact for the EMNLP 2026 paper
+"LLMs Can Predict Failure Risk, But Struggle to Predict Which Collaboration
+Protocol Pays Off: Cost-Aware Protocol Routing Across Reasoning Tasks"
+(arXiv:2608.14927).
+
+Code in this repository is released under the MIT License (see LICENSE).
+
+--------------------------------------------------------------------------
+Upstream benchmarks
+--------------------------------------------------------------------------
+
+This artifact reports measured outcomes of language-model protocol executions
+on problems drawn from the benchmarks below. We do NOT redistribute upstream
+problem text or gold answers. We release stable problem identifiers, derived
+per-protocol outcomes, and instructions for reconstructing the inputs from the
+pinned upstream releases. Each benchmark remains subject to its own license.
+
+* Omni-MATH - https://huggingface.co/datasets/KbsdJames/Omni-MATH
+  License: Apache-2.0.
+  Used as a filtered subset of 4,181 competition-mathematics problems.
+
+* JEEBench - https://huggingface.co/datasets/daman1209arora/jeebench
+  License: MIT. 515 problems.
+
+* SciBench - https://huggingface.co/datasets/xw27/scibench
+  License: MIT (see the upstream repository's LICENSE file; the Hugging Face
+  dataset card does not currently declare a license tag). 565 problems.
+
+* LAB-Bench - https://huggingface.co/datasets/futurehouse/lab-bench
+  License: CC-BY-SA-4.0. FutureHouse.
+  Pinned upstream revision: 5c77cec648430f30611808808861eb86f81d5eaa.
+  Two text-only slices are used: a strict slice (741 problems) and a broader
+  text-no-tool slice (1,542 problems). Image-dependent subsets (FigQA, TableQA)
+  are excluded because the protocols studied here are text-only.
+
+Please cite the original benchmark authors when using these identifiers.
+
+--------------------------------------------------------------------------
+Foundation models
+--------------------------------------------------------------------------
+
+The solver families evaluated here (openai/gpt-oss-120b and
+google/gemma-4-31B-it) are third-party models. This artifact does not
+redistribute, retrain, or claim ownership of those model weights.
+
+--------------------------------------------------------------------------
+Acknowledgments
+--------------------------------------------------------------------------
+
+This research used resources of the Argonne Leadership Computing Facility, a
+U.S. Department of Energy (DOE) Office of Science user facility at Argonne
+National Laboratory (ANL) operated under Contract No. DE-AC02-06CH11357.

README.md

@@ -1,3 +1,221 @@
 ---
 license: mit
+library_name: sklearn
+tags:
+  - research-artifact
+  - llm-routing
+  - multi-agent-systems
+  - cost-aware-inference
+  - protocol-selection
+  - emnlp2026
+language:
+  - en
+pipeline_tag: tabular-classification
 ---
+
+# Cost-Aware Protocol Routing — research artifact card
+
+**EMNLP 2026 Accepted Paper**
+*LLMs Can Predict Failure Risk, But Struggle to Predict Which Collaboration Protocol Pays Off: Cost-Aware Protocol Routing Across Reasoning Tasks*
+
+[Paper (arXiv)](https://arxiv.org/abs/2608.14927) ·
+[PDF](https://arxiv.org/pdf/2608.14927) ·
+[Project page](https://chihhsuan-yang.github.io/EMNLP_Cost-Aware-Protocol-Routing/) ·
+[Code](https://github.com/ChihHsuan-Yang/EMNLP_Cost-Aware-Protocol-Routing) ·
+[Dataset](https://huggingface.co/datasets/AgentsSci/EMNLP_Cost-Aware-Protocol-Routing)
+
+---
+
+## Read this first: what this repository is, and is not
+
+**This is not a language model.** It contains no foundation-model weights. The
+paper evaluates two third-party solver families — `openai/gpt-oss-120b` and
+`google/gemma-4-31B-it` — which we did **not** train, do not own, and do not
+redistribute. Their licenses and terms are their own.
+
+**This is a research-artifact card, not a checkpoint release.** It documents the
+paper's lightweight *routers*: small scikit-learn classifiers that pick which
+collaboration protocol to run. It ships their training code, configurations,
+selected hyperparameters, and per-problem held-out predictions — but **no
+serialized model file**. The reason is stated honestly below.
+
+### Why no checkpoint is published
+
+A trained metadata-only router exists in our internal results. We chose not to
+publish it:
+
+1. Its serialized estimator stores integer classes `[0, 1, 2, 3, 4]` with **no
+   embedded label mapping**. The mapping from index to protocol is recoverable
+   only by assuming sorted label-name order. That is a convention, not a stored
+   fact, and we could not verify it from the checkpoint alone. Publishing it
+   would ask users to guess which action index `3` means — and a wrong guess
+   produces confident, silently incorrect routing.
+2. It was pickled under a different scikit-learn version and already emits
+   `InconsistentVersionWarning` on load here. Pickled estimators are
+   version-fragile and are not a durable research artifact.
+3. Nothing is lost. Every number in the paper can be reproduced without it: the
+   **per-problem held-out predictions are released with string labels**, and the
+   **training code is released and tested**. A user can re-fit the router in
+   minutes on CPU, or score our exact predictions directly.
+
+If you need a checkpoint for a downstream system, train one with the released
+code and record your own label mapping.
+
+---
+
+## What the router predicts
+
+For each problem, **before any protocol is executed**, the router picks one of
+**five actions**:
+
+| Action | Meaning | Oracle rank |
+|---|---|:--:|
+| `baseline_llm` | Direct one-shot solving | 1 |
+| `single_agent` | Iterative single-agent self-correction | 2 |
+| `per` | Planner–Executor–Reviewer collaboration | 3 |
+| `broadcast` | Multi-agent deliberation | 4 |
+| `none` | Do not escalate: no observed protocol succeeded | 5 |
+
+`none` is a **router action**, not a fifth protocol. It is the correct choice
+when every protocol would have failed, so spending on escalation is wasted.
+
+The target label is the **fixed-order oracle**: the first protocol that actually
+succeeded, in the order above. That label is retrospective — it is what the
+router is trained to *predict*, not something available at inference.
+
+## Feature inputs, and the leakage boundary
+
+Routers see **only** problem-level information available before execution:
+
+- problem text (text+metadata variant only), as TF-IDF features
+- difficulty tier, source, and domain metadata
+- benchmark and condition identifiers
+
+They **never** see: gold answers, Baseline correctness, any protocol outcome,
+the oracle label, or any downstream collaboration result. This is enforced in
+code, not merely by convention — `protocol_routing.features` maintains an
+explicit forbidden-column set and raises `LeakageError` on contact, including
+for case variants. The repository's test suite proves the guard refuses
+deliberately constructed violations.
+
+## Router variants
+
+| Variant | Features | Notes |
+|---|---|---|
+| **Text + metadata** | TF-IDF over problem text, plus metadata | The paper's main learned router |
+| **Metadata only** | Metadata, no text | Ablation; isolates the contribution of text |
+| **Tier-majority** | None (train-split lookup) | Reference policy: majority oracle label within each difficulty tier, with the train-split global majority as fallback |
+
+All are multinomial logistic regression over the five actions.
+
+## Training and selection procedure
+
+**Six-setting evaluation** (the main router results): stratified 70/15/15 split
+by oracle label, seed `20260712`. Hyperparameters selected on **dev only**, then
+refit on train+dev, then evaluated once on untouched test problem identifiers.
+All routers are scored on **identical held-out ids**, which we verified.
+
+**Primary split** (Table 1): stratified 80/10/10, seed `42`, test n=423.
+
+Confidence intervals throughout are 2,000-resample **problem-level percentile
+bootstraps**. They quantify uncertainty over which problems are in the
+benchmark — *not* run-to-run variability, since each problem-protocol pair has
+exactly one realized outcome.
+
+## Supported settings
+
+Router evaluations cover **six** settings only: both solvers on OmniMath,
+LAB-Bench strict, and LAB-Bench text-no-tool. The matched four-protocol outcome
+data covers ten settings, but the router analysis does not. Please do not read
+these results as holding across all ten.
+
+## Results
+
+Held-out router performance (from `results/aggregate/heldout_router_evaluation.csv`):
+
+| Setting | n | Baseline | Router | Oracle | Oracle gap |
+|---|--:|--:|--:|--:|--:|
+| Gemma-4-31B-it OmniMath | 628 | 0.6943 | 0.7659 | 0.9570 | 0.1911 |
+| gpt-oss-120b OmniMath | 628 | 0.5685 | 0.6672 | 0.9268 | 0.2596 |
+| Gemma-4-31B-it LAB-Bench strict | 112 | 0.4464 | 0.7054 | 0.9107 | 0.2054 |
+| gpt-oss-120b LAB-Bench strict | 112 | 0.1875 | 0.5625 | 0.7679 | 0.2054 |
+| Gemma-4-31B-it LAB-Bench text-no-tool | 232 | 0.4181 | 0.7759 | 0.9612 | 0.1853 |
+| gpt-oss-120b LAB-Bench text-no-tool | 232 | 0.3060 | 0.5733 | 0.8621 | 0.2888 |
+
+The router beats Baseline everywhere. **It also leaves 18.5–28.9 points on the
+table against the retrospective oracle**, and oracle-label macro-F1 stays in the
+0.27–0.50 range. That gap is the paper's point, not a footnote.
+
+## Why this does not solve collaboration-value prediction
+
+The paper's central negative result: a model's own confidence ranks *Baseline
+failures* well (0.8847 AUROC) but is much weaker at identifying *which* protocol
+pays off (0.1674 AUPRC for PER-specific value, 0.1041 for Broadcast-specific).
+Learned routers improve on fixed policies but do not close the oracle gap.
+
+**Cost-aware selection among Single, PER, Broadcast, and None remains
+unresolved.** These artifacts are published so that others can attack that
+problem on matched data, not because the problem is solved.
+
+## Intended use and limitations
+
+**Intended:** reproducing the paper; research on cost-aware routing and
+escalation policies; a baseline to beat.
+
+**Not intended:** production routing without re-validation on your own workload.
+These routers are fit to specific benchmarks and two specific solver families.
+Protocol value varies substantially by task — the paper shows Broadcast's
+advantage over PER ranging from about 10 to 45 points across settings — so a
+router fit here should not be assumed to transfer.
+
+**Further limitations**
+
+- One realized execution per problem-protocol pair; no run-to-run variance.
+- The fixed-order oracle is retrospective and is not a deployable policy.
+- Cost is measured in logged tokens only — not latency, price, energy, or
+  parallelism.
+- No model snapshot was version-pinned at run time, so exact historical
+  execution cannot be recreated. We claim functional, not bit-for-bit,
+  reproducibility.
+- Routing errors are asymmetric in a way these metrics do not capture:
+  under-escalating loses a solvable problem, over-escalating only wastes tokens.
+
+## Reproducing
+
+```bash
+git clone https://github.com/ChihHsuan-Yang/EMNLP_Cost-Aware-Protocol-Routing.git
+cd EMNLP_Cost-Aware-Protocol-Routing
+make setup
+make test
+python scripts/train_router.py --help
+python scripts/evaluate_router.py --help
+```
+
+See `docs/REPRODUCE.md` in the repository for the full guide.
+
+## Citation
+
+```bibtex
+@article{yang2026protocolrouting,
+  title  = {{LLMs Can Predict Failure Risk, But Struggle to Predict Which
+            Collaboration Protocol Pays Off: Cost-Aware Protocol Routing
+            Across Reasoning Tasks}},
+  author = {Yang, Chih-Hsuan and Jiang, Jingyan and Yang, Cheng-Hau and
+            Vasudevan, Vikram and Zheng, Huihuo and Vishwanath, Venkatram and
+            Thakur, Rajeev},
+  journal = {arXiv preprint arXiv:2608.14927},
+  year    = {2026},
+  note    = {To appear at EMNLP 2026},
+  url     = {https://arxiv.org/abs/2608.14927}
+}
+```
+
+## Acknowledgments
+
+This research used resources of the Argonne Leadership Computing Facility, a
+U.S. Department of Energy (DOE) Office of Science user facility at Argonne
+National Laboratory (ANL) operated under Contract No. DE-AC02-06CH11357.
+
+## Contact
+
+Chih-Hsuan (Bella) Yang — bellayang@anl.gov

artifacts/heldout_router_evaluation.csv

@@ -0,0 +1,7 @@
+solver,setting,n,baseline_solve,router_solve,router_ci,oracle_solve,oracle_gap,oracle_label_macro_f1,route_baseline,route_single,route_per,route_broadcast,route_none
+Gemma-4-31B-it,OmniMath,628,0.6943,0.7659,"[0.7309,0.7978]",0.9570,0.1911,0.2676,0.4984,0.2404,0.1274,0.0557,0.0780
+gpt-oss-120b,OmniMath,628,0.5685,0.6672,"[0.6306,0.7038]",0.9268,0.2596,0.2835,0.4570,0.1990,0.1561,0.0717,0.1162
+Gemma-4-31B-it,LAB-Bench strict,112,0.4464,0.7054,"[0.6161,0.7857]",0.9107,0.2054,0.4949,0.5714,0.0357,0.0625,0.2679,0.0625
+gpt-oss-120b,LAB-Bench strict,112,0.1875,0.5625,"[0.4732,0.6607]",0.7679,0.2054,0.4986,0.1429,0.1518,0.2946,0.2500,0.1607
+Gemma-4-31B-it,LAB-Bench text-no-tool,232,0.4181,0.7759,"[0.7198,0.8276]",0.9612,0.1853,0.4854,0.3664,0.4310,0.0431,0.1379,0.0216
+gpt-oss-120b,LAB-Bench text-no-tool,232,0.3060,0.5733,"[0.5086,0.6379]",0.8621,0.2888,0.4558,0.3491,0.2974,0.0302,0.1638,0.1595

artifacts/heldout_router_paired_differences.csv

@@ -0,0 +1,7 @@
+solver,setting,n,router_minus_tier_majority_points,router_minus_tier_majority_ci_points,router_minus_baseline_points,router_minus_baseline_ci_points
+Gemma-4-31B-it,OmniMath,628,7.2,"[4.0,10.2]",7.2,"[4.0,10.2]"
+gpt-oss-120b,OmniMath,628,3.3,"[0.0,6.5]",9.9,"[6.5,13.2]"
+Gemma-4-31B-it,LAB-Bench strict,112,25.0,"[16.1,33.9]",25.9,"[17.9,34.0]"
+gpt-oss-120b,LAB-Bench strict,112,-8.0,"[-17.0,1.8]",37.5,"[28.6,46.4]"
+Gemma-4-31B-it,LAB-Bench text-no-tool,232,10.8,"[5.2,16.4]",35.8,"[29.7,42.2]"
+gpt-oss-120b,LAB-Bench text-no-tool,232,0.9,"[-5.2,6.5]",26.7,"[20.7,32.8]"

artifacts/metadata_only_router_selected_config.json

@@ -0,0 +1,55 @@
+{
+  "selected_config": {
+    "name": "plain_l2_c0.25",
+    "class_weight": null,
+    "C": 0.25
+  },
+  "selection_metric": "macro_f1 then avg_excess_cost_tokens_vs_oracle after validation-based early stopping",
+  "dev_metrics_before_refit": {
+    "n_examples": 416,
+    "accuracy": 0.569712,
+    "macro_f1": 0.267917,
+    "realized_solve_rate": 0.627404,
+    "avg_total_tokens": 51731.351,
+    "avg_model_calls": 15.312,
+    "avg_wall_time_seconds": 89.666,
+    "avg_excess_cost_tokens_vs_oracle": 19073.486,
+    "missed_success_rate": 0.323834
+  },
+  "loss_history_epochs": 30,
+  "early_stopping_metric": "dev_log_loss",
+  "selected_epoch": 1,
+  "best_dev_log_loss": 1.177944,
+  "feature_dimensions": {
+    "source_cardinality": 63,
+    "domain_cardinality": 155,
+    "total_feature_dim": 220
+  },
+  "label_counts_train": {
+    "baseline_llm": 1898,
+    "single_agent": 768,
+    "none": 244,
+    "per": 292,
+    "broadcast": 140
+  },
+  "label_counts_dev": {
+    "baseline_llm": 237,
+    "single_agent": 96,
+    "per": 36,
+    "none": 30,
+    "broadcast": 17
+  },
+  "label_counts_test": {
+    "baseline_llm": 238,
+    "none": 32,
+    "single_agent": 97,
+    "broadcast": 18,
+    "per": 38
+  },
+  "features": [
+    "source",
+    "domain",
+    "difficulty",
+    "difficulty_tier"
+  ]
+}

artifacts/router_metadata__labbench__llm_strict__gemma_4_31b.json

@@ -0,0 +1,156 @@
+{
+  "bootstrap": {
+    "resamples": 2000,
+    "seed": 20260713,
+    "unit": "problem_uid"
+  },
+  "generated_at_utc": "2026-07-14T04:10:48.673153+00:00",
+  "label_counts_by_split": {
+    "dev": {
+      "baseline_llm": 49,
+      "broadcast": 21,
+      "planner_executor_reviewer": 13,
+      "single_agent": 17,
+      "unsolved": 11
+    },
+    "test": {
+      "baseline_llm": 50,
+      "broadcast": 21,
+      "planner_executor_reviewer": 13,
+      "single_agent": 18,
+      "unsolved": 10
+    },
+    "train": {
+      "baseline_llm": 230,
+      "broadcast": 96,
+      "planner_executor_reviewer": 60,
+      "single_agent": 83,
+      "unsolved": 49
+    }
+  },
+  "routers": {
+    "metadata_bucket_majority": {
+      "bucket_rule": "difficulty_tier, else lab_bench_subset, else source, else subtask, else dataset/domain/global",
+      "global_majority": "baseline_llm",
+      "n_train_buckets": 3,
+      "router": "train-split metadata-bucket majority"
+    },
+    "metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.545351,
+          "dev_oracle_gap": 0.207207,
+          "dev_solve_rate": 0.693694
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.545351,
+          "dev_oracle_gap": 0.207207,
+          "dev_solve_rate": 0.693694
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.545351,
+          "dev_oracle_gap": 0.207207,
+          "dev_solve_rate": 0.693694
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.499965,
+          "dev_oracle_gap": 0.135135,
+          "dev_solve_rate": 0.765766
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.499965,
+          "dev_oracle_gap": 0.135135,
+          "dev_solve_rate": 0.765766
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.499965,
+          "dev_oracle_gap": 0.135135,
+          "dev_solve_rate": 0.765766
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 34
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 0.25,
+        "class_weight": "none"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    },
+    "text_metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.524082,
+          "dev_oracle_gap": 0.216216,
+          "dev_solve_rate": 0.684685
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.541176,
+          "dev_oracle_gap": 0.207207,
+          "dev_solve_rate": 0.693694
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.570964,
+          "dev_oracle_gap": 0.198198,
+          "dev_solve_rate": 0.702703
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.51383,
+          "dev_oracle_gap": 0.135135,
+          "dev_solve_rate": 0.765766
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.541161,
+          "dev_oracle_gap": 0.153153,
+          "dev_solve_rate": 0.747748
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.540774,
+          "dev_oracle_gap": 0.198198,
+          "dev_solve_rate": 0.702703
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 34,
+        "text_vocab_size": 10637
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 4.0,
+        "class_weight": "none"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    }
+  },
+  "setting": "labbench__llm_strict__gemma_4_31b",
+  "split_counts": {
+    "dev": 111,
+    "test": 112,
+    "train": 518
+  }
+}

artifacts/router_metadata__labbench__llm_strict__gpt_oss_120b.json

@@ -0,0 +1,156 @@
+{
+  "bootstrap": {
+    "resamples": 2000,
+    "seed": 20260713,
+    "unit": "problem_uid"
+  },
+  "generated_at_utc": "2026-07-14T04:10:52.280480+00:00",
+  "label_counts_by_split": {
+    "dev": {
+      "baseline_llm": 21,
+      "broadcast": 26,
+      "planner_executor_reviewer": 23,
+      "single_agent": 16,
+      "unsolved": 25
+    },
+    "test": {
+      "baseline_llm": 21,
+      "broadcast": 26,
+      "planner_executor_reviewer": 23,
+      "single_agent": 16,
+      "unsolved": 26
+    },
+    "train": {
+      "baseline_llm": 99,
+      "broadcast": 120,
+      "planner_executor_reviewer": 107,
+      "single_agent": 75,
+      "unsolved": 117
+    }
+  },
+  "routers": {
+    "metadata_bucket_majority": {
+      "bucket_rule": "difficulty_tier, else lab_bench_subset, else source, else subtask, else dataset/domain/global",
+      "global_majority": "broadcast",
+      "n_train_buckets": 3,
+      "router": "train-split metadata-bucket majority"
+    },
+    "metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.458689,
+          "dev_oracle_gap": 0.297297,
+          "dev_solve_rate": 0.477477
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.468224,
+          "dev_oracle_gap": 0.297297,
+          "dev_solve_rate": 0.477477
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.449204,
+          "dev_oracle_gap": 0.324324,
+          "dev_solve_rate": 0.45045
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.370943,
+          "dev_oracle_gap": 0.306306,
+          "dev_solve_rate": 0.468468
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.352693,
+          "dev_oracle_gap": 0.351351,
+          "dev_solve_rate": 0.423423
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.352693,
+          "dev_oracle_gap": 0.351351,
+          "dev_solve_rate": 0.423423
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 37
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 1.0,
+        "class_weight": "none"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    },
+    "text_metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.451874,
+          "dev_oracle_gap": 0.288288,
+          "dev_solve_rate": 0.486486
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.465895,
+          "dev_oracle_gap": 0.288288,
+          "dev_solve_rate": 0.486486
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.483344,
+          "dev_oracle_gap": 0.27027,
+          "dev_solve_rate": 0.504505
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.440253,
+          "dev_oracle_gap": 0.279279,
+          "dev_solve_rate": 0.495495
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.431743,
+          "dev_oracle_gap": 0.279279,
+          "dev_solve_rate": 0.495495
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.441665,
+          "dev_oracle_gap": 0.297297,
+          "dev_solve_rate": 0.477477
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 37,
+        "text_vocab_size": 10438
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 4.0,
+        "class_weight": "none"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    }
+  },
+  "setting": "labbench__llm_strict__gpt_oss_120b",
+  "split_counts": {
+    "dev": 111,
+    "test": 112,
+    "train": 518
+  }
+}

artifacts/router_metadata__labbench__text_no_tool__gemma_4_31b_text_no_tool.json

@@ -0,0 +1,156 @@
+{
+  "bootstrap": {
+    "resamples": 2000,
+    "seed": 20260713,
+    "unit": "problem_uid"
+  },
+  "generated_at_utc": "2026-07-14T04:11:33.595705+00:00",
+  "label_counts_by_split": {
+    "dev": {
+      "baseline_llm": 97,
+      "broadcast": 22,
+      "planner_executor_reviewer": 25,
+      "single_agent": 79,
+      "unsolved": 8
+    },
+    "test": {
+      "baseline_llm": 97,
+      "broadcast": 22,
+      "planner_executor_reviewer": 25,
+      "single_agent": 79,
+      "unsolved": 9
+    },
+    "train": {
+      "baseline_llm": 453,
+      "broadcast": 103,
+      "planner_executor_reviewer": 117,
+      "single_agent": 367,
+      "unsolved": 39
+    }
+  },
+  "routers": {
+    "metadata_bucket_majority": {
+      "bucket_rule": "difficulty_tier, else lab_bench_subset, else source, else subtask, else dataset/domain/global",
+      "global_majority": "baseline_llm",
+      "n_train_buckets": 6,
+      "router": "train-split metadata-bucket majority"
+    },
+    "metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.380555,
+          "dev_oracle_gap": 0.212121,
+          "dev_solve_rate": 0.753247
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.46948,
+          "dev_oracle_gap": 0.220779,
+          "dev_solve_rate": 0.744589
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.46948,
+          "dev_oracle_gap": 0.220779,
+          "dev_solve_rate": 0.744589
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.380129,
+          "dev_oracle_gap": 0.186147,
+          "dev_solve_rate": 0.779221
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.334797,
+          "dev_oracle_gap": 0.233766,
+          "dev_solve_rate": 0.731602
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.251877,
+          "dev_oracle_gap": 0.307359,
+          "dev_solve_rate": 0.658009
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 124
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 1.0,
+        "class_weight": "none"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    },
+    "text_metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.471906,
+          "dev_oracle_gap": 0.212121,
+          "dev_solve_rate": 0.753247
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.476508,
+          "dev_oracle_gap": 0.194805,
+          "dev_solve_rate": 0.770563
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.478219,
+          "dev_oracle_gap": 0.190476,
+          "dev_solve_rate": 0.774892
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.267499,
+          "dev_oracle_gap": 0.272727,
+          "dev_solve_rate": 0.692641
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.226383,
+          "dev_oracle_gap": 0.108225,
+          "dev_solve_rate": 0.857143
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.389084,
+          "dev_oracle_gap": 0.225108,
+          "dev_solve_rate": 0.74026
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 124,
+        "text_vocab_size": 12555
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 4.0,
+        "class_weight": "none"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    }
+  },
+  "setting": "labbench__text_no_tool__gemma_4_31b_text_no_tool",
+  "split_counts": {
+    "dev": 231,
+    "test": 232,
+    "train": 1079
+  }
+}

artifacts/router_metadata__labbench__text_no_tool__gpt_oss_120b_text_no_tool.json

@@ -0,0 +1,156 @@
+{
+  "bootstrap": {
+    "resamples": 2000,
+    "seed": 20260713,
+    "unit": "problem_uid"
+  },
+  "generated_at_utc": "2026-07-14T04:11:39.208933+00:00",
+  "label_counts_by_split": {
+    "dev": {
+      "baseline_llm": 71,
+      "broadcast": 37,
+      "planner_executor_reviewer": 29,
+      "single_agent": 63,
+      "unsolved": 31
+    },
+    "test": {
+      "baseline_llm": 71,
+      "broadcast": 38,
+      "planner_executor_reviewer": 28,
+      "single_agent": 63,
+      "unsolved": 32
+    },
+    "train": {
+      "baseline_llm": 329,
+      "broadcast": 176,
+      "planner_executor_reviewer": 133,
+      "single_agent": 294,
+      "unsolved": 147
+    }
+  },
+  "routers": {
+    "metadata_bucket_majority": {
+      "bucket_rule": "difficulty_tier, else lab_bench_subset, else source, else subtask, else dataset/domain/global",
+      "global_majority": "baseline_llm",
+      "n_train_buckets": 6,
+      "router": "train-split metadata-bucket majority"
+    },
+    "metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.461884,
+          "dev_oracle_gap": 0.298701,
+          "dev_solve_rate": 0.5671
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.46713,
+          "dev_oracle_gap": 0.294372,
+          "dev_solve_rate": 0.571429
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.46713,
+          "dev_oracle_gap": 0.294372,
+          "dev_solve_rate": 0.571429
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.451913,
+          "dev_oracle_gap": 0.25974,
+          "dev_solve_rate": 0.606061
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.451913,
+          "dev_oracle_gap": 0.25974,
+          "dev_solve_rate": 0.606061
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.451913,
+          "dev_oracle_gap": 0.25974,
+          "dev_solve_rate": 0.606061
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 58
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 1.0,
+        "class_weight": "none"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    },
+    "text_metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.482273,
+          "dev_oracle_gap": 0.298701,
+          "dev_solve_rate": 0.5671
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.489049,
+          "dev_oracle_gap": 0.285714,
+          "dev_solve_rate": 0.580087
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.476716,
+          "dev_oracle_gap": 0.311688,
+          "dev_solve_rate": 0.554113
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.458072,
+          "dev_oracle_gap": 0.264069,
+          "dev_solve_rate": 0.601732
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.459016,
+          "dev_oracle_gap": 0.290043,
+          "dev_solve_rate": 0.575758
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.458166,
+          "dev_oracle_gap": 0.281385,
+          "dev_solve_rate": 0.584416
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 58,
+        "text_vocab_size": 12628
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 1.0,
+        "class_weight": "none"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    }
+  },
+  "setting": "labbench__text_no_tool__gpt_oss_120b_text_no_tool",
+  "split_counts": {
+    "dev": 231,
+    "test": 232,
+    "train": 1079
+  }
+}

artifacts/router_metadata__omnimath2__competition_math_4181__gemma_4_31b.json

@@ -0,0 +1,156 @@
+{
+  "bootstrap": {
+    "resamples": 2000,
+    "seed": 20260713,
+    "unit": "problem_uid"
+  },
+  "generated_at_utc": "2026-07-14T04:10:25.417360+00:00",
+  "label_counts_by_split": {
+    "dev": {
+      "baseline_llm": 435,
+      "broadcast": 18,
+      "planner_executor_reviewer": 41,
+      "single_agent": 106,
+      "unsolved": 27
+    },
+    "test": {
+      "baseline_llm": 436,
+      "broadcast": 18,
+      "planner_executor_reviewer": 40,
+      "single_agent": 107,
+      "unsolved": 27
+    },
+    "train": {
+      "baseline_llm": 2030,
+      "broadcast": 84,
+      "planner_executor_reviewer": 187,
+      "single_agent": 498,
+      "unsolved": 127
+    }
+  },
+  "routers": {
+    "metadata_bucket_majority": {
+      "bucket_rule": "difficulty_tier, else lab_bench_subset, else source, else subtask, else dataset/domain/global",
+      "global_majority": "baseline_llm",
+      "n_train_buckets": 10,
+      "router": "train-split metadata-bucket majority"
+    },
+    "metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.179087,
+          "dev_oracle_gap": 0.256778,
+          "dev_solve_rate": 0.700159
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.178893,
+          "dev_oracle_gap": 0.253589,
+          "dev_solve_rate": 0.703349
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.182069,
+          "dev_oracle_gap": 0.251994,
+          "dev_solve_rate": 0.704944
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.217796,
+          "dev_oracle_gap": 0.221691,
+          "dev_solve_rate": 0.735247
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.216413,
+          "dev_oracle_gap": 0.226475,
+          "dev_solve_rate": 0.730463
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.218656,
+          "dev_oracle_gap": 0.221691,
+          "dev_solve_rate": 0.735247
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 363
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 4.0,
+        "class_weight": "balanced"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    },
+    "text_metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.185324,
+          "dev_oracle_gap": 0.253589,
+          "dev_solve_rate": 0.703349
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.191559,
+          "dev_oracle_gap": 0.245614,
+          "dev_solve_rate": 0.711324
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.205862,
+          "dev_oracle_gap": 0.232855,
+          "dev_solve_rate": 0.724083
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.233681,
+          "dev_oracle_gap": 0.208931,
+          "dev_solve_rate": 0.748006
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.260752,
+          "dev_oracle_gap": 0.197767,
+          "dev_solve_rate": 0.759171
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.235345,
+          "dev_oracle_gap": 0.210526,
+          "dev_solve_rate": 0.746411
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 363,
+        "text_vocab_size": 15585
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 1.0,
+        "class_weight": "balanced"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    }
+  },
+  "setting": "omnimath2__competition_math_4181__gemma_4_31b",
+  "split_counts": {
+    "dev": 627,
+    "test": 628,
+    "train": 2926
+  }
+}

artifacts/router_metadata__omnimath2__competition_math_4181__gpt_oss_120b.json

@@ -0,0 +1,156 @@
+{
+  "bootstrap": {
+    "resamples": 2000,
+    "seed": 20260713,
+    "unit": "problem_uid"
+  },
+  "generated_at_utc": "2026-07-14T04:10:45.093159+00:00",
+  "label_counts_by_split": {
+    "dev": {
+      "baseline_llm": 356,
+      "broadcast": 26,
+      "planner_executor_reviewer": 55,
+      "single_agent": 144,
+      "unsolved": 46
+    },
+    "test": {
+      "baseline_llm": 357,
+      "broadcast": 26,
+      "planner_executor_reviewer": 55,
+      "single_agent": 144,
+      "unsolved": 46
+    },
+    "train": {
+      "baseline_llm": 1663,
+      "broadcast": 123,
+      "planner_executor_reviewer": 256,
+      "single_agent": 670,
+      "unsolved": 214
+    }
+  },
+  "routers": {
+    "metadata_bucket_majority": {
+      "bucket_rule": "difficulty_tier, else lab_bench_subset, else source, else subtask, else dataset/domain/global",
+      "global_majority": "baseline_llm",
+      "n_train_buckets": 10,
+      "router": "train-split metadata-bucket majority"
+    },
+    "metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.258693,
+          "dev_oracle_gap": 0.259968,
+          "dev_solve_rate": 0.666667
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.259024,
+          "dev_oracle_gap": 0.259968,
+          "dev_solve_rate": 0.666667
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.257415,
+          "dev_oracle_gap": 0.261563,
+          "dev_solve_rate": 0.665072
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.290482,
+          "dev_oracle_gap": 0.23126,
+          "dev_solve_rate": 0.695375
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.280241,
+          "dev_oracle_gap": 0.248804,
+          "dev_solve_rate": 0.677831
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.277894,
+          "dev_oracle_gap": 0.259968,
+          "dev_solve_rate": 0.666667
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 374
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 0.25,
+        "class_weight": "balanced"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    },
+    "text_metadata_logreg": {
+      "config_search": [
+        {
+          "C": 0.25,
+          "class_weight": "none",
+          "dev_macro_f1": 0.262641,
+          "dev_oracle_gap": 0.261563,
+          "dev_solve_rate": 0.665072
+        },
+        {
+          "C": 1.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.281265,
+          "dev_oracle_gap": 0.264753,
+          "dev_solve_rate": 0.661882
+        },
+        {
+          "C": 4.0,
+          "class_weight": "none",
+          "dev_macro_f1": 0.279245,
+          "dev_oracle_gap": 0.261563,
+          "dev_solve_rate": 0.665072
+        },
+        {
+          "C": 0.25,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.305407,
+          "dev_oracle_gap": 0.220096,
+          "dev_solve_rate": 0.706539
+        },
+        {
+          "C": 1.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.323259,
+          "dev_oracle_gap": 0.221691,
+          "dev_solve_rate": 0.704944
+        },
+        {
+          "C": 4.0,
+          "class_weight": "balanced",
+          "dev_macro_f1": 0.32304,
+          "dev_oracle_gap": 0.23126,
+          "dev_solve_rate": 0.695375
+        }
+      ],
+      "feature_dimensions": {
+        "metadata_feature_count": 374,
+        "text_vocab_size": 15599
+      },
+      "refit": "train+dev before held-out test evaluation",
+      "selected_config": {
+        "C": 1.0,
+        "class_weight": "balanced"
+      },
+      "selection_metric": "dev macro-F1, then dev solve rate, then oracle gap"
+    }
+  },
+  "setting": "omnimath2__competition_math_4181__gpt_oss_120b",
+  "split_counts": {
+    "dev": 627,
+    "test": 628,
+    "train": 2926
+  }
+}