Add the Router Benchmark tab and the October 2026 board.

Router runs stay out of the model dropdown.

Co-authored-by: Cursor <cursoragent@cursor.com>

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 import pandas as pd
 import plotly.express as px
+import inspect
 import os
 import json
 from typing import Dict, List, Optional
@@ -47,6 +48,10 @@ def discover_available_runs() -> List[Dict]:
                 try:
                     with open(metadata_path, 'r') as f:
                         metadata = json.load(f)
+                    # Router runs have their own tab. Leaving them in this list
+                    # would make the model dropdown try to load summary CSVs.
+                    if metadata.get("run_type") == "router":
+                        continue
                     metadata["path"] = run_path
                     runs.append(metadata)
                 except Exception as e:
@@ -463,7 +468,171 @@ def update_leaderboard_data(selected_run_id: str) -> tuple:
     )
 
 # --- Build Gradio App ---
-with gr.Blocks(theme=gr.themes.Soft()) as app:
+def discover_router_runs() -> List[Dict]:
+    """Router runs are excluded from the model dropdown."""
+    routers = []
+    if not os.path.exists(RUNS_DIR):
+        return routers
+    for run_dir in os.listdir(RUNS_DIR):
+        run_path = os.path.join(RUNS_DIR, run_dir)
+        metadata_path = os.path.join(run_path, "metadata.json")
+        if not os.path.isdir(run_path) or not os.path.exists(metadata_path):
+            continue
+        try:
+            with open(metadata_path, "r", encoding="utf-8") as handle:
+                metadata = json.load(handle)
+        except Exception as exc:
+            print(f"Error loading router metadata for {run_dir}: {exc}")
+            continue
+        if metadata.get("run_type") != "router":
+            continue
+        metadata["path"] = run_path
+        routers.append(metadata)
+    routers.sort(key=lambda item: item.get("date", ""), reverse=True)
+    return routers
+
+
+def router_quality_cost_figure(board: pd.DataFrame):
+    headline = board[board["is_baseline"] == 0].copy()
+    headline["group"] = headline["collapsed"].map(
+        {1: "Collapsed (at least 90% one model)", 0: "Spread"}
+    )
+    figure = px.scatter(
+        headline,
+        x="cost",
+        y="accuracy",
+        color="group",
+        hover_name="label",
+        hover_data={
+            "vendor": True,
+            "n": True,
+            "cost": ":.4f",
+            "accuracy": ":.1f",
+            "group": False,
+        },
+        log_x=True,
+        color_discrete_map={
+            "Spread": "#155dfc",
+            "Collapsed (at least 90% one model)": "#d97706",
+        },
+        labels={"cost": "USD per question", "accuracy": "Accuracy (%)", "group": ""},
+    )
+    for name, color in (
+        ("Always-Sol", "#111827"),
+        ("Always-Luna", "#059669"),
+        ("Random", "#6b7280"),
+    ):
+        match = board[board["label"] == name]
+        if match.empty:
+            continue
+        accuracy = float(match.iloc[0]["accuracy"])
+        figure.add_hline(
+            y=accuracy,
+            line_dash="dash",
+            line_color=color,
+            annotation_text=f"{name} {accuracy:.1f}%",
+            annotation_position="top left",
+        )
+    figure.update_layout(
+        yaxis_title="Accuracy (%)",
+        xaxis_title="USD per question (log scale)",
+        legend_title_text="",
+        margin=dict(l=20, r=20, t=30, b=20),
+    )
+    return figure
+
+
+def router_collapse_figure(board: pd.DataFrame):
+    headline = board[board["is_baseline"] == 0].dropna(subset=["top_share"]).copy()
+    headline = headline.sort_values("top_share", ascending=True)
+    headline["share_pct"] = headline["top_share"] * 100
+    figure = px.bar(
+        headline,
+        x="share_pct",
+        y="label",
+        orientation="h",
+        labels={"share_pct": "Top-model share (%)", "label": ""},
+    )
+    figure.add_vline(x=90, line_dash="dash", line_color="#d97706")
+    figure.update_layout(margin=dict(l=20, r=20, t=30, b=20), showlegend=False)
+    return figure
+
+
+def router_same_pick_figure(same: pd.DataFrame, axis: List[str]):
+    pivot = same.pivot(index="pass_a", columns="pass_b", values="rate")
+    pivot = pivot.reindex(index=axis, columns=axis)
+    figure = px.imshow(
+        pivot,
+        zmin=0,
+        zmax=1,
+        color_continuous_scale="Blues",
+        labels=dict(color="Same-pick rate", x="", y=""),
+        aspect="auto",
+    )
+    figure.update_layout(margin=dict(l=20, r=20, t=30, b=20))
+    return figure
+
+
+def router_board_display(board: pd.DataFrame) -> pd.DataFrame:
+    view = board.sort_values(["cost", "display_order"]).copy()
+    displayed = pd.DataFrame(
+        {
+            "Pass": view["label"],
+            "Vendor": view["vendor"],
+            "Setting": view["setting"],
+            "n": view["n"],
+            "Accuracy (%)": view["accuracy"].map(lambda value: f"{value:.1f}"),
+            "95% CI": [
+                f"{low:.1f}–{high:.1f}"
+                for low, high in zip(view["ci_low"], view["ci_high"])
+            ],
+            "USD / question": view["cost"].map(lambda value: f"{value:.4f}"),
+            "Cost CI": [
+                f"{low:.4f}–{high:.4f}"
+                for low, high in zip(view["cost_ci_low"], view["cost_ci_high"])
+            ],
+            "Top-model share": view["top_share"].map(
+                lambda value: "" if pd.isna(value) else f"{value * 100:.1f}%"
+            ),
+            "Collapsed": view["collapsed"].map({1: "Yes", 0: ""}),
+            "Baseline": view["is_baseline"].map({1: "Yes", 0: ""}),
+        }
+    )
+    return displayed
+
+
+def router_intro_markdown(metadata: Dict) -> str:
+    facts = metadata.get("facts", {})
+    threshold = float(facts.get("collapse_threshold", 0.9)) * 100
+    median = facts.get("median_same_pick", 0)
+    return f"""
+## AutoBench Router Benchmark
+
+{metadata.get("description", "")}
+
+We graded **{facts.get("commercial_routers")} commercial routers** ({facts.get("headline_passes")} passes) on a frozen matrix of {facts.get("questions")} questions and {facts.get("matrix_models")} models ({facts.get("matrix_date")}).
+
+**Accuracy** is the share of questions where the chosen model is statistically as good as the best available answer.
+
+### Findings
+- The cheap-and-strong region is empty.
+- {facts.get("collapsed_passes")} of {facts.get("headline_passes")} passes collapsed, sending at least {threshold:.0f}% of questions to one model.
+- Median cross-vendor same-pick is {median:g} across {facts.get("cross_vendor_pairs")} pairs.
+
+The board is sorted by cost. Each row shows how many questions that pass actually answered (`n`). Read the report: [{metadata.get("report_url", "")}]({metadata.get("report_url", "")}).
+"""
+
+
+def open_blocks():
+    """Gradio 5 takes theme on Blocks. Gradio 6 takes it on launch."""
+    parameters = inspect.signature(gr.Blocks.__init__).parameters
+    theme_on_blocks = "theme" in parameters and parameters["theme"].kind is not inspect.Parameter.VAR_KEYWORD
+    if theme_on_blocks:
+        return gr.Blocks(theme=gr.themes.Soft())
+    return gr.Blocks()
+
+
+with open_blocks() as app:
     gr.Markdown("# AutoBench LLM Leaderboard")
     gr.Markdown(
         "Interactive leaderboard for AutoBench, where LLMs rank LLMs' responses. "
@@ -604,12 +773,57 @@ with gr.Blocks(theme=gr.themes.Soft()) as app:
             label="Domain Performance"
         )
 
+    # --- Router Benchmark (not part of the model-run dropdown) ---
+    with gr.Tab("Router Benchmark"):
+        router_runs = discover_router_runs()
+        if not router_runs:
+            gr.Markdown("No router benchmark run is in this checkout.")
+        else:
+            router_meta = router_runs[0]
+            router_board = pd.read_csv(os.path.join(router_meta["path"], "router_board.csv"))
+            router_same = pd.read_csv(os.path.join(router_meta["path"], "same_pick.csv"))
+            gr.Markdown(router_intro_markdown(router_meta))
+            gr.Markdown("### Board")
+            gr.DataFrame(value=router_board_display(router_board), interactive=False)
+            gr.Markdown(
+                "### Quality versus cost\n\n"
+                "Dashed lines are Always-Sol, Always-Luna, and Random. "
+                "Amber points collapsed onto one model."
+            )
+            gr.Plot(value=router_quality_cost_figure(router_board))
+            gr.Markdown("### Top-model share\n\nPasses at or above 90% are collapsed.")
+            gr.Plot(value=router_collapse_figure(router_board))
+            gr.Markdown(
+                "### Same-pick\n\n"
+                "Share of questions on which two passes named the same model. "
+                "This grid is the 12-pass comparison from the report. "
+                "The median of 0 is across the full set of cross-vendor pairs, not this grid alone."
+            )
+            gr.Plot(
+                value=router_same_pick_figure(
+                    router_same, router_meta.get("same_pick_axis", [])
+                )
+            )
+            gr.Markdown(
+                f"""
+### Methodology
+- The answer matrix is frozen: {router_meta.get("questions")} questions, {router_meta.get("matrix_models")} models, dated {router_meta.get("matrix_date")}.
+- Scoring rules were locked before any router was called. Each router choice is scored by lookup on that matrix. Routers are not re-judged.
+- **Accuracy** is the share of questions where the chosen model is statistically as good as the best available answer.
+- A pass is collapsed when its top model receives at least 90% of questions.
+- Same-pick is the share of questions on which two passes named the same model. The published median is the median across cross-vendor pairs.
+- [Read the report]({router_meta.get("report_url", "")})
+"""
+            )
+
     # --- Tab 5: About ---
     with gr.Tab("About AutoBench"):
         gr.Markdown("""
         ## About AutoBench
 
-        AutoBench is an LLM benchmark where Large Language Models (LLMs) evaluate and rank the responses generated by other LLMs. The questions themselves are also generated by LLMs across a diverse set of domains and ranked for quality.
+        AutoBench is the evaluation engine for the agentic era. A committee of language models writes questions, answers them, and scores the answers, so quality, cost, and speed are measured together. The same evidence grades models and, on the Router Benchmark tab, grades routers.
+
+        We don't build base models. We measure them. Neutrality is the product.
 
         ### Methodology
         1.  **Question Generation:** High-quality questions across various domains (Coding, History, Science, etc.) are generated by selected LLMs.
@@ -625,9 +839,10 @@ with gr.Blocks(theme=gr.themes.Soft()) as app:
         * **Chatbot Arena / Artificial Analysis Intelligence Index / MMLU:** Scores from other well-known benchmarks for comparison (where available).
 
         ### Data
-        This leaderboard reflects a run completed on April 23, 2025. Models included recently released models such as o4-mini, Gpt-4.1-mini, Gemini 2.5 Pro Preview, Claude 3.7 Sonnet:thikning, etc..
+        The model tabs show the latest public model run. The Router Benchmark is a separate tab and does not replace that run. Routers are scored by lookup on a frozen answer matrix: the rules were locked before any router was called.
 
         ### Links
+        * [Router Benchmark report](https://autobench.org/reports/router-benchmark-2026.pdf)
         * [AutoBench Run 2 Results](https://huggingface.co/blog/PeterKruger/autobench-2nd-run)
         * [AutoBench Blog Post](https://huggingface.co/blog/PeterKruger/autobench)
         * [Autobench Repositories](https://huggingface.co/AutoBench)
@@ -662,12 +877,16 @@ with gr.Blocks(theme=gr.themes.Soft()) as app:
     # Note: Initial data is already loaded via value parameters above
 
 # --- Launch the App ---
-print("Launching Gradio app...")
-app.launch(
-    favicon_path="static/manifest.json" if os.path.exists("static/manifest.json") else None,
-    server_name="0.0.0.0",
-    server_port=7860,
-    ssr_mode=False,
-    show_error=True
-)
-print("Gradio app launched.")
+if __name__ == "__main__":
+    print("Launching Gradio app...")
+    launch_kwargs = {
+        "favicon_path": "static/manifest.json" if os.path.exists("static/manifest.json") else None,
+        "server_name": "0.0.0.0",
+        "server_port": 7860,
+        "ssr_mode": False,
+        "show_error": True,
+    }
+    if "theme" in inspect.signature(gr.Blocks.launch).parameters:
+        launch_kwargs["theme"] = gr.themes.Soft()
+    app.launch(**launch_kwargs)
+    print("Gradio app launched.")

export_router_run.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""Export the Router Benchmark board from the paper's generated stats.
+
+Reads stats.json, collapse.dat, and isolation.dat. Does not retype results.
+Writes runs/router1_2026-10-26/ for the Hugging Face Space and the website importer.
+
+Usage (from AutoBench-Leaderboard-live):
+  python export_router_run.py
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import re
+from pathlib import Path
+
+
+# Frozen answer matrix this board is scored on. Identity of the published
+# model run, not a result copied out of the paper tables.
+MATRIX_DATE = "2026-08-29"
+MATRIX_MODELS = 20
+COLLAPSE_THRESHOLD = 0.90
+
+# key, public label, vendor, setting, is_baseline, collapse.dat label
+PASSES = [
+    ("oracle", "Oracle", "Baseline", "oracle", 1, None),
+    ("cheapest_adequate", "Cheapest-adequate", "Baseline", "cheapest adequate", 1, None),
+    ("best_static_single", "Always-Sol", "Baseline", "always Sol", 1, None),
+    ("always_luna", "Always-Luna", "Baseline", "always Luna", 1, None),
+    ("random", "Random", "Baseline", "random", 1, "Random"),
+    ("always_cheapest", "Always-cheapest", "Baseline", "always cheapest", 1, None),
+    ("orca_constrained_cheapest", "Orca cheapest", "OrcaRouter", "cheapest", 0, "Orca cheap"),
+    ("orca_constrained_balanced", "Orca balanced", "OrcaRouter", "balanced", 0, "Orca bal"),
+    ("orca_constrained_quality", "Orca quality", "OrcaRouter", "quality", 0, "Orca qual"),
+    ("azure_cost", "Azure cost", "Azure Model Router", "cost", 0, "Azure cost"),
+    ("azure_balanced", "Azure balanced", "Azure Model Router", "balanced", 0, "Azure bal"),
+    ("azure_quality", "Azure quality", "Azure Model Router", "quality", 0, "Azure qual"),
+    ("nadirclaw", "NadirClaw", "NadirClaw", "as shipped", 0, "NadirClaw"),
+    ("brick", "Brick", "Brick", "as shipped", 0, "Brick"),
+    ("cascade", "Cascade", "Cascade", "as shipped", 0, "Cascade"),
+    ("nd_subset_k0", "ND k0", "Not Diamond", "k0", 0, "ND k0"),
+    ("nd_subset_k4", "ND k4", "Not Diamond", "k4", 0, "ND k4"),
+    ("nd_subset_k5", "ND k5", "Not Diamond", "k5", 0, "ND k5"),
+    ("nd_native_k5", "ND native", "Not Diamond", "native", 0, "ND native"),
+    ("or_auto", "Auto low", "OpenRouter", "low", 0, "Auto low"),
+    ("or_auto_medium", "Auto medium", "OpenRouter", "medium", 0, "Auto med"),
+    ("or_auto_high", "Auto high", "OpenRouter", "high", 0, "Auto high"),
+    ("or_auto_max", "Auto max", "OpenRouter", "max", 0, "Auto max"),
+]
+
+BOARD_FIELDS = [
+    "pass_key",
+    "label",
+    "vendor",
+    "setting",
+    "n",
+    "accuracy",
+    "ci_low",
+    "ci_high",
+    "cost",
+    "cost_ci_low",
+    "cost_ci_high",
+    "top_share",
+    "collapsed",
+    "is_baseline",
+    "display_order",
+]
+
+
+def parse_collapse(path: Path) -> dict[str, float]:
+    shares: dict[str, float] = {}
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("idx"):
+            continue
+        match = re.match(r"^\d+\s+([0-9.]+)\s+\{(.+)\}$", line)
+        if not match:
+            raise SystemExit(f"Unparsed collapse row: {line}")
+        shares[match.group(2)] = float(match.group(1))
+    return shares
+
+
+def parse_axis(path: Path) -> list[str]:
+    text = path.read_text(encoding="utf-8")
+    match = re.search(r"xticklabels=\{([^}]+)\}", text)
+    if not match:
+        raise SystemExit(f"No xticklabels in {path}")
+    return [label.strip() for label in match.group(1).split(",")]
+
+
+def parse_isolation(path: Path, axis: list[str]) -> list[dict]:
+    rows = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("x "):
+            continue
+        x_s, y_s, rate_s = line.split()
+        x, y = int(x_s), int(y_s)
+        rows.append(
+            {
+                "axis_a": x,
+                "axis_b": y,
+                "pass_a": axis[x],
+                "pass_b": axis[y],
+                "rate": float(rate_s),
+            }
+        )
+    return rows
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--stats",
+        type=Path,
+        default=Path(
+            r"C:\DEV\AutoBench\autobench\docs\projects\router-benchmark-paper\generated\stats.json"
+        ),
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=Path(__file__).resolve().parent / "runs" / "router1_2026-10-26",
+    )
+    args = parser.parse_args()
+
+    generated = args.stats.parent
+    stats = json.loads(args.stats.read_text(encoding="utf-8"))
+    marginal = stats["marginal"]
+    shares = parse_collapse(generated / "data" / "collapse.dat")
+    axis = parse_axis(generated / "data" / "isolation_axis.tex")
+    same_pick = parse_isolation(generated / "data" / "isolation.dat", axis)
+    isolation = stats["isolation_summary"]
+
+    board = []
+    for order, (key, label, vendor, setting, is_baseline, collapse_label) in enumerate(PASSES):
+        if key not in marginal:
+            raise SystemExit(f"Missing marginal key: {key}")
+        row = marginal[key]
+        top_share = ""
+        collapsed = 0
+        if collapse_label is not None:
+            if collapse_label not in shares:
+                raise SystemExit(f"Missing collapse label: {collapse_label}")
+            top_share = shares[collapse_label]
+            collapsed = 1 if top_share >= COLLAPSE_THRESHOLD else 0
+        board.append(
+            {
+                "pass_key": key,
+                "label": label,
+                "vendor": vendor,
+                "setting": setting,
+                "n": int(row["n"]),
+                "accuracy": row["dh_pct"],
+                "ci_low": row["dh_ci"][0],
+                "ci_high": row["dh_ci"][1],
+                "cost": row["cost"],
+                "cost_ci_low": row["cost_ci"][0],
+                "cost_ci_high": row["cost_ci"][1],
+                "top_share": top_share,
+                "collapsed": collapsed,
+                "is_baseline": is_baseline,
+                "display_order": order,
+            }
+        )
+
+    headline = [row for row in board if row["is_baseline"] == 0]
+    collapsed_n = sum(row["collapsed"] for row in headline)
+    vendors = sorted({row["vendor"] for row in headline})
+    questions = int(marginal["oracle"]["n"])
+
+    metadata = {
+        "run_id": "router1_2026-10-26",
+        "run_type": "router",
+        "title": "AutoBench Router Benchmark 1 - October 2026",
+        "date": "2026-10-26",
+        "run_version": "2026-10-26",
+        "is_latest": False,
+        "description": (
+            "Commercial routers and baselines graded on a frozen answer matrix. "
+            "Accuracy is the share of questions where the chosen model is statistically "
+            "as good as the best available answer."
+        ),
+        "report_url": "https://autobench.org/reports/router-benchmark-2026.pdf",
+        "blog_url": "https://autobench.org/reports/router-benchmark-2026.pdf",
+        "matrix_date": MATRIX_DATE,
+        "matrix_models": MATRIX_MODELS,
+        "questions": questions,
+        "same_pick_axis": axis,
+        "facts": {
+            "headline_passes": len(headline),
+            "collapsed_passes": collapsed_n,
+            "collapse_threshold": COLLAPSE_THRESHOLD,
+            "commercial_routers": len(vendors),
+            "cross_vendor_pairs": isolation["n_cross_vendor"],
+            "median_same_pick": isolation["median_same_pick"],
+            "questions": questions,
+            "matrix_models": MATRIX_MODELS,
+            "matrix_date": MATRIX_DATE,
+        },
+    }
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    (args.out / "metadata.json").write_text(
+        json.dumps(metadata, indent=2) + "\n", encoding="utf-8"
+    )
+    with (args.out / "router_board.csv").open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(handle, fieldnames=BOARD_FIELDS)
+        writer.writeheader()
+        writer.writerows(board)
+    with (args.out / "same_pick.csv").open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(
+            handle, fieldnames=["axis_a", "axis_b", "pass_a", "pass_b", "rate"]
+        )
+        writer.writeheader()
+        writer.writerows(same_pick)
+
+    print(f"Wrote {args.out}")
+    print(
+        f"headline={len(headline)} collapsed={collapsed_n} "
+        f"vendors={len(vendors)} ({', '.join(vendors)}) "
+        f"pairs={isolation['n_cross_vendor']} median={isolation['median_same_pick']} "
+        f"questions={questions} same_pick_cells={len(same_pick)}"
+    )
+
+
+if __name__ == "__main__":
+    main()

runs/router1_2026-10-26/metadata.json

@@ -0,0 +1,39 @@
+{
+  "run_id": "router1_2026-10-26",
+  "run_type": "router",
+  "title": "AutoBench Router Benchmark 1 - October 2026",
+  "date": "2026-10-26",
+  "run_version": "2026-10-26",
+  "is_latest": false,
+  "description": "Commercial routers and baselines graded on a frozen answer matrix. Accuracy is the share of questions where the chosen model is statistically as good as the best available answer.",
+  "report_url": "https://autobench.org/reports/router-benchmark-2026.pdf",
+  "blog_url": "https://autobench.org/reports/router-benchmark-2026.pdf",
+  "matrix_date": "2026-08-29",
+  "matrix_models": 20,
+  "questions": 252,
+  "same_pick_axis": [
+    "Orca cheap",
+    "Orca bal",
+    "Orca qual",
+    "Azure cost",
+    "Azure bal",
+    "Azure qual",
+    "NadirClaw",
+    "Brick",
+    "Cascade",
+    "ND k5",
+    "Auto high",
+    "Auto low"
+  ],
+  "facts": {
+    "headline_passes": 17,
+    "collapsed_passes": 5,
+    "collapse_threshold": 0.9,
+    "commercial_routers": 7,
+    "cross_vendor_pairs": 59,
+    "median_same_pick": 0.0,
+    "questions": 252,
+    "matrix_models": 20,
+    "matrix_date": "2026-08-29"
+  }
+}

runs/router1_2026-10-26/router_board.csv

@@ -0,0 +1,24 @@
+pass_key,label,vendor,setting,n,accuracy,ci_low,ci_high,cost,cost_ci_low,cost_ci_high,top_share,collapsed,is_baseline,display_order
+oracle,Oracle,Baseline,oracle,252,100.0,100.0,100.0,0.03406342571592063,0.023090085942222026,0.04675808716741607,,0,1,0
+cheapest_adequate,Cheapest-adequate,Baseline,cheapest adequate,252,100.0,100.0,100.0,0.0017775440658730159,0.0012147553282936506,0.0024296202333019836,,0,1,1
+best_static_single,Always-Sol,Baseline,always Sol,248,87.5,83.5,91.5,0.07865474395161291,0.06454585211693548,0.09954083326612903,,0,1,2
+always_luna,Always-Luna,Baseline,always Luna,251,85.3,80.9,89.6,0.004334039103585658,0.0037202327948207175,0.005072818161354581,,0,1,3
+random,Random,Baseline,random,252,64.7,58.7,70.6,0.011535058849650793,0.007392945504305159,0.017047359445582142,0.0714,0,1,4
+always_cheapest,Always-cheapest,Baseline,always cheapest,252,44.4,38.5,50.4,0.00016450185134920633,0.00014635133277579365,0.0001831236469801587,,0,1,5
+orca_constrained_cheapest,Orca cheapest,OrcaRouter,cheapest,252,39.3,33.3,45.2,0.00022013774206349209,0.00019737690396825396,0.00024262488363095238,0.9921,1,0,6
+orca_constrained_balanced,Orca balanced,OrcaRouter,balanced,252,39.3,33.3,45.2,0.0002193627023809524,0.00019667757688492064,0.0002418150847222222,0.996,1,0,7
+orca_constrained_quality,Orca quality,OrcaRouter,quality,245,65.7,59.2,71.0,0.0005173967175510203,0.00042429937727755105,0.0006060875131632653,1.0,1,0,8
+azure_cost,Azure cost,Azure Model Router,cost,74,45.9,33.8,56.8,0.00912647891891892,0.00708104558918919,0.01094242530135135,0.5913,0,0,9
+azure_balanced,Azure balanced,Azure Model Router,balanced,179,82.7,78.8,89.4,0.020369913128491617,0.013021539309357542,0.03823974199790503,0.4802,0,0,10
+azure_quality,Azure quality,Azure Model Router,quality,238,85.3,80.3,89.1,0.064655818697479,0.054270396591386556,0.08091888025735294,0.4087,0,0,11
+nadirclaw,NadirClaw,NadirClaw,as shipped,249,70.3,64.3,75.9,0.05136010172289157,0.04020215791164659,0.0689510219376506,0.5873,0,0,12
+brick,Brick,Brick,as shipped,241,63.9,57.7,69.7,0.00310902156506224,0.002298716507154565,0.005002192490113071,0.8016,0,0,13
+cascade,Cascade,Cascade,as shipped,235,60.9,53.2,66.0,0.01035301790706383,0.007934312614846808,0.012938480779023402,0.5826,0,0,14
+nd_subset_k0,ND k0,Not Diamond,k0,248,87.5,83.5,91.5,0.07865474395161291,0.06454585211693548,0.09954083326612903,1.0,1,0,15
+nd_subset_k4,ND k4,Not Diamond,k4,249,80.3,75.5,85.2,0.05996616918433735,0.04669444507111647,0.07957475177908833,0.7698,0,0,16
+nd_subset_k5,ND k5,Not Diamond,k5,251,46.6,40.2,52.6,0.000941773053625498,0.0006677803593406375,0.0012532932769940238,0.8294,0,0,17
+nd_native_k5,ND native,Not Diamond,native,250,72.0,66.0,77.2,0.059749960000000005,0.0475751486,0.07856580617999999,0.6389,0,0,18
+or_auto,Auto low,OpenRouter,low,245,65.7,59.2,71.0,0.0005501390004897959,0.0004446207167918367,0.0006506275904183674,0.9841,1,0,19
+or_auto_medium,Auto medium,OpenRouter,medium,246,73.6,67.1,78.0,0.013130098215934962,0.008043599401597561,0.019080980965126015,0.7817,0,0,20
+or_auto_high,Auto high,OpenRouter,high,244,79.1,73.4,83.6,0.023005027726557378,0.018076698892231558,0.02858057378719467,0.5794,0,0,21
+or_auto_max,Auto max,OpenRouter,max,240,78.3,72.9,83.3,0.028915902712999996,0.020327376228814583,0.043068421132057284,0.7817,0,0,22

runs/router1_2026-10-26/same_pick.csv

@@ -0,0 +1,145 @@
+axis_a,axis_b,pass_a,pass_b,rate
+0,0,Orca cheap,Orca cheap,1.0
+0,1,Orca cheap,Orca bal,0.9881
+0,2,Orca cheap,Orca qual,0.0079
+0,3,Orca cheap,Azure cost,0.1389
+0,4,Orca cheap,Azure bal,0.0079
+0,5,Orca cheap,Azure qual,0.0
+0,6,Orca cheap,NadirClaw,0.0238
+0,7,Orca cheap,Brick,0.004
+0,8,Orca cheap,Cascade,0.0083
+0,9,Orca cheap,ND k5,0.0
+0,10,Orca cheap,Auto high,0.004
+0,11,Orca cheap,Auto low,0.0119
+1,0,Orca bal,Orca cheap,0.9881
+1,1,Orca bal,Orca bal,1.0
+1,2,Orca bal,Orca qual,0.004
+1,3,Orca bal,Azure cost,0.1389
+1,4,Orca bal,Azure bal,0.0079
+1,5,Orca bal,Azure qual,0.0
+1,6,Orca bal,NadirClaw,0.0238
+1,7,Orca bal,Brick,0.004
+1,8,Orca bal,Cascade,0.0
+1,9,Orca bal,ND k5,0.0
+1,10,Orca bal,Auto high,0.004
+1,11,Orca bal,Auto low,0.0079
+2,0,Orca qual,Orca cheap,0.0079
+2,1,Orca qual,Orca bal,0.004
+2,2,Orca qual,Orca qual,1.0
+2,3,Orca qual,Azure cost,0.0
+2,4,Orca qual,Azure bal,0.0
+2,5,Orca qual,Azure qual,0.0
+2,6,Orca qual,NadirClaw,0.0
+2,7,Orca qual,Brick,0.8016
+2,8,Orca qual,Cascade,0.5826
+2,9,Orca qual,ND k5,0.0
+2,10,Orca qual,Auto high,0.0
+2,11,Orca qual,Auto low,0.9841
+3,0,Azure cost,Orca cheap,0.1389
+3,1,Azure cost,Orca bal,0.1389
+3,2,Azure cost,Orca qual,0.0
+3,3,Azure cost,Azure cost,1.0
+3,4,Azure cost,Azure bal,0.1905
+3,5,Azure cost,Azure qual,0.0317
+3,6,Azure cost,NadirClaw,0.0
+3,7,Azure cost,Brick,0.0
+3,8,Azure cost,Cascade,0.0
+3,9,Azure cost,ND k5,0.0
+3,10,Azure cost,Auto high,0.0
+3,11,Azure cost,Auto low,0.0
+4,0,Azure bal,Orca cheap,0.0079
+4,1,Azure bal,Orca bal,0.0079
+4,2,Azure bal,Orca qual,0.0
+4,3,Azure bal,Azure cost,0.1905
+4,4,Azure bal,Azure bal,1.0
+4,5,Azure bal,Azure qual,0.1111
+4,6,Azure bal,NadirClaw,0.0079
+4,7,Azure bal,Brick,0.0
+4,8,Azure bal,Cascade,0.0
+4,9,Azure bal,ND k5,0.0
+4,10,Azure bal,Auto high,0.0
+4,11,Azure bal,Auto low,0.0119
+5,0,Azure qual,Orca cheap,0.0
+5,1,Azure qual,Orca bal,0.0
+5,2,Azure qual,Orca qual,0.0
+5,3,Azure qual,Azure cost,0.0317
+5,4,Azure qual,Azure bal,0.1111
+5,5,Azure qual,Azure qual,1.0
+5,6,Azure qual,NadirClaw,0.123
+5,7,Azure qual,Brick,0.0
+5,8,Azure qual,Cascade,0.0
+5,9,Azure qual,ND k5,0.0
+5,10,Azure qual,Auto high,0.0079
+5,11,Azure qual,Auto low,0.0
+6,0,NadirClaw,Orca cheap,0.0238
+6,1,NadirClaw,Orca bal,0.0238
+6,2,NadirClaw,Orca qual,0.0
+6,3,NadirClaw,Azure cost,0.0
+6,4,NadirClaw,Azure bal,0.0079
+6,5,NadirClaw,Azure qual,0.123
+6,6,NadirClaw,NadirClaw,1.0
+6,7,NadirClaw,Brick,0.0
+6,8,NadirClaw,Cascade,0.0
+6,9,NadirClaw,ND k5,0.3413
+6,10,NadirClaw,Auto high,0.004
+6,11,NadirClaw,Auto low,0.0
+7,0,Brick,Orca cheap,0.004
+7,1,Brick,Orca bal,0.004
+7,2,Brick,Orca qual,0.8016
+7,3,Brick,Azure cost,0.0
+7,4,Brick,Azure bal,0.0
+7,5,Brick,Azure qual,0.0
+7,6,Brick,NadirClaw,0.0
+7,7,Brick,Brick,1.0
+7,8,Brick,Cascade,0.4752
+7,9,Brick,ND k5,0.0
+7,10,Brick,Auto high,0.0
+7,11,Brick,Auto low,0.7897
+8,0,Cascade,Orca cheap,0.0083
+8,1,Cascade,Orca bal,0.0
+8,2,Cascade,Orca qual,0.5826
+8,3,Cascade,Azure cost,0.0
+8,4,Cascade,Azure bal,0.0
+8,5,Cascade,Azure qual,0.0
+8,6,Cascade,NadirClaw,0.0
+8,7,Cascade,Brick,0.4752
+8,8,Cascade,Cascade,1.0
+8,9,Cascade,ND k5,0.0
+8,10,Cascade,Auto high,0.0
+8,11,Cascade,Auto low,0.5702
+9,0,ND k5,Orca cheap,0.0
+9,1,ND k5,Orca bal,0.0
+9,2,ND k5,Orca qual,0.0
+9,3,ND k5,Azure cost,0.0
+9,4,ND k5,Azure bal,0.0
+9,5,ND k5,Azure qual,0.0
+9,6,ND k5,NadirClaw,0.3413
+9,7,ND k5,Brick,0.0
+9,8,ND k5,Cascade,0.0
+9,9,ND k5,ND k5,1.0
+9,10,ND k5,Auto high,0.0
+9,11,ND k5,Auto low,0.0
+10,0,Auto high,Orca cheap,0.004
+10,1,Auto high,Orca bal,0.004
+10,2,Auto high,Orca qual,0.0
+10,3,Auto high,Azure cost,0.0
+10,4,Auto high,Azure bal,0.0
+10,5,Auto high,Azure qual,0.0079
+10,6,Auto high,NadirClaw,0.004
+10,7,Auto high,Brick,0.0
+10,8,Auto high,Cascade,0.0
+10,9,Auto high,ND k5,0.0
+10,10,Auto high,Auto high,1.0
+10,11,Auto high,Auto low,0.0
+11,0,Auto low,Orca cheap,0.0119
+11,1,Auto low,Orca bal,0.0079
+11,2,Auto low,Orca qual,0.9841
+11,3,Auto low,Azure cost,0.0
+11,4,Auto low,Azure bal,0.0119
+11,5,Auto low,Azure qual,0.0
+11,6,Auto low,NadirClaw,0.0
+11,7,Auto low,Brick,0.7897
+11,8,Auto low,Cascade,0.5702
+11,9,Auto low,ND k5,0.0
+11,10,Auto low,Auto high,0.0
+11,11,Auto low,Auto low,1.0