Point the Router Benchmark at the white paper and lock ARI at β = 0.2.

app.py

@@ -48,18 +48,17 @@ def discover_available_runs() -> List[Dict]:
                 try:
                     with open(metadata_path, 'r') as f:
                         metadata = json.load(f)
-                    # Router runs have their own tab. Leaving them in this list
-                    # would make the model dropdown try to load summary CSVs.
-                    if metadata.get("run_type") == "router":
-                        continue
                     metadata["path"] = run_path
                     runs.append(metadata)
                 except Exception as e:
                     print(f"Error loading metadata for {run_dir}: {e}")
     
-    # Sort by date, newest first
+    # Model runs stay newest-first. Router benchmarks are separate runs and
+    # are listed after every model run, so they are never the default.
     runs.sort(key=lambda x: x.get("date", ""), reverse=True)
-    return runs
+    model_runs = [run for run in runs if run.get("run_type") != "router"]
+    router_runs = [run for run in runs if run.get("run_type") == "router"]
+    return model_runs + router_runs
 
 def load_run_metadata(run_id: str) -> Dict:
     """Load metadata for a specific run."""
@@ -392,27 +391,86 @@ def create_p99_latency_plot(data: Dict[str, pd.DataFrame]) -> tuple:
     )
     return fig_p99, ""
 
+def benchmark_visibility(show_router: bool) -> tuple:
+    """Show either the model tabs or the router tabs.
+
+    Hiding the wrapping column is not enough in Gradio 6: a column that
+    contains tabs stays on screen. Each tab is toggled as well.
+    """
+    model_on = not show_router
+    return (
+        gr.update(visible=model_on),
+        gr.update(visible=show_router),
+        gr.update(visible=model_on),
+        gr.update(visible=show_router),
+        *(gr.update(visible=model_on) for _ in range(6)),
+        *(gr.update(visible=show_router) for _ in range(5)),
+    )
+
+
+def format_run_info(run_metadata: Dict) -> str:
+    """One-line description of the run currently selected in the dropdown."""
+    info = (
+        f"**Current Run:** {run_metadata.get('title', 'Unknown')} "
+        f"({run_metadata.get('date', 'Unknown date')})"
+    )
+    if run_metadata.get("run_type") == "router":
+        routers = run_metadata.get("facts", {}).get("commercial_routers")
+        if routers:
+            info += f" - {routers} commercial routers"
+        return info
+    if "model_count" in run_metadata:
+        info += f" - {run_metadata['model_count']} models"
+    return info
+
+
+def load_router_views(run_metadata: Dict) -> Dict:
+    """Tables and figures for one router-benchmark run."""
+    board = pd.read_csv(os.path.join(run_metadata["path"], "router_board.csv"))
+    same = pd.read_csv(os.path.join(run_metadata["path"], "same_pick.csv"))
+    return {
+        "intro": router_intro_markdown(run_metadata),
+        "board": router_board_display(board),
+        "quality": router_quality_cost_figure(board),
+        "collapse": router_collapse_figure(board),
+        "same": router_same_pick_figure(same, run_metadata.get("same_pick_axis", [])),
+        "method": router_method_markdown(run_metadata),
+    }
+
+
 def update_leaderboard_data(selected_run_id: str) -> tuple:
-    """Update all leaderboard components when run selection changes."""
+    """Update the visible benchmark when the run selection changes."""
     if not selected_run_id:
-        # Return empty/default values for all outputs
         empty_df = pd.DataFrame()
         return (
-            empty_df, empty_df, "", empty_df, empty_df, empty_df, empty_df,  # DataFrames + benchmark intro
-            None, "", None, "", None, "",  # Plots and messages
-            "No run selected", ""  # Info message, correlations text
+            empty_df, empty_df, "", empty_df, empty_df, empty_df, empty_df,
+            None, "", None, "", None, "",
+            "No run selected", "",
+            *benchmark_visibility(False),
+            gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(),
+        )
+
+    run_metadata = load_run_metadata(selected_run_id)
+    if run_metadata.get("run_type") == "router":
+        views = load_router_views(run_metadata)
+        return (
+            gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(),
+            gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(),
+            format_run_info(run_metadata), gr.skip(),
+            *benchmark_visibility(True),
+            views["intro"], views["board"], views["quality"], views["collapse"], views["same"], views["method"],
         )
-    
-    # Load data for selected run
+
     data = load_run_data(selected_run_id)
-    run_metadata = load_run_metadata(selected_run_id)
-    
+
     if not data:
         empty_df = pd.DataFrame()
         return (
             empty_df, empty_df, "", empty_df, empty_df, empty_df, empty_df,
             None, "Error loading data", None, "Error loading data", None, "Error loading data",
-            f"Error loading run: {selected_run_id}", ""
+            f"Error loading run: {selected_run_id}", "",
+            *benchmark_visibility(False),
+            gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(),
         )
     
     # Get DataFrames
@@ -452,46 +510,18 @@ def update_leaderboard_data(selected_run_id: str) -> tuple:
     avg_latency_plot, avg_latency_msg = create_avg_latency_plot(data)
     p99_latency_plot, p99_latency_msg = create_p99_latency_plot(data)
     
-    # Create info message
-    info_msg = f"**Current Run:** {run_metadata.get('title', 'Unknown')} ({run_metadata.get('date', 'Unknown date')})"
-    if 'model_count' in run_metadata:
-        info_msg += f" - {run_metadata['model_count']} models"
-    
-    # Get correlation text and benchmark comparison intro
     correlations_text = format_correlations_text(data.get("correlations", {}))
     benchmark_intro_text = format_benchmark_comparison_intro(data, run_metadata)
-    
+
     return (
         overall_rank_display, benchmark_display, benchmark_intro_text, cost_display, avg_latency_display, p99_latency_display, domain_display,
         cost_plot, cost_msg, avg_latency_plot, avg_latency_msg, p99_latency_plot, p99_latency_msg,
-        info_msg, correlations_text
+        format_run_info(run_metadata), correlations_text,
+        *benchmark_visibility(False),
+        gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(),
     )
 
 # --- Build Gradio App ---
-def discover_router_runs() -> List[Dict]:
-    """Router runs are excluded from the model dropdown."""
-    routers = []
-    if not os.path.exists(RUNS_DIR):
-        return routers
-    for run_dir in os.listdir(RUNS_DIR):
-        run_path = os.path.join(RUNS_DIR, run_dir)
-        metadata_path = os.path.join(run_path, "metadata.json")
-        if not os.path.isdir(run_path) or not os.path.exists(metadata_path):
-            continue
-        try:
-            with open(metadata_path, "r", encoding="utf-8") as handle:
-                metadata = json.load(handle)
-        except Exception as exc:
-            print(f"Error loading router metadata for {run_dir}: {exc}")
-            continue
-        if metadata.get("run_type") != "router":
-            continue
-        metadata["path"] = run_path
-        routers.append(metadata)
-    routers.sort(key=lambda item: item.get("date", ""), reverse=True)
-    return routers
-
-
 def router_quality_cost_figure(board: pd.DataFrame):
     headline = board[board["is_baseline"] == 0].copy()
     headline["group"] = headline["collapsed"].map(
@@ -574,10 +604,13 @@ def router_same_pick_figure(same: pd.DataFrame, axis: List[str]):
 
 
 def router_board_display(board: pd.DataFrame) -> pd.DataFrame:
-    view = board.sort_values(["cost", "display_order"]).copy()
+    view = board.sort_values(
+        ["ari", "cost", "display_order"], ascending=[False, True, True]
+    ).copy()
     displayed = pd.DataFrame(
         {
             "Pass": view["label"],
+            "ARI": view["ari"].map(lambda value: f"{value:.1f}"),
             "Vendor": view["vendor"],
             "Setting": view["setting"],
             "n": view["n"],
@@ -619,7 +652,22 @@ We graded **{facts.get("commercial_routers")} commercial routers** ({facts.get("
 - {facts.get("collapsed_passes")} of {facts.get("headline_passes")} passes collapsed, sending at least {threshold:.0f}% of questions to one model.
 - Median cross-vendor same-pick is {median:g} across {facts.get("cross_vendor_pairs")} pairs.
 
-The board is sorted by cost. Each row shows how many questions that pass actually answered (`n`). Read the report: [{metadata.get("report_url", "")}]({metadata.get("report_url", "")}).
+The board is ordered by **ARI** (AutoBench Router Index) at β = 0.2, highest first. ARI runs from 0 to 100 and is higher when a pass is both accurate and cheap. A pass no better than random scores 0. Quality is the share of the gap from the random floor to a perfect answer. Cost is a log score between $0.00001 and $1 per question. β = 0.2 is the weight on cost. Each row shows how many questions that pass actually answered (`n`). Read the report: [{metadata.get("report_url", "")}]({metadata.get("report_url", "")}).
+"""
+
+
+def router_method_markdown(metadata: Dict) -> str:
+    facts = metadata.get("facts", {})
+    return f"""
+## Methodology
+
+- The answer matrix is frozen: {facts.get("questions", metadata.get("questions"))} questions, {facts.get("matrix_models", metadata.get("matrix_models"))} models, dated {facts.get("matrix_date", metadata.get("matrix_date"))}.
+- Scoring rules were locked before any router was called. Each router choice is scored by lookup on that matrix. Routers are not re-judged.
+- **Accuracy** is the share of questions where the chosen model is statistically as good as the best available answer.
+- **ARI** (AutoBench Router Index) ranks the board at β = 0.2. It is a 0–100 weighted harmonic mean of quality and cost. Quality is the share of the gap from the random floor (64.7% on this matrix) to a perfect answer. Cost is a log score between the fixed anchors $0.00001 and $1 per question. A pass at or below the random floor scores 0.
+- A pass is collapsed when its top model receives at least 90% of questions.
+- Same-pick is the share of questions on which two passes named the same model. The published median is the median across cross-vendor pairs. The same-pick tab shows the 12-pass grid from the report, which is not that full pair set.
+- [Read the report]({metadata.get("report_url", "")})
 """
 
 
@@ -632,6 +680,24 @@ def open_blocks():
     return gr.Blocks()
 
 
+router_catalog = [run for run in available_runs if run.get("run_type") == "router"]
+if router_catalog:
+    initial_router = load_router_views(router_catalog[0])
+    opening_run = router_catalog[0]
+else:
+    initial_router = {
+        "intro": "No router benchmark run is in this checkout.",
+        "board": pd.DataFrame(),
+        "quality": None,
+        "collapse": None,
+        "same": None,
+        "method": "",
+    }
+    opening_run = latest_run
+opens_on_router = opening_run.get("run_type") == "router"
+model_visible = not opens_on_router
+print(f"Opening on: {opening_run['title']}")
+
 with open_blocks() as app:
     gr.Markdown("# AutoBench LLM Leaderboard")
     gr.Markdown(
@@ -647,209 +713,194 @@ with open_blocks() as app:
             run_choices = [(f"{run['date']} - {run['title']}", run['run_id']) for run in available_runs]
             run_selector = gr.Dropdown(
                 choices=run_choices,
-                value=latest_run["run_id"],
+                value=opening_run["run_id"],
                 label="📊 Select AutoBench Run",
-                info="Choose a benchmark run to view its results"
+                info="Opens on the Router Benchmark. Model runs are listed above it."
             )
         with gr.Column(scale=2):
-            current_run_info = gr.Markdown(
-                f"**Current Run:** {latest_run['title']} ({latest_run['date']})" + 
-                (f" - {latest_run['model_count']} models" if 'model_count' in latest_run else "")
-            )
+            current_run_info = gr.Markdown(format_run_info(opening_run))
     
     gr.Markdown("---")
 
-    # --- Tab 1: Overall Ranking ---
-    with gr.Tab("Overall Ranking"):
-        gr.Markdown("## Overall Model Performance")
-        gr.Markdown("Models ranked by AutoBench score. Lower cost ($ Cents), latency (s), and fail rate (%) are better. Iterations shows the number of evaluations per model.")
-        
-        # Add correlations display
-        initial_correlations = format_correlations_text(current_data.get("correlations", {}))
-        correlations_display = gr.Markdown(value=initial_correlations)
-        
-        overall_ranking_table = gr.DataFrame(
-            current_data.get("summary_display", pd.DataFrame()).copy().rename(columns={'AutoBench': 'Rank'}) if 'AutoBench' in current_data.get("summary_display", pd.DataFrame()).columns else current_data.get("summary_display", pd.DataFrame()),
-            interactive=True,
-            label="Overall Rankings"
-        )
-
-    # --- Tab 2: Benchmark Comparison ---
-    with gr.Tab("Benchmark Comparison"):
-        gr.Markdown("## Benchmark Comparison")
-        benchmark_comparison_intro = gr.Markdown(
-            value=format_benchmark_comparison_intro(current_data, latest_run)
-        )
-        
-        benchmark_comparison_table = gr.DataFrame(
-            current_data.get("benchmark_display", pd.DataFrame()),
-            interactive=True,
-            label="Benchmark Comparison"
-        )
-
-    # --- Tab 3: Performance Plots ---
-    with gr.Tab("Performance Plots"):
-        gr.Markdown("## Performance Visualizations")
-        gr.Markdown("Exploring relationships between AutoBench Rank, Latency, and Cost.")
-
-        # Scatter Plot 1: Cost vs Rank
-        gr.Markdown("### Rank vs. Average Cost") 
-        initial_cost_plot, initial_cost_msg = create_cost_scatter_plot(current_data)
-        cost_plot = gr.Plot(value=initial_cost_plot)
-        cost_plot_msg = gr.Markdown(value=initial_cost_msg)
-
-        # Plot 2: Rank vs Average Latency
-        gr.Markdown("### Rank vs. Average Latency")
-        initial_avg_latency_plot, initial_avg_latency_msg = create_avg_latency_plot(current_data)
-        avg_latency_plot = gr.Plot(value=initial_avg_latency_plot)
-        avg_latency_plot_msg = gr.Markdown(value=initial_avg_latency_msg)
-
-        # Plot 3: Rank vs P99 Latency
-        gr.Markdown("### Rank vs. P99 Latency")
-        initial_p99_latency_plot, initial_p99_latency_msg = create_p99_latency_plot(current_data)
-        p99_latency_plot = gr.Plot(value=initial_p99_latency_plot)
-        p99_latency_plot_msg = gr.Markdown(value=initial_p99_latency_msg)
-
-    # --- Tab 4: Cost & Latency Analysis ---
-    with gr.Tab("Cost & Latency Analysis"):
-        gr.Markdown("## Performance vs. Cost/Latency Trade-offs")
-
-        # Cost Breakdown Table
-        gr.Markdown("### Cost Breakdown per Domain ($ Cents/Response)")
-        cost_df = current_data.get("cost", pd.DataFrame())
-        if not cost_df.empty and 'model_name' in cost_df.columns:
-            cols = ['model_name'] + [col for col in cost_df.columns if col != 'model_name']
-            initial_cost_display = cost_df[cols]
-        else:
-            initial_cost_display = cost_df
-        cost_breakdown_table = gr.DataFrame(
-            value=initial_cost_display,
-            interactive=True,
-            label="Cost Breakdown"
-        )
-
-        # Latency Breakdown Tables
-        gr.Markdown("### Average Latency Breakdown per Domain (Seconds)")
-        avg_latency_df = current_data.get("avg_latency", pd.DataFrame())
-        if not avg_latency_df.empty and 'model_name' in avg_latency_df.columns:
-            cols = ['model_name'] + [col for col in avg_latency_df.columns if col != 'model_name']
-            initial_avg_latency_display = avg_latency_df[cols]
-        else:
-            initial_avg_latency_display = avg_latency_df
-        avg_latency_breakdown_table = gr.DataFrame(
-            value=initial_avg_latency_display,
-            interactive=True,
-            label="Average Latency Breakdown"
-        )
-
-        gr.Markdown("### P99 Latency Breakdown per Domain (Seconds)")
-        p99_latency_df = current_data.get("p99_latency", pd.DataFrame())
-        if not p99_latency_df.empty and 'model_name' in p99_latency_df.columns:
-            cols = ['model_name'] + [col for col in p99_latency_df.columns if col != 'model_name']
-            initial_p99_latency_display = p99_latency_df[cols]
-        else:
-            initial_p99_latency_display = p99_latency_df
-        p99_latency_breakdown_table = gr.DataFrame(
-            value=initial_p99_latency_display,
-            interactive=True,
-            label="P99 Latency Breakdown"
-        )
-
-
-    # --- Tab 5: Domain Performance ---
-    with gr.Tab("Domain Performance"):
-        gr.Markdown("## Performance Across Different Domains")
-        gr.Markdown("Model ranks within specific knowledge or task areas. Higher is better.")
-        
-        domain_df = current_data.get("domain", pd.DataFrame())
-        if not domain_df.empty and 'Model Name' in domain_df.columns:
-            cols = ['Model Name'] + [col for col in domain_df.columns if col != 'Model Name']
-            initial_domain_display = domain_df[cols]
-        else:
-            initial_domain_display = domain_df
-        domain_performance_table = gr.DataFrame(
-            value=initial_domain_display,
-            interactive=True,
-            label="Domain Performance"
-        )
+    with gr.Column(visible=model_visible) as model_view:
+        with gr.Tabs(visible=model_visible) as model_tabs:
+            # --- Tab 1: Overall Ranking ---
+            with gr.Tab("Overall Ranking", visible=model_visible) as tab_overall:
+                gr.Markdown("## Overall Model Performance")
+                gr.Markdown("Models ranked by AutoBench score. Lower cost ($ Cents), latency (s), and fail rate (%) are better. Iterations shows the number of evaluations per model.")
+
+                # Add correlations display
+                initial_correlations = format_correlations_text(current_data.get("correlations", {}))
+                correlations_display = gr.Markdown(value=initial_correlations)
+
+                overall_ranking_table = gr.DataFrame(
+                    current_data.get("summary_display", pd.DataFrame()).copy().rename(columns={'AutoBench': 'Rank'}) if 'AutoBench' in current_data.get("summary_display", pd.DataFrame()).columns else current_data.get("summary_display", pd.DataFrame()),
+                    interactive=True,
+                    label="Overall Rankings"
+                )
 
-    # --- Router Benchmark (not part of the model-run dropdown) ---
-    with gr.Tab("Router Benchmark"):
-        router_runs = discover_router_runs()
-        if not router_runs:
-            gr.Markdown("No router benchmark run is in this checkout.")
-        else:
-            router_meta = router_runs[0]
-            router_board = pd.read_csv(os.path.join(router_meta["path"], "router_board.csv"))
-            router_same = pd.read_csv(os.path.join(router_meta["path"], "same_pick.csv"))
-            gr.Markdown(router_intro_markdown(router_meta))
-            gr.Markdown("### Board")
-            gr.DataFrame(value=router_board_display(router_board), interactive=False)
-            gr.Markdown(
-                "### Quality versus cost\n\n"
-                "Dashed lines are Always-Sol, Always-Luna, and Random. "
-                "Amber points collapsed onto one model."
-            )
-            gr.Plot(value=router_quality_cost_figure(router_board))
-            gr.Markdown("### Top-model share\n\nPasses at or above 90% are collapsed.")
-            gr.Plot(value=router_collapse_figure(router_board))
-            gr.Markdown(
-                "### Same-pick\n\n"
-                "Share of questions on which two passes named the same model. "
-                "This grid is the 12-pass comparison from the report. "
-                "The median of 0 is across the full set of cross-vendor pairs, not this grid alone."
-            )
-            gr.Plot(
-                value=router_same_pick_figure(
-                    router_same, router_meta.get("same_pick_axis", [])
+            # --- Tab 2: Benchmark Comparison ---
+            with gr.Tab("Benchmark Comparison", visible=model_visible) as tab_compare:
+                gr.Markdown("## Benchmark Comparison")
+                benchmark_comparison_intro = gr.Markdown(
+                    value=format_benchmark_comparison_intro(current_data, latest_run)
                 )
-            )
-            gr.Markdown(
-                f"""
-### Methodology
-- The answer matrix is frozen: {router_meta.get("questions")} questions, {router_meta.get("matrix_models")} models, dated {router_meta.get("matrix_date")}.
-- Scoring rules were locked before any router was called. Each router choice is scored by lookup on that matrix. Routers are not re-judged.
-- **Accuracy** is the share of questions where the chosen model is statistically as good as the best available answer.
-- A pass is collapsed when its top model receives at least 90% of questions.
-- Same-pick is the share of questions on which two passes named the same model. The published median is the median across cross-vendor pairs.
-- [Read the report]({router_meta.get("report_url", "")})
-"""
-            )
 
-    # --- Tab 5: About ---
-    with gr.Tab("About AutoBench"):
-        gr.Markdown("""
-        ## About AutoBench
+                benchmark_comparison_table = gr.DataFrame(
+                    current_data.get("benchmark_display", pd.DataFrame()),
+                    interactive=True,
+                    label="Benchmark Comparison"
+                )
 
-        AutoBench is the evaluation engine for the agentic era. A committee of language models writes questions, answers them, and scores the answers, so quality, cost, and speed are measured together. The same evidence grades models and, on the Router Benchmark tab, grades routers.
+            # --- Tab 3: Performance Plots ---
+            with gr.Tab("Performance Plots", visible=model_visible) as tab_plots:
+                gr.Markdown("## Performance Visualizations")
+                gr.Markdown("Exploring relationships between AutoBench Rank, Latency, and Cost.")
+
+                # Scatter Plot 1: Cost vs Rank
+                gr.Markdown("### Rank vs. Average Cost") 
+                initial_cost_plot, initial_cost_msg = create_cost_scatter_plot(current_data)
+                cost_plot = gr.Plot(value=initial_cost_plot)
+                cost_plot_msg = gr.Markdown(value=initial_cost_msg)
+
+                # Plot 2: Rank vs Average Latency
+                gr.Markdown("### Rank vs. Average Latency")
+                initial_avg_latency_plot, initial_avg_latency_msg = create_avg_latency_plot(current_data)
+                avg_latency_plot = gr.Plot(value=initial_avg_latency_plot)
+                avg_latency_plot_msg = gr.Markdown(value=initial_avg_latency_msg)
+
+                # Plot 3: Rank vs P99 Latency
+                gr.Markdown("### Rank vs. P99 Latency")
+                initial_p99_latency_plot, initial_p99_latency_msg = create_p99_latency_plot(current_data)
+                p99_latency_plot = gr.Plot(value=initial_p99_latency_plot)
+                p99_latency_plot_msg = gr.Markdown(value=initial_p99_latency_msg)
+
+            # --- Tab 4: Cost & Latency Analysis ---
+            with gr.Tab("Cost & Latency Analysis", visible=model_visible) as tab_cost:
+                gr.Markdown("## Performance vs. Cost/Latency Trade-offs")
+
+                # Cost Breakdown Table
+                gr.Markdown("### Cost Breakdown per Domain ($ Cents/Response)")
+                cost_df = current_data.get("cost", pd.DataFrame())
+                if not cost_df.empty and 'model_name' in cost_df.columns:
+                    cols = ['model_name'] + [col for col in cost_df.columns if col != 'model_name']
+                    initial_cost_display = cost_df[cols]
+                else:
+                    initial_cost_display = cost_df
+                cost_breakdown_table = gr.DataFrame(
+                    value=initial_cost_display,
+                    interactive=True,
+                    label="Cost Breakdown"
+                )
 
-        We don't build base models. We measure them. Neutrality is the product.
+                # Latency Breakdown Tables
+                gr.Markdown("### Average Latency Breakdown per Domain (Seconds)")
+                avg_latency_df = current_data.get("avg_latency", pd.DataFrame())
+                if not avg_latency_df.empty and 'model_name' in avg_latency_df.columns:
+                    cols = ['model_name'] + [col for col in avg_latency_df.columns if col != 'model_name']
+                    initial_avg_latency_display = avg_latency_df[cols]
+                else:
+                    initial_avg_latency_display = avg_latency_df
+                avg_latency_breakdown_table = gr.DataFrame(
+                    value=initial_avg_latency_display,
+                    interactive=True,
+                    label="Average Latency Breakdown"
+                )
 
-        ### Methodology
-        1.  **Question Generation:** High-quality questions across various domains (Coding, History, Science, etc.) are generated by selected LLMs.
-        2.  **Response Generation:** The models being benchmarked generate answers to these questions.
-        3.  **Ranking:** Ranking LLMs rank the responses from different models for each question, on a 1-5 scale.
-        4.  **Aggregation:** Scores are averaged across multiple questions and domains to produce the final AutoBench rank. 
+                gr.Markdown("### P99 Latency Breakdown per Domain (Seconds)")
+                p99_latency_df = current_data.get("p99_latency", pd.DataFrame())
+                if not p99_latency_df.empty and 'model_name' in p99_latency_df.columns:
+                    cols = ['model_name'] + [col for col in p99_latency_df.columns if col != 'model_name']
+                    initial_p99_latency_display = p99_latency_df[cols]
+                else:
+                    initial_p99_latency_display = p99_latency_df
+                p99_latency_breakdown_table = gr.DataFrame(
+                    value=initial_p99_latency_display,
+                    interactive=True,
+                    label="P99 Latency Breakdown"
+                )
 
-        ### Metrics
-        * **AutoBench Score (AB):** The average rank received by a model's responses across all questions/domains (higher is better).
-        * **Avg Cost (USD Cents/response):** Estimated average cost to generate one response based on model provider pricing (input+output tokens). Lower is better.
-        * **Avg Latency (s):** Average time taken by the model to generate a response. Lower is better.
-        * **P99 Latency (s):** The 99th percentile of response time, indicating worst-case latency. Lower is better.
-        * **Chatbot Arena / Artificial Analysis Intelligence Index / MMLU:** Scores from other well-known benchmarks for comparison (where available).
 
-        ### Data
-        The model tabs show the latest public model run. The Router Benchmark is a separate tab and does not replace that run. Routers are scored by lookup on a frozen answer matrix: the rules were locked before any router was called.
+            # --- Tab 5: Domain Performance ---
+            with gr.Tab("Domain Performance", visible=model_visible) as tab_domain:
+                gr.Markdown("## Performance Across Different Domains")
+                gr.Markdown("Model ranks within specific knowledge or task areas. Higher is better.")
+
+                domain_df = current_data.get("domain", pd.DataFrame())
+                if not domain_df.empty and 'Model Name' in domain_df.columns:
+                    cols = ['Model Name'] + [col for col in domain_df.columns if col != 'Model Name']
+                    initial_domain_display = domain_df[cols]
+                else:
+                    initial_domain_display = domain_df
+                domain_performance_table = gr.DataFrame(
+                    value=initial_domain_display,
+                    interactive=True,
+                    label="Domain Performance"
+                )
 
-        ### Links
-        * [Router Benchmark report](https://autobench.org/reports/router-benchmark-2026.pdf)
-        * [AutoBench Run 2 Results](https://huggingface.co/blog/PeterKruger/autobench-2nd-run)
-        * [AutoBench Blog Post](https://huggingface.co/blog/PeterKruger/autobench)
-        * [Autobench Repositories](https://huggingface.co/AutoBench)
+            # --- Tab 5: About ---
+            with gr.Tab("About AutoBench", visible=model_visible) as tab_about:
+                gr.Markdown("""
+                ## About AutoBench
+
+                AutoBench is the evaluation engine for the agentic era. A committee of language models writes questions, answers them, and scores the answers, so quality, cost, and speed are measured together. The same evidence grades models and grades routers. Choose the Router Benchmark from the run dropdown; it is listed last and opens its own tabs.
+
+                We don't build base models. We measure them. Neutrality is the product.
+
+                ### Methodology
+                1.  **Question Generation:** High-quality questions across various domains (Coding, History, Science, etc.) are generated by selected LLMs.
+                2.  **Response Generation:** The models being benchmarked generate answers to these questions.
+                3.  **Ranking:** Ranking LLMs rank the responses from different models for each question, on a 1-5 scale.
+                4.  **Aggregation:** Scores are averaged across multiple questions and domains to produce the final AutoBench rank. 
+
+                ### Metrics
+                * **AutoBench Score (AB):** The average rank received by a model's responses across all questions/domains (higher is better).
+                * **Avg Cost (USD Cents/response):** Estimated average cost to generate one response based on model provider pricing (input+output tokens). Lower is better.
+                * **Avg Latency (s):** Average time taken by the model to generate a response. Lower is better.
+                * **P99 Latency (s):** The 99th percentile of response time, indicating worst-case latency. Lower is better.
+                * **Chatbot Arena / Artificial Analysis Intelligence Index / MMLU:** Scores from other well-known benchmarks for comparison (where available).
+
+                ### Data
+                The space opens on the Router Benchmark. Model runs are listed above it in the dropdown and keep these tabs. Routers are scored by lookup on a frozen answer matrix: the rules were locked before any router was called.
+
+                ### Links
+                * [Router Benchmark report](https://autobench.org/reports/do-llm-routers-pay-for-themselves)
+                * [AutoBench Run 2 Results](https://huggingface.co/blog/PeterKruger/autobench-2nd-run)
+                * [AutoBench Blog Post](https://huggingface.co/blog/PeterKruger/autobench)
+                * [Autobench Repositories](https://huggingface.co/AutoBench)
+
+                **Disclaimer:** Benchmark results provide one perspective on model capabilities. Performance can vary based on specific tasks, prompts, and API conditions. Costs are estimates and subject to change by providers. Latency depends on server load and geographic location.
+                """)
+
+
+    with gr.Column(visible=opens_on_router) as router_view:
+        with gr.Tabs(visible=opens_on_router) as router_tabs:
+            with gr.Tab("Board", visible=opens_on_router) as tab_board:
+                router_intro = gr.Markdown(value=initial_router["intro"])
+                router_board_table = gr.DataFrame(
+                    value=initial_router["board"],
+                    interactive=True,
+                    label="Router board",
+                )
+            with gr.Tab("Quality versus cost", visible=opens_on_router) as tab_quality:
+                gr.Markdown(
+                    "Dashed lines are Always-Sol, Always-Luna, and Random. "
+                    "Amber points collapsed onto one model."
+                )
+                router_quality_plot = gr.Plot(value=initial_router["quality"])
+            with gr.Tab("Top-model share", visible=opens_on_router) as tab_share:
+                gr.Markdown("Passes at or above 90% are collapsed.")
+                router_collapse_plot = gr.Plot(value=initial_router["collapse"])
+            with gr.Tab("Same-pick", visible=opens_on_router) as tab_same:
+                gr.Markdown(
+                    "Share of questions on which two passes named the same model. "
+                    "This grid is the 12-pass comparison from the report. "
+                    "The median of 0 is across the full set of cross-vendor pairs, not this grid alone."
+                )
+                router_same_plot = gr.Plot(value=initial_router["same"])
+            with gr.Tab("Methodology", visible=opens_on_router) as tab_method:
+                router_method = gr.Markdown(value=initial_router["method"])
 
-        **Disclaimer:** Benchmark results provide one perspective on model capabilities. Performance can vary based on specific tasks, prompts, and API conditions. Costs are estimates and subject to change by providers. Latency depends on server load and geographic location.
-        """)
-    
     # --- Event Handlers ---
     # Set up reactive data loading when run selection changes
     run_selector.change(
@@ -870,7 +921,28 @@ with open_blocks() as app:
             p99_latency_plot,
             p99_latency_plot_msg,
             current_run_info,
-            correlations_display
+            correlations_display,
+            model_view,
+            router_view,
+            model_tabs,
+            router_tabs,
+            tab_overall,
+            tab_compare,
+            tab_plots,
+            tab_cost,
+            tab_domain,
+            tab_about,
+            tab_board,
+            tab_quality,
+            tab_share,
+            tab_same,
+            tab_method,
+            router_intro,
+            router_board_table,
+            router_quality_plot,
+            router_collapse_plot,
+            router_same_plot,
+            router_method,
         ]
     )
     

export_router_run.py

@@ -13,7 +13,9 @@ from __future__ import annotations
 import argparse
 import csv
 import json
+import math
 import re
+from decimal import Decimal, ROUND_HALF_UP
 from pathlib import Path
 
 
@@ -23,6 +25,14 @@ MATRIX_DATE = "2026-08-29"
 MATRIX_MODELS = 20
 COLLAPSE_THRESHOLD = 0.90
 
+# Official AutoBench Router Index. beta is locked at 0.2.
+# Quality is the share of the gap from the seed-7 random floor to a perfect answer.
+# Cost is a log score between fixed anchors of $0.00001 and $1 per question.
+ARI_BETA = 0.2
+ARI_DH_RAND = 0.646825
+ARI_C_MIN = 1e-5
+ARI_C_MAX = 1.0
+
 # key, public label, vendor, setting, is_baseline, collapse.dat label
 PASSES = [
     ("oracle", "Oracle", "Baseline", "oracle", 1, None),
@@ -66,9 +76,32 @@ BOARD_FIELDS = [
     "collapsed",
     "is_baseline",
     "display_order",
+    "ari",
 ]
 
 
+def router_index(signal: float, cost: float) -> float:
+    """ARI at the official beta. Returns one decimal, matching the published table."""
+    quality = max(0.0, (signal - ARI_DH_RAND) / (1.0 - ARI_DH_RAND))
+    if cost <= 0 or quality == 0:
+        return 0.0
+    span = math.log2(ARI_C_MAX) - math.log2(ARI_C_MIN)
+    cost_score = (math.log2(ARI_C_MAX) - math.log2(cost)) / span
+    cost_score = min(1.0, max(0.0, cost_score))
+    if cost_score == 0:
+        return 0.0
+    value = (
+        100.0
+        * (1.0 + ARI_BETA)
+        * quality
+        * cost_score
+        / (ARI_BETA * quality + cost_score)
+    )
+    return float(
+        Decimal(f"{value:.10f}").quantize(Decimal("0.1"), rounding=ROUND_HALF_UP)
+    )
+
+
 def parse_collapse(path: Path) -> dict[str, float]:
     shares: dict[str, float] = {}
     for line in path.read_text(encoding="utf-8").splitlines():
@@ -163,6 +196,7 @@ def main() -> None:
                 "collapsed": collapsed,
                 "is_baseline": is_baseline,
                 "display_order": order,
+                "ari": router_index(float(row["dh"]), float(row["cost"])),
             }
         )
 
@@ -183,8 +217,8 @@ def main() -> None:
             "Accuracy is the share of questions where the chosen model is statistically "
             "as good as the best available answer."
         ),
-        "report_url": "https://autobench.org/reports/router-benchmark-2026.pdf",
-        "blog_url": "https://autobench.org/reports/router-benchmark-2026.pdf",
+        "report_url": "https://autobench.org/reports/do-llm-routers-pay-for-themselves",
+        "blog_url": "https://autobench.org/reports/do-llm-routers-pay-for-themselves.pdf",
         "matrix_date": MATRIX_DATE,
         "matrix_models": MATRIX_MODELS,
         "questions": questions,
@@ -199,6 +233,7 @@ def main() -> None:
             "questions": questions,
             "matrix_models": MATRIX_MODELS,
             "matrix_date": MATRIX_DATE,
+            "ari_beta": ARI_BETA,
         },
     }
 

runs/router1_2026-10-26/metadata.json

@@ -6,8 +6,8 @@
   "run_version": "2026-10-26",
   "is_latest": false,
   "description": "Commercial routers and baselines graded on a frozen answer matrix. Accuracy is the share of questions where the chosen model is statistically as good as the best available answer.",
-  "report_url": "https://autobench.org/reports/router-benchmark-2026.pdf",
-  "blog_url": "https://autobench.org/reports/router-benchmark-2026.pdf",
+  "report_url": "https://autobench.org/reports/do-llm-routers-pay-for-themselves",
+  "blog_url": "https://autobench.org/reports/do-llm-routers-pay-for-themselves.pdf",
   "matrix_date": "2026-08-29",
   "matrix_models": 20,
   "questions": 252,
@@ -34,6 +34,7 @@
     "median_same_pick": 0.0,
     "questions": 252,
     "matrix_models": 20,
-    "matrix_date": "2026-08-29"
+    "matrix_date": "2026-08-29",
+    "ari_beta": 0.2
   }
 }

runs/router1_2026-10-26/router_board.csv

@@ -1,24 +1,24 @@
-pass_key,label,vendor,setting,n,accuracy,ci_low,ci_high,cost,cost_ci_low,cost_ci_high,top_share,collapsed,is_baseline,display_order
-oracle,Oracle,Baseline,oracle,252,100.0,100.0,100.0,0.03406342571592063,0.023090085942222026,0.04675808716741607,,0,1,0
-cheapest_adequate,Cheapest-adequate,Baseline,cheapest adequate,252,100.0,100.0,100.0,0.0017775440658730159,0.0012147553282936506,0.0024296202333019836,,0,1,1
-best_static_single,Always-Sol,Baseline,always Sol,248,87.5,83.5,91.5,0.07865474395161291,0.06454585211693548,0.09954083326612903,,0,1,2
-always_luna,Always-Luna,Baseline,always Luna,251,85.3,80.9,89.6,0.004334039103585658,0.0037202327948207175,0.005072818161354581,,0,1,3
-random,Random,Baseline,random,252,64.7,58.7,70.6,0.011535058849650793,0.007392945504305159,0.017047359445582142,0.0714,0,1,4
-always_cheapest,Always-cheapest,Baseline,always cheapest,252,44.4,38.5,50.4,0.00016450185134920633,0.00014635133277579365,0.0001831236469801587,,0,1,5
-orca_constrained_cheapest,Orca cheapest,OrcaRouter,cheapest,252,39.3,33.3,45.2,0.00022013774206349209,0.00019737690396825396,0.00024262488363095238,0.9921,1,0,6
-orca_constrained_balanced,Orca balanced,OrcaRouter,balanced,252,39.3,33.3,45.2,0.0002193627023809524,0.00019667757688492064,0.0002418150847222222,0.996,1,0,7
-orca_constrained_quality,Orca quality,OrcaRouter,quality,245,65.7,59.2,71.0,0.0005173967175510203,0.00042429937727755105,0.0006060875131632653,1.0,1,0,8
-azure_cost,Azure cost,Azure Model Router,cost,74,45.9,33.8,56.8,0.00912647891891892,0.00708104558918919,0.01094242530135135,0.5913,0,0,9
-azure_balanced,Azure balanced,Azure Model Router,balanced,179,82.7,78.8,89.4,0.020369913128491617,0.013021539309357542,0.03823974199790503,0.4802,0,0,10
-azure_quality,Azure quality,Azure Model Router,quality,238,85.3,80.3,89.1,0.064655818697479,0.054270396591386556,0.08091888025735294,0.4087,0,0,11
-nadirclaw,NadirClaw,NadirClaw,as shipped,249,70.3,64.3,75.9,0.05136010172289157,0.04020215791164659,0.0689510219376506,0.5873,0,0,12
-brick,Brick,Brick,as shipped,241,63.9,57.7,69.7,0.00310902156506224,0.002298716507154565,0.005002192490113071,0.8016,0,0,13
-cascade,Cascade,Cascade,as shipped,235,60.9,53.2,66.0,0.01035301790706383,0.007934312614846808,0.012938480779023402,0.5826,0,0,14
-nd_subset_k0,ND k0,Not Diamond,k0,248,87.5,83.5,91.5,0.07865474395161291,0.06454585211693548,0.09954083326612903,1.0,1,0,15
-nd_subset_k4,ND k4,Not Diamond,k4,249,80.3,75.5,85.2,0.05996616918433735,0.04669444507111647,0.07957475177908833,0.7698,0,0,16
-nd_subset_k5,ND k5,Not Diamond,k5,251,46.6,40.2,52.6,0.000941773053625498,0.0006677803593406375,0.0012532932769940238,0.8294,0,0,17
-nd_native_k5,ND native,Not Diamond,native,250,72.0,66.0,77.2,0.059749960000000005,0.0475751486,0.07856580617999999,0.6389,0,0,18
-or_auto,Auto low,OpenRouter,low,245,65.7,59.2,71.0,0.0005501390004897959,0.0004446207167918367,0.0006506275904183674,0.9841,1,0,19
-or_auto_medium,Auto medium,OpenRouter,medium,246,73.6,67.1,78.0,0.013130098215934962,0.008043599401597561,0.019080980965126015,0.7817,0,0,20
-or_auto_high,Auto high,OpenRouter,high,244,79.1,73.4,83.6,0.023005027726557378,0.018076698892231558,0.02858057378719467,0.5794,0,0,21
-or_auto_max,Auto max,OpenRouter,max,240,78.3,72.9,83.3,0.028915902712999996,0.020327376228814583,0.043068421132057284,0.7817,0,0,22
+pass_key,label,vendor,setting,n,accuracy,ci_low,ci_high,cost,cost_ci_low,cost_ci_high,top_share,collapsed,is_baseline,display_order,ari
+oracle,Oracle,Baseline,oracle,252,100.0,100.0,100.0,0.03406342571592063,0.023090085942222026,0.04675808716741607,,0,1,0,71.4
+cheapest_adequate,Cheapest-adequate,Baseline,cheapest adequate,252,100.0,100.0,100.0,0.0017775440658730159,0.0012147553282936506,0.0024296202333019836,,0,1,1,88.0
+best_static_single,Always-Sol,Baseline,always Sol,248,87.5,83.5,91.5,0.07865474395161291,0.06454585211693548,0.09954083326612903,,0,1,2,48.9
+always_luna,Always-Luna,Baseline,always Luna,251,85.3,80.9,89.6,0.004334039103585658,0.0037202327948207175,0.005072818161354581,,0,1,3,56.1
+random,Random,Baseline,random,252,64.7,58.7,70.6,0.011535058849650793,0.007392945504305159,0.017047359445582142,0.0714,0,1,4,0.0
+always_cheapest,Always-cheapest,Baseline,always cheapest,252,44.4,38.5,50.4,0.00016450185134920633,0.00014635133277579365,0.0001831236469801587,,0,1,5,0.0
+orca_constrained_cheapest,Orca cheapest,OrcaRouter,cheapest,252,39.3,33.3,45.2,0.00022013774206349209,0.00019737690396825396,0.00024262488363095238,0.9921,1,0,6,0.0
+orca_constrained_balanced,Orca balanced,OrcaRouter,balanced,252,39.3,33.3,45.2,0.0002193627023809524,0.00019667757688492064,0.0002418150847222222,0.996,1,0,7,0.0
+orca_constrained_quality,Orca quality,OrcaRouter,quality,245,65.7,59.2,71.0,0.0005173967175510203,0.00042429937727755105,0.0006060875131632653,1.0,1,0,8,3.5
+azure_cost,Azure cost,Azure Model Router,cost,74,45.9,33.8,56.8,0.00912647891891892,0.00708104558918919,0.01094242530135135,0.5913,0,0,9,0.0
+azure_balanced,Azure balanced,Azure Model Router,balanced,179,82.7,78.8,89.4,0.020369913128491617,0.013021539309357542,0.03823974199790503,0.4802,0,0,10,47.0
+azure_quality,Azure quality,Azure Model Router,quality,238,85.3,80.3,89.1,0.064655818697479,0.054270396591386556,0.08091888025735294,0.4087,0,0,11,47.0
+nadirclaw,NadirClaw,NadirClaw,as shipped,249,70.3,64.3,75.9,0.05136010172289157,0.04020215791164659,0.0689510219376506,0.5873,0,0,12,16.9
+brick,Brick,Brick,as shipped,241,63.9,57.7,69.7,0.00310902156506224,0.002298716507154565,0.005002192490113071,0.8016,0,0,13,0.0
+cascade,Cascade,Cascade,as shipped,235,60.9,53.2,66.0,0.01035301790706383,0.007934312614846808,0.012938480779023402,0.5826,0,0,14,0.0
+nd_subset_k0,ND k0,Not Diamond,k0,248,87.5,83.5,91.5,0.07865474395161291,0.06454585211693548,0.09954083326612903,1.0,1,0,15,48.9
+nd_subset_k4,ND k4,Not Diamond,k4,249,80.3,75.5,85.2,0.05996616918433735,0.04669444507111647,0.07957475177908833,0.7698,0,0,16,39.0
+nd_subset_k5,ND k5,Not Diamond,k5,251,46.6,40.2,52.6,0.000941773053625498,0.0006677803593406375,0.0012532932769940238,0.8294,0,0,17,0.0
+nd_native_k5,ND native,Not Diamond,native,250,72.0,66.0,77.2,0.059749960000000005,0.0475751486,0.07856580617999999,0.6389,0,0,18,21.3
+or_auto,Auto low,OpenRouter,low,245,65.7,59.2,71.0,0.0005501390004897959,0.0004446207167918367,0.0006506275904183674,0.9841,1,0,19,3.5
+or_auto_medium,Auto medium,OpenRouter,medium,246,73.6,67.1,78.0,0.013130098215934962,0.008043599401597561,0.019080980965126015,0.7817,0,0,20,26.7
+or_auto_high,Auto high,OpenRouter,high,244,79.1,73.4,83.6,0.023005027726557378,0.018076698892231558,0.02858057378719467,0.5794,0,0,21,39.2
+or_auto_max,Auto max,OpenRouter,max,240,78.3,72.9,83.3,0.028915902712999996,0.020327376228814583,0.043068421132057284,0.7817,0,0,22,37.1