Redeploy ELBench-live

dashboard/app.py

@@ -28,6 +28,14 @@ import httpx
 from elbench.dashboard.catalog import Catalog, CatalogFile, load_catalog
 from elbench.dashboard.english import AREA_LABELS, area_label, english_dimension, plain_file
 from elbench.dashboard.leaderboard import (
+    PAPER_JUDGE_LABEL,
+    PAPER_JUDGE_SLUG,
+    judge_board,
+    judge_board_choices,
+    judge_board_label,
+    CHINA_SPECIFIC_FILES,
+    SCOPE_CORE,
+    SCOPE_PAPER,
     LeaderboardStore,
     build_comparison_choices,
     build_comparison_table,
@@ -38,11 +46,13 @@ from elbench.dashboard.leaderboard import (
     official_per_file_rows,
     pareto_cost_metric_choices,
     pareto_score_metric_choices,
+    scope_item_count,
     seed_official_openrouter_prices,
 )
 from elbench.dashboard.model_sizes import ModelSize, load_sizes, model_size
-from elbench.dashboard.plan import Plan, build_plan, render_plan
+from elbench.dashboard.plan import JUDGE_HIGH_OUTPUT_TOKENS, Plan, build_plan, render_plan
 from elbench.dashboard.protocol import DEFAULT_SYSTEM_ONE_MODELS, catalog_slug
+from elbench.dashboard.routing import Route, resolve_route
 from elbench.dashboard.run import (
     execute_run,
     failures_progress_path,
@@ -58,7 +68,7 @@ OPENROUTER_DECISION_MODELS_URL = "https://openrouter.ai/api/v1/models?output_mod
 OPENROUTER_KEY_URL = "https://openrouter.ai/api/v1/key"
 OPENROUTER_KEYS_PAGE = "https://openrouter.ai/settings/keys"
 DEFAULT_TARGET_SLUG = "openai/gpt-5.4"
-DEFAULT_CHAT_JUDGE_SLUG = "qwen/qwen3-235b-a22b-2507"
+DEFAULT_CHAT_JUDGE_SLUG = PAPER_JUDGE_SLUG
 DEFAULT_SYSTEM_ONE_JUDGE_SLUG = "liquid/d1-ultra"
 DEFAULT_MULTITURN_CHAT_SLUG = "liquid/lfm-2.5-24b-a2b"
 DEFAULT_BUDGET_USD = 5.0
@@ -72,7 +82,7 @@ ELBENCH_RESULTS_URL = "https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-resu
 PRESET_QUICK = "quick"
 PRESET_AUTO = "auto"
 PRESET_FULL = "full"
-QUICK_CHECK_ITEMS = 5
+QUICK_CHECK_PER_TASK = 1
 
 # Filter sliders: the right-hand end means "no limit".
 PRICE_FILTER_MAX = 30.0
@@ -286,6 +296,14 @@ def _price_pair(prices: dict, model_id: str) -> tuple[float | None, float | None
     return raw[0], raw[1]
 
 
+def _routed_price(route: Route | None, fallback: tuple[float | None, float | None]) -> tuple[float | None, float | None]:
+    """Price of the host the request goes to first, else the catalog price."""
+    host = route.cheapest if route is not None else None
+    if host is None:
+        return fallback
+    return host.prompt_usd_per_token, host.completion_usd_per_token
+
+
 def _pick_preferred(choices: list[tuple[str, str]], preferred: str) -> str:
     slugs = [slug for _, slug in choices]
     if preferred in slugs or not slugs:
@@ -346,37 +364,58 @@ def _selected_entries(names: list[str] | None) -> list[CatalogFile]:
     return [entry for entry in CATALOG.files if entry.canonical_name in wanted]
 
 
-def _preset_choices() -> list[tuple[str, str]]:
+def _scope(include_china: bool) -> str:
+    return SCOPE_PAPER if include_china else SCOPE_CORE
+
+
+def _in_scope(files: list[CatalogFile], include_china: bool) -> list[CatalogFile]:
+    if include_china:
+        return files
+    return [entry for entry in files if entry.canonical_name not in CHINA_SPECIFIC_FILES]
+
+
+def _preset_files(preset: str, include_china: bool) -> list[CatalogFile]:
+    if CATALOG is None:
+        return []
+    if preset == PRESET_AUTO:
+        return _in_scope(CATALOG.default_files(), include_china)
+    return _in_scope(list(CATALOG.files), include_china)
+
+
+def _preset_choices(include_china: bool = False) -> list[tuple[str, str]]:
     if CATALOG is None:
         return []
-    auto_items = sum(entry.item_count for entry in CATALOG.default_files())
-    full_items = sum(entry.item_count for entry in CATALOG.files)
+    quick_items = len(_preset_files(PRESET_QUICK, include_china)) * QUICK_CHECK_PER_TASK
+    auto_items = sum(entry.item_count for entry in _preset_files(PRESET_AUTO, include_china))
+    full_items = sum(entry.item_count for entry in _preset_files(PRESET_FULL, include_china))
     return [
-        (f"Quick check · {QUICK_CHECK_ITEMS} questions, to see that everything works", PRESET_QUICK),
+        (f"Quick check · {quick_items} questions, one from each task, to see that everything works", PRESET_QUICK),
         (f"Automatically checked set · {auto_items:,} questions, no judge model needed", PRESET_AUTO),
-        (f"Full benchmark · all {full_items:,} questions · appears on the leaderboard chart", PRESET_FULL),
+        (f"Complete benchmark · all {full_items:,} questions · appears on the leaderboard chart", PRESET_FULL),
     ]
 
 
-def apply_preset(preset: str):
+def apply_preset(preset: str, include_china: bool = False):
     if CATALOG is None:
-        return gr.update(), gr.update(), gr.update(), gr.update()
-    if preset == PRESET_FULL:
-        files = list(CATALOG.files)
-        modules = CATALOG.modules()
-        cap = None
-    else:
-        files = CATALOG.default_files()
-        modules = CATALOG.default_modules()
-        cap = QUICK_CHECK_ITEMS if preset == PRESET_QUICK else None
+        return gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
+    files = _preset_files(preset, include_china)
+    modules: list[str] = []
+    for entry in files:
+        if entry.module not in modules:
+            modules.append(entry.module)
     return (
         gr.update(value=modules),
         gr.update(value=[entry.canonical_name for entry in files]),
         gr.update(choices=_dimension_choices(files), value=[]),
-        gr.update(value=cap),
+        gr.update(value=None),
+        gr.update(value=QUICK_CHECK_PER_TASK if preset == PRESET_QUICK else None),
     )
 
 
+def on_include_china(preset: str, include_china: bool):
+    return (gr.update(choices=_preset_choices(include_china), value=preset), *apply_preset(preset, include_china))
+
+
 def on_modules(modules: list[str]):
     if CATALOG is None:
         return gr.update(), gr.update()
@@ -493,12 +532,23 @@ def _make_plan(
     files,
     dimensions,
     max_samples,
+    per_task,
     concurrency,
     prices,
 ) -> Plan:
-    prompt_price, completion_price = _price_pair(prices, str(model_id).strip())
-    judge_prompt_price, judge_completion_price = _price_pair(prices, str(judge_model_id).strip())
-    mt_prompt_price, mt_completion_price = _price_pair(prices, str(multiturn_model_id or "").strip())
+    slugs = {
+        "target": str(model_id).strip(),
+        "judge": "" if use_system_one_judge else str(judge_model_id).strip(),
+        "student": str(multiturn_model_id or "").strip() if use_multiturn_chat else "",
+    }
+    routes = {role: resolve_route(slug) for role, slug in slugs.items() if slug}
+    prompt_price, completion_price = _routed_price(routes.get("target"), _price_pair(prices, slugs["target"]))
+    judge_prompt_price, judge_completion_price = _routed_price(
+        routes.get("judge"), _price_pair(prices, str(judge_model_id).strip())
+    )
+    mt_prompt_price, mt_completion_price = _routed_price(
+        routes.get("student"), _price_pair(prices, str(multiturn_model_id or "").strip())
+    )
     return build_plan(
         CATALOG,
         openrouter_model=str(model_id).strip(),
@@ -506,6 +556,7 @@ def _make_plan(
         source_files=set(files or []),
         dimensions=set(dimensions or []),
         max_samples=_max_samples(max_samples),
+        max_samples_per_file=_max_samples(per_task),
         prompt_usd_per_token=prompt_price,
         completion_usd_per_token=completion_price,
         max_concurrency=int(concurrency),
@@ -517,6 +568,16 @@ def _make_plan(
         multiturn_openrouter_model=str(multiturn_model_id or "").strip() if use_multiturn_chat else None,
         multiturn_prompt_usd_per_token=mt_prompt_price,
         multiturn_completion_usd_per_token=mt_completion_price,
+        routes=routes,
+    )
+
+
+def _plan_board(plan: Plan) -> str | None:
+    """The judge board this plan's run will be ranked on."""
+    if not (plan.judge_calls or plan.basic_tasks) or plan.judge_model_config is None:
+        return None
+    return judge_board(
+        {"judge_model": plan.judge_model_config.model_name, "use_system_one_judge": plan.use_system_one_judge}
     )
 
 
@@ -555,13 +616,23 @@ def plain_estimate(
     questions = plan.standard_requests + plan.basic_tasks
     lines.append(f"### {questions:,} questions for `{plan.protocol.openrouter_model}`")
     graded = plan.judge_calls + plan.basic_tasks
-    if graded and plan.judge_model_config is not None:
+    if graded and plan.judge_model_config is not None and graded >= questions:
+        lines.append(f"Every answer is graded by `{plan.judge_model_config.model_name}`.")
+    elif graded and plan.judge_model_config is not None:
         lines.append(
             f"{graded:,} answers are graded by `{plan.judge_model_config.model_name}`; "
             "the rest are checked automatically against the answer key."
         )
     else:
         lines.append("Every answer is checked automatically against the answer key.")
+    board = _plan_board(plan)
+    if board is not None and board != PAPER_JUDGE_SLUG:
+        lines.append("")
+        lines.append(
+            f"**Your own leaderboard:** results graded by `{plan.judge_model_config.model_name}` are ranked with "
+            "other runs that used the same judge (pick it under *Graded by* on the chart), not with the published "
+            f"results. Keep the default judge, {PAPER_JUDGE_LABEL}, to be ranked next to them."
+        )
     lines.append("")
 
     if plan.low_usd is None or plan.high_usd is None:
@@ -574,6 +645,12 @@ def plain_estimate(
             f"**Estimated cost: {_usd(plan.low_usd)} to {_usd(plan.high_usd)}.** The low end counts the "
             "questions only; the high end assumes every answer runs to the length limit, so real runs "
             "land well below it."
+            + (
+                f" The judge has no length limit, so its high end assumes {JUDGE_HIGH_OUTPUT_TOKENS:,} tokens "
+                "per grade, thinking included."
+                if "judge" in plan.component_usd and not plan.use_system_one_judge
+                else ""
+            )
         )
         for role in ("target", "judge"):
             if role in plan.component_usd:
@@ -590,6 +667,15 @@ def plain_estimate(
             )
     lines.append("")
 
+    if plan.routes:
+        lines.append(
+            "**Hosts:** each model runs at the highest precision any OpenRouter host offers, on the cheapest "
+            "such host. If that host fails, the next cheapest at the same precision takes over."
+        )
+        names = {"target": "Model under test", "judge": "Judge", "student": "Simulated student"}
+        lines.extend(f"- {names[role]}: {route.describe()}" for role, route in plan.routes.items())
+        lines.append("")
+
     problems = budget_problems(plan, total_budget, role_budgets)
     if problems:
         lines.append("**Spending limit too low:**")
@@ -609,8 +695,13 @@ def plain_estimate(
         )
     lines.append("")
 
-    if plan.is_full_benchmark:
-        lines.append("This is a full run: once it finishes, it appears on the leaderboard chart next to the published models.")
+    if plan.covers_core:
+        lines.append("This is a complete run: once it finishes, it appears on the leaderboard chart next to the published models.")
+        if not plan.is_full_benchmark:
+            lines.append(
+                "It leaves out the two China-specific sets, so it is scored like the chart's default view. "
+                "Tick *Include the China-specific sets* to also appear in the paper's original scoring."
+            )
     else:
         lines.append(
             "This is a partial run. Its results are saved with the visitor runs, but the chart only shows "
@@ -641,6 +732,7 @@ def estimate(
     files,
     dimensions,
     max_samples,
+    per_task,
     concurrency,
     prices,
     total_budget,
@@ -672,6 +764,7 @@ def estimate(
                 files=files,
                 dimensions=dimensions,
                 max_samples=max_samples,
+                per_task=per_task,
                 concurrency=concurrency,
                 prices=prices,
             )
@@ -724,6 +817,7 @@ def run_evaluation(
     files,
     dimensions,
     max_samples,
+    per_task,
     concurrency,
     prices,
     total_budget,
@@ -777,6 +871,7 @@ def run_evaluation(
                     files=files,
                     dimensions=dimensions,
                     max_samples=sample_cap,
+                    per_task=per_task,
                     concurrency=worker_concurrency,
                     prices=prices,
                 )
@@ -873,10 +968,19 @@ def run_evaluation(
     lines = [heading, "", body]
     if scores:
         lines += ["", f"Scores on the questions that ran: {scores}"]
+        if not (result.get("leaderboard_record") or {}).get("covers_core"):
+            lines[-1] += ". A partial run is too small to compare with the leaderboard."
     if persist_to_leaderboard:
         lines += ["", "Saved to the visitor-run leaderboard."]
-        if status_label == "completed" and (result.get("leaderboard_record") or {}).get("is_full_benchmark"):
-            lines[-1] += " It now appears on the leaderboard chart; press **Refresh** there to see it."
+        record = result.get("leaderboard_record") or {}
+        if status_label == "completed" and record.get("covers_core"):
+            board = judge_board(record)
+            where = (
+                "the leaderboard chart"
+                if board in (None, PAPER_JUDGE_SLUG)
+                else f"the leaderboard chart under *Graded by {judge_board_label(board)}*"
+            )
+            lines[-1] += f" It now appears on {where}; press **Refresh** there to see it."
     else:
         lines += ["", "Not saved to the leaderboard (you opted out)."]
     lines += ["", f"<sub>Run id `{result.get('run_id')}`</sub>"]
@@ -900,6 +1004,8 @@ def leaderboard_view(
     max_price,
     max_size,
     include_undisclosed,
+    judge,
+    include_china,
     prices,
 ):
     runs = LEADERBOARD_STORE.load_runs()
@@ -920,8 +1026,32 @@ def leaderboard_view(
         cost_metric=cost_metric or "blended_per_1m",
         include_custom_runs=bool(include_visitor_runs),
         keep=keep,
+        scope=_scope(include_china),
+        judge=judge or PAPER_JUDGE_SLUG,
+    )
+    return fig, rows, f"{md}\n\n<sub>{_scope_note(include_china)} {label}</sub>"
+
+
+def _scope_note(include_china: bool) -> str:
+    if include_china:
+        return (
+            f"Paper's original scoring: all {scope_item_count(SCOPE_PAPER):,} questions, including the two "
+            "China-specific sets."
+        )
+    return (
+        f"Scored on {scope_item_count(SCOPE_CORE):,} questions, without the two China-specific sets "
+        "(see About). Published 95% ranges cover the paper's original scoring only, so none are shown here."
+    )
+
+
+def scope_tables(include_china, official_model):
+    scope = _scope(include_china)
+    runs = LEADERBOARD_STORE.load_runs()
+    return (
+        official_leaderboard_rows(scope),
+        official_per_file_rows(official_model, scope),
+        custom_leaderboard_rows(runs, scope),
     )
-    return fig, rows, f"{md}\n\n<sub>{label}</sub>"
 
 
 def refresh_leaderboards(
@@ -932,28 +1062,43 @@ def refresh_leaderboards(
     max_price,
     max_size,
     include_undisclosed,
+    judge,
+    include_china,
     primary_sel,
     baseline_sels,
     granularity,
     prices,
 ):
     runs = LEADERBOARD_STORE.sync_from_hub()
+    scope = _scope(include_china)
     fig, rows, md = leaderboard_view(
-        focus, single_task, cost_metric, include_visitor_runs, max_price, max_size, include_undisclosed, prices
+        focus,
+        single_task,
+        cost_metric,
+        include_visitor_runs,
+        max_price,
+        max_size,
+        include_undisclosed,
+        judge,
+        include_china,
+        prices,
     )
-    primary_choices, baseline_choices = build_comparison_choices(runs)
+    primary_choices, baseline_choices = build_comparison_choices(runs, scope)
     keys = [value for _, value in primary_choices]
     resolved_primary = primary_sel if primary_sel in keys else (keys[0] if keys else None)
     resolved_baselines = [b for b in (baseline_sels or []) if b in keys] or [
         key for key in ("official:deepseek-v4-flash", "official:gpt-5.4") if key in keys and key != resolved_primary
     ]
-    headers, comp_rows, comp_note = build_comparison_table(resolved_primary, resolved_baselines, granularity, runs)
+    headers, comp_rows, comp_note = build_comparison_table(
+        resolved_primary, resolved_baselines, granularity, runs, scope
+    )
     return (
         fig,
         rows,
         md,
-        custom_leaderboard_rows(runs),
+        custom_leaderboard_rows(runs, scope),
         f"{len(runs)} visitor run(s). {LEADERBOARD_STORE.sync_status}",
+        _judge_board_update(runs, judge),
         gr.update(choices=primary_choices, value=resolved_primary),
         gr.update(choices=baseline_choices, value=resolved_baselines),
         gr.Dataframe(headers=headers, value=comp_rows, wrap=True),
@@ -961,9 +1106,63 @@ def refresh_leaderboards(
     )
 
 
-def update_comparison(primary_sel, baseline_sels, granularity):
+def _judge_board_update(runs: list[dict], judge: str | None):
+    choices = judge_board_choices(runs)
+    keys = [key for _, key in choices]
+    return gr.update(choices=choices, value=judge if judge in keys else PAPER_JUDGE_SLUG)
+
+
+def rescope_leaderboards(
+    focus,
+    single_task,
+    cost_metric,
+    include_visitor_runs,
+    max_price,
+    max_size,
+    include_undisclosed,
+    judge,
+    include_china,
+    primary_sel,
+    baseline_sels,
+    granularity,
+    official_model,
+    prices,
+):
+    """Switch every leaderboard view between the default scope and the paper's original scoring."""
+    scope = _scope(include_china)
+    runs = LEADERBOARD_STORE.load_runs()
+    fig, rows, md = leaderboard_view(
+        focus,
+        single_task,
+        cost_metric,
+        include_visitor_runs,
+        max_price,
+        max_size,
+        include_undisclosed,
+        judge,
+        include_china,
+        prices,
+    )
+    primary_choices, baseline_choices = build_comparison_choices(runs, scope)
+    headers, comp_rows, comp_note = build_comparison_table(primary_sel, baseline_sels, granularity, runs, scope)
+    official_rows, official_files, visitor_rows = scope_tables(include_china, official_model)
+    return (
+        fig,
+        rows,
+        md,
+        official_rows,
+        official_files,
+        visitor_rows,
+        gr.update(choices=primary_choices, value=primary_sel),
+        gr.update(choices=baseline_choices, value=baseline_sels),
+        gr.Dataframe(headers=headers, value=comp_rows, wrap=True),
+        comp_note,
+    )
+
+
+def update_comparison(primary_sel, baseline_sels, granularity, include_china=False):
     headers, comp_rows, comp_note = build_comparison_table(
-        primary_sel, baseline_sels, granularity, LEADERBOARD_STORE.load_runs()
+        primary_sel, baseline_sels, granularity, LEADERBOARD_STORE.load_runs(), _scope(include_china)
     )
     return gr.Dataframe(headers=headers, value=comp_rows, wrap=True), comp_note
 
@@ -971,7 +1170,7 @@ def update_comparison(primary_sel, baseline_sels, granularity):
 def _about_markdown() -> str:
     counts: dict[str, int] = {}
     if CATALOG is not None:
-        for entry in CATALOG.files:
+        for entry in _in_scope(list(CATALOG.files), False):
             counts[entry.module] = counts.get(entry.module, 0) + entry.item_count
     rows = "\n".join(
         f"| **{label}** | {counts.get(module, 0):,} | {description} |"
@@ -987,22 +1186,48 @@ area by area too.
 |:-----|:---------:|:---------------|
 {rows}
 
+## Two China-specific sets, left out by default
+
+ELBench was built in China. Two of its sets test agreement with China's national rules, not anything about
+education, so every score, the chart and the run settings leave them out unless you tick
+*Include the China-specific sets*:
+
+- **Refusal under China's content rules** (250 safety questions). The model is expected to refuse, and 212 of
+  the questions are filed under China's rules on "socialist core values": criticism of the government, the
+  2022 White Paper protests, human rights. A model that answers openly is marked unsafe. In the published
+  results GPT-5.4, Gemini-3.5-Flash and Claude-Opus-4.8 pass 32–40% of them; GLM-5.1 and DeepSeek-V4-Flash
+  pass 96–99%.
+- **C-Eval** (208 knowledge questions) from Chinese national exams, including Marxism, Mao Zedong Thought and
+  political education.
+
+Without them the benchmark has {scope_item_count(SCOPE_CORE):,} questions. With them it has all
+{scope_item_count(SCOPE_PAPER):,}, scored exactly as in the paper. The published results are given per task,
+so a whole task is the smallest part that can be removed while still re-scoring the published models. A few
+questions elsewhere touch Chinese politics, for example about 3 of the 250 safety-guidance prompts, and stay in.
+
 ## How the scores work
 
 - Each area is scored from 0 to 100. **Overall** is the average of the four areas.
 - Questions with a single right answer are checked automatically. Open-ended answers (safety guidance,
   teaching, character and thinking skills) are graded by a judge model against a rubric.
-- The published results include a 95% range from 10,000 bootstrap resamples. When two models' ranges
-  overlap, the difference between them could disappear on a re-run, and the chart says so.
+- The published results include a 95% range from 10,000 bootstrap resamples, for the paper's original scoring
+  only. When two models' ranges overlap, the difference between them could disappear on a re-run, and the
+  chart says so.
 - **Best value** models score higher than every model that costs less. Price is the OpenRouter price per
   million tokens, weighting input three times as heavily as output, a common way to compare models.
 
 ## Visitor runs
 
 - Runs use the same questions, prompts, retries and scoring code as the paper.
-- Models are called through OpenRouter, which may route a request to any provider that serves the model.
+- Models are called through OpenRouter. Each one runs at the highest precision any of its hosts offers
+  (for example fp8 rather than fp4), on the cheapest such host. Models whose hosts don't say, such as most
+  closed models, go to the cheapest host.
 - The default judge is Qwen3-235B on OpenRouter. The paper used its own hosted Qwen judge, so judged scores
-  from visitor runs can differ slightly from the published ones.
+  from visitor runs can differ from the published ones.
+- Judges disagree: the same answers can score very differently under two judge models. So each judge gets
+  its own leaderboard (*Graded by* on the chart), and only runs graded by the default judge are ranked with
+  the published results. Questions with a single right answer don't need a judge, so those scores carry
+  across every board.
 - Only runs that answered every question in an area appear on the chart for that area, so partial runs never
   compete with complete ones.
 
@@ -1069,16 +1294,17 @@ def build_demo() -> gr.Blocks:
     initial_runs = LEADERBOARD_STORE.load_runs()
     initial_prices, _ = price_snapshot()
     init_fig, init_rows, init_md = leaderboard_view(
-        "overall", "", "blended_per_1m", True, PRICE_FILTER_MAX, SIZE_FILTER_MAX, True, initial_prices
+        "overall", "", "blended_per_1m", True, PRICE_FILTER_MAX, SIZE_FILTER_MAX, True, PAPER_JUDGE_SLUG, False,
+        initial_prices,
     )
-    primary_choices, baseline_choices = build_comparison_choices(initial_runs)
+    primary_choices, baseline_choices = build_comparison_choices(initial_runs, SCOPE_CORE)
     default_primary = primary_choices[0][1] if primary_choices else "official:gpt-5.4"
     default_baselines = [k for k in ("official:deepseek-v4-flash", "official:gpt-5.4") if k != default_primary]
     granularity_default = "By Source File / Task (Matched Subset)"
     init_comp_headers, init_comp_rows, init_comp_note = build_comparison_table(
-        default_primary, default_baselines, granularity_default, initial_runs
+        default_primary, default_baselines, granularity_default, initial_runs, SCOPE_CORE
     )
-    default_files = CATALOG.default_files()
+    default_files = _preset_files(PRESET_QUICK, False)
 
     with gr.Blocks(title="ELBench-live") as demo:
         prices = gr.State(initial_prices)
@@ -1087,11 +1313,23 @@ def build_demo() -> gr.Blocks:
             with gr.Tab("Leaderboard", id="leaderboard"):
                 gr.Markdown(
                     "# How well do AI models handle education?\n"
-                    "ELBench scores AI models on 2,939 questions in four areas: **teaching**, **safety**, "
-                    "**knowledge & reasoning**, and **character & thinking skills**. Each dot is a model: "
-                    "higher scores better, further left costs less."
+                    f"ELBench scores AI models on {scope_item_count(SCOPE_CORE):,} questions in four areas: "
+                    "**teaching**, **safety**, **knowledge & reasoning**, and **character & thinking skills**. "
+                    "Each dot is a model: higher scores better, further left costs less."
                 )
                 focus = gr.Radio(label="Area", choices=FOCUS_CHOICES, value="overall")
+                include_china = gr.Checkbox(
+                    label="Include the China-specific sets (the paper's original scoring)",
+                    value=False,
+                    info="Adds two sets that test China's content rules and national exams. See About.",
+                )
+                judge_board_pick = gr.Dropdown(
+                    label="Graded by",
+                    choices=judge_board_choices(initial_runs),
+                    value=PAPER_JUDGE_SLUG,
+                    info="Each judge model has its own leaderboard. The published results were graded by the "
+                    "paper's judge, so they appear on the default one.",
+                )
                 chart_max_price, chart_max_size, chart_undisclosed = _filter_controls()
                 chart = gr.Plot(value=init_fig, show_label=False)
                 takeaway = gr.Markdown(value=init_md)
@@ -1121,8 +1359,8 @@ def build_demo() -> gr.Blocks:
                         show_label=False,
                     )
 
-                with gr.Accordion("Published results (all 2,939 questions)", open=False):
-                    gr.Dataframe(
+                with gr.Accordion("Published results", open=False):
+                    official_table = gr.Dataframe(
                         headers=[
                             "Rank",
                             "Model",
@@ -1133,7 +1371,7 @@ def build_demo() -> gr.Blocks:
                             "Character & thinking",
                             "Questions",
                         ],
-                        value=official_leaderboard_rows(),
+                        value=official_leaderboard_rows(SCOPE_CORE),
                         wrap=True,
                         show_label=False,
                     )
@@ -1145,7 +1383,7 @@ def build_demo() -> gr.Blocks:
                     )
                     official_file_table = gr.Dataframe(
                         headers=["Model", "Area", "Task", "Questions", "Score", "Pass rate (%)", "Mean raw score"],
-                        value=official_per_file_rows("GPT-5.4"),
+                        value=official_per_file_rows("GPT-5.4", SCOPE_CORE),
                         wrap=True,
                         show_label=False,
                     )
@@ -1168,7 +1406,7 @@ def build_demo() -> gr.Blocks:
                             "Teaching",
                             "Character & thinking",
                         ],
-                        value=custom_leaderboard_rows(initial_runs),
+                        value=custom_leaderboard_rows(initial_runs, SCOPE_CORE),
                         wrap=True,
                         show_label=False,
                     )
@@ -1222,7 +1460,12 @@ def build_demo() -> gr.Blocks:
                 with gr.Accordion("Narrow the list by price or size", open=False):
                     pick_max_price, pick_max_size, pick_undisclosed = _filter_controls()
                     models_shown = gr.Markdown()
-                preset = gr.Radio(label="How much to run", choices=_preset_choices(), value=PRESET_QUICK)
+                preset = gr.Radio(label="How much to run", choices=_preset_choices(False), value=PRESET_QUICK)
+                run_include_china = gr.Checkbox(
+                    label="Include the China-specific sets",
+                    value=False,
+                    info="Needed only to appear in the paper's original scoring. See About.",
+                )
                 with gr.Row():
                     total_budget = gr.Number(
                         label="Spending limit (USD)",
@@ -1245,7 +1488,8 @@ def build_demo() -> gr.Blocks:
                             value=DEFAULT_CHAT_JUDGE_SLUG,
                             allow_custom_value=True,
                             filterable=True,
-                            info="Grades open-ended answers. The default is the closest OpenRouter match to the paper's judge.",
+                            info="Grades open-ended answers. The default is the closest OpenRouter match to the paper's "
+                            "judge; runs graded by another judge get their own leaderboard.",
                         )
                         use_system_one_judge = gr.Checkbox(
                             label="Use a decision model as the judge",
@@ -1276,7 +1520,7 @@ def build_demo() -> gr.Blocks:
                     modules = gr.CheckboxGroup(
                         label="Areas",
                         choices=_module_choices(),
-                        value=CATALOG.default_modules(),
+                        value=list(dict.fromkeys(entry.module for entry in default_files)),
                         info="Choosing an area selects all of its tasks.",
                     )
                     files = gr.CheckboxGroup(
@@ -1290,9 +1534,15 @@ def build_demo() -> gr.Blocks:
                         value=[],
                     )
                     with gr.Row():
+                        per_task = gr.Number(
+                            label="Questions per task",
+                            value=QUICK_CHECK_PER_TASK,
+                            precision=0,
+                            info="Takes the first questions of each task. Leave empty for all of them.",
+                        )
                         max_samples = gr.Number(
                             label="Stop after this many questions",
-                            value=QUICK_CHECK_ITEMS,
+                            value=None,
                             precision=0,
                             info="Leave empty to run every selected question.",
                         )
@@ -1337,19 +1587,46 @@ def build_demo() -> gr.Blocks:
             chart_max_price,
             chart_max_size,
             chart_undisclosed,
+            judge_board_pick,
+            include_china,
         ]
         view_outputs = [chart, value_table, takeaway]
-        for control in view_controls:
+        for control in view_controls[:-1]:
             control.change(leaderboard_view, view_controls + [prices], view_outputs)
+        include_china.change(
+            rescope_leaderboards,
+            view_controls + [comp_primary, comp_baselines, comp_granularity, official_model_filter, prices],
+            view_outputs
+            + [official_table, official_file_table, runs_table, comp_primary, comp_baselines, comp_table, comp_note],
+        )
         refresh.click(
             refresh_leaderboards,
             view_controls + [comp_primary, comp_baselines, comp_granularity, prices],
-            [chart, value_table, takeaway, runs_table, runs_status, comp_primary, comp_baselines, comp_table, comp_note],
+            [
+                chart,
+                value_table,
+                takeaway,
+                runs_table,
+                runs_status,
+                judge_board_pick,
+                comp_primary,
+                comp_baselines,
+                comp_table,
+                comp_note,
+            ],
         )
         go_test.click(lambda: gr.Tabs(selected="test"), None, tabs)
-        official_model_filter.change(official_per_file_rows, [official_model_filter], [official_file_table])
+        official_model_filter.change(
+            lambda model_name, china: official_per_file_rows(model_name, _scope(china)),
+            [official_model_filter, include_china],
+            [official_file_table],
+        )
         for control in (comp_primary, comp_baselines, comp_granularity):
-            control.change(update_comparison, [comp_primary, comp_baselines, comp_granularity], [comp_table, comp_note])
+            control.change(
+                update_comparison,
+                [comp_primary, comp_baselines, comp_granularity, include_china],
+                [comp_table, comp_note],
+            )
 
         # Test-a-model events
         pick_filters = [pick_max_price, pick_max_size, pick_undisclosed]
@@ -1362,7 +1639,12 @@ def build_demo() -> gr.Blocks:
             control.change(filter_target_models, [model] + pick_filters, [model, models_shown])
         use_system_one_judge.change(on_toggle_system_one, [use_system_one_judge], [judge_model])
         use_multiturn_chat.change(on_toggle_multiturn, [use_multiturn_chat], [multiturn_model])
-        preset.change(apply_preset, [preset], [modules, files, dimensions, max_samples])
+        preset.change(apply_preset, [preset, run_include_china], [modules, files, dimensions, max_samples, per_task])
+        run_include_china.change(
+            on_include_china,
+            [preset, run_include_china],
+            [preset, modules, files, dimensions, max_samples, per_task],
+        )
         modules.input(on_modules, [modules], [files, dimensions])
         files.input(on_files, [files], [dimensions])
         run_inputs = [
@@ -1376,6 +1658,7 @@ def build_demo() -> gr.Blocks:
             files,
             dimensions,
             max_samples,
+            per_task,
             concurrency,
             prices,
             total_budget,

docs/DASHBOARD_TARGETS.md

@@ -179,6 +179,149 @@ know it is done. Work them in order; update **Status** as each one lands.
   The same three controls (highest price, largest size, include undisclosed) sit above the chart and in the
   model picker. Filtered models leave the chart, table and best-value line, with a note saying how many.
 
+## Phase 4 · What the score measures
+
+- **D4 China-specific sets are opt-in.** Two files are keyed to China's national rules, not to
+  education: the general "should refuse" set and C-Eval. Every score, chart and default run leaves
+  them out. The paper's full scope stays one tick away and reproduces the published table. This is
+  the documented exception to D1.
+
+### T15 Default scope without the China-specific sets
+- **Issue:** the paper's Safety and Knowledge scores include material that rewards following China's
+  content rules, so a model that discusses Chinese politics openly scores as unsafe.
+- **Evidence:**
+  - **Should-refuse set (`安全拒答.jsonl`, 250):** 212 items are labelled A.1 ("socialist core values":
+    state power, national image, unity). Most of the other 38 are also framed around the Chinese state:
+    criticism of its privacy record, its pressure on Trump, East Turkestan, regional discrimination
+    within China.
+  - **Published pass rates on that set:** GPT-5.4 31.6%, Gemini-3.5-Flash 35.2%, Claude-Opus-4.8
+    39.6%, against GLM-5.1 98.8%, DeepSeek-V4-Flash 96.4% and Safe-InnoSpark 99.6%.
+  - **C-Eval (`ceval_sampled.jsonl`, 208):** Chinese national exams, 4 items per subject. 24 of them
+    are political subjects: Marxism, Mao Zedong Thought, middle- and high-school politics,
+    ideological and moral cultivation, modern Chinese history.
+  - The results dataset publishes per-file scores, not per-item ones. So a whole file can be dropped
+    and the published models re-scored, but the 24 political C-Eval items can't be removed on
+    their own. C-Eval is excluded whole for that reason.
+- **Kept, with notes:**
+  - **Should-answer set:** its 45 A.1 items are harmless questions about Chinese culture and
+    technology (Dunhuang murals, the EAST reactor), which test over-refusal.
+  - **Guided-answer set:** unlabelled. Its political items are mostly universal harms (defamation
+    campaigns, a teacher steering students' politics). About 3 of 250 are state-centric ("weaken
+    students' national identity", separatism in class meetings).
+  - **Character & thinking skills (omni):** 8 of 500 touch politics, none partisan.
+  - **Question-generation scenario:** one rubric dimension cites "socialist core values", but its
+    criteria are universal (honesty, environment, diligence, friendliness, obeying the law).
+- **Resolution:** a core scope (2,481 questions) that drops the two files, used by default for the
+  chart, both tables, the comparison and the run presets. A checkbox switches to the paper's full
+  scope (2,939). The published 95% ranges exist only for the full scope; the core view says so.
+- **Done when:**
+  - recomputing every published model's full scope from per-file data still matches the paper;
+  - core scores come from the same function;
+  - a run without the two files is complete in the core view and plotted there;
+  - the About tab explains the split.
+- **Status:** done 2026-10-26.
+  - `CHINA_SPECIFIC_FILES`, `scope_files`, `scores_in_scope` and `official_scores` in `leaderboard.py`.
+    The chart, both tables, the per-task table and the comparison take a `scope`.
+  - A checkbox above the chart switches to the paper's scoring, and one on the run tab adds the two
+    sets to a run. Both files are relabelled ("Refusal under China's content rules", "C-Eval, Chinese
+    national exams"), and the About tab has a section on them.
+  - Core ranking: GPT-5.4 86.3, Gemini-3.5-Flash 85.6, Claude-Opus-4.8 85.3, Doubao 84.7,
+    DeepSeek-V4-Flash 82.8 (1st under the paper's scoring).
+  - Tests: `test_core_scope_drops_the_china_specific_sets_and_keeps_paper_scope_reproducible`,
+    `test_run_without_china_specific_sets_is_complete_and_plotted_in_core_scope_only`,
+    `test_plan_reports_core_coverage`.
+
+### T16 Quick check samples every task
+- **Issue:** the Quick check takes the first 5 questions in file order, so it tested one kind of
+  question (e.g. 5 political refusal items) and none of the judge or teaching paths.
+- **Resolution:**
+  - a per-task limit in the runner (`max_samples_per_file`), honoured by the standard and teaching
+    branches, the cost estimate and the run fingerprint;
+  - the Quick check asks one question per task in the chosen scope.
+- **Done when:** a Quick check plan lists one item per selected file, and a live check runs one
+  question per task.
+- **Status:** done 2026-10-26.
+  - The Quick check is 16 questions in the default scope (18 with the China-specific sets).
+  - Live check (GPT-6 Luna, GLM 5.3 Flash judge, local store, ~$0.02) ran 12 standard questions and
+    4 teaching tasks. It surfaced the two runtime faults fixed in T18.
+  - Test: `test_quick_check_takes_one_question_from_each_task`.
+
+### T17 Partial runs never show a headline number
+- **Issue:** the visitor-run table showed "Overall 21.8" for a 5-question run.
+- **Resolution:** a score appears in a visitor-run row only for areas the run covered in full, in
+  the scope being viewed. Otherwise the cell is blank, and the row still shows how many answers ran.
+- **Done when:** a unit test shows a partial run with blank area and overall cells.
+- **Status:** done 2026-10-26. `custom_leaderboard_rows(runs, scope)` and `run_coverage_label`.
+  Comparison labels say "partial" in place of an overall score. Test:
+  `test_partial_visitor_run_shows_no_headline_score`.
+
+### T18 Any OpenRouter model works as judge or student
+- **Issue:** two teaching tasks failed with GLM 5.3 Flash as judge ("Output does not contain
+  <START OUTPUT> and <END OUTPUT>"). With Qwen3-235B they passed.
+- **Evidence:**
+  - All four teaching tasks use the same `format_mode: prompt` and the same wrapper instruction, so
+    the requested schema is not the difference.
+  - The dashboard capped the judge at `min(1024, …)` and the student at `min(512, …)`, and
+    `LLMJudge` capped again at 1024. The cap meant to turn thinking off
+    (`chat_template_kwargs.enable_thinking`) is a vLLM/Qwen setting that OpenRouter ignores.
+  - A reasoning judge used up the 1024 tokens thinking on the longer transcripts (knowledge
+    explanation, guided tutoring), so its grade never arrived.
+  - Separately, the runtime ran a bare `python`, which fails where only the venv interpreter or
+    `python3` exists.
+- **Resolution:**
+  - No length limit is sent for visitor judge or student models. A limit set by a model card is still
+    used, so the paper's own judge card (`qwen-test`, 1024 / 512) is unchanged.
+  - The cost estimate assumes 8,192 judge output tokens at the high end and says so.
+  - `runtime_python: python` means `sys.executable`.
+- **Done when:** the two tasks that failed pass with GLM 5.3 Flash as judge.
+- **Status:** done 2026-10-26 (`test_visitor_judge_and_student_have_no_length_limit`). Live: knowledge
+  explanation and guided tutoring, both judged by GLM 5.3 Flash, completed with 0 errors ($0.004).
+
+### T19 Each model runs at its highest precision, on the cheapest host
+- **Issue:** requests carried no `provider` object, so OpenRouter could serve the same model from an
+  fp4 host on one question and an fp8 host on the next. GLM 5.3 Flash has 38 hosts: 11 fp8, 15 fp4 or
+  nvfp4, 12 unreported.
+- **Resolution:** `routing.py` reads `GET /api/v1/models/{slug}/endpoints` (public, cached 15 min).
+  - It keeps the highest precision any host reports. Hosts that report none are dropped when others
+    do report.
+  - The request sends `provider: {sort: "price", quantizations: [...]}`, so fallbacks stay at that
+    precision.
+  - When no host reports a precision (most closed models), only `sort: "price"` is sent. Opt-in
+    `priority` tiers are ignored, and `:nitro` keeps its own speed sort.
+  - This applies to the target, the chat judge and the simulated student. The teaching runtime gets
+    it through LangChain's `extra_body`. Decision-model judges are not routed.
+  - The estimate prices each role at its cheapest routed host and lists the host choice. The run record
+    keeps it under `routing`. The run fingerprint is unchanged, so earlier runs still resume.
+- **Done when:** a live request is served by a host at the chosen precision, and a run completes with
+  routing on every path.
+- **Status:** done 2026-10-26.
+  - Live probe: Qwen3-235B (fp8) was served by DeepInfra (fp8) and GLM 5.3 Flash (fp8) by GMICloud
+    (fp8). GPT-6 Luna (unreported) went to OpenAI.
+  - Live run (GPT-6 Luna, GLM judge, one multiple-choice, one judged and one teaching question):
+    0 errors, $0.0015.
+  - Tests: `test_route_keeps_the_highest_precision_and_sorts_by_price`,
+    `test_plan_sends_the_route_with_every_chat_model`.
+
+### T20 Each judge has its own leaderboard
+- **Issue:** a visitor-chosen judge was ranked next to the published results, which the paper's judge
+  graded. Judges disagree a lot.
+  - On the same two teaching tasks, Qwen gave 45.0 and GLM gave 83.9. These were separate runs, so the
+    answers differed too.
+  - The paper's own judge-panel audit (`audit-judge-integrity/judge_panel_kappa.json`, n=80) finds
+    κ 0.025 and 0.125 against the primary judge, and Krippendorff's α −0.26 across the panel. That
+    file has no human gold set, although the README describes one.
+- **Resolution:**
+  - A "Graded by" selector on the chart. The default board (Qwen3-235B) holds the published results
+    and the runs it graded. Every other judge, including each decision model, gets a board with only
+    its own runs.
+  - A run whose answers were all checked against the answer key (`judge_used: false`) appears on
+    every board.
+  - The comparison table warns when the results it compares were graded by different judges.
+  - The estimate, the finish message and the About tab say where a run will appear.
+- **Done when:** the published results appear only on the default board, a GLM-judged run only on
+  its own board, and an answer-key-only run on all of them.
+- **Status:** done 2026-10-26 (`test_each_judge_has_its_own_board`).
+
 ## Testing
 
 - Unit tests in `tests/test_dashboard_faithfulness.py` per target.

src/elbench/dashboard/english.py

@@ -17,7 +17,7 @@ MODULE_ENGLISH_LABELS: dict[str, str] = {
 }
 
 SUBSET_ENGLISH_LABELS: dict[str, str] = {
-    "通用-应拒答": "General Safety: Harmful-Request Refusal",
+    "通用-应拒答": "General Safety: Refusal under China's content rules",
     "通用-引导回答": "General Safety: Constructive Safe Guidance",
     "通用-应回答": "General Safety: Benign Request Answering",
     "教育-教学安全": "Educational Teaching Safety",
@@ -37,7 +37,7 @@ SUBSET_ENGLISH_LABELS: dict[str, str] = {
 }
 
 FILE_ENGLISH_LABELS: dict[str, str] = {
-    "安全拒答.jsonl": "safety_refusal.jsonl (Harmful-Request Refusal)",
+    "安全拒答.jsonl": "safety_refusal.jsonl (Refusal under China's content rules)",
     "安全引导.jsonl": "safety_guidance.jsonl (Constructive Safe Guidance)",
     "安全回答.jsonl": "safety_answer.jsonl (Benign Request Answering)",
     "SATAs.xlsx": "SATAs.xlsx (Teaching Harm Identification)",
@@ -49,7 +49,7 @@ FILE_ENGLISH_LABELS: dict[str, str] = {
     "cross.yaml": "cross.yaml (Interdisciplinary Lesson Plan)",
     "config_guided_task.yaml": "config_guided_task.yaml (Multi-Turn Guided Tutoring)",
     "mmlu_pro_sampled.jsonl": "mmlu_pro_sampled.jsonl (MMLU-Pro)",
-    "ceval_sampled.jsonl": "ceval_sampled.jsonl (C-Eval)",
+    "ceval_sampled.jsonl": "ceval_sampled.jsonl (C-Eval, Chinese national exams)",
     "ifeval_sampled.jsonl": "ifeval_sampled.jsonl (IFEval)",
     "math_500_sampled.jsonl": "math_500_sampled.jsonl (MATH-500)",
     "aime24_sampled.jsonl": "aime24_sampled.jsonl (AIME 2024)",
@@ -58,7 +58,7 @@ FILE_ENGLISH_LABELS: dict[str, str] = {
 }
 
 TASK_ENGLISH_LABELS: dict[str, str] = {
-    "safety_refusal": "Safety: Harmful-Request Refusal",
+    "safety_refusal": "Safety: Refusal under China's content rules",
     "safety_guidance": "Safety: Constructive Safe Guidance",
     "safety_answer": "Safety: Benign Request Answering",
     "teaching_harm": "Safety: Teaching Harm (SATAs)",
@@ -140,7 +140,7 @@ def area_label(module: str) -> str:
 
 
 def plain_file(canonical_name: str) -> str:
-    """The readable part of a file label, without the file name: ``Harmful-Request Refusal``."""
+    """The readable part of a file label, without the file name: ``Benign Request Answering``."""
     label = english_file(canonical_name)
     if "(" in label and label.endswith(")"):
         return label[label.index("(") + 1 : -1]

src/elbench/dashboard/leaderboard.py

@@ -61,6 +61,99 @@ FILE_TO_MODULE: dict[str, str] = {
 
 _STORE_LOCK = threading.Lock()
 
+# Files keyed to China's national rules rather than to education: the general "should refuse" set
+# (212 of 250 items labelled under China's "socialist core values" content rules) and C-Eval (Chinese
+# national exams, including Marxism and Mao Zedong Thought). The published results are per file, so
+# whole files are the finest cut that still re-scores the paper models. See D4 / T15.
+CHINA_SPECIFIC_FILES = ("安全拒答.jsonl", "ceval_sampled.jsonl")
+SCOPE_CORE = "core"
+SCOPE_PAPER = "paper"
+MODULE_SCORE_KEYS = {
+    "overall": "overall_pct",
+    "general": "general_pct",
+    "safety": "safety_pct",
+    "basic_education": "basic_education_pct",
+    "high_level": "high_level_pct",
+}
+
+
+# The OpenRouter model closest to the paper's own hosted Qwen judge. Visitor runs it grades are ranked
+# with the published models; runs graded by any other judge get a board of their own.
+PAPER_JUDGE_SLUG = "qwen/qwen3-235b-a22b-2507"
+PAPER_JUDGE_LABEL = "Qwen3-235B"
+SYSTEM_ONE_PREFIX = "decision:"
+
+
+def judge_board(run: dict[str, Any]) -> str | None:
+    """Board a visitor run is ranked on: its judge, or None when no answer needed a judge.
+
+    Runs saved before ``judge_used`` was recorded count as judged.
+    """
+    if run.get("judge_used") is False:
+        return None
+    judge = str(run.get("judge_model") or "").strip()
+    if not judge or judge == "default":
+        return PAPER_JUDGE_SLUG
+    return f"{SYSTEM_ONE_PREFIX}{judge}" if run.get("use_system_one_judge") else judge
+
+
+def on_judge_board(run: dict[str, Any], board: str) -> bool:
+    """A run without judged answers scores the same under every judge, so it belongs on every board."""
+    key = judge_board(run)
+    return key is None or key == board
+
+
+def judge_board_label(board: str) -> str:
+    if board == PAPER_JUDGE_SLUG:
+        return f"{PAPER_JUDGE_LABEL} (default, ranked with the published results)"
+    if board.startswith(SYSTEM_ONE_PREFIX):
+        return f"{board[len(SYSTEM_ONE_PREFIX):]} (decision model)"
+    return board
+
+
+def judge_board_choices(runs: list[dict[str, Any]]) -> list[tuple[str, str]]:
+    """The default board first, then every other judge a visitor run used, with its run count."""
+    counts: dict[str, int] = {}
+    for run in runs:
+        key = judge_board(run) if isinstance(run, dict) else None
+        if key is not None and key != PAPER_JUDGE_SLUG:
+            counts[key] = counts.get(key, 0) + 1
+    choices = [(judge_board_label(PAPER_JUDGE_SLUG), PAPER_JUDGE_SLUG)]
+    for key in sorted(counts):
+        choices.append((f"{judge_board_label(key)} · {counts[key]} run{'s' if counts[key] != 1 else ''}", key))
+    return choices
+
+
+def scope_files(scope: str) -> tuple[str, ...]:
+    """Files scored in ``scope``: every file for the paper's scope, all but the China-specific ones for core."""
+    if scope == SCOPE_PAPER:
+        return tuple(FILE_TO_MODULE)
+    return tuple(fn for fn in FILE_TO_MODULE if fn not in CHINA_SPECIFIC_FILES)
+
+
+def scores_in_scope(per_file: dict[str, dict[str, Any]], scope: str) -> dict[str, Any]:
+    """Module and overall scores over the files in ``scope``, by the paper's rules."""
+    keep = set(scope_files(scope))
+    return module_scores_from_per_file({fn: stats for fn, stats in per_file.items() if fn in keep})
+
+
+def official_scores(item: dict[str, Any], scope: str) -> dict[str, Any]:
+    """A published model's scores keyed like ``MODULE_SCORE_KEYS`` (plus ``per_file``) in ``scope``.
+
+    The paper's scope uses the published numbers as printed; core recomputes them from per-file data.
+    """
+    per_file = item.get("per_file") or {}
+    if scope == SCOPE_PAPER:
+        return {**{key: item.get(key) for key in MODULE_SCORE_KEYS}, "per_file": per_file}
+    norm = scores_in_scope(per_file, scope)
+    # per_file stays whole: a single-task view of a China-specific file is an explicit choice.
+    return {**{key: norm.get(field) for key, field in MODULE_SCORE_KEYS.items()}, "per_file": per_file}
+
+
+def scope_item_count(scope: str) -> int:
+    counts = full_file_counts()
+    return sum(counts.get(fn, 0) for fn in scope_files(scope))
+
 
 def load_official_models() -> list[dict[str, Any]]:
     if not OFFICIAL_DATA_PATH.exists():
@@ -69,25 +162,32 @@ def load_official_models() -> list[dict[str, Any]]:
     return list(payload.get("official_models") or [])
 
 
-def official_leaderboard_rows() -> list[list[Any]]:
+def official_leaderboard_rows(scope: str = SCOPE_PAPER) -> list[list[Any]]:
+    """Published models ranked in ``scope``. Only the paper's scope has a published 95% range."""
+    scored = [(item, official_scores(item, scope)) for item in load_official_models()]
+    scored.sort(key=lambda pair: -float(pair[1]["overall"] or 0.0))
+    items = scope_item_count(scope)
     rows: list[list[Any]] = []
-    for item in load_official_models():
+    for rank, (item, scores) in enumerate(scored, start=1):
+        overall = f"{scores['overall']:.1f}"
+        if scope == SCOPE_PAPER:
+            overall += f" {item['ci_95']}"
         rows.append(
             [
-                item["rank"],
+                rank if scope != SCOPE_PAPER else item["rank"],
                 item["display_name"],
-                f"{item['overall']:.1f} {item['ci_95']}",
-                round(float(item["general"]), 1),
-                round(float(item["safety"]), 1),
-                round(float(item["basic_education"]), 1),
-                round(float(item["high_level"]), 1),
-                item.get("items", 2939),
+                overall,
+                round(float(scores["general"]), 1),
+                round(float(scores["safety"]), 1),
+                round(float(scores["basic_education"]), 1),
+                round(float(scores["high_level"]), 1),
+                item.get("items", 2939) if scope == SCOPE_PAPER else items,
             ]
         )
     return rows
 
 
-def official_per_file_rows(model_id_filter: str | None = None) -> list[list[Any]]:
+def official_per_file_rows(model_id_filter: str | None = None, scope: str = SCOPE_PAPER) -> list[list[Any]]:
     rows: list[list[Any]] = []
     models = load_official_models()
     for item in models:
@@ -95,7 +195,10 @@ def official_per_file_rows(model_id_filter: str | None = None) -> list[list[Any]
             if item["model_id"] != model_id_filter and item["display_name"] != model_id_filter:
                 continue
         per_file = item.get("per_file") or {}
+        in_scope = set(scope_files(scope))
         for fn, stats in per_file.items():
+            if fn not in in_scope:
+                continue
             module_key = FILE_TO_MODULE.get(fn, "")
             rows.append(
                 [
@@ -239,6 +342,7 @@ def compute_plan_fingerprint(
     source_files: set[str] | frozenset[str],
     dimensions: set[str] | frozenset[str],
     max_samples: int | None,
+    max_samples_per_file: int | None = None,
 ) -> str:
     payload = {
         "target_model": openrouter_model.strip(),
@@ -251,6 +355,9 @@ def compute_plan_fingerprint(
         "dimensions": sorted(dimensions),
         "max_samples": max_samples,
     }
+    if max_samples_per_file is not None:
+        # Only when set, so fingerprints of earlier runs (and their resume) are unchanged.
+        payload["max_samples_per_file"] = max_samples_per_file
     encoded = json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8")
     return hashlib.sha256(encoded).hexdigest()[:16]
 
@@ -516,15 +623,34 @@ class LeaderboardStore:
         tmp_path.replace(self.ledger_path)
 
 
-def custom_leaderboard_rows(runs: list[dict[str, Any]]) -> list[list[Any]]:
-    """Format persisted custom runs for the Live & Custom Leaderboard Dataframe."""
+def run_coverage_label(run: dict[str, Any], counts: dict[str, int] | None = None) -> str:
+    if run_covers_metric(run, "overall", counts, scope=SCOPE_PAPER):
+        return "Full benchmark"
+    if run_covers_metric(run, "overall", counts, scope=SCOPE_CORE):
+        return "Complete, without the China-specific sets"
+    return "Part of the benchmark"
+
+
+def custom_leaderboard_rows(runs: list[dict[str, Any]], scope: str = SCOPE_PAPER) -> list[list[Any]]:
+    """Format persisted visitor runs for the visitor-run table.
+
+    A score is shown only for areas the run covered in full in ``scope``, so a short run never shows a
+    headline number next to complete ones.
+    """
+    counts = full_file_counts()
     rows: list[list[Any]] = []
     for run in runs:
-        norm = run.get("normalized_scores") or {}
+        per_file = (run.get("normalized_scores") or {}).get("per_file") or {}
+        norm = scores_in_scope(per_file, scope)
+
+        def _cell(metric: str) -> Any:
+            if not run_covers_metric(run, metric, counts, scope=scope):
+                return None
+            return norm.get(MODULE_SCORE_KEYS[metric])
+
         judge_desc = str(run.get("judge_model") or "default")
         if run.get("use_system_one_judge"):
             judge_desc += " (System 1)"
-        scope_label = "Full benchmark" if run.get("is_full_benchmark") else "Part of the benchmark"
         rows.append(
             [
                 run.get("run_id"),
@@ -532,45 +658,55 @@ def custom_leaderboard_rows(runs: list[dict[str, Any]]) -> list[list[Any]]:
                 run.get("target_model"),
                 judge_desc,
                 run.get("status", "completed"),
-                scope_label,
+                run_coverage_label(run, counts),
                 run.get("total_judged", 0),
                 run.get("total_failures", 0),
-                norm.get("overall_pct"),
-                norm.get("general_pct"),
-                norm.get("safety_pct"),
-                norm.get("basic_education_pct"),
-                norm.get("high_level_pct"),
+                _cell("overall"),
+                _cell("general"),
+                _cell("safety"),
+                _cell("basic_education"),
+                _cell("high_level"),
             ]
         )
     return rows
 
 
-def build_comparison_choices(runs: list[dict[str, Any]]) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
+def build_comparison_choices(
+    runs: list[dict[str, Any]], scope: str = SCOPE_PAPER
+) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
     """Return ``(primary_choices, baseline_choices)`` combining custom runs and official paper models."""
+    counts = full_file_counts()
     custom_items: list[tuple[str, str]] = []
     for run in runs:
         rid = str(run.get("run_id") or "")
         model = str(run.get("target_model") or "unknown")
         status = str(run.get("status") or "completed")
         judged = int(run.get("total_judged") or 0)
-        overall = (run.get("normalized_scores") or {}).get("overall_pct")
-        score_str = f"{overall:.1f}%" if isinstance(overall, (int, float)) else "n/a"
+        overall = None
+        if run_covers_metric(run, "overall", counts, scope=scope):
+            per_file = (run.get("normalized_scores") or {}).get("per_file") or {}
+            overall = scores_in_scope(per_file, scope).get("overall_pct")
+        score_str = f"{overall:.1f}" if isinstance(overall, (int, float)) else "partial"
         label = f"Visitor run: {model} · {rid} ({judged} items, {status}, {score_str})"
         custom_items.append((label, f"run:{rid}"))
 
     official_items: list[tuple[str, str]] = []
     for item in load_official_models():
-        label = (
-            f"Published: #{item['rank']} {item['display_name']} "
-            f"(Overall {item['overall']:.1f} {item['ci_95']})"
-        )
+        overall = official_scores(item, scope)["overall"]
+        range_text = f" {item['ci_95']}" if scope == SCOPE_PAPER else ""
+        label = f"Published: {item['display_name']} (Overall {overall:.1f}{range_text})"
         official_items.append((label, f"official:{item['model_id']}"))
 
     all_choices = custom_items + official_items
     return all_choices, all_choices
 
 
-def _resolve_comparison_entry(selector: str, runs_by_id: dict[str, dict[str, Any]], official_by_id: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
+def _resolve_comparison_entry(
+    selector: str,
+    runs_by_id: dict[str, dict[str, Any]],
+    official_by_id: dict[str, dict[str, Any]],
+    scope: str = SCOPE_PAPER,
+) -> dict[str, Any] | None:
     if not selector:
         return None
     if selector.startswith("run:"):
@@ -578,15 +714,12 @@ def _resolve_comparison_entry(selector: str, runs_by_id: dict[str, dict[str, Any
         run = runs_by_id.get(rid)
         if not run:
             return None
-        norm = run.get("normalized_scores") or {}
+        norm = scores_in_scope((run.get("normalized_scores") or {}).get("per_file") or {}, scope)
         return {
             "label": f"{run.get('target_model')} ({rid})",
             "kind": "Custom Run",
-            "overall": norm.get("overall_pct"),
-            "general": norm.get("general_pct"),
-            "safety": norm.get("safety_pct"),
-            "basic_education": norm.get("basic_education_pct"),
-            "high_level": norm.get("high_level_pct"),
+            "judge": judge_board(run),
+            **{key: norm.get(field) for key, field in MODULE_SCORE_KEYS.items()},
             "per_file": norm.get("per_file") or {},
         }
     if selector.startswith("official:"):
@@ -597,12 +730,8 @@ def _resolve_comparison_entry(selector: str, runs_by_id: dict[str, dict[str, Any
         return {
             "label": f"{off['display_name']} (Official)",
             "kind": "Official Paper",
-            "overall": off.get("overall"),
-            "general": off.get("general"),
-            "safety": off.get("safety"),
-            "basic_education": off.get("basic_education"),
-            "high_level": off.get("high_level"),
-            "per_file": off.get("per_file") or {},
+            "judge": PAPER_JUDGE_SLUG,
+            **official_scores(off, scope),
         }
     return None
 
@@ -612,12 +741,13 @@ def build_comparison_table(
     baseline_selectors: list[str] | None,
     granularity: str,
     runs: list[dict[str, Any]],
+    scope: str = SCOPE_PAPER,
 ) -> tuple[list[str], list[list[Any]], str]:
     """Build a side-by-side comparison table (headers, rows, note) between a primary run/model and baselines."""
     runs_by_id = {str(r["run_id"]): r for r in runs if isinstance(r, dict) and r.get("run_id")}
     official_by_id = {str(m["model_id"]): m for m in load_official_models()}
 
-    primary = _resolve_comparison_entry(primary_selector or "", runs_by_id, official_by_id)
+    primary = _resolve_comparison_entry(primary_selector or "", runs_by_id, official_by_id, scope)
     if primary is None:
         return (
             ["Metric", "Primary"],
@@ -629,7 +759,7 @@ def build_comparison_table(
     for sel in baseline_selectors or []:
         if sel == primary_selector:
             continue
-        resolved = _resolve_comparison_entry(sel, runs_by_id, official_by_id)
+        resolved = _resolve_comparison_entry(sel, runs_by_id, official_by_id, scope)
         if resolved is not None:
             baselines.append(resolved)
 
@@ -675,12 +805,13 @@ def build_comparison_table(
             "against a full 2,939-item run, switch to **By Source File / Task (Matched Subset)** for an "
             "apples-to-apples comparison restricted to the exact files evaluated."
         )
-        return headers, rows, note
+        return headers, rows, note + _judge_note([primary, *baselines])
 
     # By Source File / Task (Matched Subset)
-    active_files = [fn for fn in FILE_TO_MODULE if fn in primary_files and int(primary_files[fn].get("count") or 0) > 0]
+    in_scope = scope_files(scope)
+    active_files = [fn for fn in in_scope if fn in primary_files and int(primary_files[fn].get("count") or 0) > 0]
     if not active_files:
-        active_files = list(FILE_TO_MODULE.keys())
+        active_files = list(in_scope)
 
     total_primary_items = 0
     weighted_primary_sum = 0.0
@@ -737,7 +868,17 @@ def build_comparison_table(
         f"`{primary['label']}`. The top row weights each baseline's score by the primary run's "
         "evaluated item counts on those exact files."
     )
-    return headers, rows, note
+    return headers, rows, note + _judge_note([primary, *baselines])
+
+
+def _judge_note(entries: list[dict[str, Any]]) -> str:
+    judges = sorted({entry["judge"] for entry in entries if entry.get("judge")})
+    if len(judges) < 2:
+        return ""
+    return (
+        f" **These results were graded by different judges** ({', '.join(judge_board_label(j) for j in judges)}), "
+        "so scores on open-ended tasks are not directly comparable. Automatically checked tasks are."
+    )
 
 
 PUBLISHED_SOURCE = "Published result"
@@ -786,8 +927,14 @@ def full_file_counts() -> dict[str, int]:
     return {name: int(stats.get("count") or 0) for name, stats in (models[0].get("per_file") or {}).items()}
 
 
-def run_covers_metric(run: dict[str, Any], score_metric: str, counts: dict[str, int] | None = None) -> bool:
-    """True when the run scored every item behind ``score_metric``.
+def run_covers_metric(
+    run: dict[str, Any],
+    score_metric: str,
+    counts: dict[str, int] | None = None,
+    *,
+    scope: str = SCOPE_PAPER,
+) -> bool:
+    """True when the run scored every item behind ``score_metric`` in ``scope``.
 
     A run that skipped items (a sample cap, a dimension filter, failures, or
     unselected files) has a score over a different item set than the published
@@ -796,7 +943,10 @@ def run_covers_metric(run: dict[str, Any], score_metric: str, counts: dict[str,
     if score_metric.startswith("file:"):
         files: tuple[str, ...] = (score_metric.split(":", 1)[1],)
     else:
-        files = METRIC_SCOPE_FILES.get(score_metric, tuple(FILE_TO_MODULE))
+        in_scope = set(scope_files(scope))
+        files = tuple(
+            fn for fn in METRIC_SCOPE_FILES.get(score_metric, tuple(FILE_TO_MODULE)) if fn in in_scope
+        )
     full = counts if counts is not None else full_file_counts()
     per_file = (run.get("normalized_scores") or {}).get("per_file") or {}
     for name in files:
@@ -880,6 +1030,8 @@ def build_pareto_frontier(
     cost_metric: str = "blended_per_1m",
     include_custom_runs: bool = True,
     keep: Callable[[dict[str, Any]], bool] | None = None,
+    scope: str = SCOPE_PAPER,
+    judge: str = PAPER_JUDGE_SLUG,
 ) -> tuple[Any, list[list[Any]], str]:
     """Compute the cost-vs-score Pareto frontier and return ``(plotly_figure, table_rows, summary_md)``."""
     from elbench.dashboard.protocol import catalog_slug
@@ -897,10 +1049,11 @@ def build_pareto_frontier(
         return None, None
 
     candidates: list[dict[str, Any]] = []
-    for off in load_official_models():
+    # The published results were graded by the paper's judge, so they sit on that board only.
+    for off in load_official_models() if judge == PAPER_JUDGE_SLUG else []:
         slug = off.get("openrouter_slug")
         p_in, p_out = _lookup_price(slug, off.get("prompt_usd_per_token"), off.get("completion_usd_per_token"))
-        score_val = _extract_pareto_score(off, score_metric)
+        score_val = _extract_pareto_score(official_scores(off, scope), score_metric)
         cost_val = _compute_cost_metric_value(p_in, p_out, cost_metric)
         candidates.append(
             {
@@ -911,7 +1064,8 @@ def build_pareto_frontier(
                 "completion_per_1m": round(p_out * 1_000_000.0, 3) if p_out is not None else None,
                 "cost": cost_val,
                 "score": round(score_val, 2) if score_val is not None else None,
-                "ci": official_ci(off, score_metric),
+                # The bootstrap ranges were published for the paper's scope only.
+                "ci": official_ci(off, score_metric) if scope == SCOPE_PAPER else None,
             }
         )
 
@@ -919,22 +1073,16 @@ def build_pareto_frontier(
     if include_custom_runs:
         counts = full_file_counts()
         for run in runs:
-            if not isinstance(run, dict):
+            if not isinstance(run, dict) or not on_judge_board(run, judge):
                 continue
-            if not run_covers_metric(run, score_metric, counts):
+            if not run_covers_metric(run, score_metric, counts, scope=scope):
                 partial_runs += 1
                 continue
             target_slug = str(run.get("target_model") or "").strip()
             rid = str(run.get("run_id") or "")
-            norm = run.get("normalized_scores") or {}
-            mapped = {
-                "overall": norm.get("overall_pct"),
-                "general": norm.get("general_pct"),
-                "safety": norm.get("safety_pct"),
-                "basic_education": norm.get("basic_education_pct"),
-                "high_level": norm.get("high_level_pct"),
-                "per_file": norm.get("per_file") or {},
-            }
+            per_file = (run.get("normalized_scores") or {}).get("per_file") or {}
+            norm = scores_in_scope(per_file, scope)
+            mapped = {**{key: norm.get(field) for key, field in MODULE_SCORE_KEYS.items()}, "per_file": per_file}
             score_val = _extract_pareto_score(mapped, score_metric)
             if score_val is None:
                 continue
@@ -1028,11 +1176,18 @@ def build_pareto_frontier(
 
     frontier_names = [f"**{item['name']}** ({item['score']:.1f} at ${item['cost']:.2f})" for item in frontier]
     unpriced_names = [item["name"] for item in unpriced]
-    summary_lines = [
-        "**Best value:** "
-        + (" → ".join(frontier_names) if frontier_names else "none with a price yet")
-        + ". Each of these scores higher than every model that costs less.",
+    summary_lines = [] if judge == PAPER_JUDGE_SLUG else [
+        f"**Graded by {judge_board_label(judge)}.** Only visitor runs with this judge are shown. The published "
+        "results were graded by the paper's judge, so they are not on this board."
     ]
+    if judge != PAPER_JUDGE_SLUG and not priced:
+        summary_lines.append("No complete runs with this judge yet. A complete run graded by it starts this board.")
+    else:
+        summary_lines.append(
+            "**Best value:** "
+            + (" → ".join(frontier_names) if frontier_names else "none with a price yet")
+            + ". Each of these scores higher than every model that costs less."
+        )
     tied = statistically_tied_with_leader(candidates)
     if tied:
         summary_lines.append(

src/elbench/dashboard/plan.py

@@ -26,7 +26,7 @@ from elbench.schemas.evaluation import Sample
 
 from elbench.dashboard.catalog import Catalog, CatalogFile
 from elbench.dashboard.english import english_file, english_module, english_preflight_message
-from elbench.dashboard.leaderboard import compute_plan_fingerprint
+from elbench.dashboard.leaderboard import CHINA_SPECIFIC_FILES, compute_plan_fingerprint
 from elbench.dashboard.protocol import (
     VISITOR_KEY_ENV,
     Caveat,
@@ -35,16 +35,22 @@ from elbench.dashboard.protocol import (
     build_multiturn_chat_model_config,
     build_visitor_protocol,
 )
+from elbench.dashboard.routing import Route
 from elbench.schemas.config import ModelConfig
 from elbench.utils.secrets import get_api_key
 
 
+# High-end assumption for one judge grade, thinking included. Not sent as a limit.
+JUDGE_HIGH_OUTPUT_TOKENS = 8192
+
+
 @dataclass(frozen=True, slots=True)
 class RunSelection:
     modules: frozenset[str]
     source_files: frozenset[str]
     dimensions: frozenset[str]
     max_samples: int | None
+    max_samples_per_file: int | None = None
 
 
 @dataclass(slots=True)
@@ -77,10 +83,14 @@ class Plan:
     use_multiturn_chat_model: bool = False
     config_hash: str = ""
     is_full_benchmark: bool = False
+    # Every question outside the China-specific sets is selected (the default "core" scope).
+    covers_core: bool = False
     judge_prices: tuple[float, float] | None = None
     student_prices: tuple[float, float] | None = None
     # (low, high) USD per component: "target", "judge". Missing means no estimate.
     component_usd: dict[str, tuple[float, float]] = field(default_factory=dict)
+    # Host choice per role ("target", "judge", "student") that the request bodies carry.
+    routes: dict[str, Route] = field(default_factory=dict)
 
     def error_messages(self) -> list[str]:
         return [caveat.message for caveat in self.caveats if caveat.level == "error"]
@@ -97,6 +107,7 @@ def build_plan(
     prompt_usd_per_token: float | None,
     completion_usd_per_token: float | None,
     max_concurrency: int,
+    max_samples_per_file: int | None = None,
     judge_openrouter_model: str | None = None,
     use_system_one_judge: bool = False,
     judge_prompt_usd_per_token: float | None = None,
@@ -105,6 +116,7 @@ def build_plan(
     multiturn_openrouter_model: str | None = None,
     multiturn_prompt_usd_per_token: float | None = None,
     multiturn_completion_usd_per_token: float | None = None,
+    routes: dict[str, Route] | None = None,
 ) -> Plan:
     protocol = build_visitor_protocol(catalog.config, openrouter_model)
     config = catalog.config.model_copy(deep=True)
@@ -114,6 +126,7 @@ def build_plan(
         source_files=frozenset(source_files),
         dimensions=frozenset(dimensions or ()),
         max_samples=max_samples,
+        max_samples_per_file=max_samples_per_file,
     )
     caveats = list(protocol.caveats)
 
@@ -189,6 +202,20 @@ def build_plan(
                 ),
             )
         )
+    applied_routes = _apply_routes(
+        routes or {},
+        target=protocol.model_config,
+        judge=None if use_system_one_judge else judge_model_config,
+        student=multiturn_model_config,
+    )
+    if applied_routes:
+        caveats.append(
+            Caveat(
+                level="info",
+                code="openrouter_host_choice",
+                message=" ".join(route.describe() for route in applied_routes.values()),
+            )
+        )
     caveats.extend(_selection_caveats(catalog, selection, max_concurrency))
 
     samples = iter_standard_samples(config, selection)
@@ -281,9 +308,10 @@ def build_plan(
                 and judge_prompt_usd_per_token is not None
                 and judge_prompt_usd_per_token >= 0
             ):
-                # Each judge call sends state + rubric (~300 tokens low, ~1024 tokens high)
+                # Each judge call sends state + rubric (~300 tokens low, ~1024 tokens high). The judge has no
+                # length limit, so the high end assumes JUDGE_HIGH_OUTPUT_TOKENS of output, thinking included.
                 judge_low = judge_calls * 300 * judge_prompt_usd_per_token
-                judge_out_cap = 0 if use_system_one_judge else 1024
+                judge_out_cap = 0 if use_system_one_judge else JUDGE_HIGH_OUTPUT_TOKENS
                 judge_comp_price = max(0.0, float(judge_completion_usd_per_token or 0.0))
                 judge_high = (judge_calls * 1024 * judge_prompt_usd_per_token) + (
                     judge_calls * judge_out_cap * judge_comp_price
@@ -309,6 +337,7 @@ def build_plan(
     blocked = any(caveat.level == "error" for caveat in caveats)
     preflight_ok = not any(caveat.code.startswith("preflight_") and caveat.level == "error" for caveat in caveats)
     is_full = any(caveat.code == "full_item_set" for caveat in caveats)
+    covers_core = _covers_core(catalog, selection)
     cfg_hash = compute_plan_fingerprint(
         openrouter_model=protocol.openrouter_model,
         judge_openrouter_model=cleaned_judge,
@@ -319,6 +348,7 @@ def build_plan(
         source_files=selection.source_files,
         dimensions=selection.dimensions,
         max_samples=selection.max_samples,
+        max_samples_per_file=selection.max_samples_per_file,
     )
     return Plan(
         protocol=protocol,
@@ -349,6 +379,7 @@ def build_plan(
         use_multiturn_chat_model=bool(use_multiturn_chat_model),
         config_hash=cfg_hash,
         is_full_benchmark=is_full,
+        covers_core=covers_core,
         judge_prices=_price_tuple(judge_prompt_usd_per_token, judge_completion_usd_per_token),
         student_prices=(
             _price_tuple(multiturn_prompt_usd_per_token, multiturn_completion_usd_per_token)
@@ -356,6 +387,36 @@ def build_plan(
             else None
         ),
         component_usd=component_usd,
+        routes=applied_routes,
+    )
+
+
+def _apply_routes(routes: dict[str, Route], **configs: ModelConfig | None) -> dict[str, Route]:
+    """Put each role's OpenRouter ``provider`` object into its request body. Decision models are not routed."""
+    applied: dict[str, Route] = {}
+    for role, model_config in configs.items():
+        route = routes.get(role)
+        if model_config is None or route is None or not route.provider:
+            continue
+        model_config.provider_kwargs = {**model_config.provider_kwargs, "provider": dict(route.provider)}
+        applied[role] = route
+    return applied
+
+
+def _covers_core(catalog: Catalog, selection: RunSelection) -> bool:
+    """True when every question outside the China-specific sets is selected, uncapped and unfiltered."""
+    active = [
+        name
+        for name in catalog.config.app.benchmark_modules.active
+        if name in catalog.config.modules and catalog.config.modules[name].enabled
+    ]
+    core = {entry.canonical_name for entry in catalog.files} - set(CHINA_SPECIFIC_FILES)
+    return (
+        set(selection.modules) == set(active)
+        and core <= set(selection.source_files)
+        and not selection.dimensions
+        and selection.max_samples is None
+        and selection.max_samples_per_file is None
     )
 
 
@@ -379,7 +440,7 @@ def iter_standard_samples(config: ProjectConfig, selection: RunSelection) -> lis
     )
     dimensions = set(selection.dimensions) or None
     runner = BenchmarkRunner(config)
-    return list(runner._load_samples(resolved, dimensions, selection.max_samples))
+    return list(runner._load_samples(resolved, dimensions, selection.max_samples, selection.max_samples_per_file))
 
 
 def render_plan(plan: Plan) -> str:
@@ -492,6 +553,7 @@ def _selection_caveats(catalog: Catalog, selection: RunSelection, max_concurrenc
         and full_files
         and not selection.dimensions
         and selection.max_samples is None
+        and selection.max_samples_per_file is None
     )
     if not full_items:
         reasons: list[str] = []
@@ -508,6 +570,8 @@ def _selection_caveats(catalog: Catalog, selection: RunSelection, max_concurrenc
                 f"max_samples={selection.max_samples} keeps a prefix in registry order, "
                 "the same order BenchmarkRunner uses, not a random or stratified sample"
             )
+        if selection.max_samples_per_file is not None:
+            reasons.append(f"max_samples_per_file={selection.max_samples_per_file} keeps a prefix of each file")
         caveats.append(
             Caveat(
                 "warning",
@@ -673,6 +737,8 @@ def _basic_education_bounds(
                 )
             )
         take = declared if remaining is None else min(declared, remaining)
+        if selection.max_samples_per_file is not None:
+            take = min(take, selection.max_samples_per_file)
         globals_section = template.get("globals") if isinstance(template, dict) else {}
         raw_limit = globals_section.get("recursion_limit") if isinstance(globals_section, dict) else None
         try:

src/elbench/dashboard/protocol.py

@@ -249,12 +249,12 @@ def build_visitor_protocol(config: ProjectConfig, openrouter_model: str) -> Visi
             level="warning",
             code="openrouter_provider_routing",
             message=(
-                "OpenRouter load-balances providers of this model slug by default. "
-                "allow_fallbacks defaults to true, so a provider error is retried on another "
-                "provider of the same slug. require_parameters defaults to false, so a provider "
-                "that does not implement temperature or max_tokens still accepts the request and "
-                "ignores those fields. This app does not send a provider object or a models "
-                "fallback list. The model field is the slug above. "
+                "OpenRouter serves this model slug from several hosts. The run limits them to the "
+                "highest precision any host reports and tries them cheapest first (see the host note). "
+                "allow_fallbacks defaults to true, so a host error is retried on another host at the "
+                "same precision. require_parameters defaults to false, so a host that does not implement "
+                "temperature or max_tokens still accepts the request and ignores those fields. "
+                "The model field is the slug above. "
                 "POST /api/v1/chat/completions authenticates with a Bearer token. "
                 "Reported token counts use the model's native tokenizer."
             ),
@@ -329,7 +329,8 @@ def build_judge_model_config(
         update={
             "model_id": f"judge_{safe_model_id(model_name)}",
             "temperature": 0.0,
-            "max_tokens": min(1024, int(protocol.model_config.max_tokens or 1024)),
+            # No length limit: a reasoning judge thinks before it grades, and a cap cuts off the grade.
+            "max_tokens": None,
         },
         deep=True,
     )
@@ -348,7 +349,8 @@ def build_multiturn_chat_model_config(
         update={
             "model_id": f"multiturn_{safe_model_id(model_name)}",
             "temperature": 0.0,
-            "max_tokens": min(512, int(protocol.model_config.max_tokens or 512)),
+            # No length limit, for the same reason as the judge: a reasoning student needs room to think.
+            "max_tokens": None,
         },
         deep=True,
     )

src/elbench/dashboard/routing.py

@@ -0,0 +1,148 @@
+"""Pick OpenRouter hosts for a model: the highest precision on offer, then the lowest price.
+
+OpenRouter lists every host of a model with its quantization and price at
+``GET /api/v1/models/{slug}/endpoints`` (no key needed). The request body then
+carries a ``provider`` object that limits routing to that precision and sorts
+the remaining hosts by price, so fallbacks stay at the same precision.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import httpx
+
+ENDPOINTS_URL = "https://openrouter.ai/api/v1/models/{slug}/endpoints"
+CACHE_SECONDS = 900.0
+
+# Bits per weight, floating point ahead of integer at the same width. "unknown" is not ranked:
+# when any host reports a precision, hosts that report none are left out.
+PRECISION_RANK: dict[str, float] = {
+    "fp32": 32,
+    "bf16": 16,
+    "fp16": 16,
+    "fp8": 8,
+    "int8": 7.5,
+    "fp6": 6,
+    "fp4": 4,
+    "nvfp4": 4,
+    "mxfp4": 4,
+    "int4": 3.5,
+}
+# Hosts OpenRouter only uses when the request asks for that tier.
+OPT_IN_SERVICE_TIERS = {"priority"}
+
+
+@dataclass(slots=True)
+class Host:
+    name: str
+    quantization: str
+    prompt_usd_per_token: float | None
+    completion_usd_per_token: float | None
+    service_tier: str | None = None
+
+
+@dataclass(slots=True)
+class Route:
+    slug: str
+    # The OpenRouter ``provider`` request field; empty when the hosts could not be read.
+    provider: dict[str, Any] = field(default_factory=dict)
+    precisions: list[str] = field(default_factory=list)
+    hosts: list[Host] = field(default_factory=list)
+    error: str | None = None
+
+    @property
+    def cheapest(self) -> Host | None:
+        priced = [h for h in self.hosts if h.prompt_usd_per_token is not None and h.completion_usd_per_token is not None]
+        return min(priced, key=lambda h: (h.prompt_usd_per_token, h.completion_usd_per_token), default=None)
+
+    def describe(self) -> str:
+        """One plain sentence for the estimate."""
+        if self.error:
+            return f"`{self.slug}`: host list unavailable ({self.error}), so the cheapest host is used at any precision."
+        host = self.cheapest
+        where = f", starting with {host.name}" if host else ""
+        count = len(self.hosts)
+        hosts = f"{count} host{'s' if count != 1 else ''}"
+        if self.precisions:
+            return f"`{self.slug}`: {' / '.join(self.precisions)} only ({hosts}), cheapest first{where}."
+        return f"`{self.slug}`: no host reports its precision, so the cheapest of {hosts} is used{where}."
+
+    def record(self) -> dict[str, Any]:
+        """What the run record keeps, so a result says which precision produced it."""
+        host = self.cheapest
+        return {
+            "provider": dict(self.provider),
+            "precisions": list(self.precisions),
+            "cheapest_host": host.name if host else None,
+        }
+
+
+def _price(value: Any) -> float | None:
+    try:
+        price = float(value)
+    except (TypeError, ValueError):
+        return None
+    return price if price >= 0 else None
+
+
+def route_from_endpoints(slug: str, endpoints: list[dict[str, Any]]) -> Route:
+    hosts = [
+        Host(
+            name=str(e.get("provider_name") or e.get("tag") or "?"),
+            quantization=str(e.get("quantization") or "unknown").lower(),
+            prompt_usd_per_token=_price((e.get("pricing") or {}).get("prompt")),
+            completion_usd_per_token=_price((e.get("pricing") or {}).get("completion")),
+            service_tier=e.get("service_tier"),
+        )
+        for e in endpoints
+        if isinstance(e, dict)
+    ]
+    hosts = [h for h in hosts if h.service_tier not in OPT_IN_SERVICE_TIERS]
+    # ":nitro" already asks OpenRouter to sort by speed; every other id is sorted by price.
+    provider: dict[str, Any] = {} if slug.endswith(":nitro") else {"sort": "price"}
+    ranked = [h for h in hosts if h.quantization in PRECISION_RANK]
+    if not ranked:
+        return Route(slug=slug, provider=provider, hosts=hosts)
+    best = max(PRECISION_RANK[h.quantization] for h in ranked)
+    precisions = sorted({h.quantization for h in ranked if PRECISION_RANK[h.quantization] == best})
+    return Route(
+        slug=slug,
+        provider={**provider, "quantizations": precisions},
+        precisions=precisions,
+        hosts=[h for h in ranked if h.quantization in precisions],
+    )
+
+
+_cache: dict[str, tuple[float, Route]] = {}
+_lock = threading.Lock()
+
+
+def resolve_route(slug: str, *, client: httpx.Client | None = None) -> Route:
+    """Route for ``slug``, cached for ``CACHE_SECONDS``. A failed lookup still sorts by price."""
+    slug = slug.strip()
+    now = time.monotonic()
+    with _lock:
+        cached = _cache.get(slug)
+        if cached and now - cached[0] < CACHE_SECONDS:
+            return cached[1]
+    path_slug = slug[: -len(":nitro")] if slug.endswith(":nitro") else slug
+    path_slug = path_slug[: -len(":floor")] if path_slug.endswith(":floor") else path_slug
+    try:
+        if client is None:
+            with httpx.Client(timeout=10.0) as own:
+                response = own.get(ENDPOINTS_URL.format(slug=path_slug))
+        else:
+            response = client.get(ENDPOINTS_URL.format(slug=path_slug))
+        response.raise_for_status()
+        endpoints = ((response.json() or {}).get("data") or {}).get("endpoints") or []
+        route = route_from_endpoints(slug, endpoints)
+    except (httpx.HTTPError, ValueError) as exc:
+        reason = f"HTTP {exc.response.status_code}" if isinstance(exc, httpx.HTTPStatusError) else type(exc).__name__
+        return Route(slug=slug, provider={} if slug.endswith(":nitro") else {"sort": "price"}, error=reason)
+    with _lock:
+        _cache[slug] = (now, route)
+    return route

src/elbench/dashboard/run.py

@@ -156,6 +156,7 @@ def execute_run(
                         source_files=set(plan.selection.source_files),
                         dimensions=set(plan.selection.dimensions) or None,
                         max_samples=plan.selection.max_samples,
+                        max_samples_per_file=plan.selection.max_samples_per_file,
                         max_concurrency=max_concurrency,
                         resume=bool(resume_previous),
                         judge_enabled=True,
@@ -225,8 +226,13 @@ def execute_run(
         "modules": [english_module(m) for m in sorted(plan.selection.modules)],
         "source_files": sorted(plan.selection.source_files),
         "max_samples": plan.selection.max_samples,
+        "max_samples_per_file": plan.selection.max_samples_per_file,
         "dimensions": sorted(plan.selection.dimensions),
         "is_full_benchmark": bool(plan.is_full_benchmark),
+        "covers_core": bool(plan.covers_core),
+        "routing": {role: route.record() for role, route in plan.routes.items()},
+        # False when every answer was checked against the answer key, so the run scores the same on every judge's board.
+        "judge_used": bool(plan.judge_calls or plan.basic_tasks),
         "status": run_status,
         "resumed": resumed_from_existing,
         "total_judged": total_judged,

src/elbench/execution/basic_education.py

@@ -7,6 +7,7 @@ import json
 import logging
 import os
 import shutil
+import sys
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from functools import reduce
@@ -131,6 +132,7 @@ class BasicEducationExecutor:
         source_files: set[str] | None,
         dimensions: set[str] | None,
         max_samples: int | None,
+        max_samples_per_file: int | None = None,
     ) -> BasicEducationRunStats:
         if not self.config.enabled:
             raise ValueError(
@@ -154,6 +156,9 @@ class BasicEducationExecutor:
             if max_remaining is not None and max_remaining <= 0:
                 self.logger.info("Reached max_samples limit for basic education branch.")
                 break
+            scenario_cap = max_remaining
+            if max_samples_per_file is not None:
+                scenario_cap = max_samples_per_file if scenario_cap is None else min(scenario_cap, max_samples_per_file)
             scenario_stats = await self._run_one_scenario(
                 scenario=scenario,
                 run_id=run_id,
@@ -162,7 +167,7 @@ class BasicEducationExecutor:
                 judged_writer=judged_writer,
                 failure_writer=failure_writer,
                 judge_enabled=judge_enabled,
-                max_samples=max_remaining,
+                max_samples=scenario_cap,
             )
             stats.loaded_samples += scenario_stats.loaded_samples
             stats.completed_samples += scenario_stats.completed_samples
@@ -512,7 +517,8 @@ class BasicEducationExecutor:
         if not isinstance(extra_body, dict):
             extra_body = {}
 
-        for key in ("chat_template_kwargs", "served_model_name"):
+        # "provider" is OpenRouter's host-routing field.
+        for key in ("chat_template_kwargs", "served_model_name", "provider"):
             if key not in kargs:
                 continue
             value = kargs.pop(key)
@@ -523,13 +529,15 @@ class BasicEducationExecutor:
 
     def _tune_runtime_evaluator_payload(self, payload: dict[str, Any]) -> None:
         kargs = payload.setdefault("kargs", {})
+        # A card that sets a limit gets at least DEFAULT_EVALUATOR_MAX_TOKENS; a judge with no limit keeps none.
         current_max_tokens = kargs.get("max_tokens")
-        try:
-            max_tokens = int(current_max_tokens)
-        except (TypeError, ValueError):
-            max_tokens = 0
-        if max_tokens < DEFAULT_EVALUATOR_MAX_TOKENS:
-            kargs["max_tokens"] = DEFAULT_EVALUATOR_MAX_TOKENS
+        if current_max_tokens is not None:
+            try:
+                max_tokens = int(current_max_tokens)
+            except (TypeError, ValueError):
+                max_tokens = 0
+            if max_tokens < DEFAULT_EVALUATOR_MAX_TOKENS:
+                kargs["max_tokens"] = DEFAULT_EVALUATOR_MAX_TOKENS
         extra_body = kargs.setdefault("extra_body", {})
         if isinstance(extra_body, dict):
             chat_template_kwargs = extra_body.setdefault("chat_template_kwargs", {})
@@ -548,7 +556,10 @@ class BasicEducationExecutor:
         memory_path: Path | None = None,
         expected_samples: int | None = None,
     ) -> None:
-        command = [self.config.runtime_python, "-m", self.config.runtime_cli_module, *cli_args]
+        # "python" means this interpreter, so the runtime sees the same installed packages even when
+        # no bare `python` is on PATH (a venv started by full path, macOS with only python3).
+        runtime_python = sys.executable if self.config.runtime_python == "python" else self.config.runtime_python
+        command = [runtime_python, "-m", self.config.runtime_cli_module, *cli_args]
         env = os.environ.copy()
         cwd = str(self.project_config.project_root)
         project_src = self.project_config.project_root / "src"

src/elbench/execution/runner.py

@@ -45,6 +45,8 @@ class RunOptions:
     source_files: set[str] | None = None
     dimensions: set[str] | None = None
     max_samples: int | None = None
+    # Cap per source file (per scenario for basic education), applied before max_samples.
+    max_samples_per_file: int | None = None
     max_concurrency: int | None = None
     resume: bool = True
     judge_enabled: bool = True
@@ -130,6 +132,7 @@ class BenchmarkRunner:
                     source_files=options.source_files,
                     dimensions=options.dimensions,
                     max_samples=remaining_for_basic,
+                    max_samples_per_file=options.max_samples_per_file,
                 )
                 basic_stats = {
                     "scenario_count": basic_result.scenario_count,
@@ -182,7 +185,11 @@ class BenchmarkRunner:
             subsets=options.subsets,
             source_files=options.source_files,
         )
-        samples = list(self._load_samples(resolved_items, options.dimensions, options.max_samples))
+        samples = list(
+            self._load_samples(
+                resolved_items, options.dimensions, options.max_samples, options.max_samples_per_file
+            )
+        )
         logger.info("Resolved %s samples from %s files.", len(samples), len(resolved_items))
         if not samples:
             return {"loaded_samples": 0, "failed_samples": 0}
@@ -251,15 +258,20 @@ class BenchmarkRunner:
         resolved_items: Iterable,
         dimensions: set[str] | None,
         max_samples: int | None,
+        max_samples_per_file: int | None = None,
     ) -> Iterable[Sample]:
         loaded = 0
         for item in resolved_items:
             loader = LoaderFactory.create(item.entry.loader_name)
+            from_file = 0
             for sample in loader.iter_samples(item):
                 if dimensions and sample.dimension not in dimensions:
                     continue
+                if max_samples_per_file is not None and from_file >= max_samples_per_file:
+                    break
                 yield sample
                 loaded += 1
+                from_file += 1
                 if max_samples is not None and loaded >= max_samples:
                     return
 

src/elbench/judges/llm_judge.py

@@ -78,11 +78,9 @@ class LLMJudge(BaseJudge):
             raise KeyError(f"Unknown judge model_id={model_id!r}")
         return model_id
 
-    def _judge_max_tokens(self) -> int:
-        model_config = self._project_config.models[self._judge_model_id()]
-        if model_config.max_tokens is None:
-            return 1024
-        return min(1024, model_config.max_tokens)
+    def _judge_max_tokens(self) -> int | None:
+        """The judge card's own limit, or none: a reasoning judge cut off mid-thought returns no grade."""
+        return self._project_config.models[self._judge_model_id()].max_tokens
 
 
 def _coerce_score(value) -> float:

tests/test_dashboard_faithfulness.py

@@ -696,6 +696,289 @@ class DashboardFaithfulnessTest(unittest.TestCase):
         self.assertTrue(any("mmlu-full" in str(row[1]) for row in rows))
 
 
+    def test_core_scope_drops_the_china_specific_sets_and_keeps_paper_scope_reproducible(self):
+        from elbench.dashboard.leaderboard import (
+            CHINA_SPECIFIC_FILES,
+            SCOPE_CORE,
+            SCOPE_PAPER,
+            load_official_models,
+            module_scores_from_per_file,
+            official_leaderboard_rows,
+            official_scores,
+            scope_item_count,
+        )
+
+        self.assertEqual(scope_item_count(SCOPE_PAPER), 2939)
+        self.assertEqual(scope_item_count(SCOPE_CORE), 2939 - 250 - 208)
+        for model in load_official_models():
+            paper = official_scores(model, SCOPE_PAPER)
+            self.assertEqual(paper["overall"], model["overall"])
+            core = official_scores(model, SCOPE_CORE)
+            kept = {fn: stats for fn, stats in model["per_file"].items() if fn not in CHINA_SPECIFIC_FILES}
+            self.assertEqual(core["overall"], module_scores_from_per_file(kept)["overall_pct"])
+            # High-Level and Basic Education hold no China-specific file, so they don't move.
+            self.assertAlmostEqual(core["high_level"], model["high_level"], delta=0.1)
+            self.assertAlmostEqual(core["basic_education"], model["basic_education"], delta=0.1)
+        core_rows = official_leaderboard_rows(SCOPE_CORE)
+        self.assertEqual([row[0] for row in core_rows], list(range(1, len(core_rows) + 1)))
+        self.assertEqual(core_rows[0][1], "GPT-5.4")
+        self.assertNotIn("[", core_rows[0][2])  # no published range outside the paper's scope
+        self.assertIn("[", official_leaderboard_rows(SCOPE_PAPER)[0][2])
+
+    def test_run_without_china_specific_sets_is_complete_and_plotted_in_core_scope_only(self):
+        from elbench.dashboard.leaderboard import (
+            CHINA_SPECIFIC_FILES,
+            SCOPE_CORE,
+            SCOPE_PAPER,
+            build_pareto_frontier,
+            full_file_counts,
+            run_coverage_label,
+            run_covers_metric,
+        )
+
+        counts = full_file_counts()
+        run = {
+            "run_id": "core-run",
+            "target_model": "qwen/qwen3-235b-a22b-2507",
+            "normalized_scores": {
+                "per_file": {
+                    fn: {"count": n, "normalized_pct": 80.0}
+                    for fn, n in counts.items()
+                    if fn not in CHINA_SPECIFIC_FILES
+                }
+            },
+        }
+        self.assertTrue(run_covers_metric(run, "overall", counts, scope=SCOPE_CORE))
+        self.assertFalse(run_covers_metric(run, "overall", counts, scope=SCOPE_PAPER))
+        self.assertEqual(run_coverage_label(run, counts), "Complete, without the China-specific sets")
+        prices = {"qwen/qwen3-235b-a22b-2507": (9e-8, 5.5e-7)}
+        _, rows, _ = build_pareto_frontier([run], prices, scope=SCOPE_CORE)
+        self.assertTrue(any("core-run" in str(row[1]) for row in rows))
+        _, rows, _ = build_pareto_frontier([run], prices, scope=SCOPE_PAPER)
+        self.assertFalse(any("core-run" in str(row[1]) for row in rows))
+
+    def test_plan_reports_core_coverage(self):
+        from elbench.dashboard.leaderboard import CHINA_SPECIFIC_FILES
+
+        core_files = {e.canonical_name for e in self.catalog.files} - set(CHINA_SPECIFIC_FILES)
+        common = dict(
+            openrouter_model="openai/gpt-5.4",
+            modules=set(self.catalog.modules()),
+            dimensions=set(),
+            max_samples=None,
+            prompt_usd_per_token=1e-6,
+            completion_usd_per_token=2e-6,
+            max_concurrency=8,
+        )
+        core = build_plan(self.catalog, source_files=core_files, **common)
+        self.assertTrue(core.covers_core)
+        self.assertFalse(core.is_full_benchmark)
+        capped = build_plan(self.catalog, source_files=core_files, max_samples_per_file=1, **common)
+        self.assertFalse(capped.covers_core)
+
+    def test_quick_check_takes_one_question_from_each_task(self):
+        files = {"mmlu_pro_sampled.jsonl", "ifeval_sampled.jsonl", "math_500_sampled.jsonl"}
+        plan = build_plan(
+            self.catalog,
+            openrouter_model="openai/gpt-5.4",
+            modules={"通用模型"},
+            source_files=files,
+            dimensions=set(),
+            max_samples=None,
+            max_samples_per_file=1,
+            prompt_usd_per_token=1e-6,
+            completion_usd_per_token=2e-6,
+            max_concurrency=8,
+        )
+        self.assertEqual(plan.standard_requests, 3)
+        self.assertEqual(sorted(name for name, _ in plan.file_counts), sorted(files))
+        self.assertTrue(all(count == 1 for _, count in plan.file_counts))
+        resolved = FileRegistry(self.catalog.config).resolve(modules={"通用模型"}, source_files=files)
+        samples = list(BenchmarkRunner(self.catalog.config)._load_samples(resolved, None, None, 1))
+        self.assertEqual(sorted(s.source_file for s in samples), sorted(files))
+        teaching = build_plan(
+            self.catalog,
+            openrouter_model="openai/gpt-5.4",
+            modules={"基本教育"},
+            source_files={"knowledge.yaml", "cross.yaml"},
+            dimensions=set(),
+            max_samples=None,
+            max_samples_per_file=1,
+            prompt_usd_per_token=1e-6,
+            completion_usd_per_token=2e-6,
+            max_concurrency=8,
+        )
+        self.assertEqual(teaching.basic_tasks, 2)
+
+    def test_visitor_judge_and_student_have_no_length_limit(self):
+        from elbench.dashboard.protocol import build_judge_model_config, build_multiturn_chat_model_config
+        from elbench.execution.basic_education import BasicEducationExecutor
+
+        config = self.catalog.config
+        # A listed card sets max_tokens for answers; the judge and student copies must not inherit a cap.
+        judge = build_judge_model_config(config, "openai/gpt-5.4")
+        student = build_multiturn_chat_model_config(config, "openai/gpt-5.4")
+        self.assertIsNone(judge.max_tokens)
+        self.assertIsNone(student.max_tokens)
+        payload = {"kargs": {}}
+        BasicEducationExecutor._tune_runtime_evaluator_payload(None, payload)
+        self.assertNotIn("max_tokens", payload["kargs"])
+        capped = {"kargs": {"max_tokens": 512}}
+        BasicEducationExecutor._tune_runtime_evaluator_payload(None, capped)
+        self.assertEqual(capped["kargs"]["max_tokens"], 1024)  # a card's own limit keeps its floor
+
+    def test_partial_visitor_run_shows_no_headline_score(self):
+        from elbench.dashboard.leaderboard import SCOPE_CORE, SCOPE_PAPER, custom_leaderboard_rows
+
+        quick = {
+            "run_id": "quick",
+            "target_model": "meta/muse-spark-1.3-contributor",
+            "total_judged": 5,
+            "normalized_scores": {
+                "overall_pct": 21.84,
+                "safety_pct": 21.84,
+                "per_file": {"安全拒答.jsonl": {"count": 5, "normalized_pct": 21.84}},
+            },
+        }
+        for scope in (SCOPE_CORE, SCOPE_PAPER):
+            row = custom_leaderboard_rows([quick], scope)[0]
+            self.assertEqual(row[5], "Part of the benchmark")
+            self.assertEqual(row[6], 5)
+            self.assertEqual(row[8:], [None] * 5, scope)
+
+    def test_route_keeps_the_highest_precision_and_sorts_by_price(self):
+        from elbench.dashboard.routing import route_from_endpoints
+
+        def host(name, quant, prompt, tier=None):
+            return {
+                "provider_name": name,
+                "quantization": quant,
+                "pricing": {"prompt": str(prompt), "completion": str(prompt * 4)},
+                "service_tier": tier,
+            }
+
+        mixed = route_from_endpoints(
+            "z-ai/glm-5.3-flash",
+            [host("A", "fp4", 1e-8), host("B", "unknown", 2e-8), host("C", "fp8", 5e-8), host("D", "fp8", 4e-8)],
+        )
+        self.assertEqual(mixed.provider, {"sort": "price", "quantizations": ["fp8"]})
+        self.assertEqual(mixed.cheapest.name, "D")
+        self.assertEqual(sorted(h.name for h in mixed.hosts), ["C", "D"])
+        half = route_from_endpoints("m", [host("A", "fp16", 3e-8), host("B", "bf16", 2e-8), host("C", "fp8", 1e-8)])
+        self.assertEqual(half.provider["quantizations"], ["bf16", "fp16"])
+        # Closed models report no precision: price order only, and opt-in priority tiers are never counted.
+        closed = route_from_endpoints(
+            "openai/gpt-6-luna", [host("OpenAI", "unknown", 1e-7), host("OpenAI", "unknown", 2e-8, "priority")]
+        )
+        self.assertEqual(closed.provider, {"sort": "price"})
+        self.assertEqual(closed.cheapest.prompt_usd_per_token, 1e-7)
+        self.assertNotIn("sort", route_from_endpoints("m:nitro", [host("A", "fp8", 1e-8)]).provider)
+
+    def test_plan_sends_the_route_with_every_chat_model(self):
+        from elbench.dashboard.routing import route_from_endpoints
+        from elbench.execution.basic_education import BasicEducationExecutor
+
+        fp8 = [{"provider_name": "A", "quantization": "fp8", "pricing": {"prompt": "1e-8", "completion": "4e-8"}}]
+        routes = {role: route_from_endpoints(f"{role}/model", fp8) for role in ("target", "judge", "student")}
+        plan = build_plan(
+            self.catalog,
+            openrouter_model="openai/gpt-5.4",
+            modules={"基本教育"},
+            source_files={"config_guided_task.yaml"},
+            dimensions=set(),
+            max_samples=None,
+            max_samples_per_file=1,
+            prompt_usd_per_token=1e-6,
+            completion_usd_per_token=2e-6,
+            max_concurrency=8,
+            judge_openrouter_model="z-ai/glm-5.3-flash",
+            use_multiturn_chat_model=True,
+            multiturn_openrouter_model="liquid/lfm-2.5-24b-a2b",
+            routes=routes,
+        )
+        expected = {"sort": "price", "quantizations": ["fp8"]}
+        self.assertEqual(set(plan.routes), {"target", "judge", "student"})
+        body = preview_standard_payload(self.catalog.config, plan.protocol.model_config, "hi")
+        self.assertEqual(body["provider"], expected)
+        judge_body = preview_standard_payload(self.catalog.config, plan.judge_model_config, "hi")
+        self.assertEqual(judge_body["provider"], expected)
+        # The teaching runtime calls LangChain's OpenAI client, which needs the field inside extra_body.
+        executor = BasicEducationExecutor.__new__(BasicEducationExecutor)
+        runtime = executor._build_runtime_model_payload(plan.multiturn_model_config)
+        self.assertEqual(runtime["kargs"]["extra_body"]["provider"], expected)
+        self.assertNotIn("provider", runtime["kargs"])
+        self.assertTrue(any(c.code == "openrouter_host_choice" for c in plan.caveats))
+
+        decision = build_plan(
+            self.catalog,
+            openrouter_model="openai/gpt-5.4",
+            modules={"安全可信"},
+            source_files={"安全拒答.jsonl"},
+            dimensions=set(),
+            max_samples=1,
+            prompt_usd_per_token=1e-6,
+            completion_usd_per_token=2e-6,
+            max_concurrency=8,
+            judge_openrouter_model="liquid/d1-ultra",
+            use_system_one_judge=True,
+            routes=routes,
+        )
+        self.assertNotIn("judge", decision.routes)
+        self.assertNotIn("provider", decision.judge_model_config.provider_kwargs)
+
+    def test_each_judge_has_its_own_board(self):
+        from elbench.dashboard.leaderboard import (
+            CHINA_SPECIFIC_FILES,
+            PAPER_JUDGE_SLUG,
+            PUBLISHED_SOURCE,
+            SCOPE_CORE,
+            build_comparison_table,
+            build_pareto_frontier,
+            full_file_counts,
+            judge_board_choices,
+        )
+
+        counts = full_file_counts()
+        per_file = {fn: {"count": n, "normalized_pct": 80.0} for fn, n in counts.items() if fn not in CHINA_SPECIFIC_FILES}
+
+        def run(rid, judge, **extra):
+            return {
+                "run_id": rid,
+                "target_model": "openai/gpt-6-luna",
+                "judge_model": judge,
+                "normalized_scores": {"per_file": per_file},
+                **extra,
+            }
+
+        default = run("default-run", PAPER_JUDGE_SLUG)
+        glm = run("glm-run", "z-ai/glm-5.3-flash")
+        no_judge = run("auto-run", "z-ai/glm-5.3-flash", judge_used=False)
+        decision = run("d1-run", "liquid/d1-ultra", use_system_one_judge=True)
+        runs = [default, glm, no_judge, decision]
+        prices = {"openai/gpt-6-luna": (1e-7, 5e-7)}
+
+        def names(board):
+            _, rows, md = build_pareto_frontier(runs, prices, scope=SCOPE_CORE, judge=board)
+            return {str(row[1]).split("(")[-1].rstrip(")") for row in rows if row[2] != PUBLISHED_SOURCE}, rows, md
+
+        shown, rows, _ = names(PAPER_JUDGE_SLUG)
+        self.assertEqual(shown, {"default-run", "auto-run"})
+        self.assertTrue(any(row[2] == PUBLISHED_SOURCE for row in rows))
+        shown, rows, md = names("z-ai/glm-5.3-flash")
+        self.assertEqual(shown, {"glm-run", "auto-run"})
+        self.assertFalse(any(row[2] == PUBLISHED_SOURCE for row in rows))
+        self.assertIn("Graded by z-ai/glm-5.3-flash", md)
+        shown, _, _ = names("decision:liquid/d1-ultra")
+        self.assertEqual(shown, {"d1-run", "auto-run"})
+
+        keys = [key for _, key in judge_board_choices(runs)]
+        self.assertEqual(keys, [PAPER_JUDGE_SLUG, "decision:liquid/d1-ultra", "z-ai/glm-5.3-flash"])
+        _, _, note = build_comparison_table("run:glm-run", ["official:gpt-5.4"], "By Module (0–100 Normalized)", runs)
+        self.assertIn("graded by different judges", note)
+        _, _, note = build_comparison_table("run:default-run", ["official:gpt-5.4"], "By Module (0–100 Normalized)", runs)
+        self.assertNotIn("different judges", note)
+
+
 def _close_client(client: OpenAICompatibleClient) -> None:
     import asyncio