Disable invalid live compare until benchmark parity

demo_controller.py

@@ -313,6 +313,14 @@ def _build_unavailable_result(request: DemoRequest, logs: list[str]) -> dict[str
             f"Live run blocked: selected context {int(request.context_length)} exceeds the current live limit "
             f"of {max_live_context} tokens. Lower Context length or raise DOTCACHE_SPACE_MAX_LIVE_CONTEXT."
         )
+    elif any("Live benchmark parity is not wired in this Space yet" in entry for entry in recent_logs):
+        summary = "Live compare is disabled until the Space uses the same benchmark selector artifacts and profile settings as the paper runs."
+        blocked_reason = "benchmark_parity"
+        candidate_text = (
+            "Live run blocked: this Space is not yet executing the benchmark-faithful Qwen lane. "
+            "Preset-backed compare is still the valid paper/benchmark view until the learned selector artifacts "
+            "and benchmark profile wiring are packaged for live execution."
+        )
     elif any("is not wired in this v1 Space yet" in entry for entry in recent_logs):
         summary = "This model is listed for the demo, but its live execution lane is not wired yet."
         blocked_reason = "model_family_not_wired"

engines/live_request.py

@@ -69,6 +69,12 @@ def resolve_live_runtime_settings(
             "Live execution for this gated Llama model requires HF_TOKEN in the Space settings. "
             "Preset-backed compare still works without it."
         )
+    if model.family == "qwen35":
+        raise ValueError(
+            "Live benchmark parity is not wired in this Space yet. The current live lane does not load the "
+            "benchmark selector artifact or benchmark-exact profile settings, so live compare is disabled until "
+            "that path is packaged correctly. Preset-backed compare remains the valid paper/benchmark view."
+        )
 
     context_length = int(request.get("context_length") or 0)
     if context_length <= 0:

space_app.py

@@ -293,14 +293,19 @@ def _is_live_result(result: dict[str, Any]) -> bool:
     }
 
 
+def _is_paper_fixture(result: dict[str, Any]) -> bool:
+    return bool(result.get("comparison", {}).get("paper_fixture"))
+
+
 def _hero_headline(result: dict[str, Any]) -> str:
     comparison = result["comparison"]
     if _is_live_result(result):
         speed_value, _ = _speed_display(comparison["speedup"])
-        memory_value, _ = _memory_display(comparison["memory_reduction"])
         _, agreement_label, _ = _agreement_state(comparison["agreement"])
+        candidate_kv = _format_bytes(result["candidate"]["kv_bytes"])
+        baseline_kv = _format_bytes(result["baseline"]["kv_bytes"])
         return (
-            f"Live run estimate: DotCache is {speed_value}, reports {memory_value} resident KV, "
+            f"Live Space run: DotCache is {speed_value}, with {candidate_kv} resident KV vs {baseline_kv} dense, "
             f"and shows {agreement_label.lower()} vs the reference run."
         )
     paper_headline = str(comparison.get("paper_headline") or "").strip()
@@ -363,12 +368,19 @@ def _render_loading_state(label: str) -> str:
 
 
 def _render_hero(result: dict[str, Any], response: Any) -> str:
+    baseline = result["baseline"]
+    candidate = result["candidate"]
     comparison = result["comparison"]
+    is_live = _is_live_result(result)
+    is_paper = _is_paper_fixture(result)
     if _is_blocked_result(result):
         reason = comparison.get("blocked_reason")
         if reason == "hw_limit":
             subtitle = "This request hit the current hardware/runtime limit before the reference or systems row could execute."
             status_value = "Hardware limit hit"
+        elif reason == "benchmark_parity":
+            subtitle = "Live compare is disabled until this Space is wired to the same selector artifacts and benchmark profile settings used for the paper runs."
+            status_value = "Benchmark parity required"
         elif reason == "model_family_not_wired":
             subtitle = "This model is available in the demo list, but its live execution lane has not been wired yet."
             status_value = "Live lane not wired"
@@ -405,12 +417,28 @@ def _render_hero(result: dict[str, Any], response: Any) -> str:
             "</div>"
         )
     speed_value, _ = _speed_display(comparison["speedup"])
-    memory_value, _ = _memory_display(comparison["memory_reduction"])
     agreement_badge = _agreement_badge_text(comparison["agreement"])
+    if is_live:
+        memory_value = _format_bytes(candidate["kv_bytes"])
+        memory_label = "Runtime resident KV"
+    elif is_paper:
+        memory_value = str(comparison.get("paper_metric_badge") or "Not reported")
+        memory_label = "Paper metric"
+    else:
+        memory_value, _ = _memory_display(comparison["memory_reduction"])
+        memory_label = "KV footprint"
+    subtitle = str(comparison.get("paper_subtitle") or "").strip()
+    if is_live:
+        subtitle = (
+            "Live prompt comparison using the current Space runtime. Resident KV bytes here are runtime telemetry "
+            "and are not directly comparable to the paper preset tables."
+        )
+    elif not subtitle:
+        subtitle = "Reference vs systems long-context inference, summarized from the current run."
     hero_stats = "".join(
         [
             _hero_stat("⚡", speed_value, "Speed vs reference"),
-            _hero_stat("💾", memory_value, "KV footprint"),
+            _hero_stat("💾", memory_value, memory_label),
             _hero_stat("🎯", agreement_badge, f"{100.0 * comparison['agreement']:.0f}% agreement vs reference"),
         ]
     )
@@ -422,7 +450,7 @@ def _render_hero(result: dict[str, Any], response: Any) -> str:
         f"<h1>{html.escape(_hero_headline(result))}</h1>"
         f"{_hero_about_link()}"
         "</div>"
-        f"<p class='hero-subtitle'>{html.escape(str(comparison.get('paper_subtitle') or ('Live prompt comparison using the current runtime estimates.' if _is_live_result(result) else 'Reference vs systems long-context inference, summarized from the current run.')))}</p>"
+        f"<p class='hero-subtitle'>{html.escape(subtitle)}</p>"
         "</div>"
         f"<div class='hero-stats'>{hero_stats}</div>"
         f"{_render_badge_html(response)}"
@@ -435,6 +463,7 @@ def _render_kpis(result: dict[str, Any]) -> str:
     candidate = result["candidate"]
     comparison = result["comparison"]
     is_live = _is_live_result(result)
+    is_paper = _is_paper_fixture(result)
     if _is_blocked_result(result):
         cards = [
             _kpi_card("⚡ SPEEDUP", "Not run", "No comparison was executed", tone="neutral"),
@@ -452,17 +481,33 @@ def _render_kpis(result: dict[str, Any]) -> str:
     agreement_primary, agreement_secondary, agreement_tone = _agreement_metric_text(comparison["agreement"])
     latency_tone = "good" if candidate["latency_ms_per_token"] <= baseline["latency_ms_per_token"] else "bad"
     memory_size_tone = "good" if candidate["kv_bytes"] <= baseline["kv_bytes"] else "bad"
-    memory_title = "💾 EST. KV RESIDENCY" if is_live else "💾 MEMORY REDUCTION"
-    memory_primary = (
-        _format_ratio(comparison["memory_reduction"])
-        if not is_live
-        else _format_bytes(candidate["kv_bytes"])
-    )
-    memory_secondary = (
-        memory_text.replace("better", "smaller than dense").replace("worse", "larger than dense")
-        if not is_live
-        else f"runtime-reported vs reference {_format_bytes(baseline['kv_bytes'])}"
-    )
+    if is_live:
+        memory_title = "💾 EST. KV RESIDENCY"
+        memory_primary = _format_bytes(candidate["kv_bytes"])
+        memory_secondary = f"runtime-reported vs reference {_format_bytes(baseline['kv_bytes'])}"
+        memory_tone_for_card = "neutral"
+        raw_kv_title = "📦 RAW KV SIZE"
+        raw_kv_primary = _format_bytes(candidate["kv_bytes"])
+        raw_kv_secondary = f"dense {_format_bytes(baseline['kv_bytes'])}"
+        raw_kv_tone = memory_size_tone
+    elif is_paper:
+        memory_title = "💾 KV MEMORY"
+        memory_primary = "Not reported"
+        memory_secondary = "This paper row does not publish a KV-memory reduction number."
+        memory_tone_for_card = "neutral"
+        raw_kv_title = "📄 PAPER METRIC"
+        raw_kv_primary = str(comparison.get("paper_metric_badge") or "Paper-backed row")
+        raw_kv_secondary = "Preset speed and quality come from the paper; KV bytes are not reported there."
+        raw_kv_tone = "neutral"
+    else:
+        memory_title = "💾 MEMORY REDUCTION"
+        memory_primary = _format_ratio(comparison["memory_reduction"])
+        memory_secondary = memory_text.replace("better", "smaller than dense").replace("worse", "larger than dense")
+        memory_tone_for_card = memory_tone
+        raw_kv_title = "📦 RAW KV SIZE"
+        raw_kv_primary = _format_bytes(candidate["kv_bytes"])
+        raw_kv_secondary = f"dense {_format_bytes(baseline['kv_bytes'])}"
+        raw_kv_tone = memory_size_tone
     cards = [
         _kpi_card(
             "⚡ SPEEDUP",
@@ -474,7 +519,7 @@ def _render_kpis(result: dict[str, Any]) -> str:
             memory_title,
             memory_primary,
             memory_secondary,
-            tone="neutral" if is_live else memory_tone,
+            tone=memory_tone_for_card,
         ),
         _kpi_card(
             "🎯 AGREEMENT",
@@ -489,10 +534,10 @@ def _render_kpis(result: dict[str, Any]) -> str:
             tone=latency_tone,
         ),
         _kpi_card(
-            "📦 RAW KV SIZE",
-            _format_bytes(candidate["kv_bytes"]),
-            f"dense {_format_bytes(baseline['kv_bytes'])}",
-            tone=memory_size_tone,
+            raw_kv_title,
+            raw_kv_primary,
+            raw_kv_secondary,
+            tone=raw_kv_tone,
         ),
         _kpi_card(
             "🧪 TOKENS / SEC",
@@ -559,6 +604,23 @@ def _render_inline_trends(result: dict[str, Any]) -> str:
         )
     baseline = result["baseline"]
     candidate = result["candidate"]
+    if _is_paper_fixture(result):
+        return (
+            "<div class='trend-grid'>"
+            + _mini_trend_chart(
+                "Speed",
+                baseline["tok_per_sec"],
+                candidate["tok_per_sec"],
+                formatter=lambda value: f"{_format_float(value, 1)} tok/s",
+                better_when_higher=True,
+            )
+            + "<div class='trend-card'>"
+            "<div class='trend-title'>KV Memory</div>"
+            "<div class='trend-delta neutral'>Not reported in this paper row</div>"
+            "<p>Preset mode uses the paper's speed and quality row only, so placeholder KV bytes are hidden here.</p>"
+            "</div>"
+            "</div>"
+        )
     charts = [
         _mini_trend_chart(
             "Speed",
@@ -598,6 +660,11 @@ def _render_story_sentence(result: dict[str, Any]) -> str:
                 f"This request did not run because the selected context ({comparison.get('blocked_context_length', 0)}) "
                 f"exceeds the current live hardware limit of {comparison.get('max_live_context', 0)} tokens."
             )
+        if comparison.get("blocked_reason") == "benchmark_parity":
+            return (
+                "This request did not run because the current Space live lane is not yet using the same selector "
+                "artifacts and benchmark profile settings as the paper-backed presets."
+            )
         if comparison.get("blocked_reason") == "model_family_not_wired":
             return "This model can appear in the demo list, but its custom live runner has not been wired into the v1 Space yet."
         if comparison.get("blocked_reason") == "auth_required":
@@ -607,6 +674,19 @@ def _render_story_sentence(result: dict[str, Any]) -> str:
         if comparison.get("blocked_reason") == "cache_download_limit":
             return "This request did not run because the Space could not fetch the selected model into its local runtime cache."
         return "This request did not run because the current runtime blocked execution before inference started."
+    if _is_live_result(result):
+        speed_value, _ = _speed_display(comparison["speedup"])
+        _, agreement_label, _ = _agreement_state(comparison["agreement"])
+        candidate_kv = _format_bytes(result["candidate"]["kv_bytes"])
+        baseline_kv = _format_bytes(result["baseline"]["kv_bytes"])
+        return (
+            f"Live Space run: DotCache is {speed_value}, with {candidate_kv} resident KV vs {baseline_kv} dense, "
+            f"and shows {agreement_label.lower()} compared to the reference run."
+        )
+    if _is_paper_fixture(result):
+        speed_value, _ = _speed_display(comparison["speedup"])
+        _, agreement_label, _ = _agreement_state(comparison["agreement"])
+        return f"This paper row reports {speed_value} with {agreement_label.lower()}; KV-memory reduction is not reported for this preset."
     speed_value, _ = _speed_display(comparison["speedup"])
     memory_value, _ = _memory_display(comparison["memory_reduction"])
     _, agreement_label, _ = _agreement_state(comparison["agreement"])
@@ -615,11 +695,6 @@ def _render_story_sentence(result: dict[str, Any]) -> str:
             f"DotCache preserves output quality while running {speed_value} "
             f"and using {memory_value} KV memory."
         )
-    if _is_live_result(result):
-        return (
-            f"Live run estimate: DotCache is {speed_value}, reports {memory_value} resident KV, "
-            f"and shows {agreement_label.lower()} compared to the reference run."
-        )
     if comparison["agreement"] >= 0.85:
         return (
             f"DotCache is {speed_value} and uses "
@@ -641,6 +716,8 @@ def _render_delta_strip(result: dict[str, Any]) -> str:
                 f"Live run blocked at {comparison.get('blocked_context_length', 0)} tokens "
                 f"(current cap {comparison.get('max_live_context', 0)})"
             )
+        elif comparison.get("blocked_reason") == "benchmark_parity":
+            blocked_text = "Live run disabled until benchmark-parity selector artifacts and profiles are wired"
         elif comparison.get("blocked_reason") == "model_family_not_wired":
             blocked_text = "Live run blocked because this model family is not wired into the v1 runner yet"
         elif comparison.get("blocked_reason") == "auth_required":
@@ -821,6 +898,15 @@ def _chart_bar(label: str, dense_value: float, candidate_value: float, suffix: s
     )
 
 
+def _chart_note_block(label: str, body: str) -> str:
+    return (
+        "<div class='chart-block'>"
+        f"<div class='chart-label'>{html.escape(label)}</div>"
+        f"<div class='chart-empty'>{html.escape(body)}</div>"
+        "</div>"
+    )
+
+
 def _render_charts_tab(result: dict[str, Any]) -> str:
     baseline = result["baseline"]
     candidate = result["candidate"]
@@ -836,6 +922,33 @@ def _render_charts_tab(result: dict[str, Any]) -> str:
             "</div>"
             "</div>"
         )
+    if _is_paper_fixture(result):
+        return (
+            "<div class='charts-grid'>"
+            + _chart_bar("Tokens / sec", baseline["tok_per_sec"], candidate["tok_per_sec"], "", baseline_label, candidate_label)
+            + _chart_bar(
+                "Latency / token",
+                baseline["latency_ms_per_token"],
+                candidate["latency_ms_per_token"],
+                " ms",
+                baseline_label,
+                candidate_label,
+            )
+            + _chart_note_block(
+                "KV memory",
+                "Not reported in this paper row. The preset uses paper-backed speed and quality only.",
+            )
+            + _chart_bar(
+                "Agreement",
+                1.0,
+                comparison["agreement"],
+                "",
+                baseline_label,
+                candidate_label,
+            )
+            + "</div>"
+        )
+    memory_chart_label = "Resident KV" if _is_live_result(result) else "KV memory"
     return (
         "<div class='charts-grid'>"
         + _chart_bar("Tokens / sec", baseline["tok_per_sec"], candidate["tok_per_sec"], "", baseline_label, candidate_label)
@@ -848,7 +961,7 @@ def _render_charts_tab(result: dict[str, Any]) -> str:
             candidate_label,
         )
         + _chart_bar(
-            "KV memory",
+            memory_chart_label,
             baseline["kv_bytes"] / (1024.0 * 1024.0),
             candidate["kv_bytes"] / (1024.0 * 1024.0),
             " MiB",
@@ -1548,8 +1661,18 @@ def _render_arena_vote_status(state: dict[str, Any] | None, history: list[dict[s
 def _arena_reveal_sentence(result: dict[str, Any]) -> str:
     comparison = result["comparison"]
     speed_value, _ = _speed_display(comparison["speedup"])
-    memory_value, _ = _memory_display(comparison["memory_reduction"])
     agreement = float(comparison["agreement"])
+    if _is_live_result(result):
+        candidate_kv = _format_bytes(result["candidate"]["kv_bytes"])
+        baseline_kv = _format_bytes(result["baseline"]["kv_bytes"])
+        if agreement >= 0.95:
+            return f"DotCache matched the dense response closely while running {speed_value}, with {candidate_kv} resident KV vs {baseline_kv} dense."
+        if agreement >= 0.85:
+            return f"This live round traded some speed for lower resident KV, with only minor wording differences after reveal."
+        if agreement >= 0.70:
+            return f"This live round cut resident KV, but moderate output drift appeared in the revealed answer."
+        return f"This live round shows a tradeoff: lower resident KV on this Space runtime, but answer quality diverged noticeably."
+    memory_value, _ = _memory_display(comparison["memory_reduction"])
     if agreement >= 0.95:
         return f"DotCache matched the dense response closely while running {speed_value} and using {memory_value} KV memory."
     if agreement >= 0.85:
@@ -1563,6 +1686,8 @@ def _arena_compact_summary(result: dict[str, Any]) -> str:
     comparison = result["comparison"]
     _, agreement_label, _ = _agreement_state(float(comparison["agreement"]))
     speed_value, _ = _speed_display(comparison["speedup"])
+    if _is_live_result(result):
+        return f"{agreement_label} • {speed_value} • {_format_bytes(result['candidate']['kv_bytes'])} resident KV"
     memory_value, _ = _memory_display(comparison["memory_reduction"])
     return f"{agreement_label} • {speed_value} • {memory_value} KV"