Spaces:

switchboardllm
/

switchboard

Running

App Files Files Community

switchboardllm commited on 5 days ago

Commit

99c2f3d

verified ·

1 Parent(s): 4007720

Add evaluation page, plain-language how it works, FAQ and raw outputs

Browse files

Files changed (6) hide show

app.js +23 -0
data/outputs-v0.1.jsonl +0 -0
data/outputs-v0.jsonl +0 -0
evaluation.html +137 -0
index.html +194 -169
style.css +89 -0

app.js CHANGED Viewed

@@ -116,6 +116,28 @@ function renderResults(r) {
     "Small or mid counts as good enough when that answer scored 4 or more. Method and code are in the model card.";
 }
 /* ---------- Demo placeholder types real sample prompts ---------- */
 function startTyper(lines) {
@@ -344,6 +366,7 @@ async function init() {
         `${prettyModel(tiers[t].model)}, $${money(tiers[t].price_out)} per 1M output tokens`;
     }
     renderResults(results.value);
   } else {
     $("results-intro").textContent = "The evaluation results could not be loaded. See the model card on Hugging Face.";
     console.error(results.reason);

     "Small or mid counts as good enough when that answer scored 4 or more. Method and code are in the model card.";
 }
+/* ---------- Plain-language numbers: the two-bills example and FAQ figures ---------- */
+const BILL_ANSWERS = 1000;
+const BILL_TOKENS_EACH = 500;
+function renderPlainNumbers(r) {
+  const fills = { cost_saved: pct(r.cost_saved), under_routed: pct(r.under_routed) };
+  document.querySelectorAll("[data-fill]").forEach((el) => {
+    if (fills[el.dataset.fill]) el.textContent = fills[el.dataset.fill];
+  });
+  const millions = (BILL_ANSWERS * BILL_TOKENS_EACH) / 1e6;
+  const allFrontier = millions * r.tiers.frontier.price_out;
+  const routed = allFrontier * (1 - r.cost_saved);
+  const usd = (x) => `$${x.toFixed(2)}`;
+  $("bill-all").textContent = usd(allFrontier);
+  $("bill-routed").textContent = usd(routed);
+  $("bill-routed-bar").style.width = `${(1 - r.cost_saved) * 100}%`;
+  $("bill-note").textContent =
+    `Switchboard ${r.version} cut the bill by ${pct(r.cost_saved)} on ${r.n.toLocaleString()} test questions. ` +
+    `It also sent ${pct(r.under_routed)} of them to a cheaper AI than they needed.`;
+}
 /* ---------- Demo placeholder types real sample prompts ---------- */
 function startTyper(lines) {
         `${prettyModel(tiers[t].model)}, $${money(tiers[t].price_out)} per 1M output tokens`;
     }
     renderResults(results.value);
+    renderPlainNumbers(results.value);
   } else {
     $("results-intro").textContent = "The evaluation results could not be loaded. See the model card on Hugging Face.";
     console.error(results.reason);

data/outputs-v0.1.jsonl ADDED Viewed

The diff for this file is too large to render. See raw diff

data/outputs-v0.jsonl ADDED Viewed

The diff for this file is too large to render. See raw diff

evaluation.html ADDED Viewed

	@@ -0,0 +1,137 @@

+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Evaluation · Switchboard</title>
+  <meta name="description" content="How Switchboard was measured: routing quality on 10,000 held-out prompts, how each build compares to the full model, speed, and the raw outputs.">
+  <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 24 24%22%3E%3Ccircle cx=%225%22 cy=%2212%22 r=%223%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%225%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%2212%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%2219%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3C/svg%3E">
+  <link rel="preconnect" href="https://fonts.googleapis.com">
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+  <link href="https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,500;9..144,600;9..144,700&family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet">
+  <link rel="stylesheet" href="style.css?v=9">
+</head>
+<body>
+  <header class="top">
+    <div class="top-inner">
+      <a class="brand" href="index.html">
+        <svg class="brand-mark" viewBox="0 0 24 24" aria-hidden="true"><circle cx="5" cy="12" r="3"/><circle cx="19" cy="5" r="2.4"/><circle cx="19" cy="12" r="2.4"/><circle cx="19" cy="19" r="2.4"/><path d="M8 12c4 0 5-7 8.6-7"/></svg>
+        Switchboard
+      </a>
+      <nav class="nav">
+        <a href="index.html#demo">Demo</a>
+        <a href="index.html#how">How it works</a>
+        <a href="evaluation.html" aria-current="page">Evaluation</a>
+        <a href="index.html#faq">FAQ</a>
+        <a href="https://huggingface.co/switchboardllm/switchboard-router-0.5b">Model card</a>
+      </nav>
+      <a class="btn btn-primary btn-sm" href="index.html#demo">Try the router</a>
+    </div>
+  </header>
+  <main>
+    <section class="hero hero-sm">
+      <div class="wrap narrow center">
+        <p class="badge"><span class="dot" aria-hidden="true"></span>Updated October 12, 2026</p>
+        <h1>What we measured so far</h1>
+        <p class="lede">Every version of Switchboard is tested on the same 10,000 prompts it never saw during training. We publish the numbers that look bad too, and you can download every answer below.</p>
+      </div>
+    </section>
+    <section class="band white">
+      <div class="wrap">
+        <h2 class="center">Routing quality</h2>
+        <p class="section-lede center">10,000 held-out prompts in their natural mix: 6,931 easy, 1,716 medium and 1,353 that need a frontier model. Full-precision models, greedy decoding.</p>
+        <div class="metrics">
+<article class="metric"><header><h3>Cost saved</h3><span class="better">Higher is better</span></header><p class="plain">How much cheaper the bill is than sending every prompt to Claude Opus 5.5.</p><div class="ebars"><div class="ebar ours"><span class="ebar-label">Switchboard v0.1</span><span class="ebar-track"><span class="ebar-fill" style="width:64.5%"></span></span><span class="ebar-value">64.5%</span></div>
+<div class="ebar"><span class="ebar-label">Switchboard v0</span><span class="ebar-track"><span class="ebar-fill" style="width:67.8%"></span></span><span class="ebar-value">67.8%</span></div>
+<div class="ebar"><span class="ebar-label">Perfect router</span><span class="ebar-track"><span class="ebar-fill" style="width:76.2%"></span></span><span class="ebar-value">76.2%</span></div>
+<div class="ebar"><span class="ebar-label">Always small</span><span class="ebar-track"><span class="ebar-fill" style="width:97.5%"></span></span><span class="ebar-value">97.5%</span></div></div></article>
+<article class="metric"><header><h3>Prompts sent too low</h3><span class="better">Lower is better</span></header><p class="plain">Prompts that needed the frontier model but went to a cheaper one. This is the mistake you'd notice.</p><div class="ebars"><div class="ebar ours"><span class="ebar-label">Switchboard v0.1</span><span class="ebar-track"><span class="ebar-fill" style="width:27.6%"></span></span><span class="ebar-value">5.5%</span></div>
+<div class="ebar"><span class="ebar-label">Switchboard v0</span><span class="ebar-track"><span class="ebar-fill" style="width:35.5%"></span></span><span class="ebar-value">7.1%</span></div>
+<div class="ebar"><span class="ebar-label">Random router, same budget</span><span class="ebar-track"><span class="ebar-fill" style="width:49.9%"></span></span><span class="ebar-value">10.0%</span></div>
+<div class="ebar"><span class="ebar-label">Always small</span><span class="ebar-track"><span class="ebar-fill" style="width:67.7%"></span></span><span class="ebar-value">13.5%</span></div></div></article>
+<article class="metric"><header><h3>Hard prompts caught</h3><span class="better">Higher is better</span></header><p class="plain">Of the prompts that needed the frontier model, the share Switchboard sent there.</p><div class="ebars"><div class="ebar ours"><span class="ebar-label">Switchboard v0.1</span><span class="ebar-track"><span class="ebar-fill" style="width:59.1%"></span></span><span class="ebar-value">59.1%</span></div>
+<div class="ebar"><span class="ebar-label">Switchboard v0</span><span class="ebar-track"><span class="ebar-fill" style="width:47.5%"></span></span><span class="ebar-value">47.5%</span></div>
+<div class="ebar"><span class="ebar-label">Random router, same budget</span><span class="ebar-track"><span class="ebar-fill" style="width:26.2%"></span></span><span class="ebar-value">26.2%</span></div></div></article>
+<article class="metric"><header><h3>Ranking hard above easy (AUC)</h3><span class="better">Higher is better</span></header><p class="plain">How well the model orders prompts by difficulty, whatever the cut-off. 0.5 is a coin flip, 1.0 is perfect.</p><div class="ebars"><div class="ebar ours"><span class="ebar-label">Switchboard v0.1</span><span class="ebar-track"><span class="ebar-fill" style="width:78.7%"></span></span><span class="ebar-value">0.787</span></div>
+<div class="ebar"><span class="ebar-label">Switchboard v0</span><span class="ebar-track"><span class="ebar-fill" style="width:78.5%"></span></span><span class="ebar-value">0.785</span></div>
+<div class="ebar"><span class="ebar-label">Coin flip</span><span class="ebar-track"><span class="ebar-fill" style="width:50.0%"></span></span><span class="ebar-value">0.500</span></div></div></article>
+<article class="metric"><header><h3>Unreadable answers</h3><span class="better">Lower is better</span></header><p class="plain">Answers that weren't valid JSON, out of 10,000. Those would fall back to frontier.</p><div class="ebars"><div class="ebar ours"><span class="ebar-label">Switchboard v0.1</span><span class="ebar-track"><span class="ebar-fill" style="width:0.0%"></span></span><span class="ebar-value">0</span></div>
+<div class="ebar"><span class="ebar-label">Switchboard v0</span><span class="ebar-track"><span class="ebar-fill" style="width:0.0%"></span></span><span class="ebar-value">0</span></div></div></article>
+        </div>
+        <p class="fineprint center">The random router sends the same share of prompts to frontier as Switchboard v0.1, picked at random. A perfect router knows every answer in advance.</p>
+      </div>
+    </section>
+    <section class="band cream">
+      <div class="wrap narrow">
+        <h2 class="center">Does every build route the same way?</h2>
+        <p class="section-lede center">Smaller files make the model faster to download, but they can change its decisions. We checked each build against the full model on 347 test prompts: 300 picked at random from the 10,000 above, plus 47 borderline ones that cover all three tiers.</p>
+        <div class="table-wrap">
+          <table class="etable">
+            <thead><tr><th>Build</th><th class="num">Size</th><th class="num">Same decision</th><th></th></tr></thead>
+            <tbody>
+<tr><td class="build"><b>GGUF F16</b><small>Ollama 0.40.3 via ollama run hf.co/..., temperature 0 from the repo params</small><small class="note">347 in order, 346 when the prompts were sent in random order (Ollama reuses cached prompt prefixes)</small></td><td class="num">994 MB</td><td class="num"><b>347 of 347</b><small>100.0%</small></td><td><span class="tag-yes">Shipped</span></td></tr>
+<tr><td class="build"><b>Browser ONNX, 8-bit</b><small>Transformers.js 4.2.0, WebGPU, desktop Chrome</small><small class="note">299 of 300 random, 45 of 47 borderline; domain 347 of 347</small></td><td class="num">660 MB</td><td class="num"><b>344 of 347</b><small>99.1%</small></td><td><span class="tag-yes">Shipped</span></td></tr>
+<tr><td class="build"><b>GGUF Q8_0</b><small>Ollama 0.40.3, GPU</small><small class="note">338 of 347 on CPU</small></td><td class="num">531 MB</td><td class="num"><b>333 of 347</b><small>96.0%</small></td><td><span class="tag-no">Not shipped</span></td></tr>
+<tr><td class="build"><b>GGUF Q4_K_M</b><small>Ollama 0.40.3, GPU</small></td><td class="num">398 MB</td><td class="num"><b>325 of 347</b><small>93.7%</small></td><td><span class="tag-no">Not shipped</span></td></tr>
+<tr><td class="build"><b>GGUF Q6_K</b><small>Ollama 0.40.3, GPU</small></td><td class="num">506 MB</td><td class="num"><b>320 of 347</b><small>92.2%</small></td><td><span class="tag-no">Not shipped</span></td></tr>
+<tr><td class="build"><b>Browser ONNX, 4-bit layers</b><small>onnxruntime 1.24, CPU</small><small class="note">measured on the 300 random prompts only</small></td><td class="num">463 MB</td><td class="num"><b>197 of 300</b><small>65.7%</small></td><td><span class="tag-no">Not shipped</span></td></tr>
+            </tbody>
+          </table>
+        </div>
+        <p class="fineprint">For a router, a changed decision is a wrong answer, so we only ship builds that match the full model. 4-bit builds of a 0.5B model lose too much precision on the borderline prompts.</p>
+      </div>
+    </section>
+    <section class="band white">
+      <div class="wrap narrow">
+        <h2 class="center">Speed</h2>
+        <p class="section-lede center">Time to get one routing decision, measured on one desktop PC.</p>
+        <div class="table-wrap">
+          <table class="etable">
+            <thead><tr><th>Setup</th><th class="num">Median</th><th class="num">Slowest 10%</th></tr></thead>
+            <tbody><tr><td>GGUF F16 in Ollama, desktop CPU</td><td class="num">281 ms</td><td class="num">336 ms</td></tr><tr><td>Browser ONNX, WebGPU in desktop Chrome<small>after the one-time 660 MB download</small></td><td class="num">400 ms</td><td class="num">460 ms</td></tr></tbody>
+          </table>
+        </div>
+      </div>
+    </section>
+    <section class="band cream" id="raw">
+      <div class="wrap narrow">
+        <h2 class="center">Every answer, downloadable</h2>
+        <p class="section-lede center">One JSON line per prompt: the prompt, its label, the Mixtral score behind the label, Switchboard's raw output and decision, and whether the prompt was served well.</p>
+        <ul class="downloads">
+<li><a href="data/outputs-v0.1.jsonl" download>Raw outputs v0.1</a> <small>6.0 MB, 10,000 lines</small></li>
+<li><a href="data/outputs-v0.jsonl" download>Raw outputs v0</a> <small>6.0 MB, 10,000 lines</small></li>
+        </ul>
+      </div>
+    </section>
+    <section class="band white" id="method">
+      <div class="wrap narrow">
+        <h2 class="center">How we tested</h2>
+        <dl class="method">
+          <dt>Prompts</dt>
+          <dd>The validation split of <a href="https://huggingface.co/datasets/routellm/gpt4_dataset">routellm/gpt4_dataset</a> (Apache-2.0, from LMSYS): 10,000 real prompts from chat logs, instruction sets and coding questions. None were used in training.</dd>
+          <dt>What "needs frontier" means</dt>
+          <dd>Every prompt comes with a 2023 Mixtral-8x7B answer that a GPT-4 judge scored from 1 to 5. A score of 5 means a small model is enough, 4 means mid, and 1 to 3 means frontier. A 2026 small model is stronger than Mixtral-8x7B, so this label is cautious: some prompts marked frontier would be fine on a smaller model.</dd>
+          <dt>Served well</dt>
+          <dd>A prompt counts as served well if Switchboard sent it to frontier, or sent it lower and the Mixtral answer scored 4 or 5.</dd>
+          <dt>Cost</dt>
+          <dd>Output price per 1M tokens on Surplus: Claude Haiku 5.5 ($0.50) for small, Claude Sonnet 5.5 ($10) for mid, Claude Opus 5.5 ($20) for frontier. Savings compare the routed bill with sending every prompt to Opus.</dd>
+          <dt>Settings</dt>
+          <dd>Greedy decoding (temperature 0), at most 24 new tokens, prompts cut to 512 tokens, the default chat template with no extra instructions.</dd>
+          <dt>Builds</dt>
+          <dd>The full model in fp32 is the reference. Each build answered the same prompts and we counted how often its tier matched. GGUF builds ran in Ollama 0.40.3, the browser build in Transformers.js 4.2.0 with WebGPU in desktop Chrome.</dd>
+          <dt>Where it ran</dt>
+          <dd>One desktop PC with an RTX 5070 Ti, for training, evaluation and the speed numbers.</dd>
+          <dt>Limits</dt>
+          <dd>The labels come from models and a judge, not from people. Most prompts are English. Each number is one run, so differences of a few tenths of a point don't mean much. On prompts that sit right between two tiers, tiny numeric differences can flip the decision.</dd>
+        </dl>
+      </div>
+    </section>
+  </main>
+</body>
+</html>

index.html CHANGED Viewed

@@ -1,169 +1,194 @@
-<!doctype html>
-<html lang="en">
-<head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <title>Switchboard</title>
-  <meta name="description" content="A 0.5B open-weights router that picks the cheapest model that can handle each prompt. Try it in your browser.">
-  <meta property="og:title" content="Switchboard">
-  <meta property="og:description" content="Pick the cheapest model that can handle the prompt.">
-  <link rel="preconnect" href="https://fonts.googleapis.com">
-  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-  <link href="https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,500;9..144,600;9..144,700&family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet">
-  <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 24 24%22%3E%3Ccircle cx=%225%22 cy=%2212%22 r=%223%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%225%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%2212%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%2219%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3C/svg%3E">
-  <link rel="stylesheet" href="style.css?v=6">
-</head>
-<body>
-  <header class="top">
-    <div class="top-inner">
-      <a class="brand" href="#">
-        <svg class="brand-mark" viewBox="0 0 24 24" aria-hidden="true"><circle cx="5" cy="12" r="3"/><circle cx="19" cy="5" r="2.4"/><circle cx="19" cy="12" r="2.4"/><circle cx="19" cy="19" r="2.4"/><path d="M8 12c4 0 5-7 8.6-7"/></svg>
-        Switchboard
-      </a>
-      <nav class="nav">
-        <a href="#demo">Demo</a>
-        <a href="#how">How it works</a>
-        <a href="#results">Results</a>
-        <a href="https://huggingface.co/switchboardllm/switchboard-router-0.5b">Model card</a>
-      </nav>
-      <a class="btn btn-primary btn-sm" href="#demo">Try the router</a>
-    </div>
-  </header>
-  <main>
-    <section class="hero">
-      <div class="wrap center">
-        <p class="badge"><span class="dot" aria-hidden="true"></span>Open weights. Runs in your browser.</p>
-        <h1>Pick the cheapest model<br><span class="accent">that can handle the prompt</span>.</h1>
-        <p class="lede">Most prompts don't need a frontier model. Switchboard is a 0.5B router that reads each prompt and sends it to a small, mid or frontier model, so you only pay frontier prices when the prompt needs it.</p>
-        <div class="cta-row">
-          <a class="btn btn-primary" href="#demo">Route a prompt</a>
-          <a class="btn btn-green" href="https://huggingface.co/switchboardllm/switchboard-router-0.5b">Get the weights <span class="tag">Apache 2.0</span></a>
-        </div>
-        <p class="spec">0.5B parameters · one line of JSON out · about 15 tokens per decision</p>
-      </div>
-      <div class="wrap">
-        <figure class="board-card">
-          <canvas id="board" class="board-canvas" aria-label="Animation: prompts flowing into the router and out to the small, mid or frontier line"></canvas>
-          <figcaption class="board-note">Real prompts from the held-out set and the tier Switchboard v0.1 actually picked. We show an equal number per tier, so the mix isn't to scale.</figcaption>
-        </figure>
-      </div>
-    </section>
-    <section class="tiers-strip">
-      <div class="wrap">
-        <p class="strip-title">Three tiers, one decision per prompt</p>
-        <ul class="tier-cards" id="tier-cards">
-          <li class="tier-card" data-tier="small"><span class="lamp" aria-hidden="true"></span><b>Small</b><span class="tier-model" data-model="small"></span></li>
-          <li class="tier-card" data-tier="mid"><span class="lamp" aria-hidden="true"></span><b>Mid</b><span class="tier-model" data-model="mid"></span></li>
-          <li class="tier-card" data-tier="frontier"><span class="lamp" aria-hidden="true"></span><b>Frontier</b><span class="tier-model" data-model="frontier"></span></li>
-        </ul>
-      </div>
-    </section>
-    <section class="band white" id="demo">
-      <div class="wrap">
-        <h2 class="center">Try it on your own prompt</h2>
-        <p class="section-lede center">The real model runs in this tab. Nothing you type leaves your device.</p>
-        <div class="demo">
-          <label for="prompt" class="field-label">Prompt</label>
-          <textarea id="prompt" rows="4" maxlength="2000" placeholder="Paste any prompt you'd send to an LLM"></textarea>
-          <div class="samples" role="group" aria-label="Sample prompts">
-            <button type="button" class="chip" data-prompt="What is the capital of France?">Capital of France</button>
-            <button type="button" class="chip" data-prompt="Write a haiku about rain">Haiku about rain</button>
-            <button type="button" class="chip" data-prompt="Design a rate limiter for 10M requests per second across 5 regions, with failure modes and consistency trade-offs">Global rate limiter</button>
-            <button type="button" class="chip" data-prompt="Unscramble the following words: eglar, ngareo, kobo">Unscramble words</button>
-          </div>
-          <div class="demo-actions">
-            <button type="button" id="route" class="btn btn-primary">Route prompt</button>
-            <p id="status" class="status" role="status" aria-live="polite">The first route downloads the router once (660 MB). After that it runs offline in this tab.</p>
-          </div>
-          <div class="meter" id="meter" hidden><span id="meter-fill"></span></div>
-          <div class="readout" id="readout" hidden>
-            <div class="readout-tiers" aria-hidden="true">
-              <span data-pick="small">Small</span><span data-pick="mid">Mid</span><span data-pick="frontier">Frontier</span>
-            </div>
-            <pre id="json"></pre>
-            <p id="verdict"></p>
-          </div>
-        </div>
-      </div>
-    </section>
-    <section class="band cream">
-      <div class="wrap">
-        <h2 class="center">Why route at all</h2>
-        <div class="grid3">
-          <div>
-            <h3>Most prompts are easy</h3>
-            <p>Greetings, lookups, short rewrites and simple code make up most traffic. A small model answers them just as well, at a few percent of the price.</p>
-          </div>
-          <div>
-            <h3>The router costs almost nothing</h3>
-            <p>Switchboard is 0.5B parameters and writes about 15 tokens per decision. On Surplus that's roughly $0.000005 per prompt.</p>
-          </div>
-          <div>
-            <h3>A bad call fails safe</h3>
-            <p>If the output can't be parsed, send the prompt to frontier. A broken routing call costs money, never quality.</p>
-          </div>
-        </div>
-      </div>
-    </section>
-    <section class="band white" id="how">
-      <div class="wrap">
-        <h2 class="center">Three steps</h2>
-        <ol class="steps">
-          <li><span class="step-n">1</span><h3>Send the prompt to Switchboard</h3><p>One short call before the real one.</p></li>
-          <li><span class="step-n">2</span><h3>Read one line of JSON</h3><p><code>{"tier":"small","domain":"code"}</code></p></li>
-          <li><span class="step-n">3</span><h3>Call the model for that tier</h3><p>Easy prompts go small. Hard prompts still get the frontier model.</p></li>
-        </ol>
-      </div>
-    </section>
-    <section class="band cream" id="results">
-      <div class="wrap">
-        <h2 class="center">Results</h2>
-        <p class="section-lede center" id="results-intro">Loading the latest evaluation.</p>
-        <div class="stats" id="figures"></div>
-        <p class="fineprint center" id="fineprint"></p>
-      </div>
-    </section>
-    <section class="band white">
-      <div class="wrap narrow">
-        <h2 class="center">Run it yourself</h2>
-        <p class="section-lede center">Open weights under Apache 2.0. It runs anywhere Transformers runs.</p>
-        <pre class="code"><code>from transformers import AutoModelForCausalLM, AutoTokenizer
-import json
-repo = "switchboardllm/switchboard-router-0.5b"
-tok = AutoTokenizer.from_pretrained(repo)
-model = AutoModelForCausalLM.from_pretrained(repo)
-msgs = [{"role": "user", "content": "Write a haiku about rain"}]
-ids = tok.apply_chat_template(msgs, add_generation_prompt=True, return_tensors="pt")
-out = model.generate(ids, max_new_tokens=24, do_sample=False)
-print(json.loads(tok.decode(out[0, ids.shape[1]:], skip_special_tokens=True)))</code></pre>
-      </div>
-    </section>
-    <section class="closing">
-      <canvas id="closing-board" class="closing-canvas" aria-hidden="true"></canvas>
-      <div class="wrap center closing-inner">
-        <h2>Coming to the Surplus API</h2>
-        <p class="section-lede">After launch, call Switchboard on the same Surplus key as every other model, at $0.03 per million tokens.</p>
-        <div class="cta-row">
-          <a class="btn btn-primary" href="#demo">Try the router</a>
-          <a class="btn btn-ghost" href="https://huggingface.co/switchboardllm/switchboard-router-0.5b">Read the model card</a>
-        </div>
-      </div>
-    </section>
-  </main>
-  <script type="module" src="app.js?v=12"></script>
-</body>
-</html>

+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Switchboard</title>
+  <meta name="description" content="A 0.5B open-weights router that picks the cheapest model that can handle each prompt. Try it in your browser.">
+  <meta property="og:title" content="Switchboard">
+  <meta property="og:description" content="Pick the cheapest model that can handle the prompt.">
+  <link rel="preconnect" href="https://fonts.googleapis.com">
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+  <link href="https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,500;9..144,600;9..144,700&family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet">
+  <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 24 24%22%3E%3Ccircle cx=%225%22 cy=%2212%22 r=%223%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%225%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%2212%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3Ccircle cx=%2219%22 cy=%2219%22 r=%222.4%22 fill=%22%23ea580c%22/%3E%3C/svg%3E">
+  <link rel="stylesheet" href="style.css?v=9">
+</head>
+<body>
+  <header class="top">
+    <div class="top-inner">
+      <a class="brand" href="#">
+        <svg class="brand-mark" viewBox="0 0 24 24" aria-hidden="true"><circle cx="5" cy="12" r="3"/><circle cx="19" cy="5" r="2.4"/><circle cx="19" cy="12" r="2.4"/><circle cx="19" cy="19" r="2.4"/><path d="M8 12c4 0 5-7 8.6-7"/></svg>
+        Switchboard
+      </a>
+      <nav class="nav">
+        <a href="#demo">Demo</a>
+        <a href="#how">How it works</a>
+        <a href="evaluation.html">Evaluation</a>
+        <a href="#faq">FAQ</a>
+        <a href="https://huggingface.co/switchboardllm/switchboard-router-0.5b">Model card</a>
+      </nav>
+      <a class="btn btn-primary btn-sm" href="#demo">Try the router</a>
+    </div>
+  </header>
+  <main>
+    <section class="hero">
+      <div class="wrap center">
+        <p class="badge"><span class="dot" aria-hidden="true"></span>Open weights. Runs in your browser.</p>
+        <h1>Pick the cheapest model<br><span class="accent">that can handle the prompt</span>.</h1>
+        <p class="lede">Most prompts don't need a frontier model. Switchboard is a 0.5B router that reads each prompt and sends it to a small, mid or frontier model, so you only pay frontier prices when the prompt needs it.</p>
+        <div class="cta-row">
+          <a class="btn btn-primary" href="#demo">Route a prompt</a>
+          <a class="btn btn-green" href="https://huggingface.co/switchboardllm/switchboard-router-0.5b">Get the weights <span class="tag">Apache 2.0</span></a>
+        </div>
+        <p class="spec">0.5B parameters · one line of JSON out · about 15 tokens per decision</p>
+      </div>
+      <div class="wrap">
+        <figure class="board-card">
+          <canvas id="board" class="board-canvas" aria-label="Animation: prompts flowing into the router and out to the small, mid or frontier line"></canvas>
+          <figcaption class="board-note">Real prompts from the held-out set and the tier Switchboard v0.1 actually picked. We show an equal number per tier, so the mix isn't to scale.</figcaption>
+        </figure>
+      </div>
+    </section>
+    <section class="tiers-strip">
+      <div class="wrap">
+        <p class="strip-title">Three tiers, one decision per prompt</p>
+        <ul class="tier-cards" id="tier-cards">
+          <li class="tier-card" data-tier="small"><span class="lamp" aria-hidden="true"></span><b>Small</b><span class="tier-model" data-model="small"></span></li>
+          <li class="tier-card" data-tier="mid"><span class="lamp" aria-hidden="true"></span><b>Mid</b><span class="tier-model" data-model="mid"></span></li>
+          <li class="tier-card" data-tier="frontier"><span class="lamp" aria-hidden="true"></span><b>Frontier</b><span class="tier-model" data-model="frontier"></span></li>
+        </ul>
+      </div>
+    </section>
+    <section class="band white" id="demo">
+      <div class="wrap">
+        <h2 class="center">Try it on your own prompt</h2>
+        <p class="section-lede center">The real model runs in this tab. Nothing you type leaves your device.</p>
+        <div class="demo">
+          <label for="prompt" class="field-label">Prompt</label>
+          <textarea id="prompt" rows="4" maxlength="2000" placeholder="Paste any prompt you'd send to an LLM"></textarea>
+          <div class="samples" role="group" aria-label="Sample prompts">
+            <button type="button" class="chip" data-prompt="What is the capital of France?">Capital of France</button>
+            <button type="button" class="chip" data-prompt="Write a haiku about rain">Haiku about rain</button>
+            <button type="button" class="chip" data-prompt="Design a rate limiter for 10M requests per second across 5 regions, with failure modes and consistency trade-offs">Global rate limiter</button>
+            <button type="button" class="chip" data-prompt="Unscramble the following words: eglar, ngareo, kobo">Unscramble words</button>
+          </div>
+          <div class="demo-actions">
+            <button type="button" id="route" class="btn btn-primary">Route prompt</button>
+            <p id="status" class="status" role="status" aria-live="polite">The first route downloads the router once (660 MB). After that it runs offline in this tab.</p>
+          </div>
+          <div class="meter" id="meter" hidden><span id="meter-fill"></span></div>
+          <div class="readout" id="readout" hidden>
+            <div class="readout-tiers" aria-hidden="true">
+              <span data-pick="small">Small</span><span data-pick="mid">Mid</span><span data-pick="frontier">Frontier</span>
+            </div>
+            <pre id="json"></pre>
+            <p id="verdict"></p>
+          </div>
+        </div>
+      </div>
+    </section>
+    <section class="band cream" id="how">
+      <div class="wrap">
+        <h2 class="center">How it works</h2>
+        <p class="analogy">Think of an old telephone switchboard. An operator listened to each caller and plugged the line into the right place. Switchboard does the same with your questions to AI: it reads each one and connects it to an AI that's good enough for it, without paying for the most expensive one every time.</p>
+        <div class="explainer">
+          <div class="story">
+            <div class="story-step"><span class="step-n">1</span><div><h3>You ask something</h3><p>A question, a request to write or fix something, anything you'd type into a chatbot.</p></div></div>
+            <div class="story-step"><span class="step-n">2</span><div><h3>Switchboard reads it first</h3><p>A small AI decides how hard the question is. "What's the capital of France?" is easy. A tricky math proof is hard. This takes less than half a second and costs almost nothing.</p></div></div>
+            <div class="story-step"><span class="step-n">3</span><div><h3>It goes to the right AI</h3><p>Easy questions go to a small, cheap AI. Medium ones go to a mid-size AI. Only the hard ones go to the most powerful, most expensive AI.</p></div></div>
+            <div class="story-step"><span class="step-n">4</span><div><h3>You pay less for the same answers</h3><p>Most everyday questions are easy, so most of them stop being billed at the top price. When Switchboard can't read its own answer, it plays it safe and picks the powerful AI.</p></div></div>
+          </div>
+          <aside class="bill" aria-labelledby="bill-title">
+            <h3 id="bill-title">Same 1,000 questions, two bills</h3>
+            <p>Each answer about 500 tokens (roughly 375 words), priced on Surplus.</p>
+            <div class="bill-row"><span>Every question to Claude Opus 5.5</span><b id="bill-all">…</b><span class="bill-bar"><i style="width:100%"></i></span></div>
+            <div class="bill-row ours"><span>With Switchboard picking the AI</span><b id="bill-routed">…</b><span class="bill-bar"><i id="bill-routed-bar" style="width:0%"></i></span></div>
+            <p class="fineprint" id="bill-note">Based on Switchboard's results on 10,000 test questions.</p>
+          </aside>
+        </div>
+      </div>
+    </section>
+    <section class="band white" id="results">
+      <div class="wrap">
+        <h2 class="center">Results</h2>
+        <p class="section-lede center" id="results-intro">Loading the latest evaluation.</p>
+        <div class="stats" id="figures"></div>
+        <p class="fineprint center" id="fineprint"></p>
+        <div class="cta-row"><a class="btn btn-ghost" href="evaluation.html">See the full evaluation</a></div>
+      </div>
+    </section>
+    <section class="band cream" id="start">
+      <div class="wrap narrow">
+        <h2 class="center">Up and running in a few minutes</h2>
+        <p class="section-lede center">Run it on your own computer. No GPU, account or API key needed.</p>
+        <ol class="steps">
+          <li><span class="step-n">1</span><h3>Get Ollama</h3><p>Download it free from <a href="https://ollama.com">ollama.com</a> and open it once.</p></li>
+          <li><span class="step-n">2</span><h3>Ask Switchboard</h3><p>Paste the line below into a terminal. The first run downloads the model once (994 MB).</p></li>
+          <li><span class="step-n">3</span><h3>Read the answer</h3><p>The tier tells you which size of AI to use. Here, a small one is enough.</p></li>
+        </ol>
+        <pre class="code run-code"><code><span class="prompt">$</span> ollama run hf.co/switchboardllm/switchboard-router-0.5b-GGUF "Write a haiku about rain"
+<span class="answer">{"tier":"small","domain":"writing"}</span></code></pre>
+        <h3 class="facts-title">What's inside Switchboard</h3>
+        <div class="table-wrap">
+          <table class="facts">
+            <tr><th>Size</th><td>0.5 billion parameters. 994 MB to run on your computer, 660 MB in the browser.</td></tr>
+            <tr><th>Built on</th><td>Qwen2.5-0.5B-Instruct (Apache 2.0), fully fine-tuned</td></tr>
+            <tr><th>Trained on</th><td>63,749 real prompts from routellm/gpt4_dataset (Apache 2.0)</td></tr>
+            <tr><th>Training time</th><td>55 minutes on one RTX 5070 Ti</td></tr>
+            <tr><th>Answers with</th><td>One line of JSON: a tier (small, mid or frontier) and a topic (code, math, writing or general)</td></tr>
+            <tr><th>Runs on</th><td>Ollama, llama.cpp, Transformers, and your browser</td></tr>
+            <tr><th>License</th><td>Apache 2.0, free for commercial use</td></tr>
+          </table>
+        </div>
+      </div>
+    </section>
+    <section class="band white" id="faq">
+      <div class="wrap">
+        <h2 class="center">Questions &amp; answers</h2>
+        <div class="faq-groups">
+          <div class="faq-group">
+            <h3>Switchboard</h3>
+            <details class="faq-item"><summary><span class="faq-n">01</span>What is Switchboard?</summary><p>A small, open AI model that reads a prompt and decides how capable an AI needs to be to answer it: small, mid or frontier. You then send the prompt to an AI of that size.</p></details>
+            <details class="faq-item"><summary><span class="faq-n">02</span>Why not always use the best AI?</summary><p>Price. On Surplus, an answer from Claude Opus 5.5 costs 40 times more than the same answer from Claude Haiku 5.5. Most everyday prompts get an equally good answer from the smaller model.</p></details>
+            <details class="faq-item"><summary><span class="faq-n">03</span>Which base model is it built on?</summary><p>Qwen2.5-0.5B-Instruct from Alibaba, fully fine-tuned on routing examples. It's small on purpose: a router sits in front of every request, so it has to be fast and almost free.</p></details>
+            <details class="faq-item"><summary><span class="faq-n">04</span>Who is behind it?</summary><p>An independent project. It is not affiliated with Anthropic, OpenAI, Alibaba, Robinhood, Surplus or the Switchboard oracle network. Model names appear only to describe where prompts can be sent.</p></details>
+          </div>
+          <div class="faq-group">
+            <h3>Model &amp; use</h3>
+            <details class="faq-item"><summary><span class="faq-n">05</span>Does it run on my computer?</summary><p>Yes. The 994 MB build runs with Ollama or llama.cpp on Windows, macOS and Linux, on a normal CPU. The demo on this page runs in your browser.</p></details>
+            <details class="faq-item"><summary><span class="faq-n">06</span>Does my prompt leave my device?</summary><p>Not in the demo on this page, and not when you run it with Ollama. If you call Switchboard through a hosted API, the prompt goes to that server like any other API call.</p></details>
+            <details class="faq-item"><summary><span class="faq-n">07</span>How good is it?</summary><p>On 10,000 test prompts it cut the bill by <span data-fill="cost_saved">…</span> compared with always using Claude Opus 5.5, and sent <span data-fill="under_routed">…</span> of prompts to a cheaper tier than they needed. All the numbers, including the weak spots, are on the <a href="evaluation.html">evaluation page</a>.</p></details>
+            <details class="faq-item"><summary><span class="faq-n">08</span>What if it picks the wrong tier?</summary><p>Too high costs a bit more. Too low can mean a weaker answer, which is the mistake we work hardest to reduce. If its answer can't be read, it falls back to the frontier tier, so a broken routing call never sends you to a weaker AI.</p></details>
+            <details class="faq-item"><summary><span class="faq-n">09</span>Is it free?</summary><p>Yes. The model is Apache 2.0, and running it yourself costs nothing. After the Surplus launch you can also call it on the Surplus API at $0.03 per million tokens, about $0.000005 per routing decision.</p></details>
+            <details class="faq-item"><summary><span class="faq-n">10</span>What data was it trained on?</summary><p>Real prompts from <a href="https://huggingface.co/datasets/routellm/gpt4_dataset">routellm/gpt4_dataset</a> (Apache 2.0, from LMSYS). Each prompt's label comes from how well a 2023 open model answered it, as scored by a GPT-4 judge. The model card has the details.</p></details>
+          </div>
+        </div>
+      </div>
+    </section>
+    <section class="closing">
+      <canvas id="closing-board" class="closing-canvas" aria-hidden="true"></canvas>
+      <div class="wrap center closing-inner">
+        <h2>Coming to the Surplus API</h2>
+        <p class="section-lede">After launch, call Switchboard on the same Surplus key as every other model, at $0.03 per million tokens.</p>
+        <div class="cta-row">
+          <a class="btn btn-primary" href="#demo">Try the router</a>
+          <a class="btn btn-ghost" href="https://huggingface.co/switchboardllm/switchboard-router-0.5b">Read the model card</a>
+        </div>
+      </div>
+    </section>
+  </main>
+  <script type="module" src="app.js?v=13"></script>
+</body>
+</html>

style.css CHANGED Viewed

@@ -266,3 +266,92 @@ code { font-family: var(--mono); font-size: 0.86em; }
   .word { animation: none; opacity: 1; transform: none; }
   * { transition: none !important; }
 }

   .word { animation: none; opacity: 1; transform: none; }
   * { transition: none !important; }
 }
+/* ---------- Evaluation page ---------- */
+.nav a[aria-current="page"] { color: var(--ink-strong); font-weight: 600; }
+.hero-sm { padding: 64px 0 52px; }
+.hero-sm h1 { max-width: 20ch; }
+.metrics { display: grid; grid-template-columns: repeat(2, 1fr); gap: 16px; }
+.metric { padding: 22px; border-radius: 16px; background: #fffdf7; border: 1px solid var(--line); }
+.metric header { display: flex; justify-content: space-between; align-items: baseline; gap: 12px; }
+.metric h3 { margin: 0; }
+.better { font-size: 12.5px; color: var(--muted); white-space: nowrap; }
+.plain { margin: 6px 0 16px; font-size: 14.5px; color: var(--body); }
+.ebars { display: grid; gap: 10px; }
+.ebar { display: grid; grid-template-columns: 170px 1fr 64px; align-items: center; gap: 12px; font-size: 14px; }
+.ebar-label { color: var(--body); }
+.ebar-track { height: 12px; border-radius: 999px; background: #f1ebdc; overflow: hidden; }
+.ebar-fill { display: block; height: 100%; border-radius: 999px; background: #cfc5ad; }
+.ebar.ours .ebar-fill { background: linear-gradient(90deg, var(--amber), var(--orange)); }
+.ebar.ours .ebar-label, .ebar.ours .ebar-value { color: var(--ink-strong); font-weight: 600; }
+.ebar-value { text-align: right; font-variant-numeric: tabular-nums; }
+.table-wrap { overflow-x: auto; border-radius: 16px; border: 1px solid var(--line); background: var(--white); }
+.etable { width: 100%; border-collapse: collapse; font-size: 14.5px; }
+.etable th { text-align: left; font-weight: 600; font-size: 13px; color: var(--muted); padding: 12px 16px; border-bottom: 1px solid var(--line); }
+.etable td { padding: 14px 16px; border-bottom: 1px solid var(--line); vertical-align: top; }
+.etable tr:last-child td { border-bottom: 0; }
+.etable td small { display: block; margin-top: 3px; color: var(--muted); font-size: 12.5px; }
+.etable .num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
+.tag-yes, .tag-no { display: inline-block; padding: 3px 10px; border-radius: 999px; font-size: 12px; font-weight: 600; white-space: nowrap; }
+.tag-yes { background: #e6f2ea; color: var(--green-deep); }
+.tag-no { background: #f3eee2; color: var(--muted); }
+.downloads { list-style: none; margin: 0 auto; padding: 0; max-width: 520px; display: grid; gap: 10px; }
+.downloads li { display: flex; justify-content: space-between; align-items: baseline; gap: 12px; padding: 14px 18px; border-radius: 14px; background: var(--white); border: 1px solid var(--line); }
+.downloads a { font-weight: 600; color: var(--orange-deep); }
+.downloads small { color: var(--muted); }
+.method { margin: 0; display: grid; grid-template-columns: 190px 1fr; gap: 0 28px; }
+.method dt { padding: 16px 0; border-top: 1px solid var(--line); font-family: var(--serif); font-weight: 600; font-size: 17px; }
+.method dd { margin: 0; padding: 16px 0; border-top: 1px solid var(--line); color: var(--body); font-size: 15px; line-height: 1.6; }
+.method a { color: var(--orange-deep); }
+/* ---------- Plain-language explainer ---------- */
+.explainer { display: grid; grid-template-columns: 1.1fr 0.9fr; gap: 40px; align-items: start; margin-top: 36px; }
+.story { display: grid; gap: 16px; }
+.story-step { display: grid; grid-template-columns: 40px 1fr; gap: 14px; }
+.story-step .step-n { margin: 0; }
+.story-step h3 { margin: 4px 0 4px; }
+.story-step p { margin: 0; color: var(--body); line-height: 1.6; }
+.bill { padding: 24px; border-radius: 18px; background: #fffdf8; border: 1px solid var(--line); box-shadow: 0 24px 50px -38px #3b2f1666; }
+.bill h3 { margin: 0 0 4px; }
+.bill > p { margin: 0 0 18px; font-size: 14px; color: var(--muted); }
+.bill-row { display: grid; grid-template-columns: 1fr auto; gap: 8px 12px; align-items: baseline; padding: 14px 0; border-top: 1px solid var(--line); }
+.bill-row span { color: var(--body); }
+.bill-row b { font-family: var(--serif); font-size: 30px; font-weight: 600; color: var(--ink-strong); font-variant-numeric: tabular-nums; }
+.bill-row.ours b { color: var(--orange-deep); }
+.bill-bar { grid-column: 1 / -1; height: 10px; border-radius: 999px; background: #f1ebdc; overflow: hidden; }
+.bill-bar i { display: block; height: 100%; background: #cfc5ad; border-radius: 999px; }
+.bill-row.ours .bill-bar i { background: linear-gradient(90deg, var(--amber), var(--orange)); }
+.bill .fineprint { margin: 14px 0 0; text-align: left; }
+.analogy { max-width: 64ch; margin: 0 auto; font-size: 17px; color: var(--body); line-height: 1.65; text-align: center; }
+/* ---------- Facts and FAQ ---------- */
+.facts { width: 100%; border-collapse: collapse; font-size: 15px; }
+.facts th { text-align: left; width: 34%; padding: 13px 18px; font-weight: 500; color: var(--muted); border-bottom: 1px solid var(--line); vertical-align: top; }
+.facts td { padding: 13px 18px; border-bottom: 1px solid var(--line); }
+.facts tr:last-child th, .facts tr:last-child td { border-bottom: 0; }
+.faq-groups { display: grid; grid-template-columns: 1fr 1fr; gap: 36px; margin-top: 32px; }
+.faq-group h3 { margin: 0 0 10px; font-size: 15px; font-family: var(--sans); font-weight: 600; color: var(--orange-deep); }
+.faq-item { border-top: 1px solid var(--line); }
+.faq-item:last-child { border-bottom: 1px solid var(--line); }
+.faq-item summary { display: grid; grid-template-columns: 34px 1fr 20px; gap: 6px; align-items: baseline; padding: 16px 0; cursor: pointer; list-style: none; font-weight: 600; color: var(--ink); }
+.faq-item summary::-webkit-details-marker { display: none; }
+.faq-item summary::after { content: "+"; justify-self: end; color: var(--muted); font-weight: 400; font-size: 20px; line-height: 1; }
+.faq-item[open] summary::after { content: "–"; }
+.faq-n { font: 500 13px var(--mono); color: var(--muted); }
+.faq-item p { margin: 0 0 16px 40px; color: var(--body); line-height: 1.6; font-size: 15px; }
+.faq-item a { color: var(--orange-deep); }
+@media (max-width: 860px) {
+  .metrics, .explainer, .faq-groups { grid-template-columns: 1fr; }
+  .ebar { grid-template-columns: 120px 1fr 56px; }
+  .method { grid-template-columns: 1fr; }
+  .method dd { border-top: 0; padding-top: 0; }
+}
+.facts-title { margin: 44px 0 14px; text-align: center; }
+.run-code { margin-top: 18px; }
+.run-code .prompt { color: #8fa39a; }
+.run-code .answer { color: #fbbf24; }
+.etable td.build { width: 56%; }
+.etable td.build small.note { color: var(--body); margin-top: 6px; }