Spaces:
Running
Running
Ergebnisse aktualisiert (7 Läufe)
Browse files- README.md +3 -3
- index.html +25 -25
- methodik.html +0 -124
- methodology.html +124 -0
- runs/15.html +37 -37
- runs/16.html +37 -37
- runs/17.html +37 -37
- runs/19.html +37 -37
- runs/20.html +37 -37
- runs/21.html +37 -37
- runs/22.html +37 -37
README.md
CHANGED
|
@@ -10,6 +10,6 @@ pinned: false
|
|
| 10 |
|
| 11 |
# Local LLM Benchmarks
|
| 12 |
|
| 13 |
-
|
| 14 |
-
`bench publish`.
|
| 15 |
-
|
|
|
|
| 10 |
|
| 11 |
# Local LLM Benchmarks
|
| 12 |
|
| 13 |
+
Results for local language models (knowledge, math, code, German) as a static page, built with
|
| 14 |
+
`bench publish`. Only the latest suite version is shown; scores from different suite versions are not
|
| 15 |
+
comparable.
|
index.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html lang="
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
@@ -10,27 +10,27 @@
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
-
<a href="index.html" class="active">
|
| 14 |
-
<a href="
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
-
<h1>
|
| 19 |
-
<p class="lead">
|
| 20 |
<section id="v3-quick">
|
| 21 |
-
<h2>Suite v2b ·
|
| 22 |
<div class="table-wrap">
|
| 23 |
<table class="sortable">
|
| 24 |
<thead><tr>
|
| 25 |
-
<th data-sort="text">
|
| 26 |
-
<th data-sort="num" class="num">
|
| 27 |
-
<th data-sort="num" class="num">
|
| 28 |
</tr></thead>
|
| 29 |
<tbody>
|
| 30 |
<tr>
|
| 31 |
<td data-value="qwen3.8-flash-next-ple4">
|
| 32 |
<a href="runs/15.html" class="strong">qwen3.8-flash-next-ple4</a>
|
| 33 |
-
<div class="sub">NVFP4 · Reasoning off ·
|
| 34 |
</td>
|
| 35 |
<td class="num strong" data-value="71.45393635604162">71.5</td>
|
| 36 |
<td class="num strong" data-value="77.86868686868686" title="mmlu_pro: 86.3 ± 2.0 (n=300) gpqa_diamond_gen: 77.3 ± 3.0 (n=198) musr_gen: 70.0 ± 2.9 (n=250) ">77.9</td>
|
|
@@ -41,12 +41,12 @@
|
|
| 41 |
<td class="num" data-value="26.376464212349354">26</td>
|
| 42 |
<td class="num" data-value="2041.4640201348918">2041</td>
|
| 43 |
<td class="num" data-value="1973.583644023165">1974</td>
|
| 44 |
-
<td data-value="2026-09-17">2026-09-17<
|
| 45 |
</tr>
|
| 46 |
<tr>
|
| 47 |
<td data-value="gemma-4-12b-it">
|
| 48 |
<a href="runs/21.html" class="strong">gemma-4-12b-it</a>
|
| 49 |
-
<div class="sub">Q8 · Reasoning off ·
|
| 50 |
</td>
|
| 51 |
<td class="num strong" data-value="65.82462114883167">65.8</td>
|
| 52 |
<td class="num" data-value="67.48821548821549" title="mmlu_pro: 79.3 ± 2.3 (n=300) gpqa_diamond_gen: 63.1 ± 3.4 (n=198) musr_gen: 60.0 ± 3.1 (n=250) ">67.5</td>
|
|
@@ -57,12 +57,12 @@
|
|
| 57 |
<td class="num" data-value="16.51787775562755">17</td>
|
| 58 |
<td class="num" data-value="1359.0489942173106">1359</td>
|
| 59 |
<td class="num" data-value="2952.064286917448">2952</td>
|
| 60 |
-
<td data-value="2026-09-18">2026-09-18<
|
| 61 |
</tr>
|
| 62 |
<tr>
|
| 63 |
<td data-value="gemma-4-12b-it-heretic">
|
| 64 |
<a href="runs/22.html" class="strong">gemma-4-12b-it-heretic</a>
|
| 65 |
-
<div class="sub">Q8 · Reasoning off ·
|
| 66 |
</td>
|
| 67 |
<td class="num strong" data-value="65.82311637680058">65.8</td>
|
| 68 |
<td class="num" data-value="68.44713804713804" title="mmlu_pro: 80.0 ± 2.3 (n=300) gpqa_diamond_gen: 64.1 ± 3.4 (n=198) musr_gen: 61.2 ± 3.1 (n=250) ">68.4</td>
|
|
@@ -73,12 +73,12 @@
|
|
| 73 |
<td class="num" data-value="16.51271607105954">17</td>
|
| 74 |
<td class="num" data-value="1358.107397713914">1358</td>
|
| 75 |
<td class="num" data-value="2954.110997961834">2954</td>
|
| 76 |
-
<td data-value="2026-09-19">2026-09-19<
|
| 77 |
</tr>
|
| 78 |
<tr>
|
| 79 |
<td data-value="qwen3.8-27b-mtp">
|
| 80 |
<a href="runs/20.html" class="strong">qwen3.8-27b-mtp</a>
|
| 81 |
-
<div class="sub">NVFP4 · Reasoning off ·
|
| 82 |
</td>
|
| 83 |
<td class="num strong" data-value="64.20301055669476">64.2</td>
|
| 84 |
<td class="num" data-value="69.85656565656565" title="mmlu_pro: 80.3 ± 2.3 (n=300) gpqa_diamond_gen: 63.6 ± 3.4 (n=198) musr_gen: 65.6 ± 3.0 (n=250) ">69.9</td>
|
|
@@ -89,12 +89,12 @@
|
|
| 89 |
<td class="num" data-value="15.103805505442727">15</td>
|
| 90 |
<td class="num" data-value="2550.5619640606633">2551</td>
|
| 91 |
<td class="num" data-value="1577.6915270835161">1578</td>
|
| 92 |
-
<td data-value="2026-09-18">2026-09-18<
|
| 93 |
</tr>
|
| 94 |
<tr>
|
| 95 |
<td data-value="gemma-4-12b-it-heretic">
|
| 96 |
<a href="runs/17.html" class="strong">gemma-4-12b-it-heretic</a>
|
| 97 |
-
<div class="sub">NVFP4 · Reasoning off ·
|
| 98 |
</td>
|
| 99 |
<td class="num strong" data-value="57.7166917293233">57.7</td>
|
| 100 |
<td class="num" data-value="63.70909090909091" title="mmlu_pro: 76.7 ± 2.4 (n=300) gpqa_diamond_gen: 56.1 ± 3.5 (n=198) musr_gen: 58.4 ± 3.1 (n=250) ">63.7</td>
|
|
@@ -105,12 +105,12 @@
|
|
| 105 |
<td class="num" data-value="21.360681311630678">21</td>
|
| 106 |
<td class="num" data-value="5258.027383235864">5258</td>
|
| 107 |
<td class="num" data-value="763.404164230451">763</td>
|
| 108 |
-
<td data-value="2026-09-18">2026-09-18<
|
| 109 |
</tr>
|
| 110 |
<tr>
|
| 111 |
<td data-value="nemotron-3.5-lightning-dspark">
|
| 112 |
<a href="runs/16.html" class="strong">nemotron-3.5-lightning-dspark</a>
|
| 113 |
-
<div class="sub">NVFP4 · Reasoning off ·
|
| 114 |
</td>
|
| 115 |
<td class="num strong" data-value="53.13354497354497">53.1</td>
|
| 116 |
<td class="num" data-value="55.756902356902366" title="mmlu_pro: 71.0 ± 2.6 (n=300) gpqa_diamond_gen: 57.1 ± 3.5 (n=198) musr_gen: 39.2 ± 3.1 (n=250) ">55.8</td>
|
|
@@ -121,12 +121,12 @@
|
|
| 121 |
<td class="num" data-value="83.7975869946743">84</td>
|
| 122 |
<td class="num" data-value="7236.849600555504">7237</td>
|
| 123 |
<td class="num" data-value="554.1085170116276">554</td>
|
| 124 |
-
<td data-value="2026-09-17">2026-09-17<
|
| 125 |
</tr>
|
| 126 |
<tr>
|
| 127 |
<td data-value="qwen3.5-9b-heretic-v2-mtp">
|
| 128 |
<a href="runs/19.html" class="strong">qwen3.5-9b-heretic-v2-mtp</a>
|
| 129 |
-
<div class="sub">NVFP4 · Reasoning off ·
|
| 130 |
</td>
|
| 131 |
<td class="num strong" data-value="51.662634870003295">51.7</td>
|
| 132 |
<td class="num" data-value="65.07138047138046" title="mmlu_pro: 76.3 ± 2.5 (n=300) gpqa_diamond_gen: 58.1 ± 3.5 (n=198) musr_gen: 60.8 ± 3.1 (n=250) ">65.1</td>
|
|
@@ -137,14 +137,14 @@
|
|
| 137 |
<td class="num" data-value="31.530246994028012">32</td>
|
| 138 |
<td class="num" data-value="6290.925546819461">6291</td>
|
| 139 |
<td class="num" data-value="639.6515059750527">640</td>
|
| 140 |
-
<td data-value="2026-09-18">2026-09-18<
|
| 141 |
</tr>
|
| 142 |
</tbody>
|
| 143 |
</table>
|
| 144 |
</div>
|
| 145 |
</section>
|
| 146 |
-
<p class="hint">Scores 0
|
| 147 |
-
<footer class="page-foot">
|
| 148 |
</main>
|
| 149 |
<script src="app.js"></script>
|
| 150 |
</body>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
+
<a href="index.html" class="active">Overview</a>
|
| 14 |
+
<a href="methodology.html">Methodology</a>
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
+
<h1>Overview</h1>
|
| 19 |
+
<p class="lead">Local language models compared: knowledge, math, code and German, plus speed. Each model variant is shown with its most recent completed run. How the numbers are measured is described under <a href="methodology.html">Methodology</a>.</p>
|
| 20 |
<section id="v3-quick">
|
| 21 |
+
<h2>Suite v2b · profile quick</h2>
|
| 22 |
<div class="table-wrap">
|
| 23 |
<table class="sortable">
|
| 24 |
<thead><tr>
|
| 25 |
+
<th data-sort="text">Model</th>
|
| 26 |
+
<th data-sort="num" class="num">Overall</th>
|
| 27 |
+
<th data-sort="num" class="num">Knowledge & reasoning</th><th data-sort="num" class="num">Math</th><th data-sort="num" class="num">Instruction following</th><th data-sort="num" class="num">Code</th><th data-sort="num" class="num">German</th><th data-sort="num" class="num">gen t/s</th><th data-sort="num" class="num">pp t/s 4k</th><th data-sort="num" class="num">TTFT 4k ms</th> <th data-sort="text">Date</th>
|
| 28 |
</tr></thead>
|
| 29 |
<tbody>
|
| 30 |
<tr>
|
| 31 |
<td data-value="qwen3.8-flash-next-ple4">
|
| 32 |
<a href="runs/15.html" class="strong">qwen3.8-flash-next-ple4</a>
|
| 33 |
+
<div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
|
| 34 |
</td>
|
| 35 |
<td class="num strong" data-value="71.45393635604162">71.5</td>
|
| 36 |
<td class="num strong" data-value="77.86868686868686" title="mmlu_pro: 86.3 ± 2.0 (n=300) gpqa_diamond_gen: 77.3 ± 3.0 (n=198) musr_gen: 70.0 ± 2.9 (n=250) ">77.9</td>
|
|
|
|
| 41 |
<td class="num" data-value="26.376464212349354">26</td>
|
| 42 |
<td class="num" data-value="2041.4640201348918">2041</td>
|
| 43 |
<td class="num" data-value="1973.583644023165">1974</td>
|
| 44 |
+
<td data-value="2026-09-17">2026-09-17</td>
|
| 45 |
</tr>
|
| 46 |
<tr>
|
| 47 |
<td data-value="gemma-4-12b-it">
|
| 48 |
<a href="runs/21.html" class="strong">gemma-4-12b-it</a>
|
| 49 |
+
<div class="sub">Q8 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
|
| 50 |
</td>
|
| 51 |
<td class="num strong" data-value="65.82462114883167">65.8</td>
|
| 52 |
<td class="num" data-value="67.48821548821549" title="mmlu_pro: 79.3 ± 2.3 (n=300) gpqa_diamond_gen: 63.1 ± 3.4 (n=198) musr_gen: 60.0 ± 3.1 (n=250) ">67.5</td>
|
|
|
|
| 57 |
<td class="num" data-value="16.51787775562755">17</td>
|
| 58 |
<td class="num" data-value="1359.0489942173106">1359</td>
|
| 59 |
<td class="num" data-value="2952.064286917448">2952</td>
|
| 60 |
+
<td data-value="2026-09-18">2026-09-18</td>
|
| 61 |
</tr>
|
| 62 |
<tr>
|
| 63 |
<td data-value="gemma-4-12b-it-heretic">
|
| 64 |
<a href="runs/22.html" class="strong">gemma-4-12b-it-heretic</a>
|
| 65 |
+
<div class="sub">Q8 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
|
| 66 |
</td>
|
| 67 |
<td class="num strong" data-value="65.82311637680058">65.8</td>
|
| 68 |
<td class="num" data-value="68.44713804713804" title="mmlu_pro: 80.0 ± 2.3 (n=300) gpqa_diamond_gen: 64.1 ± 3.4 (n=198) musr_gen: 61.2 ± 3.1 (n=250) ">68.4</td>
|
|
|
|
| 73 |
<td class="num" data-value="16.51271607105954">17</td>
|
| 74 |
<td class="num" data-value="1358.107397713914">1358</td>
|
| 75 |
<td class="num" data-value="2954.110997961834">2954</td>
|
| 76 |
+
<td data-value="2026-09-19">2026-09-19</td>
|
| 77 |
</tr>
|
| 78 |
<tr>
|
| 79 |
<td data-value="qwen3.8-27b-mtp">
|
| 80 |
<a href="runs/20.html" class="strong">qwen3.8-27b-mtp</a>
|
| 81 |
+
<div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
|
| 82 |
</td>
|
| 83 |
<td class="num strong" data-value="64.20301055669476">64.2</td>
|
| 84 |
<td class="num" data-value="69.85656565656565" title="mmlu_pro: 80.3 ± 2.3 (n=300) gpqa_diamond_gen: 63.6 ± 3.4 (n=198) musr_gen: 65.6 ± 3.0 (n=250) ">69.9</td>
|
|
|
|
| 89 |
<td class="num" data-value="15.103805505442727">15</td>
|
| 90 |
<td class="num" data-value="2550.5619640606633">2551</td>
|
| 91 |
<td class="num" data-value="1577.6915270835161">1578</td>
|
| 92 |
+
<td data-value="2026-09-18">2026-09-18</td>
|
| 93 |
</tr>
|
| 94 |
<tr>
|
| 95 |
<td data-value="gemma-4-12b-it-heretic">
|
| 96 |
<a href="runs/17.html" class="strong">gemma-4-12b-it-heretic</a>
|
| 97 |
+
<div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
|
| 98 |
</td>
|
| 99 |
<td class="num strong" data-value="57.7166917293233">57.7</td>
|
| 100 |
<td class="num" data-value="63.70909090909091" title="mmlu_pro: 76.7 ± 2.4 (n=300) gpqa_diamond_gen: 56.1 ± 3.5 (n=198) musr_gen: 58.4 ± 3.1 (n=250) ">63.7</td>
|
|
|
|
| 105 |
<td class="num" data-value="21.360681311630678">21</td>
|
| 106 |
<td class="num" data-value="5258.027383235864">5258</td>
|
| 107 |
<td class="num" data-value="763.404164230451">763</td>
|
| 108 |
+
<td data-value="2026-09-18">2026-09-18</td>
|
| 109 |
</tr>
|
| 110 |
<tr>
|
| 111 |
<td data-value="nemotron-3.5-lightning-dspark">
|
| 112 |
<a href="runs/16.html" class="strong">nemotron-3.5-lightning-dspark</a>
|
| 113 |
+
<div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
|
| 114 |
</td>
|
| 115 |
<td class="num strong" data-value="53.13354497354497">53.1</td>
|
| 116 |
<td class="num" data-value="55.756902356902366" title="mmlu_pro: 71.0 ± 2.6 (n=300) gpqa_diamond_gen: 57.1 ± 3.5 (n=198) musr_gen: 39.2 ± 3.1 (n=250) ">55.8</td>
|
|
|
|
| 121 |
<td class="num" data-value="83.7975869946743">84</td>
|
| 122 |
<td class="num" data-value="7236.849600555504">7237</td>
|
| 123 |
<td class="num" data-value="554.1085170116276">554</td>
|
| 124 |
+
<td data-value="2026-09-17">2026-09-17</td>
|
| 125 |
</tr>
|
| 126 |
<tr>
|
| 127 |
<td data-value="qwen3.5-9b-heretic-v2-mtp">
|
| 128 |
<a href="runs/19.html" class="strong">qwen3.5-9b-heretic-v2-mtp</a>
|
| 129 |
+
<div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
|
| 130 |
</td>
|
| 131 |
<td class="num strong" data-value="51.662634870003295">51.7</td>
|
| 132 |
<td class="num" data-value="65.07138047138046" title="mmlu_pro: 76.3 ± 2.5 (n=300) gpqa_diamond_gen: 58.1 ± 3.5 (n=198) musr_gen: 60.8 ± 3.1 (n=250) ">65.1</td>
|
|
|
|
| 137 |
<td class="num" data-value="31.530246994028012">32</td>
|
| 138 |
<td class="num" data-value="6290.925546819461">6291</td>
|
| 139 |
<td class="num" data-value="639.6515059750527">640</td>
|
| 140 |
+
<td data-value="2026-09-18">2026-09-18</td>
|
| 141 |
</tr>
|
| 142 |
</tbody>
|
| 143 |
</table>
|
| 144 |
</div>
|
| 145 |
</section>
|
| 146 |
+
<p class="hint">Scores run from 0 to 100; the best value per category is shown in bold. Hovering a category cell lists its tasks with standard error (±) and number of items (n). Speed: generation in tokens/s, prompt processing at 4k context in tokens/s, time to first token at 4k context in ms. Click a column header to sort.</p>
|
| 147 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 148 |
</main>
|
| 149 |
<script src="app.js"></script>
|
| 150 |
</body>
|
methodik.html
DELETED
|
@@ -1,124 +0,0 @@
|
|
| 1 |
-
<!doctype html>
|
| 2 |
-
<html lang="de">
|
| 3 |
-
<head>
|
| 4 |
-
<meta charset="utf-8">
|
| 5 |
-
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
-
<title>Methodik – Local LLM Benchmarks</title>
|
| 7 |
-
<link rel="stylesheet" href="style.css">
|
| 8 |
-
</head>
|
| 9 |
-
<body>
|
| 10 |
-
<header class="top">
|
| 11 |
-
<a class="brand" href="index.html">Local LLM Benchmarks</a>
|
| 12 |
-
<nav>
|
| 13 |
-
<a href="index.html">Übersicht</a>
|
| 14 |
-
<a href="methodik.html" class="active">Methodik</a>
|
| 15 |
-
</nav>
|
| 16 |
-
</header>
|
| 17 |
-
<main>
|
| 18 |
-
<h1>Methodik</h1>
|
| 19 |
-
<section class="prose">
|
| 20 |
-
<p>Jedes Modell bearbeitet dieselben öffentlichen Aufgabensammlungen (Tasks). Ein Task liefert einen Score von 0 bis 100.
|
| 21 |
-
Der Score einer Kategorie ist das gewichtete Mittel ihrer Tasks, der Gesamtscore das ungewichtete Mittel der Kategorien.
|
| 22 |
-
Fehlt ein Task, bleiben Kategorie und Gesamt leer.</p>
|
| 23 |
-
<ul>
|
| 24 |
-
<li><strong>Profil quick:</strong> feste Stichprobe je Task (Spalte quick_n), für alle Modelle dieselben Aufgaben.</li>
|
| 25 |
-
<li><strong>Profil full:</strong> alle Aufgaben des Tasks.</li>
|
| 26 |
-
<li><strong>Suite-Version:</strong> ändern sich Tasks, Prompts oder Metriken, entsteht eine neue Version. Scores verschiedener Versionen sind nicht vergleichbar.</li>
|
| 27 |
-
</ul>
|
| 28 |
-
<h2>Unsicherheit</h2>
|
| 29 |
-
<p>Zu jedem Task-Score gehört ein Standardfehler (±). Das 95-%-Intervall reicht etwa 1,96 Standardfehler um den Score.
|
| 30 |
-
Überlappen sich die Intervalle zweier Modelle, ist der Unterschied nicht belastbar – gerade im Profil quick mit
|
| 31 |
-
kleineren Stichproben.</p>
|
| 32 |
-
<h2>Geschwindigkeit</h2>
|
| 33 |
-
<p>Gemessen werden Generierung (Tokens/s), Prompt-Verarbeitung (Tokens/s) und die Zeit bis zum ersten Token bei
|
| 34 |
-
verschiedenen Kontextlängen, jeweils als Median mehrerer Wiederholungen mit p10 und p90. Je nach Lauf wird direkt am
|
| 35 |
-
Inferenz-Server oder über den AI-Hub gemessen; über den AI-Hub gemessene Werte enthalten Queue- und Proxyzeit und
|
| 36 |
-
sind daher etwas langsamer. Der Messweg steht auf der Seite des Laufs. Geschwindigkeit hängt stark von Hardware und
|
| 37 |
-
Inferenz-Software ab.</p>
|
| 38 |
-
</section>
|
| 39 |
-
|
| 40 |
-
<h2>Suite v2b</h2>
|
| 41 |
-
<div class="table-wrap">
|
| 42 |
-
<table>
|
| 43 |
-
<thead><tr><th>Kategorie</th><th>Task</th><th>Metrik</th><th>Art</th><th class="num">Gewicht</th><th class="num">quick_n</th></tr></thead>
|
| 44 |
-
<tbody>
|
| 45 |
-
<tr>
|
| 46 |
-
<td>Wissen & Reasoning</td>
|
| 47 |
-
<td>mmlu_pro</td>
|
| 48 |
-
<td>exact_match,custom-extract</td>
|
| 49 |
-
<td>Text</td>
|
| 50 |
-
<td class="num">1</td>
|
| 51 |
-
<td class="num">300</td>
|
| 52 |
-
</tr>
|
| 53 |
-
<tr>
|
| 54 |
-
<td>Wissen & Reasoning</td>
|
| 55 |
-
<td>gpqa_diamond_gen</td>
|
| 56 |
-
<td>exact_match,letter</td>
|
| 57 |
-
<td>Text</td>
|
| 58 |
-
<td class="num">1</td>
|
| 59 |
-
<td class="num">198</td>
|
| 60 |
-
</tr>
|
| 61 |
-
<tr>
|
| 62 |
-
<td>Wissen & Reasoning</td>
|
| 63 |
-
<td>musr_gen</td>
|
| 64 |
-
<td>exact_match,letter</td>
|
| 65 |
-
<td>Text</td>
|
| 66 |
-
<td class="num">1</td>
|
| 67 |
-
<td class="num">250</td>
|
| 68 |
-
</tr>
|
| 69 |
-
<tr>
|
| 70 |
-
<td>Mathe</td>
|
| 71 |
-
<td>omni_math_gen<div class="sub wrap">Omni-MATH-Rule ist die regelbasiert prüfbare Teilmenge von Omni-MATH und systematisch leichter als das volle Omni-MATH. Die Werte sind deshalb nicht mit veröffentlichten Omni-MATH-Zahlen vergleichbar. Der Datensatz ist seit Oktober 2024 öffentlich und liegt vor den Trainings-Cutoffs aller gemessenen Modelle; das betrifft alle Modelle gleich.</div></td>
|
| 72 |
-
<td>exact_match,boxed</td>
|
| 73 |
-
<td>Text</td>
|
| 74 |
-
<td class="num">1</td>
|
| 75 |
-
<td class="num">250</td>
|
| 76 |
-
</tr>
|
| 77 |
-
<tr>
|
| 78 |
-
<td>Instruktionen</td>
|
| 79 |
-
<td>ifbench</td>
|
| 80 |
-
<td>prompt_level_strict_acc,none</td>
|
| 81 |
-
<td>Text</td>
|
| 82 |
-
<td class="num">1</td>
|
| 83 |
-
<td class="num">300</td>
|
| 84 |
-
</tr>
|
| 85 |
-
<tr>
|
| 86 |
-
<td>Instruktionen</td>
|
| 87 |
-
<td>ifeval</td>
|
| 88 |
-
<td>prompt_level_strict_acc,none</td>
|
| 89 |
-
<td>Text</td>
|
| 90 |
-
<td class="num">0.33</td>
|
| 91 |
-
<td class="num">150</td>
|
| 92 |
-
</tr>
|
| 93 |
-
<tr>
|
| 94 |
-
<td>Code</td>
|
| 95 |
-
<td>livecodebench_gen<div class="sub wrap">Kontaminiert, und zwar ungleich: LiveCodeBench v6 enthält Aufgaben von Mai 2023 bis April 2025, ein neueres Release gibt es nicht. Die Trainings-Cutoffs der gemessenen Modelle liegen teils dahinter (Stand September 2026: Gemma 4 Januar 2025, Nemotron 3.5 Lightning September 2025, Qwen3.8 nicht veröffentlicht). Ein Modell mit späterem Cutoff kann einen größeren Teil der Aufgaben im Training gesehen haben; Abstände zwischen Modellen sind hier nur eingeschränkt aussagekräftig. Ausgaben werden exakt wie im offiziellen LiveCodeBench verglichen: 12 Aufgaben mit Fließkomma-Ausgabe sind dadurch für alle Modelle praktisch unlösbar (auch in den veröffentlichten LiveCodeBench-Zahlen); das senkt den erreichbaren Höchstwert für alle gleich. Liste und Abgleich mit dem offiziellen Auswerter im Kalibrierungsbericht (docs/superpowers/reports/2026-09-16-lcb-kalibrierung.md). Profil full lädt den vollen Datensatz und braucht dafür rund 9 GB RAM.</div></td>
|
| 96 |
-
<td>pass@1</td>
|
| 97 |
-
<td>Code (ausgeführt in Sandbox)</td>
|
| 98 |
-
<td class="num">1</td>
|
| 99 |
-
<td class="num">200</td>
|
| 100 |
-
</tr>
|
| 101 |
-
<tr>
|
| 102 |
-
<td>Deutsch</td>
|
| 103 |
-
<td>mmlu_prox_de</td>
|
| 104 |
-
<td>exact_match,custom-extract</td>
|
| 105 |
-
<td>Text</td>
|
| 106 |
-
<td class="num">1</td>
|
| 107 |
-
<td class="num">300</td>
|
| 108 |
-
</tr>
|
| 109 |
-
<tr>
|
| 110 |
-
<td>Deutsch</td>
|
| 111 |
-
<td>include_de_gen<div class="sub wrap">Nur 154 Aufgaben (Test und Validation von INCLUDE-German, alle werden gemessen). Das 95-%-Intervall beträgt rund ±8 Punkte; kleinere Abstände zwischen Modellen sind Rauschen. Deshalb halbes Gewicht neben mmlu_prox_de.</div></td>
|
| 112 |
-
<td>exact_match,letter</td>
|
| 113 |
-
<td>Text</td>
|
| 114 |
-
<td class="num">0.5</td>
|
| 115 |
-
<td class="num">154</td>
|
| 116 |
-
</tr>
|
| 117 |
-
</tbody>
|
| 118 |
-
</table>
|
| 119 |
-
</div>
|
| 120 |
-
<footer class="page-foot">Stand 2026-09-20 · Scores 0–100 · erzeugt mit <code>bench publish</code></footer>
|
| 121 |
-
</main>
|
| 122 |
-
<script src="app.js"></script>
|
| 123 |
-
</body>
|
| 124 |
-
</html>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
methodology.html
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<title>Methodology – Local LLM Benchmarks</title>
|
| 7 |
+
<link rel="stylesheet" href="style.css">
|
| 8 |
+
</head>
|
| 9 |
+
<body>
|
| 10 |
+
<header class="top">
|
| 11 |
+
<a class="brand" href="index.html">Local LLM Benchmarks</a>
|
| 12 |
+
<nav>
|
| 13 |
+
<a href="index.html">Overview</a>
|
| 14 |
+
<a href="methodology.html" class="active">Methodology</a>
|
| 15 |
+
</nav>
|
| 16 |
+
</header>
|
| 17 |
+
<main>
|
| 18 |
+
<h1>Methodology</h1>
|
| 19 |
+
<section class="prose">
|
| 20 |
+
<p>Every model works through the same public task collections. A task yields a score from 0 to 100.
|
| 21 |
+
A category score is the weighted mean of its tasks, the overall score the unweighted mean of the categories.
|
| 22 |
+
If a task is missing, its category and the overall score stay empty.</p>
|
| 23 |
+
<ul>
|
| 24 |
+
<li><strong>Profile quick:</strong> a fixed sample per task (column quick_n) — the same items for every model.</li>
|
| 25 |
+
<li><strong>Profile full:</strong> all items of the task.</li>
|
| 26 |
+
<li><strong>Suite version:</strong> whenever tasks, prompts or metrics change, a new version starts. Scores from different versions are not comparable, so only the latest version is published here.</li>
|
| 27 |
+
</ul>
|
| 28 |
+
<h2>Uncertainty</h2>
|
| 29 |
+
<p>Every task score comes with a standard error (±). The 95% interval spans roughly 1.96 standard errors around the
|
| 30 |
+
score. Where the intervals of two models overlap, the difference does not hold up — particularly in profile quick
|
| 31 |
+
with its smaller samples.</p>
|
| 32 |
+
<h2>Speed</h2>
|
| 33 |
+
<p>Measured are generation (tokens/s), prompt processing (tokens/s) and the time to the first token at various
|
| 34 |
+
context lengths, each as the median of several repetitions with p10 and p90. Depending on the run, the measurement
|
| 35 |
+
is taken directly at the inference server or through the AI hub; values measured through the AI hub include queue
|
| 36 |
+
and proxy time and are therefore somewhat slower. The route is stated on the run's own page. Speed depends heavily
|
| 37 |
+
on hardware and inference software.</p>
|
| 38 |
+
</section>
|
| 39 |
+
|
| 40 |
+
<h2>Suite v2b</h2>
|
| 41 |
+
<div class="table-wrap">
|
| 42 |
+
<table>
|
| 43 |
+
<thead><tr><th>Category</th><th>Task</th><th>Metric</th><th>Kind</th><th class="num">Weight</th><th class="num">quick_n</th></tr></thead>
|
| 44 |
+
<tbody>
|
| 45 |
+
<tr>
|
| 46 |
+
<td>Knowledge & reasoning</td>
|
| 47 |
+
<td>mmlu_pro</td>
|
| 48 |
+
<td>exact_match,custom-extract</td>
|
| 49 |
+
<td>Text</td>
|
| 50 |
+
<td class="num">1</td>
|
| 51 |
+
<td class="num">300</td>
|
| 52 |
+
</tr>
|
| 53 |
+
<tr>
|
| 54 |
+
<td>Knowledge & reasoning</td>
|
| 55 |
+
<td>gpqa_diamond_gen</td>
|
| 56 |
+
<td>exact_match,letter</td>
|
| 57 |
+
<td>Text</td>
|
| 58 |
+
<td class="num">1</td>
|
| 59 |
+
<td class="num">198</td>
|
| 60 |
+
</tr>
|
| 61 |
+
<tr>
|
| 62 |
+
<td>Knowledge & reasoning</td>
|
| 63 |
+
<td>musr_gen</td>
|
| 64 |
+
<td>exact_match,letter</td>
|
| 65 |
+
<td>Text</td>
|
| 66 |
+
<td class="num">1</td>
|
| 67 |
+
<td class="num">250</td>
|
| 68 |
+
</tr>
|
| 69 |
+
<tr>
|
| 70 |
+
<td>Math</td>
|
| 71 |
+
<td>omni_math_gen<div class="sub wrap">Omni-MATH-Rule is the rule-checkable subset of Omni-MATH and systematically easier than full Omni-MATH. Its scores are therefore not comparable to published Omni-MATH numbers. The dataset has been public since October 2024, which is before the training cutoff of every model measured here; that affects all models alike.</div></td>
|
| 72 |
+
<td>exact_match,boxed</td>
|
| 73 |
+
<td>Text</td>
|
| 74 |
+
<td class="num">1</td>
|
| 75 |
+
<td class="num">250</td>
|
| 76 |
+
</tr>
|
| 77 |
+
<tr>
|
| 78 |
+
<td>Instruction following</td>
|
| 79 |
+
<td>ifbench</td>
|
| 80 |
+
<td>prompt_level_strict_acc,none</td>
|
| 81 |
+
<td>Text</td>
|
| 82 |
+
<td class="num">1</td>
|
| 83 |
+
<td class="num">300</td>
|
| 84 |
+
</tr>
|
| 85 |
+
<tr>
|
| 86 |
+
<td>Instruction following</td>
|
| 87 |
+
<td>ifeval</td>
|
| 88 |
+
<td>prompt_level_strict_acc,none</td>
|
| 89 |
+
<td>Text</td>
|
| 90 |
+
<td class="num">0.33</td>
|
| 91 |
+
<td class="num">150</td>
|
| 92 |
+
</tr>
|
| 93 |
+
<tr>
|
| 94 |
+
<td>Code</td>
|
| 95 |
+
<td>livecodebench_gen<div class="sub wrap">Contaminated, and unevenly so: LiveCodeBench v6 covers problems from May 2023 to April 2025, and there is no newer release. Some of the models measured here have a later training cutoff (as of September 2026: Gemma 4 January 2025, Nemotron 3.5 Lightning September 2025, Qwen3.8 not published). A model with a later cutoff may have seen more of these problems during training, so gaps between models say less here than elsewhere. Outputs are compared exactly as in official LiveCodeBench: this makes 12 problems with floating-point output practically unsolvable for every model (they are in the published LiveCodeBench numbers too), which lowers the reachable maximum equally for all. The list and the comparison against the official evaluator are in the project's calibration report. Profile full loads the complete dataset and needs about 9 GB of RAM for it.</div></td>
|
| 96 |
+
<td>pass@1</td>
|
| 97 |
+
<td>Code (run in a sandbox)</td>
|
| 98 |
+
<td class="num">1</td>
|
| 99 |
+
<td class="num">200</td>
|
| 100 |
+
</tr>
|
| 101 |
+
<tr>
|
| 102 |
+
<td>German</td>
|
| 103 |
+
<td>mmlu_prox_de</td>
|
| 104 |
+
<td>exact_match,custom-extract</td>
|
| 105 |
+
<td>Text</td>
|
| 106 |
+
<td class="num">1</td>
|
| 107 |
+
<td class="num">300</td>
|
| 108 |
+
</tr>
|
| 109 |
+
<tr>
|
| 110 |
+
<td>German</td>
|
| 111 |
+
<td>include_de_gen<div class="sub wrap">Only 154 items (the test and validation splits of INCLUDE-German, all of them measured). The 95% interval is about ±8 points, so smaller gaps between models are noise. Hence half the weight next to mmlu_prox_de.</div></td>
|
| 112 |
+
<td>exact_match,letter</td>
|
| 113 |
+
<td>Text</td>
|
| 114 |
+
<td class="num">0.5</td>
|
| 115 |
+
<td class="num">154</td>
|
| 116 |
+
</tr>
|
| 117 |
+
</tbody>
|
| 118 |
+
</table>
|
| 119 |
+
</div>
|
| 120 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 121 |
+
</main>
|
| 122 |
+
<script src="app.js"></script>
|
| 123 |
+
</body>
|
| 124 |
+
</html>
|
runs/15.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html lang="
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
@@ -10,35 +10,35 @@
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
-
<a href="../index.html">
|
| 14 |
-
<a href="../
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
-
<p class="sub"><a href="../index.html#v3-quick">←
|
| 19 |
<h1>qwen3.8-flash-next-ple4</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
-
<dt>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
-
<dt>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
-
<dt>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
-
<dt>
|
| 28 |
-
<dt>
|
| 29 |
-
<dt>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
-
<h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
-
<thead><tr><th>
|
| 36 |
<tbody>
|
| 37 |
-
<tr><td>
|
| 38 |
-
<tr><td>
|
| 39 |
-
<tr><td>
|
| 40 |
<tr><td>Code</td><td class="num">75.5</td></tr>
|
| 41 |
-
<tr><td>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
@@ -46,45 +46,45 @@
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
-
<thead><tr><th>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
-
<td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">86.3</td>
|
| 55 |
<td class="num">± 2.0</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
-
<td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">77.3</td>
|
| 62 |
<td class="num">± 3.0</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
-
<td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">70.0</td>
|
| 69 |
<td class="num">± 2.9</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
-
<td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">73.2</td>
|
| 76 |
<td class="num">± 2.8</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
-
<td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">36.0</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
-
<td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">90.7</td>
|
| 90 |
<td class="num">± 2.4</td>
|
|
@@ -98,14 +98,14 @@
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
-
<td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">85.7</td>
|
| 104 |
<td class="num">± 2.0</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
-
<td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">72.1</td>
|
| 111 |
<td class="num">± 3.6</td>
|
|
@@ -114,15 +114,15 @@
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
-
<p class="hint">±
|
| 118 |
|
| 119 |
-
<h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
-
<thead><tr><th>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
-
<td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">26.4</td>
|
|
@@ -131,7 +131,7 @@
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
-
<td>Prompt
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">1051.1</td>
|
|
@@ -140,7 +140,7 @@
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
-
<td>Prompt
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4029</td>
|
| 146 |
<td class="num">2041.5</td>
|
|
@@ -149,7 +149,7 @@
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
-
<td>Prompt
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">16012</td>
|
| 155 |
<td class="num">2266.7</td>
|
|
@@ -158,7 +158,7 @@
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
-
<td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">503.3</td>
|
|
@@ -167,7 +167,7 @@
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
-
<td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4029</td>
|
| 173 |
<td class="num">1973.6</td>
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
-
<td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">16012</td>
|
| 182 |
<td class="num">7064.0</td>
|
|
@@ -187,8 +187,8 @@
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
-
<p class="hint">
|
| 191 |
-
<footer class="page-foot">
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
+
<a href="../index.html">Overview</a>
|
| 14 |
+
<a href="../methodology.html">Methodology</a>
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
+
<p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
|
| 19 |
<h1>qwen3.8-flash-next-ple4</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
+
<dt>Quantization</dt><dd>NVFP4</dd>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
+
<dt>Profile</dt><dd>quick</dd>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
+
<dt>Date</dt><dd>2026-09-17 – 2026-09-18</dd>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
+
<dt>Inference software</dt><dd>vLLM 0.26.0</dd>
|
| 28 |
+
<dt>Measured</dt><dd>directly at the inference server</dd>
|
| 29 |
+
<dt>Overall</dt><dd class="strong">71.5</dd>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
+
<h2>Categories</h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
+
<thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
|
| 36 |
<tbody>
|
| 37 |
+
<tr><td>Knowledge & reasoning</td><td class="num">77.9</td></tr>
|
| 38 |
+
<tr><td>Math</td><td class="num">73.2</td></tr>
|
| 39 |
+
<tr><td>Instruction following</td><td class="num">49.6</td></tr>
|
| 40 |
<tr><td>Code</td><td class="num">75.5</td></tr>
|
| 41 |
+
<tr><td>German</td><td class="num">81.1</td></tr>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
+
<thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
+
<td>Knowledge & reasoning</td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">86.3</td>
|
| 55 |
<td class="num">± 2.0</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
+
<td>Knowledge & reasoning</td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">77.3</td>
|
| 62 |
<td class="num">± 3.0</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
+
<td>Knowledge & reasoning</td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">70.0</td>
|
| 69 |
<td class="num">± 2.9</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
+
<td>Math</td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">73.2</td>
|
| 76 |
<td class="num">± 2.8</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
+
<td>Instruction following</td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">36.0</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
+
<td>Instruction following</td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">90.7</td>
|
| 90 |
<td class="num">± 2.4</td>
|
|
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
+
<td>German</td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">85.7</td>
|
| 104 |
<td class="num">± 2.0</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
+
<td>German</td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">72.1</td>
|
| 111 |
<td class="num">± 3.6</td>
|
|
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
+
<p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
|
| 118 |
|
| 119 |
+
<h2>Speed</h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
+
<thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
+
<td>Generation [t/s]</td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">26.4</td>
|
|
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
+
<td>Prompt processing [t/s]</td>
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">1051.1</td>
|
|
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
+
<td>Prompt processing [t/s]</td>
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4029</td>
|
| 146 |
<td class="num">2041.5</td>
|
|
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
+
<td>Prompt processing [t/s]</td>
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">16012</td>
|
| 155 |
<td class="num">2266.7</td>
|
|
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
+
<td>Time to first token [ms]</td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">503.3</td>
|
|
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
+
<td>Time to first token [ms]</td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4029</td>
|
| 173 |
<td class="num">1973.6</td>
|
|
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
+
<td>Time to first token [ms]</td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">16012</td>
|
| 182 |
<td class="num">7064.0</td>
|
|
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
+
<p class="hint">Measured directly at the inference server.</p>
|
| 191 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
runs/16.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html lang="
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
@@ -10,35 +10,35 @@
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
-
<a href="../index.html">
|
| 14 |
-
<a href="../
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
-
<p class="sub"><a href="../index.html#v3-quick">←
|
| 19 |
<h1>nemotron-3.5-lightning-dspark</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
-
<dt>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
-
<dt>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
-
<dt>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
-
<dt>
|
| 28 |
-
<dt>
|
| 29 |
-
<dt>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
-
<h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
-
<thead><tr><th>
|
| 36 |
<tbody>
|
| 37 |
-
<tr><td>
|
| 38 |
-
<tr><td>
|
| 39 |
-
<tr><td>
|
| 40 |
<tr><td>Code</td><td class="num">53.0</td></tr>
|
| 41 |
-
<tr><td>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
@@ -46,45 +46,45 @@
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
-
<thead><tr><th>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
-
<td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">71.0</td>
|
| 55 |
<td class="num">± 2.6</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
-
<td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">57.1</td>
|
| 62 |
<td class="num">± 3.5</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
-
<td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">39.2</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
-
<td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">40.4</td>
|
| 76 |
<td class="num">± 3.1</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
-
<td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">35.0</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
-
<td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">79.3</td>
|
| 90 |
<td class="num">± 3.3</td>
|
|
@@ -98,14 +98,14 @@
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
-
<td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">72.0</td>
|
| 104 |
<td class="num">± 2.6</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
-
<td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">67.5</td>
|
| 111 |
<td class="num">± 3.8</td>
|
|
@@ -114,15 +114,15 @@
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
-
<p class="hint">±
|
| 118 |
|
| 119 |
-
<h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
-
<thead><tr><th>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
-
<td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">83.8</td>
|
|
@@ -131,7 +131,7 @@
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
-
<td>Prompt
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">528</td>
|
| 137 |
<td class="num">3738.2</td>
|
|
@@ -140,7 +140,7 @@
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
-
<td>Prompt
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4010</td>
|
| 146 |
<td class="num">7236.8</td>
|
|
@@ -149,7 +149,7 @@
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
-
<td>Prompt
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">15960</td>
|
| 155 |
<td class="num">7177.3</td>
|
|
@@ -158,7 +158,7 @@
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
-
<td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">528</td>
|
| 164 |
<td class="num">141.2</td>
|
|
@@ -167,7 +167,7 @@
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
-
<td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4010</td>
|
| 173 |
<td class="num">554.1</td>
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
-
<td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">15960</td>
|
| 182 |
<td class="num">2223.7</td>
|
|
@@ -187,8 +187,8 @@
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
-
<p class="hint">
|
| 191 |
-
<footer class="page-foot">
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
+
<a href="../index.html">Overview</a>
|
| 14 |
+
<a href="../methodology.html">Methodology</a>
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
+
<p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
|
| 19 |
<h1>nemotron-3.5-lightning-dspark</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
+
<dt>Quantization</dt><dd>NVFP4</dd>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
+
<dt>Profile</dt><dd>quick</dd>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
+
<dt>Date</dt><dd>2026-09-17 – 2026-09-18</dd>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
+
<dt>Inference software</dt><dd>vLLM 0.26.0</dd>
|
| 28 |
+
<dt>Measured</dt><dd>directly at the inference server</dd>
|
| 29 |
+
<dt>Overall</dt><dd class="strong">53.1</dd>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
+
<h2>Categories</h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
+
<thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
|
| 36 |
<tbody>
|
| 37 |
+
<tr><td>Knowledge & reasoning</td><td class="num">55.8</td></tr>
|
| 38 |
+
<tr><td>Math</td><td class="num">40.4</td></tr>
|
| 39 |
+
<tr><td>Instruction following</td><td class="num">46.0</td></tr>
|
| 40 |
<tr><td>Code</td><td class="num">53.0</td></tr>
|
| 41 |
+
<tr><td>German</td><td class="num">70.5</td></tr>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
+
<thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
+
<td>Knowledge & reasoning</td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">71.0</td>
|
| 55 |
<td class="num">± 2.6</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
+
<td>Knowledge & reasoning</td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">57.1</td>
|
| 62 |
<td class="num">± 3.5</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
+
<td>Knowledge & reasoning</td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">39.2</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
+
<td>Math</td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">40.4</td>
|
| 76 |
<td class="num">± 3.1</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
+
<td>Instruction following</td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">35.0</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
+
<td>Instruction following</td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">79.3</td>
|
| 90 |
<td class="num">± 3.3</td>
|
|
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
+
<td>German</td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">72.0</td>
|
| 104 |
<td class="num">± 2.6</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
+
<td>German</td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">67.5</td>
|
| 111 |
<td class="num">± 3.8</td>
|
|
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
+
<p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
|
| 118 |
|
| 119 |
+
<h2>Speed</h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
+
<thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
+
<td>Generation [t/s]</td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">83.8</td>
|
|
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
+
<td>Prompt processing [t/s]</td>
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">528</td>
|
| 137 |
<td class="num">3738.2</td>
|
|
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
+
<td>Prompt processing [t/s]</td>
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4010</td>
|
| 146 |
<td class="num">7236.8</td>
|
|
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
+
<td>Prompt processing [t/s]</td>
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">15960</td>
|
| 155 |
<td class="num">7177.3</td>
|
|
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
+
<td>Time to first token [ms]</td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">528</td>
|
| 164 |
<td class="num">141.2</td>
|
|
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
+
<td>Time to first token [ms]</td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4010</td>
|
| 173 |
<td class="num">554.1</td>
|
|
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
+
<td>Time to first token [ms]</td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">15960</td>
|
| 182 |
<td class="num">2223.7</td>
|
|
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
+
<p class="hint">Measured directly at the inference server.</p>
|
| 191 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
runs/17.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html lang="
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
@@ -10,35 +10,35 @@
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
-
<a href="../index.html">
|
| 14 |
-
<a href="../
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
-
<p class="sub"><a href="../index.html#v3-quick">←
|
| 19 |
<h1>gemma-4-12b-it-heretic</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
-
<dt>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
-
<dt>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
-
<dt>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
-
<dt>
|
| 28 |
-
<dt>
|
| 29 |
-
<dt>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
-
<h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
-
<thead><tr><th>
|
| 36 |
<tbody>
|
| 37 |
-
<tr><td>
|
| 38 |
-
<tr><td>
|
| 39 |
-
<tr><td>
|
| 40 |
<tr><td>Code</td><td class="num">62.0</td></tr>
|
| 41 |
-
<tr><td>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
@@ -46,45 +46,45 @@
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
-
<thead><tr><th>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
-
<td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">76.7</td>
|
| 55 |
<td class="num">± 2.4</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
-
<td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">56.1</td>
|
| 62 |
<td class="num">± 3.5</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
-
<td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">58.4</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
-
<td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">48.4</td>
|
| 76 |
<td class="num">± 3.2</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
-
<td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">33.0</td>
|
| 83 |
<td class="num">± 2.7</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
-
<td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">88.7</td>
|
| 90 |
<td class="num">± 2.6</td>
|
|
@@ -98,14 +98,14 @@
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
-
<td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">70.0</td>
|
| 104 |
<td class="num">± 2.7</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
-
<td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">63.0</td>
|
| 111 |
<td class="num">± 3.9</td>
|
|
@@ -114,15 +114,15 @@
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
-
<p class="hint">±
|
| 118 |
|
| 119 |
-
<h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
-
<thead><tr><th>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
-
<td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">21.4</td>
|
|
@@ -131,7 +131,7 @@
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
-
<td>Prompt
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">528</td>
|
| 137 |
<td class="num">5070.5</td>
|
|
@@ -140,7 +140,7 @@
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
-
<td>Prompt
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4014</td>
|
| 146 |
<td class="num">5258.0</td>
|
|
@@ -149,7 +149,7 @@
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
-
<td>Prompt
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">15985</td>
|
| 155 |
<td class="num">3514.1</td>
|
|
@@ -158,7 +158,7 @@
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
-
<td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">528</td>
|
| 164 |
<td class="num">103.9</td>
|
|
@@ -167,7 +167,7 @@
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
-
<td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4014</td>
|
| 173 |
<td class="num">763.4</td>
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
-
<td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">15985</td>
|
| 182 |
<td class="num">4548.8</td>
|
|
@@ -187,8 +187,8 @@
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
-
<p class="hint">
|
| 191 |
-
<footer class="page-foot">
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
+
<a href="../index.html">Overview</a>
|
| 14 |
+
<a href="../methodology.html">Methodology</a>
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
+
<p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
|
| 19 |
<h1>gemma-4-12b-it-heretic</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
+
<dt>Quantization</dt><dd>NVFP4</dd>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
+
<dt>Profile</dt><dd>quick</dd>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
+
<dt>Date</dt><dd>2026-09-18</dd>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
+
<dt>Inference software</dt><dd>vLLM 0.26.0</dd>
|
| 28 |
+
<dt>Measured</dt><dd>directly at the inference server</dd>
|
| 29 |
+
<dt>Overall</dt><dd class="strong">57.7</dd>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
+
<h2>Categories</h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
+
<thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
|
| 36 |
<tbody>
|
| 37 |
+
<tr><td>Knowledge & reasoning</td><td class="num">63.7</td></tr>
|
| 38 |
+
<tr><td>Math</td><td class="num">48.4</td></tr>
|
| 39 |
+
<tr><td>Instruction following</td><td class="num">46.8</td></tr>
|
| 40 |
<tr><td>Code</td><td class="num">62.0</td></tr>
|
| 41 |
+
<tr><td>German</td><td class="num">67.7</td></tr>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
+
<thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
+
<td>Knowledge & reasoning</td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">76.7</td>
|
| 55 |
<td class="num">± 2.4</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
+
<td>Knowledge & reasoning</td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">56.1</td>
|
| 62 |
<td class="num">± 3.5</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
+
<td>Knowledge & reasoning</td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">58.4</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
+
<td>Math</td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">48.4</td>
|
| 76 |
<td class="num">± 3.2</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
+
<td>Instruction following</td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">33.0</td>
|
| 83 |
<td class="num">± 2.7</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
+
<td>Instruction following</td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">88.7</td>
|
| 90 |
<td class="num">± 2.6</td>
|
|
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
+
<td>German</td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">70.0</td>
|
| 104 |
<td class="num">± 2.7</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
+
<td>German</td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">63.0</td>
|
| 111 |
<td class="num">± 3.9</td>
|
|
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
+
<p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
|
| 118 |
|
| 119 |
+
<h2>Speed</h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
+
<thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
+
<td>Generation [t/s]</td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">21.4</td>
|
|
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
+
<td>Prompt processing [t/s]</td>
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">528</td>
|
| 137 |
<td class="num">5070.5</td>
|
|
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
+
<td>Prompt processing [t/s]</td>
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4014</td>
|
| 146 |
<td class="num">5258.0</td>
|
|
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
+
<td>Prompt processing [t/s]</td>
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">15985</td>
|
| 155 |
<td class="num">3514.1</td>
|
|
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
+
<td>Time to first token [ms]</td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">528</td>
|
| 164 |
<td class="num">103.9</td>
|
|
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
+
<td>Time to first token [ms]</td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4014</td>
|
| 173 |
<td class="num">763.4</td>
|
|
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
+
<td>Time to first token [ms]</td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">15985</td>
|
| 182 |
<td class="num">4548.8</td>
|
|
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
+
<p class="hint">Measured directly at the inference server.</p>
|
| 191 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
runs/19.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html lang="
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
@@ -10,35 +10,35 @@
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
-
<a href="../index.html">
|
| 14 |
-
<a href="../
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
-
<p class="sub"><a href="../index.html#v3-quick">←
|
| 19 |
<h1>qwen3.5-9b-heretic-v2-mtp</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
-
<dt>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
-
<dt>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
-
<dt>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
-
<dt>
|
| 28 |
-
<dt>
|
| 29 |
-
<dt>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
-
<h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
-
<thead><tr><th>
|
| 36 |
<tbody>
|
| 37 |
-
<tr><td>
|
| 38 |
-
<tr><td>
|
| 39 |
-
<tr><td>
|
| 40 |
<tr><td>Code</td><td class="num">46.5</td></tr>
|
| 41 |
-
<tr><td>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
@@ -46,45 +46,45 @@
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
-
<thead><tr><th>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
-
<td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">76.3</td>
|
| 55 |
<td class="num">± 2.5</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
-
<td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">58.1</td>
|
| 62 |
<td class="num">± 3.5</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
-
<td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">60.8</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
-
<td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">35.6</td>
|
| 76 |
<td class="num">± 3.0</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
-
<td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">29.7</td>
|
| 83 |
<td class="num">± 2.6</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
-
<td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">88.0</td>
|
| 90 |
<td class="num">± 2.7</td>
|
|
@@ -98,14 +98,14 @@
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
-
<td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">69.3</td>
|
| 104 |
<td class="num">± 2.7</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
-
<td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">62.3</td>
|
| 111 |
<td class="num">± 3.9</td>
|
|
@@ -114,15 +114,15 @@
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
-
<p class="hint">±
|
| 118 |
|
| 119 |
-
<h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
-
<thead><tr><th>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
-
<td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">31.5</td>
|
|
@@ -131,7 +131,7 @@
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
-
<td>Prompt
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">4737.2</td>
|
|
@@ -140,7 +140,7 @@
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
-
<td>Prompt
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4024</td>
|
| 146 |
<td class="num">6290.9</td>
|
|
@@ -149,7 +149,7 @@
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
-
<td>Prompt
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">16001</td>
|
| 155 |
<td class="num">5096.4</td>
|
|
@@ -158,7 +158,7 @@
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
-
<td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">111.7</td>
|
|
@@ -167,7 +167,7 @@
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
-
<td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4024</td>
|
| 173 |
<td class="num">639.7</td>
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
-
<td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">16001</td>
|
| 182 |
<td class="num">3139.6</td>
|
|
@@ -187,8 +187,8 @@
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
-
<p class="hint">
|
| 191 |
-
<footer class="page-foot">
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
+
<a href="../index.html">Overview</a>
|
| 14 |
+
<a href="../methodology.html">Methodology</a>
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
+
<p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
|
| 19 |
<h1>qwen3.5-9b-heretic-v2-mtp</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
+
<dt>Quantization</dt><dd>NVFP4</dd>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
+
<dt>Profile</dt><dd>quick</dd>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
+
<dt>Date</dt><dd>2026-09-18</dd>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
+
<dt>Inference software</dt><dd>vLLM 0.26.0</dd>
|
| 28 |
+
<dt>Measured</dt><dd>directly at the inference server</dd>
|
| 29 |
+
<dt>Overall</dt><dd class="strong">51.7</dd>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
+
<h2>Categories</h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
+
<thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
|
| 36 |
<tbody>
|
| 37 |
+
<tr><td>Knowledge & reasoning</td><td class="num">65.1</td></tr>
|
| 38 |
+
<tr><td>Math</td><td class="num">35.6</td></tr>
|
| 39 |
+
<tr><td>Instruction following</td><td class="num">44.1</td></tr>
|
| 40 |
<tr><td>Code</td><td class="num">46.5</td></tr>
|
| 41 |
+
<tr><td>German</td><td class="num">67.0</td></tr>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
+
<thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
+
<td>Knowledge & reasoning</td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">76.3</td>
|
| 55 |
<td class="num">± 2.5</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
+
<td>Knowledge & reasoning</td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">58.1</td>
|
| 62 |
<td class="num">± 3.5</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
+
<td>Knowledge & reasoning</td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">60.8</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
+
<td>Math</td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">35.6</td>
|
| 76 |
<td class="num">± 3.0</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
+
<td>Instruction following</td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">29.7</td>
|
| 83 |
<td class="num">± 2.6</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
+
<td>Instruction following</td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">88.0</td>
|
| 90 |
<td class="num">± 2.7</td>
|
|
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
+
<td>German</td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">69.3</td>
|
| 104 |
<td class="num">± 2.7</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
+
<td>German</td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">62.3</td>
|
| 111 |
<td class="num">± 3.9</td>
|
|
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
+
<p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
|
| 118 |
|
| 119 |
+
<h2>Speed</h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
+
<thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
+
<td>Generation [t/s]</td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">31.5</td>
|
|
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
+
<td>Prompt processing [t/s]</td>
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">4737.2</td>
|
|
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
+
<td>Prompt processing [t/s]</td>
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4024</td>
|
| 146 |
<td class="num">6290.9</td>
|
|
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
+
<td>Prompt processing [t/s]</td>
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">16001</td>
|
| 155 |
<td class="num">5096.4</td>
|
|
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
+
<td>Time to first token [ms]</td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">111.7</td>
|
|
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
+
<td>Time to first token [ms]</td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4024</td>
|
| 173 |
<td class="num">639.7</td>
|
|
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
+
<td>Time to first token [ms]</td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">16001</td>
|
| 182 |
<td class="num">3139.6</td>
|
|
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
+
<p class="hint">Measured directly at the inference server.</p>
|
| 191 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
runs/20.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html lang="
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
@@ -10,35 +10,35 @@
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
-
<a href="../index.html">
|
| 14 |
-
<a href="../
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
-
<p class="sub"><a href="../index.html#v3-quick">←
|
| 19 |
<h1>qwen3.8-27b-mtp</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
-
<dt>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
-
<dt>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
-
<dt>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
-
<dt>
|
| 28 |
-
<dt>
|
| 29 |
-
<dt>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
-
<h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
-
<thead><tr><th>
|
| 36 |
<tbody>
|
| 37 |
-
<tr><td>
|
| 38 |
-
<tr><td>
|
| 39 |
-
<tr><td>
|
| 40 |
<tr><td>Code</td><td class="num">65.0</td></tr>
|
| 41 |
-
<tr><td>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
@@ -46,45 +46,45 @@
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
-
<thead><tr><th>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
-
<td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">80.3</td>
|
| 55 |
<td class="num">± 2.3</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
-
<td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">63.6</td>
|
| 62 |
<td class="num">± 3.4</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
-
<td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">65.6</td>
|
| 69 |
<td class="num">± 3.0</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
-
<td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">60.4</td>
|
| 76 |
<td class="num">± 3.1</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
-
<td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">38.3</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
-
<td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">83.3</td>
|
| 90 |
<td class="num">± 3.1</td>
|
|
@@ -98,14 +98,14 @@
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
-
<td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">79.0</td>
|
| 104 |
<td class="num">± 2.4</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
-
<td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">70.8</td>
|
| 111 |
<td class="num">± 3.7</td>
|
|
@@ -114,15 +114,15 @@
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
-
<p class="hint">±
|
| 118 |
|
| 119 |
-
<h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
-
<thead><tr><th>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
-
<td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">15.1</td>
|
|
@@ -131,7 +131,7 @@
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
-
<td>Prompt
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">1836.6</td>
|
|
@@ -140,7 +140,7 @@
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
-
<td>Prompt
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4024</td>
|
| 146 |
<td class="num">2550.6</td>
|
|
@@ -149,7 +149,7 @@
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
-
<td>Prompt
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">16001</td>
|
| 155 |
<td class="num">1734.3</td>
|
|
@@ -158,7 +158,7 @@
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
-
<td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">288.0</td>
|
|
@@ -167,7 +167,7 @@
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
-
<td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4024</td>
|
| 173 |
<td class="num">1577.7</td>
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
-
<td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">16001</td>
|
| 182 |
<td class="num">9226.4</td>
|
|
@@ -187,8 +187,8 @@
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
-
<p class="hint">
|
| 191 |
-
<footer class="page-foot">
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
+
<a href="../index.html">Overview</a>
|
| 14 |
+
<a href="../methodology.html">Methodology</a>
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
+
<p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
|
| 19 |
<h1>qwen3.8-27b-mtp</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
+
<dt>Quantization</dt><dd>NVFP4</dd>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
+
<dt>Profile</dt><dd>quick</dd>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
+
<dt>Date</dt><dd>2026-09-18</dd>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
+
<dt>Inference software</dt><dd>vLLM 0.26.0</dd>
|
| 28 |
+
<dt>Measured</dt><dd>directly at the inference server</dd>
|
| 29 |
+
<dt>Overall</dt><dd class="strong">64.2</dd>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
+
<h2>Categories</h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
+
<thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
|
| 36 |
<tbody>
|
| 37 |
+
<tr><td>Knowledge & reasoning</td><td class="num">69.9</td></tr>
|
| 38 |
+
<tr><td>Math</td><td class="num">60.4</td></tr>
|
| 39 |
+
<tr><td>Instruction following</td><td class="num">49.5</td></tr>
|
| 40 |
<tr><td>Code</td><td class="num">65.0</td></tr>
|
| 41 |
+
<tr><td>German</td><td class="num">76.3</td></tr>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
+
<thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
+
<td>Knowledge & reasoning</td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">80.3</td>
|
| 55 |
<td class="num">± 2.3</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
+
<td>Knowledge & reasoning</td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">63.6</td>
|
| 62 |
<td class="num">± 3.4</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
+
<td>Knowledge & reasoning</td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">65.6</td>
|
| 69 |
<td class="num">± 3.0</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
+
<td>Math</td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">60.4</td>
|
| 76 |
<td class="num">± 3.1</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
+
<td>Instruction following</td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">38.3</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
+
<td>Instruction following</td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">83.3</td>
|
| 90 |
<td class="num">± 3.1</td>
|
|
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
+
<td>German</td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">79.0</td>
|
| 104 |
<td class="num">± 2.4</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
+
<td>German</td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">70.8</td>
|
| 111 |
<td class="num">± 3.7</td>
|
|
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
+
<p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
|
| 118 |
|
| 119 |
+
<h2>Speed</h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
+
<thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
+
<td>Generation [t/s]</td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">15.1</td>
|
|
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
+
<td>Prompt processing [t/s]</td>
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">1836.6</td>
|
|
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
+
<td>Prompt processing [t/s]</td>
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4024</td>
|
| 146 |
<td class="num">2550.6</td>
|
|
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
+
<td>Prompt processing [t/s]</td>
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">16001</td>
|
| 155 |
<td class="num">1734.3</td>
|
|
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
+
<td>Time to first token [ms]</td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">288.0</td>
|
|
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
+
<td>Time to first token [ms]</td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4024</td>
|
| 173 |
<td class="num">1577.7</td>
|
|
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
+
<td>Time to first token [ms]</td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">16001</td>
|
| 182 |
<td class="num">9226.4</td>
|
|
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
+
<p class="hint">Measured directly at the inference server.</p>
|
| 191 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
runs/21.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html lang="
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
@@ -10,35 +10,35 @@
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
-
<a href="../index.html">
|
| 14 |
-
<a href="../
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
-
<p class="sub"><a href="../index.html#v3-quick">←
|
| 19 |
<h1>gemma-4-12b-it</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
-
<dt>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
-
<dt>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
-
<dt>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
-
<dt>
|
| 28 |
-
<dt>
|
| 29 |
-
<dt>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
-
<h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
-
<thead><tr><th>
|
| 36 |
<tbody>
|
| 37 |
-
<tr><td>
|
| 38 |
-
<tr><td>
|
| 39 |
-
<tr><td>
|
| 40 |
<tr><td>Code</td><td class="num">75.0</td></tr>
|
| 41 |
-
<tr><td>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
@@ -46,45 +46,45 @@
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
-
<thead><tr><th>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
-
<td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">79.3</td>
|
| 55 |
<td class="num">± 2.3</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
-
<td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">63.1</td>
|
| 62 |
<td class="num">± 3.4</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
-
<td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">60.0</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
-
<td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">60.8</td>
|
| 76 |
<td class="num">± 3.1</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
-
<td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">38.3</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
-
<td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">90.7</td>
|
| 90 |
<td class="num">± 2.4</td>
|
|
@@ -98,14 +98,14 @@
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
-
<td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">78.3</td>
|
| 104 |
<td class="num">± 2.4</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
-
<td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">66.9</td>
|
| 111 |
<td class="num">± 3.8</td>
|
|
@@ -114,15 +114,15 @@
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
-
<p class="hint">±
|
| 118 |
|
| 119 |
-
<h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
-
<thead><tr><th>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
-
<td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">16.5</td>
|
|
@@ -131,7 +131,7 @@
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
-
<td>Prompt
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">851.2</td>
|
|
@@ -140,7 +140,7 @@
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
-
<td>Prompt
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4012</td>
|
| 146 |
<td class="num">1359.0</td>
|
|
@@ -149,7 +149,7 @@
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
-
<td>Prompt
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">15971</td>
|
| 155 |
<td class="num">1408.5</td>
|
|
@@ -158,7 +158,7 @@
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
-
<td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">621.5</td>
|
|
@@ -167,7 +167,7 @@
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
-
<td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4012</td>
|
| 173 |
<td class="num">2952.1</td>
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
-
<td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">15971</td>
|
| 182 |
<td class="num">11338.7</td>
|
|
@@ -187,8 +187,8 @@
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
-
<p class="hint">
|
| 191 |
-
<footer class="page-foot">
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
+
<a href="../index.html">Overview</a>
|
| 14 |
+
<a href="../methodology.html">Methodology</a>
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
+
<p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
|
| 19 |
<h1>gemma-4-12b-it</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
+
<dt>Quantization</dt><dd>Q8</dd>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
+
<dt>Profile</dt><dd>quick</dd>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
+
<dt>Date</dt><dd>2026-09-18 – 2026-09-19</dd>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
+
<dt>Inference software</dt><dd>vLLM 0.26.0</dd>
|
| 28 |
+
<dt>Measured</dt><dd>directly at the inference server</dd>
|
| 29 |
+
<dt>Overall</dt><dd class="strong">65.8</dd>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
+
<h2>Categories</h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
+
<thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
|
| 36 |
<tbody>
|
| 37 |
+
<tr><td>Knowledge & reasoning</td><td class="num">67.5</td></tr>
|
| 38 |
+
<tr><td>Math</td><td class="num">60.8</td></tr>
|
| 39 |
+
<tr><td>Instruction following</td><td class="num">51.3</td></tr>
|
| 40 |
<tr><td>Code</td><td class="num">75.0</td></tr>
|
| 41 |
+
<tr><td>German</td><td class="num">74.5</td></tr>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
+
<thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
+
<td>Knowledge & reasoning</td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">79.3</td>
|
| 55 |
<td class="num">± 2.3</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
+
<td>Knowledge & reasoning</td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">63.1</td>
|
| 62 |
<td class="num">± 3.4</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
+
<td>Knowledge & reasoning</td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">60.0</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
+
<td>Math</td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">60.8</td>
|
| 76 |
<td class="num">± 3.1</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
+
<td>Instruction following</td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">38.3</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
+
<td>Instruction following</td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">90.7</td>
|
| 90 |
<td class="num">± 2.4</td>
|
|
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
+
<td>German</td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">78.3</td>
|
| 104 |
<td class="num">± 2.4</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
+
<td>German</td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">66.9</td>
|
| 111 |
<td class="num">± 3.8</td>
|
|
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
+
<p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
|
| 118 |
|
| 119 |
+
<h2>Speed</h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
+
<thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
+
<td>Generation [t/s]</td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">16.5</td>
|
|
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
+
<td>Prompt processing [t/s]</td>
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">851.2</td>
|
|
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
+
<td>Prompt processing [t/s]</td>
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4012</td>
|
| 146 |
<td class="num">1359.0</td>
|
|
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
+
<td>Prompt processing [t/s]</td>
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">15971</td>
|
| 155 |
<td class="num">1408.5</td>
|
|
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
+
<td>Time to first token [ms]</td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">621.5</td>
|
|
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
+
<td>Time to first token [ms]</td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4012</td>
|
| 173 |
<td class="num">2952.1</td>
|
|
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
+
<td>Time to first token [ms]</td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">15971</td>
|
| 182 |
<td class="num">11338.7</td>
|
|
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
+
<p class="hint">Measured directly at the inference server.</p>
|
| 191 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
runs/22.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html lang="
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
@@ -10,35 +10,35 @@
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
-
<a href="../index.html">
|
| 14 |
-
<a href="../
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
-
<p class="sub"><a href="../index.html#v3-quick">←
|
| 19 |
<h1>gemma-4-12b-it-heretic</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
-
<dt>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
-
<dt>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
-
<dt>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
-
<dt>
|
| 28 |
-
<dt>
|
| 29 |
-
<dt>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
-
<h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
-
<thead><tr><th>
|
| 36 |
<tbody>
|
| 37 |
-
<tr><td>
|
| 38 |
-
<tr><td>
|
| 39 |
-
<tr><td>
|
| 40 |
<tr><td>Code</td><td class="num">75.0</td></tr>
|
| 41 |
-
<tr><td>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
@@ -46,45 +46,45 @@
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
-
<thead><tr><th>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
-
<td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">80.0</td>
|
| 55 |
<td class="num">± 2.3</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
-
<td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">64.1</td>
|
| 62 |
<td class="num">± 3.4</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
-
<td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">61.2</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
-
<td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">59.2</td>
|
| 76 |
<td class="num">± 3.1</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
-
<td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">35.7</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
-
<td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">93.3</td>
|
| 90 |
<td class="num">± 2.0</td>
|
|
@@ -98,14 +98,14 @@
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
-
<td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">80.0</td>
|
| 104 |
<td class="num">± 2.3</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
-
<td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">69.5</td>
|
| 111 |
<td class="num">± 3.7</td>
|
|
@@ -114,15 +114,15 @@
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
-
<p class="hint">±
|
| 118 |
|
| 119 |
-
<h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
-
<thead><tr><th>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
-
<td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">16.5</td>
|
|
@@ -131,7 +131,7 @@
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
-
<td>Prompt
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">854.0</td>
|
|
@@ -140,7 +140,7 @@
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
-
<td>Prompt
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4012</td>
|
| 146 |
<td class="num">1358.1</td>
|
|
@@ -149,7 +149,7 @@
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
-
<td>Prompt
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">15971</td>
|
| 155 |
<td class="num">1386.1</td>
|
|
@@ -158,7 +158,7 @@
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
-
<td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">619.5</td>
|
|
@@ -167,7 +167,7 @@
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
-
<td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4012</td>
|
| 173 |
<td class="num">2954.1</td>
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
-
<td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">15971</td>
|
| 182 |
<td class="num">11521.9</td>
|
|
@@ -187,8 +187,8 @@
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
-
<p class="hint">
|
| 191 |
-
<footer class="page-foot">
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
|
| 10 |
<header class="top">
|
| 11 |
<a class="brand" href="../index.html">Local LLM Benchmarks</a>
|
| 12 |
<nav>
|
| 13 |
+
<a href="../index.html">Overview</a>
|
| 14 |
+
<a href="../methodology.html">Methodology</a>
|
| 15 |
</nav>
|
| 16 |
</header>
|
| 17 |
<main>
|
| 18 |
+
<p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
|
| 19 |
<h1>gemma-4-12b-it-heretic</h1>
|
| 20 |
<dl class="meta">
|
| 21 |
+
<dt>Quantization</dt><dd>Q8</dd>
|
| 22 |
<dt>Reasoning</dt><dd>off</dd>
|
| 23 |
+
<dt>Profile</dt><dd>quick</dd>
|
| 24 |
<dt>Suite</dt><dd>v2b</dd>
|
| 25 |
+
<dt>Date</dt><dd>2026-09-19</dd>
|
| 26 |
<dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
|
| 27 |
+
<dt>Inference software</dt><dd>vLLM 0.26.0</dd>
|
| 28 |
+
<dt>Measured</dt><dd>directly at the inference server</dd>
|
| 29 |
+
<dt>Overall</dt><dd class="strong">65.8</dd>
|
| 30 |
</dl>
|
| 31 |
|
| 32 |
+
<h2>Categories</h2>
|
| 33 |
<div class="table-wrap">
|
| 34 |
<table>
|
| 35 |
+
<thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
|
| 36 |
<tbody>
|
| 37 |
+
<tr><td>Knowledge & reasoning</td><td class="num">68.4</td></tr>
|
| 38 |
+
<tr><td>Math</td><td class="num">59.2</td></tr>
|
| 39 |
+
<tr><td>Instruction following</td><td class="num">50.0</td></tr>
|
| 40 |
<tr><td>Code</td><td class="num">75.0</td></tr>
|
| 41 |
+
<tr><td>German</td><td class="num">76.5</td></tr>
|
| 42 |
</tbody>
|
| 43 |
</table>
|
| 44 |
</div>
|
|
|
|
| 46 |
<h2>Tasks</h2>
|
| 47 |
<div class="table-wrap">
|
| 48 |
<table>
|
| 49 |
+
<thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
|
| 50 |
<tbody>
|
| 51 |
<tr>
|
| 52 |
+
<td>Knowledge & reasoning</td>
|
| 53 |
<td>mmlu_pro</td>
|
| 54 |
<td class="num">80.0</td>
|
| 55 |
<td class="num">± 2.3</td>
|
| 56 |
<td class="num">300</td>
|
| 57 |
</tr>
|
| 58 |
<tr>
|
| 59 |
+
<td>Knowledge & reasoning</td>
|
| 60 |
<td>gpqa_diamond_gen</td>
|
| 61 |
<td class="num">64.1</td>
|
| 62 |
<td class="num">± 3.4</td>
|
| 63 |
<td class="num">198</td>
|
| 64 |
</tr>
|
| 65 |
<tr>
|
| 66 |
+
<td>Knowledge & reasoning</td>
|
| 67 |
<td>musr_gen</td>
|
| 68 |
<td class="num">61.2</td>
|
| 69 |
<td class="num">± 3.1</td>
|
| 70 |
<td class="num">250</td>
|
| 71 |
</tr>
|
| 72 |
<tr>
|
| 73 |
+
<td>Math</td>
|
| 74 |
<td>omni_math_gen</td>
|
| 75 |
<td class="num">59.2</td>
|
| 76 |
<td class="num">± 3.1</td>
|
| 77 |
<td class="num">250</td>
|
| 78 |
</tr>
|
| 79 |
<tr>
|
| 80 |
+
<td>Instruction following</td>
|
| 81 |
<td>ifbench</td>
|
| 82 |
<td class="num">35.7</td>
|
| 83 |
<td class="num">± 2.8</td>
|
| 84 |
<td class="num">300</td>
|
| 85 |
</tr>
|
| 86 |
<tr>
|
| 87 |
+
<td>Instruction following</td>
|
| 88 |
<td>ifeval</td>
|
| 89 |
<td class="num">93.3</td>
|
| 90 |
<td class="num">± 2.0</td>
|
|
|
|
| 98 |
<td class="num">200</td>
|
| 99 |
</tr>
|
| 100 |
<tr>
|
| 101 |
+
<td>German</td>
|
| 102 |
<td>mmlu_prox_de</td>
|
| 103 |
<td class="num">80.0</td>
|
| 104 |
<td class="num">± 2.3</td>
|
| 105 |
<td class="num">300</td>
|
| 106 |
</tr>
|
| 107 |
<tr>
|
| 108 |
+
<td>German</td>
|
| 109 |
<td>include_de_gen</td>
|
| 110 |
<td class="num">69.5</td>
|
| 111 |
<td class="num">± 3.7</td>
|
|
|
|
| 114 |
</tbody>
|
| 115 |
</table>
|
| 116 |
</div>
|
| 117 |
+
<p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
|
| 118 |
|
| 119 |
+
<h2>Speed</h2>
|
| 120 |
<div class="table-wrap">
|
| 121 |
<table>
|
| 122 |
+
<thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
|
| 123 |
<tbody>
|
| 124 |
<tr>
|
| 125 |
+
<td>Generation [t/s]</td>
|
| 126 |
<td class="num">–</td>
|
| 127 |
<td class="num">–</td>
|
| 128 |
<td class="num">16.5</td>
|
|
|
|
| 131 |
<td class="num">5</td>
|
| 132 |
</tr>
|
| 133 |
<tr>
|
| 134 |
+
<td>Prompt processing [t/s]</td>
|
| 135 |
<td class="num">512</td>
|
| 136 |
<td class="num">529</td>
|
| 137 |
<td class="num">854.0</td>
|
|
|
|
| 140 |
<td class="num">5</td>
|
| 141 |
</tr>
|
| 142 |
<tr>
|
| 143 |
+
<td>Prompt processing [t/s]</td>
|
| 144 |
<td class="num">4096</td>
|
| 145 |
<td class="num">4012</td>
|
| 146 |
<td class="num">1358.1</td>
|
|
|
|
| 149 |
<td class="num">5</td>
|
| 150 |
</tr>
|
| 151 |
<tr>
|
| 152 |
+
<td>Prompt processing [t/s]</td>
|
| 153 |
<td class="num">16384</td>
|
| 154 |
<td class="num">15971</td>
|
| 155 |
<td class="num">1386.1</td>
|
|
|
|
| 158 |
<td class="num">5</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
+
<td>Time to first token [ms]</td>
|
| 162 |
<td class="num">512</td>
|
| 163 |
<td class="num">529</td>
|
| 164 |
<td class="num">619.5</td>
|
|
|
|
| 167 |
<td class="num">5</td>
|
| 168 |
</tr>
|
| 169 |
<tr>
|
| 170 |
+
<td>Time to first token [ms]</td>
|
| 171 |
<td class="num">4096</td>
|
| 172 |
<td class="num">4012</td>
|
| 173 |
<td class="num">2954.1</td>
|
|
|
|
| 176 |
<td class="num">5</td>
|
| 177 |
</tr>
|
| 178 |
<tr>
|
| 179 |
+
<td>Time to first token [ms]</td>
|
| 180 |
<td class="num">16384</td>
|
| 181 |
<td class="num">15971</td>
|
| 182 |
<td class="num">11521.9</td>
|
|
|
|
| 187 |
</tbody>
|
| 188 |
</table>
|
| 189 |
</div>
|
| 190 |
+
<p class="hint">Measured directly at the inference server.</p>
|
| 191 |
+
<footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
|
| 192 |
</main>
|
| 193 |
<script src="../app.js"></script>
|
| 194 |
</body>
|