Kaleto commited on
Commit
b4891e7
·
verified ·
1 Parent(s): 34e99ea

Ergebnisse aktualisiert (9 Läufe)

Browse files
Files changed (13) hide show
  1. app.js +21 -0
  2. index.html +48 -11
  3. methodology.html +1 -1
  4. runs/15.html +1 -1
  5. runs/16.html +1 -1
  6. runs/17.html +1 -1
  7. runs/18.html +133 -0
  8. runs/19.html +1 -1
  9. runs/20.html +1 -1
  10. runs/21.html +1 -1
  11. runs/22.html +1 -1
  12. runs/23.html +195 -0
  13. style.css +7 -5
app.js CHANGED
@@ -33,6 +33,27 @@
33
  });
34
  });
35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  // Läufe für den Vergleich auswählen (mindestens 2).
37
  var picks = document.querySelectorAll(".compare-pick");
38
  var button = document.getElementById("compare-button");
 
33
  });
34
  });
35
 
36
+ // Zeilen nach Reasoning-Modus filtern. Die Schalter stehen nur im HTML, wenn eine Tabelle mehrere Modi
37
+ // zeigt; ohne JavaScript bleiben alle Zeilen sichtbar.
38
+ document.querySelectorAll("[data-filter-table]").forEach(function (group) {
39
+ var table = document.getElementById(group.dataset.filterTable);
40
+ if (!table || !table.tBodies[0]) return;
41
+ var buttons = Array.prototype.slice.call(group.querySelectorAll("[data-reasoning-filter]"));
42
+ buttons.forEach(function (button) {
43
+ button.addEventListener("click", function () {
44
+ var wanted = button.dataset.reasoningFilter;
45
+ buttons.forEach(function (other) {
46
+ var picked = other === button;
47
+ other.classList.toggle("active", picked);
48
+ other.setAttribute("aria-pressed", picked ? "true" : "false");
49
+ });
50
+ Array.prototype.slice.call(table.tBodies[0].rows).forEach(function (row) {
51
+ row.hidden = wanted !== "all" && (row.dataset.reasoning || "default") !== wanted;
52
+ });
53
+ });
54
+ });
55
+ });
56
+
57
  // Läufe für den Vergleich auswählen (mindestens 2).
58
  var picks = document.querySelectorAll(".compare-pick");
59
  var button = document.getElementById("compare-button");
index.html CHANGED
@@ -19,15 +19,36 @@
19
  <p class="lead">Local language models compared: knowledge, math, code and German, plus speed. Each model variant is shown with its most recent completed run. How the numbers are measured is described under <a href="methodology.html">Methodology</a>.</p>
20
  <section id="v3-quick">
21
  <h2>Suite v2b · profile quick</h2>
 
 
 
 
 
22
  <div class="table-wrap">
23
- <table class="sortable">
24
  <thead><tr>
25
  <th data-sort="text">Model</th>
26
  <th data-sort="num" class="num">Overall</th>
27
  <th data-sort="num" class="num">Knowledge &amp; reasoning</th><th data-sort="num" class="num">Math</th><th data-sort="num" class="num">Instruction following</th><th data-sort="num" class="num">Code</th><th data-sort="num" class="num">German</th><th data-sort="num" class="num">gen t/s</th><th data-sort="num" class="num">pp t/s 4k</th><th data-sort="num" class="num">TTFT 4k ms</th> <th data-sort="text">Date</th>
28
  </tr></thead>
29
  <tbody>
30
- <tr>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  <td data-value="qwen3.8-flash-next-ple4">
32
  <a href="runs/15.html" class="strong">qwen3.8-flash-next-ple4</a>
33
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
@@ -36,14 +57,14 @@
36
  <td class="num strong" data-value="77.86868686868686" title="mmlu_pro: 86.3 ± 2.0 (n=300)&#10;gpqa_diamond_gen: 77.3 ± 3.0 (n=198)&#10;musr_gen: 70.0 ± 2.9 (n=250)&#10;">77.9</td>
37
  <td class="num strong" data-value="73.2" title="omni_math_gen: 73.2 ± 2.8 (n=250)&#10;">73.2</td>
38
  <td class="num" data-value="49.56390977443609" title="ifbench: 36.0 ± 2.8 (n=300)&#10;ifeval: 90.7 ± 2.4 (n=150)&#10;">49.6</td>
39
- <td class="num strong" data-value="75.5" title="livecodebench_gen: 75.5 ± 3.0 (n=200)&#10;">75.5</td>
40
  <td class="num strong" data-value="81.13708513708514" title="mmlu_prox_de: 85.7 ± 2.0 (n=300)&#10;include_de_gen: 72.1 ± 3.6 (n=154)&#10;">81.1</td>
41
  <td class="num" data-value="26.376464212349354">26</td>
42
  <td class="num" data-value="2041.4640201348918">2041</td>
43
  <td class="num" data-value="1973.583644023165">1974</td>
44
  <td data-value="2026-09-17">2026-09-17</td>
45
  </tr>
46
- <tr>
47
  <td data-value="gemma-4-12b-it">
48
  <a href="runs/21.html" class="strong">gemma-4-12b-it</a>
49
  <div class="sub">Q8 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
@@ -51,7 +72,7 @@
51
  <td class="num strong" data-value="65.82462114883167">65.8</td>
52
  <td class="num" data-value="67.48821548821549" title="mmlu_pro: 79.3 ± 2.3 (n=300)&#10;gpqa_diamond_gen: 63.1 ± 3.4 (n=198)&#10;musr_gen: 60.0 ± 3.1 (n=250)&#10;">67.5</td>
53
  <td class="num" data-value="60.8" title="omni_math_gen: 60.8 ± 3.1 (n=250)&#10;">60.8</td>
54
- <td class="num strong" data-value="51.318295739348365" title="ifbench: 38.3 ± 2.8 (n=300)&#10;ifeval: 90.7 ± 2.4 (n=150)&#10;">51.3</td>
55
  <td class="num" data-value="75.0" title="livecodebench_gen: 75.0 ± 3.1 (n=200)&#10;">75.0</td>
56
  <td class="num" data-value="74.51659451659451" title="mmlu_prox_de: 78.3 ± 2.4 (n=300)&#10;include_de_gen: 66.9 ± 3.8 (n=154)&#10;">74.5</td>
57
  <td class="num" data-value="16.51787775562755">17</td>
@@ -59,7 +80,7 @@
59
  <td class="num" data-value="2952.064286917448">2952</td>
60
  <td data-value="2026-09-18">2026-09-18</td>
61
  </tr>
62
- <tr>
63
  <td data-value="gemma-4-12b-it-heretic">
64
  <a href="runs/22.html" class="strong">gemma-4-12b-it-heretic</a>
65
  <div class="sub">Q8 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
@@ -75,7 +96,7 @@
75
  <td class="num" data-value="2954.110997961834">2954</td>
76
  <td data-value="2026-09-19">2026-09-19</td>
77
  </tr>
78
- <tr>
79
  <td data-value="qwen3.8-27b-mtp">
80
  <a href="runs/20.html" class="strong">qwen3.8-27b-mtp</a>
81
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
@@ -91,7 +112,23 @@
91
  <td class="num" data-value="1577.6915270835161">1578</td>
92
  <td data-value="2026-09-18">2026-09-18</td>
93
  </tr>
94
- <tr>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  <td data-value="gemma-4-12b-it-heretic">
96
  <a href="runs/17.html" class="strong">gemma-4-12b-it-heretic</a>
97
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
@@ -107,7 +144,7 @@
107
  <td class="num" data-value="763.404164230451">763</td>
108
  <td data-value="2026-09-18">2026-09-18</td>
109
  </tr>
110
- <tr>
111
  <td data-value="nemotron-3.5-lightning-dspark">
112
  <a href="runs/16.html" class="strong">nemotron-3.5-lightning-dspark</a>
113
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
@@ -123,7 +160,7 @@
123
  <td class="num" data-value="554.1085170116276">554</td>
124
  <td data-value="2026-09-17">2026-09-17</td>
125
  </tr>
126
- <tr>
127
  <td data-value="qwen3.5-9b-heretic-v2-mtp">
128
  <a href="runs/19.html" class="strong">qwen3.5-9b-heretic-v2-mtp</a>
129
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
@@ -144,7 +181,7 @@
144
  </div>
145
  </section>
146
  <p class="hint">Scores run from 0 to 100; the best value per category is shown in bold. Hovering a category cell lists its tasks with standard error (±) and number of items (n). Speed: generation in tokens/s, prompt processing at 4k context in tokens/s, time to first token at 4k context in ms. Click a column header to sort.</p>
147
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
148
  </main>
149
  <script src="app.js"></script>
150
  </body>
 
19
  <p class="lead">Local language models compared: knowledge, math, code and German, plus speed. Each model variant is shown with its most recent completed run. How the numbers are measured is described under <a href="methodology.html">Methodology</a>.</p>
20
  <section id="v3-quick">
21
  <h2>Suite v2b · profile quick</h2>
22
+ <p class="segmented" role="group" aria-label="Filter by reasoning mode" data-filter-table="table-v3-quick">
23
+ <button type="button" class="active" aria-pressed="true" data-reasoning-filter="all">All runs</button>
24
+ <button type="button" aria-pressed="false" data-reasoning-filter="off">Reasoning off</button>
25
+ <button type="button" aria-pressed="false" data-reasoning-filter="on">Reasoning on</button>
26
+ </p>
27
  <div class="table-wrap">
28
+ <table class="sortable" id="table-v3-quick">
29
  <thead><tr>
30
  <th data-sort="text">Model</th>
31
  <th data-sort="num" class="num">Overall</th>
32
  <th data-sort="num" class="num">Knowledge &amp; reasoning</th><th data-sort="num" class="num">Math</th><th data-sort="num" class="num">Instruction following</th><th data-sort="num" class="num">Code</th><th data-sort="num" class="num">German</th><th data-sort="num" class="num">gen t/s</th><th data-sort="num" class="num">pp t/s 4k</th><th data-sort="num" class="num">TTFT 4k ms</th> <th data-sort="text">Date</th>
33
  </tr></thead>
34
  <tbody>
35
+ <tr data-reasoning="on">
36
+ <td data-value="gpt-oss-120b">
37
+ <a href="runs/18.html" class="strong">gpt-oss-120b</a>
38
+ <div class="sub">MXFP4 · Reasoning on · direct · NVIDIA DGX Spark, 128 GB RAM</div>
39
+ </td>
40
+ <td class="num strong" data-value="73.08253056884635">73.1</td>
41
+ <td class="num" data-value="68.52727272727272" title="mmlu_pro: 71.0 ± 2.6 (n=300)&#10;gpqa_diamond_gen: 68.2 ± 3.3 (n=198)&#10;musr_gen: 66.4 ± 3.0 (n=250)&#10;">68.5</td>
42
+ <td class="num" data-value="70.39999999999999" title="omni_math_gen: 70.4 ± 2.9 (n=250)&#10;">70.4</td>
43
+ <td class="num strong" data-value="66.1203007518797" title="ifbench: 60.0 ± 2.8 (n=300)&#10;ifeval: 84.7 ± 3.0 (n=150)&#10;">66.1</td>
44
+ <td class="num strong" data-value="83.0" title="livecodebench_gen: 83.0 ± 2.7 (n=200)&#10;">83.0</td>
45
+ <td class="num" data-value="77.36507936507935" title="mmlu_prox_de: 80.3 ± 2.3 (n=300)&#10;include_de_gen: 71.4 ± 3.7 (n=154)&#10;">77.4</td>
46
+ <td class="num" data-value="">–</td>
47
+ <td class="num" data-value="">–</td>
48
+ <td class="num" data-value="">–</td>
49
+ <td data-value="2026-09-18">2026-09-18</td>
50
+ </tr>
51
+ <tr data-reasoning="off">
52
  <td data-value="qwen3.8-flash-next-ple4">
53
  <a href="runs/15.html" class="strong">qwen3.8-flash-next-ple4</a>
54
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
 
57
  <td class="num strong" data-value="77.86868686868686" title="mmlu_pro: 86.3 ± 2.0 (n=300)&#10;gpqa_diamond_gen: 77.3 ± 3.0 (n=198)&#10;musr_gen: 70.0 ± 2.9 (n=250)&#10;">77.9</td>
58
  <td class="num strong" data-value="73.2" title="omni_math_gen: 73.2 ± 2.8 (n=250)&#10;">73.2</td>
59
  <td class="num" data-value="49.56390977443609" title="ifbench: 36.0 ± 2.8 (n=300)&#10;ifeval: 90.7 ± 2.4 (n=150)&#10;">49.6</td>
60
+ <td class="num" data-value="75.5" title="livecodebench_gen: 75.5 ± 3.0 (n=200)&#10;">75.5</td>
61
  <td class="num strong" data-value="81.13708513708514" title="mmlu_prox_de: 85.7 ± 2.0 (n=300)&#10;include_de_gen: 72.1 ± 3.6 (n=154)&#10;">81.1</td>
62
  <td class="num" data-value="26.376464212349354">26</td>
63
  <td class="num" data-value="2041.4640201348918">2041</td>
64
  <td class="num" data-value="1973.583644023165">1974</td>
65
  <td data-value="2026-09-17">2026-09-17</td>
66
  </tr>
67
+ <tr data-reasoning="off">
68
  <td data-value="gemma-4-12b-it">
69
  <a href="runs/21.html" class="strong">gemma-4-12b-it</a>
70
  <div class="sub">Q8 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
 
72
  <td class="num strong" data-value="65.82462114883167">65.8</td>
73
  <td class="num" data-value="67.48821548821549" title="mmlu_pro: 79.3 ± 2.3 (n=300)&#10;gpqa_diamond_gen: 63.1 ± 3.4 (n=198)&#10;musr_gen: 60.0 ± 3.1 (n=250)&#10;">67.5</td>
74
  <td class="num" data-value="60.8" title="omni_math_gen: 60.8 ± 3.1 (n=250)&#10;">60.8</td>
75
+ <td class="num" data-value="51.318295739348365" title="ifbench: 38.3 ± 2.8 (n=300)&#10;ifeval: 90.7 ± 2.4 (n=150)&#10;">51.3</td>
76
  <td class="num" data-value="75.0" title="livecodebench_gen: 75.0 ± 3.1 (n=200)&#10;">75.0</td>
77
  <td class="num" data-value="74.51659451659451" title="mmlu_prox_de: 78.3 ± 2.4 (n=300)&#10;include_de_gen: 66.9 ± 3.8 (n=154)&#10;">74.5</td>
78
  <td class="num" data-value="16.51787775562755">17</td>
 
80
  <td class="num" data-value="2952.064286917448">2952</td>
81
  <td data-value="2026-09-18">2026-09-18</td>
82
  </tr>
83
+ <tr data-reasoning="off">
84
  <td data-value="gemma-4-12b-it-heretic">
85
  <a href="runs/22.html" class="strong">gemma-4-12b-it-heretic</a>
86
  <div class="sub">Q8 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
 
96
  <td class="num" data-value="2954.110997961834">2954</td>
97
  <td data-value="2026-09-19">2026-09-19</td>
98
  </tr>
99
+ <tr data-reasoning="off">
100
  <td data-value="qwen3.8-27b-mtp">
101
  <a href="runs/20.html" class="strong">qwen3.8-27b-mtp</a>
102
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
 
112
  <td class="num" data-value="1577.6915270835161">1578</td>
113
  <td data-value="2026-09-18">2026-09-18</td>
114
  </tr>
115
+ <tr data-reasoning="off">
116
+ <td data-value="qwen3.8-27b-abliterated">
117
+ <a href="runs/23.html" class="strong">qwen3.8-27b-abliterated</a>
118
+ <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
119
+ </td>
120
+ <td class="num strong" data-value="61.428784587731954">61.4</td>
121
+ <td class="num" data-value="71.50235690235691" title="mmlu_pro: 81.0 ± 2.3 (n=300)&#10;gpqa_diamond_gen: 70.7 ± 3.2 (n=198)&#10;musr_gen: 62.8 ± 3.1 (n=250)&#10;">71.5</td>
122
+ <td class="num" data-value="57.99999999999999" title="omni_math_gen: 58.0 ± 3.1 (n=250)&#10;">58.0</td>
123
+ <td class="num" data-value="46.07518796992481" title="ifbench: 34.0 ± 2.7 (n=300)&#10;ifeval: 82.7 ± 3.1 (n=150)&#10;">46.1</td>
124
+ <td class="num" data-value="59.5" title="livecodebench_gen: 59.5 ± 3.5 (n=200)&#10;">59.5</td>
125
+ <td class="num" data-value="72.06637806637806" title="mmlu_prox_de: 74.3 ± 2.5 (n=300)&#10;include_de_gen: 67.5 ± 3.8 (n=154)&#10;">72.1</td>
126
+ <td class="num" data-value="15.903617023850842">16</td>
127
+ <td class="num" data-value="2472.0116600865786">2472</td>
128
+ <td class="num" data-value="1627.824036986567">1628</td>
129
+ <td data-value="2026-09-20">2026-09-20</td>
130
+ </tr>
131
+ <tr data-reasoning="off">
132
  <td data-value="gemma-4-12b-it-heretic">
133
  <a href="runs/17.html" class="strong">gemma-4-12b-it-heretic</a>
134
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
 
144
  <td class="num" data-value="763.404164230451">763</td>
145
  <td data-value="2026-09-18">2026-09-18</td>
146
  </tr>
147
+ <tr data-reasoning="off">
148
  <td data-value="nemotron-3.5-lightning-dspark">
149
  <a href="runs/16.html" class="strong">nemotron-3.5-lightning-dspark</a>
150
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
 
160
  <td class="num" data-value="554.1085170116276">554</td>
161
  <td data-value="2026-09-17">2026-09-17</td>
162
  </tr>
163
+ <tr data-reasoning="off">
164
  <td data-value="qwen3.5-9b-heretic-v2-mtp">
165
  <a href="runs/19.html" class="strong">qwen3.5-9b-heretic-v2-mtp</a>
166
  <div class="sub">NVFP4 · Reasoning off · direct · NVIDIA DGX Spark, 128 GB RAM</div>
 
181
  </div>
182
  </section>
183
  <p class="hint">Scores run from 0 to 100; the best value per category is shown in bold. Hovering a category cell lists its tasks with standard error (±) and number of items (n). Speed: generation in tokens/s, prompt processing at 4k context in tokens/s, time to first token at 4k context in ms. Click a column header to sort.</p>
184
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
185
  </main>
186
  <script src="app.js"></script>
187
  </body>
methodology.html CHANGED
@@ -117,7 +117,7 @@ on hardware and inference software.</p>
117
  </tbody>
118
  </table>
119
  </div>
120
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
121
  </main>
122
  <script src="app.js"></script>
123
  </body>
 
117
  </tbody>
118
  </table>
119
  </div>
120
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
121
  </main>
122
  <script src="app.js"></script>
123
  </body>
runs/15.html CHANGED
@@ -188,7 +188,7 @@
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
 
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
runs/16.html CHANGED
@@ -188,7 +188,7 @@
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
 
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
runs/17.html CHANGED
@@ -188,7 +188,7 @@
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
 
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
runs/18.html ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>gpt-oss-120b – Local LLM Benchmarks</title>
7
+ <link rel="stylesheet" href="../style.css">
8
+ </head>
9
+ <body>
10
+ <header class="top">
11
+ <a class="brand" href="../index.html">Local LLM Benchmarks</a>
12
+ <nav>
13
+ <a href="../index.html">Overview</a>
14
+ <a href="../methodology.html">Methodology</a>
15
+ </nav>
16
+ </header>
17
+ <main>
18
+ <p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
19
+ <h1>gpt-oss-120b</h1>
20
+ <dl class="meta">
21
+ <dt>Quantization</dt><dd>MXFP4</dd>
22
+ <dt>Reasoning</dt><dd>on</dd>
23
+ <dt>Profile</dt><dd>quick</dd>
24
+ <dt>Suite</dt><dd>v2b</dd>
25
+ <dt>Date</dt><dd>2026-09-18 – 2026-09-19</dd>
26
+ <dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
27
+ <dt>Inference software</dt><dd>vLLM 0.26.0</dd>
28
+ <dt>Measured</dt><dd>directly at the inference server</dd>
29
+ <dt>Overall</dt><dd class="strong">73.1</dd>
30
+ </dl>
31
+
32
+ <h2>Categories</h2>
33
+ <div class="table-wrap">
34
+ <table>
35
+ <thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
36
+ <tbody>
37
+ <tr><td>Knowledge &amp; reasoning</td><td class="num">68.5</td></tr>
38
+ <tr><td>Math</td><td class="num">70.4</td></tr>
39
+ <tr><td>Instruction following</td><td class="num">66.1</td></tr>
40
+ <tr><td>Code</td><td class="num">83.0</td></tr>
41
+ <tr><td>German</td><td class="num">77.4</td></tr>
42
+ </tbody>
43
+ </table>
44
+ </div>
45
+
46
+ <h2>Tasks</h2>
47
+ <div class="table-wrap">
48
+ <table>
49
+ <thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
50
+ <tbody>
51
+ <tr>
52
+ <td>Knowledge &amp; reasoning</td>
53
+ <td>mmlu_pro</td>
54
+ <td class="num">71.0</td>
55
+ <td class="num">± 2.6</td>
56
+ <td class="num">300</td>
57
+ </tr>
58
+ <tr>
59
+ <td>Knowledge &amp; reasoning</td>
60
+ <td>gpqa_diamond_gen</td>
61
+ <td class="num">68.2</td>
62
+ <td class="num">± 3.3</td>
63
+ <td class="num">198</td>
64
+ </tr>
65
+ <tr>
66
+ <td>Knowledge &amp; reasoning</td>
67
+ <td>musr_gen</td>
68
+ <td class="num">66.4</td>
69
+ <td class="num">± 3.0</td>
70
+ <td class="num">250</td>
71
+ </tr>
72
+ <tr>
73
+ <td>Math</td>
74
+ <td>omni_math_gen</td>
75
+ <td class="num">70.4</td>
76
+ <td class="num">± 2.9</td>
77
+ <td class="num">250</td>
78
+ </tr>
79
+ <tr>
80
+ <td>Instruction following</td>
81
+ <td>ifbench</td>
82
+ <td class="num">60.0</td>
83
+ <td class="num">± 2.8</td>
84
+ <td class="num">300</td>
85
+ </tr>
86
+ <tr>
87
+ <td>Instruction following</td>
88
+ <td>ifeval</td>
89
+ <td class="num">84.7</td>
90
+ <td class="num">± 3.0</td>
91
+ <td class="num">150</td>
92
+ </tr>
93
+ <tr>
94
+ <td>Code</td>
95
+ <td>livecodebench_gen</td>
96
+ <td class="num">83.0</td>
97
+ <td class="num">± 2.7</td>
98
+ <td class="num">200</td>
99
+ </tr>
100
+ <tr>
101
+ <td>German</td>
102
+ <td>mmlu_prox_de</td>
103
+ <td class="num">80.3</td>
104
+ <td class="num">± 2.3</td>
105
+ <td class="num">300</td>
106
+ </tr>
107
+ <tr>
108
+ <td>German</td>
109
+ <td>include_de_gen</td>
110
+ <td class="num">71.4</td>
111
+ <td class="num">± 3.7</td>
112
+ <td class="num">154</td>
113
+ </tr>
114
+ </tbody>
115
+ </table>
116
+ </div>
117
+ <p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
118
+
119
+ <h2>Speed</h2>
120
+ <div class="table-wrap">
121
+ <table>
122
+ <thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
123
+ <tbody>
124
+ <tr><td colspan="7" class="muted">No speed measurement.</td></tr>
125
+ </tbody>
126
+ </table>
127
+ </div>
128
+ <p class="hint">Measured directly at the inference server.</p>
129
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
130
+ </main>
131
+ <script src="../app.js"></script>
132
+ </body>
133
+ </html>
runs/19.html CHANGED
@@ -188,7 +188,7 @@
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
 
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
runs/20.html CHANGED
@@ -188,7 +188,7 @@
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
 
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
runs/21.html CHANGED
@@ -188,7 +188,7 @@
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
 
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
runs/22.html CHANGED
@@ -188,7 +188,7 @@
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
- <footer class="page-foot">As of 2026-09-20 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
 
188
  </table>
189
  </div>
190
  <p class="hint">Measured directly at the inference server.</p>
191
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
192
  </main>
193
  <script src="../app.js"></script>
194
  </body>
runs/23.html ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>qwen3.8-27b-abliterated – Local LLM Benchmarks</title>
7
+ <link rel="stylesheet" href="../style.css">
8
+ </head>
9
+ <body>
10
+ <header class="top">
11
+ <a class="brand" href="../index.html">Local LLM Benchmarks</a>
12
+ <nav>
13
+ <a href="../index.html">Overview</a>
14
+ <a href="../methodology.html">Methodology</a>
15
+ </nav>
16
+ </header>
17
+ <main>
18
+ <p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
19
+ <h1>qwen3.8-27b-abliterated</h1>
20
+ <dl class="meta">
21
+ <dt>Quantization</dt><dd>NVFP4</dd>
22
+ <dt>Reasoning</dt><dd>off</dd>
23
+ <dt>Profile</dt><dd>quick</dd>
24
+ <dt>Suite</dt><dd>v2b</dd>
25
+ <dt>Date</dt><dd>2026-09-20 – 2026-09-21</dd>
26
+ <dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
27
+ <dt>Inference software</dt><dd>vLLM 0.26.0</dd>
28
+ <dt>Measured</dt><dd>directly at the inference server</dd>
29
+ <dt>Overall</dt><dd class="strong">61.4</dd>
30
+ </dl>
31
+
32
+ <h2>Categories</h2>
33
+ <div class="table-wrap">
34
+ <table>
35
+ <thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
36
+ <tbody>
37
+ <tr><td>Knowledge &amp; reasoning</td><td class="num">71.5</td></tr>
38
+ <tr><td>Math</td><td class="num">58.0</td></tr>
39
+ <tr><td>Instruction following</td><td class="num">46.1</td></tr>
40
+ <tr><td>Code</td><td class="num">59.5</td></tr>
41
+ <tr><td>German</td><td class="num">72.1</td></tr>
42
+ </tbody>
43
+ </table>
44
+ </div>
45
+
46
+ <h2>Tasks</h2>
47
+ <div class="table-wrap">
48
+ <table>
49
+ <thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
50
+ <tbody>
51
+ <tr>
52
+ <td>Knowledge &amp; reasoning</td>
53
+ <td>mmlu_pro</td>
54
+ <td class="num">81.0</td>
55
+ <td class="num">± 2.3</td>
56
+ <td class="num">300</td>
57
+ </tr>
58
+ <tr>
59
+ <td>Knowledge &amp; reasoning</td>
60
+ <td>gpqa_diamond_gen</td>
61
+ <td class="num">70.7</td>
62
+ <td class="num">± 3.2</td>
63
+ <td class="num">198</td>
64
+ </tr>
65
+ <tr>
66
+ <td>Knowledge &amp; reasoning</td>
67
+ <td>musr_gen</td>
68
+ <td class="num">62.8</td>
69
+ <td class="num">± 3.1</td>
70
+ <td class="num">250</td>
71
+ </tr>
72
+ <tr>
73
+ <td>Math</td>
74
+ <td>omni_math_gen</td>
75
+ <td class="num">58.0</td>
76
+ <td class="num">± 3.1</td>
77
+ <td class="num">250</td>
78
+ </tr>
79
+ <tr>
80
+ <td>Instruction following</td>
81
+ <td>ifbench</td>
82
+ <td class="num">34.0</td>
83
+ <td class="num">± 2.7</td>
84
+ <td class="num">300</td>
85
+ </tr>
86
+ <tr>
87
+ <td>Instruction following</td>
88
+ <td>ifeval</td>
89
+ <td class="num">82.7</td>
90
+ <td class="num">± 3.1</td>
91
+ <td class="num">150</td>
92
+ </tr>
93
+ <tr>
94
+ <td>Code</td>
95
+ <td>livecodebench_gen</td>
96
+ <td class="num">59.5</td>
97
+ <td class="num">± 3.5</td>
98
+ <td class="num">200</td>
99
+ </tr>
100
+ <tr>
101
+ <td>German</td>
102
+ <td>mmlu_prox_de</td>
103
+ <td class="num">74.3</td>
104
+ <td class="num">± 2.5</td>
105
+ <td class="num">300</td>
106
+ </tr>
107
+ <tr>
108
+ <td>German</td>
109
+ <td>include_de_gen</td>
110
+ <td class="num">67.5</td>
111
+ <td class="num">± 3.8</td>
112
+ <td class="num">154</td>
113
+ </tr>
114
+ </tbody>
115
+ </table>
116
+ </div>
117
+ <p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>
118
+
119
+ <h2>Speed</h2>
120
+ <div class="table-wrap">
121
+ <table>
122
+ <thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
123
+ <tbody>
124
+ <tr>
125
+ <td>Generation [t/s]</td>
126
+ <td class="num">–</td>
127
+ <td class="num">–</td>
128
+ <td class="num">15.9</td>
129
+ <td class="num">15.1</td>
130
+ <td class="num">16.0</td>
131
+ <td class="num">5</td>
132
+ </tr>
133
+ <tr>
134
+ <td>Prompt processing [t/s]</td>
135
+ <td class="num">512</td>
136
+ <td class="num">529</td>
137
+ <td class="num">1805.5</td>
138
+ <td class="num">1685.0</td>
139
+ <td class="num">1818.2</td>
140
+ <td class="num">5</td>
141
+ </tr>
142
+ <tr>
143
+ <td>Prompt processing [t/s]</td>
144
+ <td class="num">4096</td>
145
+ <td class="num">4024</td>
146
+ <td class="num">2472.0</td>
147
+ <td class="num">2175.2</td>
148
+ <td class="num">2483.2</td>
149
+ <td class="num">5</td>
150
+ </tr>
151
+ <tr>
152
+ <td>Prompt processing [t/s]</td>
153
+ <td class="num">16384</td>
154
+ <td class="num">16001</td>
155
+ <td class="num">1685.4</td>
156
+ <td class="num">1489.9</td>
157
+ <td class="num">1688.7</td>
158
+ <td class="num">5</td>
159
+ </tr>
160
+ <tr>
161
+ <td>Time to first token [ms]</td>
162
+ <td class="num">512</td>
163
+ <td class="num">529</td>
164
+ <td class="num">293.0</td>
165
+ <td class="num">291.0</td>
166
+ <td class="num">314.3</td>
167
+ <td class="num">5</td>
168
+ </tr>
169
+ <tr>
170
+ <td>Time to first token [ms]</td>
171
+ <td class="num">4096</td>
172
+ <td class="num">4024</td>
173
+ <td class="num">1627.8</td>
174
+ <td class="num">1620.5</td>
175
+ <td class="num">1872.0</td>
176
+ <td class="num">5</td>
177
+ </tr>
178
+ <tr>
179
+ <td>Time to first token [ms]</td>
180
+ <td class="num">16384</td>
181
+ <td class="num">16001</td>
182
+ <td class="num">9493.9</td>
183
+ <td class="num">9475.4</td>
184
+ <td class="num">10858.2</td>
185
+ <td class="num">5</td>
186
+ </tr>
187
+ </tbody>
188
+ </table>
189
+ </div>
190
+ <p class="hint">Measured directly at the inference server.</p>
191
+ <footer class="page-foot">As of 2026-09-21 · scores 0–100 · built with <code>bench publish</code></footer>
192
+ </main>
193
+ <script src="../app.js"></script>
194
+ </body>
195
+ </html>
style.css CHANGED
@@ -70,11 +70,13 @@ h1 { font-size: 20px; margin: 8px 0 16px; overflow-wrap: anywhere; }
70
  h2 { font-size: 16px; margin: 24px 0 8px; }
71
  h3 { font-size: 13px; margin: 8px 0 4px; color: var(--ink-2); }
72
  .toolbar { display: flex; flex-wrap: wrap; gap: 12px; align-items: center; justify-content: space-between; margin: 12px 0; }
73
- .segmented a { display: inline-block; padding: 6px 12px; border: 1px solid var(--border); color: var(--ink-2); }
74
- .segmented a:hover { text-decoration: none; color: var(--ink); }
75
- .segmented a:first-child { border-radius: 6px 0 0 6px; }
76
- .segmented a:last-child { border-radius: 0 6px 6px 0; }
77
- .segmented a.active { background: var(--surface); color: var(--ink); font-weight: 600; }
 
 
78
 
79
  button { font: inherit; padding: 6px 12px; border-radius: 6px; border: 1px solid var(--border); background: var(--surface); color: var(--ink); cursor: pointer; }
80
  button:disabled { opacity: 0.5; cursor: default; }
 
70
  h2 { font-size: 16px; margin: 24px 0 8px; }
71
  h3 { font-size: 13px; margin: 8px 0 4px; color: var(--ink-2); }
72
  .toolbar { display: flex; flex-wrap: wrap; gap: 12px; align-items: center; justify-content: space-between; margin: 12px 0; }
73
+ .segmented a, .segmented button { display: inline-block; padding: 6px 12px; border: 1px solid var(--border); color: var(--ink-2); }
74
+ .segmented button { background: none; border-radius: 0; font: inherit; }
75
+ .segmented a:hover, .segmented button:hover { text-decoration: none; color: var(--ink); }
76
+ .segmented a:first-child, .segmented button:first-child { border-radius: 6px 0 0 6px; }
77
+ .segmented a:last-child, .segmented button:last-child { border-radius: 0 6px 6px 0; }
78
+ .segmented a.active, .segmented button.active { background: var(--surface); color: var(--ink); font-weight: 600; }
79
+ [hidden] { display: none !important; }
80
 
81
  button { font: inherit; padding: 6px 12px; border-radius: 6px; border: 1px solid var(--border); background: var(--surface); color: var(--ink); cursor: pointer; }
82
  button:disabled { opacity: 0.5; cursor: default; }