runs/19.html

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>qwen3.5-9b-heretic-v2-mtp – Local LLM Benchmarks</title>
<link rel="stylesheet" href="../style.css">
</head>
<body>
<header class="top">
  <a class="brand" href="../index.html">Local LLM Benchmarks</a>
  <nav>
    <a href="../index.html">Overview</a>
    <a href="../methodology.html">Methodology</a>
  </nav>
</header>
<main>
<p class="sub"><a href="../index.html#v3-quick">← Overview</a></p>
<h1>qwen3.5-9b-heretic-v2-mtp</h1>
<dl class="meta">
  <dt>Quantization</dt><dd>NVFP4</dd>
  <dt>Reasoning</dt><dd>off</dd>
  <dt>Profile</dt><dd>quick</dd>
  <dt>Suite</dt><dd>v2b</dd>
  <dt>Date</dt><dd>2026-09-18</dd>
  <dt>Hardware</dt><dd>NVIDIA DGX Spark, 128 GB RAM</dd>
  <dt>Inference software</dt><dd>vLLM 0.26.0</dd>
  <dt>Measured</dt><dd>directly at the inference server</dd>
  <dt>Overall</dt><dd class="strong">51.7</dd>
</dl>

<h2>Categories</h2>
<div class="table-wrap">
<table>
<thead><tr><th>Category</th><th class="num">Score</th></tr></thead>
<tbody>
<tr><td>Knowledge &amp; reasoning</td><td class="num">65.1</td></tr>
<tr><td>Math</td><td class="num">35.6</td></tr>
<tr><td>Instruction following</td><td class="num">44.1</td></tr>
<tr><td>Code</td><td class="num">46.5</td></tr>
<tr><td>German</td><td class="num">67.0</td></tr>
</tbody>
</table>
</div>

<h2>Tasks</h2>
<div class="table-wrap">
<table>
<thead><tr><th>Category</th><th>Task</th><th class="num">Score</th><th class="num">±</th><th class="num">n</th></tr></thead>
<tbody>
<tr>
  <td>Knowledge &amp; reasoning</td>
  <td>mmlu_pro</td>
  <td class="num">76.3</td>
  <td class="num">± 2.5</td>
  <td class="num">300</td>
</tr>
<tr>
  <td>Knowledge &amp; reasoning</td>
  <td>gpqa_diamond_gen</td>
  <td class="num">58.1</td>
  <td class="num">± 3.5</td>
  <td class="num">198</td>
</tr>
<tr>
  <td>Knowledge &amp; reasoning</td>
  <td>musr_gen</td>
  <td class="num">60.8</td>
  <td class="num">± 3.1</td>
  <td class="num">250</td>
</tr>
<tr>
  <td>Math</td>
  <td>omni_math_gen</td>
  <td class="num">35.6</td>
  <td class="num">± 3.0</td>
  <td class="num">250</td>
</tr>
<tr>
  <td>Instruction following</td>
  <td>ifbench</td>
  <td class="num">29.7</td>
  <td class="num">± 2.6</td>
  <td class="num">300</td>
</tr>
<tr>
  <td>Instruction following</td>
  <td>ifeval</td>
  <td class="num">88.0</td>
  <td class="num">± 2.7</td>
  <td class="num">150</td>
</tr>
<tr>
  <td>Code</td>
  <td>livecodebench_gen</td>
  <td class="num">46.5</td>
  <td class="num">± 3.5</td>
  <td class="num">200</td>
</tr>
<tr>
  <td>German</td>
  <td>mmlu_prox_de</td>
  <td class="num">69.3</td>
  <td class="num">± 2.7</td>
  <td class="num">300</td>
</tr>
<tr>
  <td>German</td>
  <td>include_de_gen</td>
  <td class="num">62.3</td>
  <td class="num">± 3.9</td>
  <td class="num">154</td>
</tr>
</tbody>
</table>
</div>
<p class="hint">± is the standard error; the 95% interval spans roughly 1.96 standard errors around the score.</p>

<h2>Speed</h2>
<div class="table-wrap">
<table>
<thead><tr><th>Measure</th><th class="num">Context (target)</th><th class="num">Context (actual)</th><th class="num">Median</th><th class="num">p10</th><th class="num">p90</th><th class="num">n</th></tr></thead>
<tbody>
<tr>
  <td>Generation [t/s]</td>
  <td class="num">–</td>
  <td class="num">–</td>
  <td class="num">31.5</td>
  <td class="num">31.0</td>
  <td class="num">32.0</td>
  <td class="num">5</td>
</tr>
<tr>
  <td>Prompt processing [t/s]</td>
  <td class="num">512</td>
  <td class="num">529</td>
  <td class="num">4737.2</td>
  <td class="num">4672.5</td>
  <td class="num">4758.4</td>
  <td class="num">5</td>
</tr>
<tr>
  <td>Prompt processing [t/s]</td>
  <td class="num">4096</td>
  <td class="num">4024</td>
  <td class="num">6290.9</td>
  <td class="num">6277.2</td>
  <td class="num">6307.3</td>
  <td class="num">5</td>
</tr>
<tr>
  <td>Prompt processing [t/s]</td>
  <td class="num">16384</td>
  <td class="num">16001</td>
  <td class="num">5096.4</td>
  <td class="num">5083.6</td>
  <td class="num">5104.6</td>
  <td class="num">5</td>
</tr>
<tr>
  <td>Time to first token [ms]</td>
  <td class="num">512</td>
  <td class="num">529</td>
  <td class="num">111.7</td>
  <td class="num">111.2</td>
  <td class="num">113.1</td>
  <td class="num">5</td>
</tr>
<tr>
  <td>Time to first token [ms]</td>
  <td class="num">4096</td>
  <td class="num">4024</td>
  <td class="num">639.7</td>
  <td class="num">638.0</td>
  <td class="num">641.1</td>
  <td class="num">5</td>
</tr>
<tr>
  <td>Time to first token [ms]</td>
  <td class="num">16384</td>
  <td class="num">16001</td>
  <td class="num">3139.6</td>
  <td class="num">3134.6</td>
  <td class="num">3147.6</td>
  <td class="num">5</td>
</tr>
</tbody>
</table>
</div>
<p class="hint">Measured directly at the inference server.</p>
<footer class="page-foot">As of 2026-10-10 · scores 0–100 · built with <code>bench publish</code></footer>
</main>
<script src="../chart.umd.min.js"></script>
<script src="../app.js"></script>
</body>
</html>