Model card: benchmark tables with models as rows
Browse files
README.md
CHANGED
|
@@ -29,6 +29,7 @@ time-to-final-output (TTFO) is about 2x shorter. At `reasoning_effort="low"` it
|
|
| 29 |
.tl-table td.tl strong, .tl-table td.tl { color:#F6F6F3; }
|
| 30 |
.tl-table strong { color:#F6F6F3; }
|
| 31 |
.tl-table .up { color:#8DA2F5; font-weight:700; }
|
|
|
|
| 32 |
.tl-table .avg td { border-top:2px solid #3E61EB; font-weight:700; }
|
| 33 |
@media (max-width: 640px) {
|
| 34 |
.tl-table { display:block !important; overflow-x:auto !important; table-layout:auto !important; }
|
|
@@ -90,17 +91,18 @@ tokenizer and `reasoning_effort` control.
|
|
| 90 |
|
| 91 |
<table class="tl-table">
|
| 92 |
<thead>
|
| 93 |
-
<tr><th
|
| 94 |
-
<tr><th class="tl">Thinklite-27B</th><th>ThinkingCap-Qwen3.8-27B</th><th>Swift-1.5-Qwen3.8-27b</th><th>Qwen3.8-27B</th><th class="tl">Thinklite-27B</th><th>ThinkingCap-Qwen3.8-27B</th><th>Swift-1.5-Qwen3.8-27b</th></tr>
|
| 95 |
</thead>
|
| 96 |
<tbody>
|
| 97 |
-
<tr><td class="
|
| 98 |
-
<tr><td class="name">
|
| 99 |
-
<tr><td class="name">
|
| 100 |
-
<tr><td class="name">
|
| 101 |
-
<tr><td class="name">
|
| 102 |
-
<tr><td class="
|
| 103 |
-
<tr class="
|
|
|
|
|
|
|
| 104 |
</tbody>
|
| 105 |
</table>
|
| 106 |
|
|
@@ -124,17 +126,18 @@ tokenizer and `reasoning_effort` control.
|
|
| 124 |
|
| 125 |
<table class="tl-table">
|
| 126 |
<thead>
|
| 127 |
-
<tr><th
|
| 128 |
-
<tr><th class="tl">Thinklite-27B</th><th>ThinkingCap-Qwen3.8-27B</th><th>Swift-1.5-Qwen3.8-27b</th><th>Qwen3.8-27B</th><th class="tl">Thinklite-27B</th><th>ThinkingCap-Qwen3.8-27B</th><th>Swift-1.5-Qwen3.8-27b</th></tr>
|
| 129 |
</thead>
|
| 130 |
<tbody>
|
| 131 |
-
<tr><td class="
|
| 132 |
-
<tr><td class="name">
|
| 133 |
-
<tr><td class="name">
|
| 134 |
-
<tr><td class="name">
|
| 135 |
-
<tr><td class="name">
|
| 136 |
-
<tr><td class="
|
| 137 |
-
<tr class="
|
|
|
|
|
|
|
| 138 |
</tbody>
|
| 139 |
</table>
|
| 140 |
|
|
|
|
| 29 |
.tl-table td.tl strong, .tl-table td.tl { color:#F6F6F3; }
|
| 30 |
.tl-table strong { color:#F6F6F3; }
|
| 31 |
.tl-table .up { color:#8DA2F5; font-weight:700; }
|
| 32 |
+
.tl-table .section { padding:10px 16px; text-align:left; color:#8DA2F5; background:#101010; font-weight:700; letter-spacing:.06em; text-transform:uppercase; font-size:12px; }
|
| 33 |
.tl-table .avg td { border-top:2px solid #3E61EB; font-weight:700; }
|
| 34 |
@media (max-width: 640px) {
|
| 35 |
.tl-table { display:block !important; overflow-x:auto !important; table-layout:auto !important; }
|
|
|
|
| 91 |
|
| 92 |
<table class="tl-table">
|
| 93 |
<thead>
|
| 94 |
+
<tr><th style="width:26%;text-align:left;padding-left:16px;">Model</th><th>GSM8K</th><th>AIME 2025</th><th>MMLU-Pro</th><th>IFEval</th><th>LiveCodeBench v6</th><th>BFCL v4</th><th>Average</th></tr>
|
|
|
|
| 95 |
</thead>
|
| 96 |
<tbody>
|
| 97 |
+
<tr><td class="section" colspan="8">Accuracy (%)</td></tr>
|
| 98 |
+
<tr><td class="name tl">Thinklite-27B</td><td class="tl">96.8</td><td class="tl">95.0</td><td class="tl">84.4</td><td class="tl">89.1</td><td class="tl"><strong>84.7</strong></td><td class="tl">88.9</td><td class="tl">89.82</td></tr>
|
| 99 |
+
<tr><td class="name">ThinkingCap-Qwen3.8-27B</td><td>97.0</td><td><strong>100.0</strong></td><td>86.5</td><td>90.9</td><td><strong>84.7</strong></td><td><strong>89.0</strong></td><td>91.37</td></tr>
|
| 100 |
+
<tr><td class="name">Swift-1.5-Qwen3.8-27b</td><td>96.9</td><td><strong>100.0</strong></td><td><strong>86.6</strong></td><td>90.9</td><td><strong>84.7</strong></td><td>87.6</td><td>91.12</td></tr>
|
| 101 |
+
<tr><td class="name">Qwen3.8-27B</td><td><strong>97.2</strong></td><td>93.3</td><td>86.3</td><td><strong>91.5</strong></td><td>84.0</td><td>88.7</td><td>90.17</td></tr>
|
| 102 |
+
<tr><td class="section" colspan="8">Reasoning tokens vs Qwen3.8-27B</td></tr>
|
| 103 |
+
<tr><td class="name tl">Thinklite-27B</td><td class="tl">↓ 63%</td><td class="tl"><strong>↓ 43%</strong></td><td class="tl"><strong>↓ 69%</strong></td><td class="tl">↓ 41%</td><td class="tl"><strong>↓ 38%</strong></td><td class="tl">↓ 28%</td><td class="tl">↓ 47.0%</td></tr>
|
| 104 |
+
<tr><td class="name">ThinkingCap-Qwen3.8-27B</td><td><strong>↓ 68%</strong></td><td>↓ 33%</td><td>↓ 58%</td><td><strong>↓ 51%</strong></td><td>↓ 20%</td><td><strong>↓ 44%</strong></td><td>↓ 45.8%</td></tr>
|
| 105 |
+
<tr><td class="name">Swift-1.5-Qwen3.8-27b</td><td>↓ 40%</td><td>↓ 34%</td><td>↓ 44%</td><td>↓ 44%</td><td>↓ 32%</td><td>↓ 20%</td><td>↓ 35.6%</td></tr>
|
| 106 |
</tbody>
|
| 107 |
</table>
|
| 108 |
|
|
|
|
| 126 |
|
| 127 |
<table class="tl-table">
|
| 128 |
<thead>
|
| 129 |
+
<tr><th style="width:26%;text-align:left;padding-left:16px;">Model</th><th>GSM8K</th><th>AIME 2025</th><th>MMLU-Pro</th><th>IFEval</th><th>LiveCodeBench v6</th><th>BFCL v4</th><th>Average</th></tr>
|
|
|
|
| 130 |
</thead>
|
| 131 |
<tbody>
|
| 132 |
+
<tr><td class="section" colspan="8">Accuracy (%)</td></tr>
|
| 133 |
+
<tr><td class="name tl">Thinklite-27B</td><td class="tl">96.7</td><td class="tl">81.7</td><td class="tl">83.3</td><td class="tl">84.3</td><td class="tl">71.0</td><td class="tl">91.2</td><td class="tl">84.70</td></tr>
|
| 134 |
+
<tr><td class="name">ThinkingCap-Qwen3.8-27B</td><td>97.2</td><td>86.7</td><td><strong>84.8</strong></td><td>85.8</td><td>72.5</td><td><strong>91.5</strong></td><td>86.40</td></tr>
|
| 135 |
+
<tr><td class="name">Swift-1.5-Qwen3.8-27b</td><td>97.1</td><td><strong>90.0</strong></td><td>84.7</td><td>86.0</td><td><strong>73.3</strong></td><td>91.2</td><td>87.04</td></tr>
|
| 136 |
+
<tr><td class="name">Qwen3.8-27B</td><td><strong>97.4</strong></td><td><strong>90.0</strong></td><td>84.4</td><td><strong>86.1</strong></td><td>72.5</td><td>90.8</td><td>86.88</td></tr>
|
| 137 |
+
<tr><td class="section" colspan="8">Reasoning tokens vs Qwen3.8-27B</td></tr>
|
| 138 |
+
<tr><td class="name tl">Thinklite-27B</td><td class="tl"><strong>↓ 21%</strong></td><td class="tl"><strong>↓ 46%</strong></td><td class="tl"><strong>↓ 23%</strong></td><td class="tl"><strong>↓ 41%</strong></td><td class="tl"><strong>↓ 17%</strong></td><td class="tl"><strong>↓ 19%</strong></td><td class="tl">↓ 28.0%</td></tr>
|
| 139 |
+
<tr><td class="name">ThinkingCap-Qwen3.8-27B</td><td>↓ 12%</td><td>↓ 23%</td><td>↓ 18%</td><td>↓ 19%</td><td>↓ 15%</td><td>↓ 12%</td><td>↓ 16.4%</td></tr>
|
| 140 |
+
<tr><td class="name">Swift-1.5-Qwen3.8-27b</td><td>↓ 5%</td><td>↓ 22%</td><td>↓ 13%</td><td>↓ 15%</td><td>↓ 13%</td><td>↓ 7%</td><td>↓ 12.7%</td></tr>
|
| 141 |
</tbody>
|
| 142 |
</table>
|
| 143 |
|