Upload folder using huggingface_hub
Browse files- models.json +2 -0
- plots/wall_vs_active_perfect_country.png +2 -2
- plots/wall_vs_total_perfect_country.png +2 -2
- results_full42.csv +87 -87
- scripts/aggregate_results.py +38 -1
- scripts/render_table.py +6 -2
models.json
CHANGED
|
@@ -764,6 +764,8 @@
|
|
| 764 |
"size_gb": 38.8,
|
| 765 |
"link": "https://huggingface.co/Myric/Ripple-65B-Preview-APEX-GGUF",
|
| 766 |
"link_note": "Tested target quant: prithivMLmods Q4_K_M (exact file); card carries the DFlash2 drafter, serving recipe, and provenance notes",
|
|
|
|
|
|
|
| 767 |
"box": "gb10",
|
| 768 |
"annotate": true,
|
| 769 |
"offset": [
|
|
|
|
| 764 |
"size_gb": 38.8,
|
| 765 |
"link": "https://huggingface.co/Myric/Ripple-65B-Preview-APEX-GGUF",
|
| 766 |
"link_note": "Tested target quant: prithivMLmods Q4_K_M (exact file); card carries the DFlash2 drafter, serving recipe, and provenance notes",
|
| 767 |
+
"base_country": "US",
|
| 768 |
+
"post_country": "US",
|
| 769 |
"box": "gb10",
|
| 770 |
"annotate": true,
|
| 771 |
"offset": [
|
plots/wall_vs_active_perfect_country.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/wall_vs_total_perfect_country.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
results_full42.csv
CHANGED
|
@@ -1,87 +1,87 @@
|
|
| 1 |
-
label,group,box,model_path,general,hard,frontier,tier4a,valid_total,void,passed_total,wall_h,tokens_out,tokens_per_task,
|
| 2 |
-
bolt-e4b-q4,bolt-e4b-q4,gb10,Bolt-E4B-Q4_0.gguf,19/25,1/5,2/9,3/3,25/42,0,25,0.8,133003,3166.7,46.12,0.6797
|
| 3 |
-
bonsai2-pq2-oc2-r1,bonsai2-pq2-oc2,gb10,Ternary-Bonsai-2-27B-PQ2_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,6.12,474702,11578.1,21.56,1.0
|
| 4 |
-
flint-5bpw,flint-5bpw,gb10,Recursive-Flint-APEX-5bpw.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.28,184694,4397.5,40.08,1.0
|
| 5 |
-
gemma4-12b-qat-google-r1,gemma4-12b-qat-google,gb10,gemma-4-12b-it-qat-q4_0.gguf,24/24,5/5,7/7,3/3,39/39,3,39,7.86,379608,9733.5,13.41,1.0
|
| 6 |
-
gemma4-12b-qat-google-r2,gemma4-12b-qat-google,gb10,gemma-4-12b-it-qat-q4_0.gguf,24/25,5/5,8/8,3/3,40/41,1,40,6.92,312468,7621.2,12.54,0.9756
|
| 7 |
-
gemma4-12b-qat-grid-r1,gemma4-12b-qat-grid,gb10,gemma-4-12b-it-qat-grid-q4_0-all.gguf,24/25,5/5,9/9,3/3,41/42,0,41,5.12,249114,5931.3,13.52,0.9762
|
| 8 |
-
glimmer-heretic-q4km,glimmer-heretic-q4km,gb10,glimmer-heretic-Q4_K_M.gguf,25/25,5/5,9/9,3/3,42/42,0,42,6.47,205580,4894.8,8.82,1.0
|
| 9 |
-
glimmer-udq4kxl-oc2-r1,glimmer-udq4kxl-oc2,gb10,Muse-Glimmer-30B-UD-Q4_K_XL.gguf,25/25,5/5,9/9,3/3,42/42,0,42,6.27,210880,5021.0,9.35,1.0
|
| 10 |
-
glm47-flash-apex-iquality,glm47-flash-apex-iquality,gb10,GLM-4.7-Flash-APEX-i-quality.gguf,24/25,3/5,6/9,3/3,36/42,0,36,4.23,429484,10225.8,28.2,0.9391
|
| 11 |
-
k2horizon-v2,k2horizon-v2,gb10,,25/25,5/5,9/9,3/3,42/42,0,42,4.91,109090,2597.4,6.18,1.0
|
| 12 |
-
kolibri1-apex-2.4bpw,kolibri1-apex-2.4bpw,gb10,Kolibri-1-APEX-2.4bpw.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.21,355575,8466.1,44.67,1.0
|
| 13 |
-
kolibri1-uniform-q4k,kolibri1-uniform-q4k,gb10,Kolibri-1-uniform-Q4_K.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.53,375157,8932.3,41.15,1.0
|
| 14 |
-
kolibri1-uniform-q4k-low,kolibri1-uniform-q4k-low,gb10,Kolibri-1-uniform-Q4_K.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.13,303509,7226.4,39.5,1.0
|
| 15 |
-
lagunas21-iquality-v2-oc2-r1,lagunas21-iquality-v2-oc2,gb10,Laguna-S-2.1-APEX-i-quality-v2.gguf,25/25,5/5,7/7,3/3,40/40,2,40,3.15,222316,5557.9,19.63,1.0
|
| 16 |
-
lagunaxs21-apex-dynamic-v2-oc2-r1,lagunaxs21-apex-dynamic-v2-oc2,gb10,Laguna-XS-2.1-APEX-dynamic-v2.gguf,25/25,5/5,8/9,3/3,41/42,0,41,2.09,301294,7173.7,40.05,0.9762
|
| 17 |
-
lagunaxs21-apex-dynamic-v2-oc2-r2,lagunaxs21-apex-dynamic-v2-oc2,gb10,Laguna-XS-2.1-APEX-dynamic-v2.gguf,25/25,5/5,8/9,3/3,41/42,0,41,2.69,379902,9045.3,39.27,0.9762
|
| 18 |
-
lagunaxs21-apex-iquality-v2-oc2-r1,lagunaxs21-apex-iquality-v2-oc2,gb10,Laguna-XS-2.1-APEX-i-quality-v2.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.03,175549,4179.7,47.3,1.0
|
| 19 |
-
lagunaxs21-apex-iquality-v2-oc2-r2,lagunaxs21-apex-iquality-v2-oc2,gb10,Laguna-XS-2.1-APEX-i-quality-v2.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.89,303993,7237.9,44.56,1.0
|
| 20 |
-
lfm25-24b-a2b-bf16src-iq3_xxs-v2-r1,lfm25-24b-a2b-bf16src-iq3_xxs-v2,gb10,LFM2.5-24B-A2B-bf16-IQ3_XXS.gguf,25/25,3/4,5/9,3/3,36/41,1,36,2.69,682432,16644.7,70.55,0.9555
|
| 21 |
-
lfm25-24b-a2b-bf16src-iq3_xxs-v2-r2,lfm25-24b-a2b-bf16src-iq3_xxs-v2,gb10,LFM2.5-24B-A2B-bf16-IQ3_XXS.gguf,25/25,4/4,7/9,3/3,39/41,1,39,1.73,489402,11936.6,78.44,0.9804
|
| 22 |
-
lfm25-24b-a2b-q8_0-v2-r1,lfm25-24b-a2b-q8_0-v2,gb10,LFM2.5-24B-A2B-Q8_0.gguf,25/25,3/3,8/8,3/3,39/39,3,39,1.65,229133,5875.2,38.68,1.0
|
| 23 |
-
lfm25-24b-a2b-qad-q4_0-v2-r1,lfm25-24b-a2b-qad-q4_0-v2,gb10,LFM2.5-24B-A2B-QAD-Q4_0.gguf,25/25,5/5,7/9,3/3,40/42,0,40,1.96,522574,12442.2,73.9,0.969
|
| 24 |
-
lfm25-24b-a2b-qad-q4_0-v2-r2,lfm25-24b-a2b-qad-q4_0-v2,gb10,LFM2.5-24B-A2B-QAD-Q4_0.gguf,25/25,3/3,8/8,3/3,39/39,3,39,1.48,347283,8904.7,65.16,1.0
|
| 25 |
-
lfm25-24b-a2b-qad-q4_0-v2-r3,lfm25-24b-a2b-qad-q4_0-v2,gb10,LFM2.5-24B-A2B-QAD-Q4_0.gguf,25/25,5/5,7/9,3/3,40/42,0,40,1.59,427171,10170.7,74.64,0.9746
|
| 26 |
-
lfm25-24b-a2b-qad-q4_0-v2-r4,lfm25-24b-a2b-qad-q4_0-v2,gb10,LFM2.5-24B-A2B-QAD-Q4_0.gguf,25/25,3/4,6/8,3/3,37/40,2,37,2.03,479900,11997.5,65.77,0.9703
|
| 27 |
-
lfm3-1.7b-q4,lfm3-1.7b-q4,gb10,LFM3-1.7B-Q4_0.gguf,23/24,3/4,4/7,2/3,32/38,4,32,2.28,700365,18430.7,85.31,0.8949
|
| 28 |
-
mellum21-12b-a2.5b-thinking-q8_0-v2-r1,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,1.11,206172,5028.6,51.79,1.0
|
| 29 |
-
mellum21-12b-a2.5b-thinking-q8_0-v2-r2,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf,25/25,4/4,8/9,3/3,40/41,1,40,1.53,255666,6235.8,46.4,0.9934
|
| 30 |
-
minicpm5-2b-q8,minicpm5-2b-q8,gb10,MiniCPM5-2B-Q8_0.gguf,23/23,3/3,6/6,3/3,35/35,7,35,1.9,390175,11147.9,57.09,1.0
|
| 31 |
-
nemotron35-bolt-74b-nvidia24gb-oc2-r1,nemotron35-bolt-74b-nvidia24gb-oc2,gb10,NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf,24/24,5/5,7/7,3/3,39/39,3,39,2.74,164349,4214.1,16.68,1.0
|
| 32 |
-
nemotron35-super-vl-iquality-oc2-r1,nemotron35-super-vl-iquality-oc2,gb10,NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.07,119938,2855.7,10.85,1.0
|
| 33 |
-
nemotron3super-iquality-oc2-r1,nemotron3super-iquality-oc2,gb10,Nemotron-3-Super-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.4,165619,3943.3,13.52,1.0
|
| 34 |
-
ornith122-icompact-r1,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.79,132233,3148.4,20.56,1.0
|
| 35 |
-
ornith122-icompact-r2,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.83,146077,3478.0,22.14,1.0
|
| 36 |
-
ornith122-iquality-r1,ornith122-iquality,gb10,Ornith-1.5-122B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.86,134646,3205.9,20.07,1.0
|
| 37 |
-
ornith122-iquality-r2,ornith122-iquality,gb10,Ornith-1.5-122B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.75,120711,2874.1,19.17,1.0
|
| 38 |
-
ornith35-apex-iquality,ornith35-apex-iquality,gb10,Ornith-1.5-35B-A3B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.15,211193,5028.4,50.84,1.0
|
| 39 |
-
qwen38-27b-q4km,qwen38-27b-q4km,gb10,Qwen3.8-27B-Q4_K_M.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.71,93914,2236.0,9.64,1.0
|
| 40 |
-
qwen38-flash-next-middle,qwen38-flash-next-middle,gb10,qwen3.8-flash-next-MIDDLE.gguf,24/24,5/5,9/9,3/3,41/41,1,41,4.24,244781,5970.3,16.06,1.0
|
| 41 |
-
ripple65b-dflash16,ripple65b-dflash16,gb10,Ripple-65B-Preview.Q4_K_M.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.68,91901,2188.1,9.53,1.0
|
| 42 |
-
scout-iquality-proxy-oc2-r4,scout-iquality-proxy-oc2,gb10,Llama-4-Scout-17B-16E-Instruct-APEX-i-quality.gguf,0/0,0/0,0/1,0/0,0/1,41,0,1.69,90,90.0,0.01,0.5004
|
| 43 |
-
spark-x2.5-4b-bf16,spark-x2.5-4b-bf16,gb10,spark-x2.5-4b-BF16.gguf,22/22,3/3,4/4,3/3,32/32,10,32,14.54,574489,17952.8,10.97,1.0
|
| 44 |
-
spark-x2.5-4b-q6_k,spark-x2.5-4b-q6_k,gb10,,22/22,3/3,2/2,3/3,30/30,12,30,6.64,387490,12916.3,16.21,1.0
|
| 45 |
-
spark-x2.6-4b-q6_k,spark-x2.6-4b-q6_k,gb10,,25/25,4/4,7/7,3/3,39/39,3,39,2.5,224657,5760.4,24.98,1.0
|
| 46 |
-
thor-gemma4-26b-r1,thor-gemma4-26b,thor,,25/25,5/5,9/9,3/3,42/42,0,42,2.31,252002,6000.0,30.35,1.0
|
| 47 |
-
thor-gemma4-26b-r2,thor-gemma4-26b,thor,,25/25,5/5,8/8,3/3,41/41,1,41,3.58,437605,10673.3,33.96,1.0
|
| 48 |
-
thor-gptoss20b-r1,thor-gptoss20b,thor,,25/25,4/5,8/9,3/3,40/42,0,40,1.53,277208,6600.2,50.26,0.9524
|
| 49 |
-
thor-gptoss20b-r2,thor-gptoss20b,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.77,321411,7652.6,50.34,0.9887
|
| 50 |
-
thor-kolibri-iq-high-r1,thor-kolibri-iq-high,thor,,25/25,5/5,9/9,3/3,42/42,0,42,6.12,347312,8269.3,15.75,1.0
|
| 51 |
-
thor-kolibri-iq-low-r1,thor-kolibri-iq-low,thor,,25/25,5/5,9/9,3/3,42/42,0,42,4.15,235190,5599.8,15.75,1.0
|
| 52 |
-
thor-lightning-r1,thor-lightning,thor,,25/25,5/5,9/9,3/3,42/42,0,42,4.42,554855,13210.8,34.88,1.0
|
| 53 |
-
thor-lightning-r2,thor-lightning,thor,,24/24,5/5,8/9,3/3,40/41,1,40,3.71,433310,10568.5,32.42,0.9756
|
| 54 |
-
thor-mellum21-apex-3.1bpw-v2-r1,thor-mellum21-apex-3.1bpw-v2,thor,,24/25,1/5,8/9,3/3,36/42,0,36,1.16,410727,9779.2,98.32,0.8777
|
| 55 |
-
thor-mellum21-apex-3.1bpw-v2-r2,thor-mellum21-apex-3.1bpw-v2,thor,,25/25,5/5,7/9,3/3,40/42,0,40,1.33,450630,10729.3,93.8,0.979
|
| 56 |
-
thor-mellum21-apex-3.7bpw-v2-r1,thor-mellum21-apex-3.7bpw-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,0.83,233584,5561.5,77.93,0.9806
|
| 57 |
-
thor-mellum21-apex-3.7bpw-v2-r2,thor-mellum21-apex-3.7bpw-v2,thor,,25/25,5/5,9/9,3/3,42/42,0,42,0.84,251673,5992.2,83.0,1.0
|
| 58 |
-
thor-mellum21-apex-4.3bpw-v2-r1,thor-mellum21-apex-4.3bpw-v2,thor,,25/25,5/5,7/9,3/3,40/42,0,40,1.92,293682,6992.4,42.52,0.9669
|
| 59 |
-
thor-mellum21-apex-4.3bpw-v2-r2,thor-mellum21-apex-4.3bpw-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.0,266940,6355.7,73.8,0.9915
|
| 60 |
-
thor-mellum21-apex-4.9bpw-v2-r1,thor-mellum21-apex-4.9bpw-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.1,284856,6782.3,71.74,0.9956
|
| 61 |
-
thor-mellum21-apex-4.9bpw-v2-r2,thor-mellum21-apex-4.9bpw-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,0.88,231263,5506.3,72.87,0.9952
|
| 62 |
-
thor-mellum21-q4_k_m-stock-v2-r1,thor-mellum21-q4_k_m-stock-v2,thor,,25/25,5/5,7/9,3/3,40/42,0,40,1.05,246107,5859.7,65.35,0.9955
|
| 63 |
-
thor-mellum21-q4_k_m-stock-v2-r2,thor-mellum21-q4_k_m-stock-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.0,248624,5919.6,68.89,0.9956
|
| 64 |
-
thor-mellum21-q6_k_l-bartowski-v2-r1,thor-mellum21-q6_k_l-bartowski-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.09,226675,5397.0,57.61,0.9826
|
| 65 |
-
thor-mellum21-q6_k_l-bartowski-v2-r2,thor-mellum21-q6_k_l-bartowski-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.01,208741,4970.0,57.34,0.9762
|
| 66 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.15,wk-gemma4-26b-lora-run2-tok5M-s0.15,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,0.89,76153,1857.4,23.77,1.0
|
| 67 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.15-think,wk-gemma4-26b-lora-run2-tok5M-s0.15-think,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.94,188312,4483.6,27.02,1.0
|
| 68 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.2,wk-gemma4-26b-lora-run2-tok5M-s0.2,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.95,100577,2394.7,29.56,0.998
|
| 69 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.25,wk-gemma4-26b-lora-run2-tok5M-s0.25,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,0.6,52265,1244.4,24.17,1.0
|
| 70 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.25-r2,wk-gemma4-26b-lora-run2-tok5M-s0.25,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.65,71879,1711.4,30.85,0.9968
|
| 71 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.25-r3,wk-gemma4-26b-lora-run2-tok5M-s0.25,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.63,65175,1551.8,28.83,0.9881
|
| 72 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.25-think,wk-gemma4-26b-lora-run2-tok5M-s0.25-think,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,1.84,190589,4537.8,28.78,0.9762
|
| 73 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.3,wk-gemma4-26b-lora-run2-tok5M-s0.3,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/8,2/3,40/41,1,40,1.17,104072,2538.3,24.64,0.9919
|
| 74 |
-
wk-gemma4-26b-lora-run2-tok5M-s0.5,wk-gemma4-26b-lora-run2-tok5M-s0.5,gb10,gemma4-26b-a4b-it-Q8_0.gguf,24/25,5/5,8/9,2/3,39/42,0,39,0.89,101100,2407.1,31.63,0.9616
|
| 75 |
-
wk-gemma4-26b-lora-run4-tok5M-s0.1,wk-gemma4-26b-lora-run4-tok5M-s0.1,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,1.1,129632,3086.5,32.84,0.9911
|
| 76 |
-
wk-gemma4-26b-lora-run4-tok5M-s0.15,wk-gemma4-26b-lora-run4-tok5M-s0.15,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.54,58164,1384.9,30.08,0.9802
|
| 77 |
-
wk-gemma4-26b-lora-run4-tok5M-s0.15-r2,wk-gemma4-26b-lora-run4-tok5M-s0.15,gb10,gemma4-26b-a4b-it-Q8_0.gguf,24/25,5/5,9/9,3/3,41/42,0,41,0.95,110012,2619.3,32.33,0.9762
|
| 78 |
-
wk-gemma4-26b-lora-run4-tok5M-s0.15-think,wk-gemma4-26b-lora-run4-tok5M-s0.15-think,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,2.73,220567,5379.7,22.48,1.0
|
| 79 |
-
wk-gemma4-26b-lora-run4-tok5M-s0.25,wk-gemma4-26b-lora-run4-tok5M-s0.25,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,0.89,103233,2457.9,32.08,1.0
|
| 80 |
-
wk-gemma4-26b-lora-run4-tok5M-s0.5,wk-gemma4-26b-lora-run4-tok5M-s0.5,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,7/9,3/3,40/42,0,40,1.14,130602,3109.6,31.88,0.9915
|
| 81 |
-
wk-gemma4-26b-lora-run4-tok5M-s0.75,wk-gemma4-26b-lora-run4-tok5M-s0.75,gb10,gemma4-26b-a4b-it-Q8_0.gguf,15/25,3/5,6/8,2/3,26/41,1,26,1.39,168131,4100.8,33.62,0.7092
|
| 82 |
-
wk-gemma4-26b-posttrain-alpha0.75,wk-gemma4-26b-posttrain-alpha0.75,gb10,gemma4-26b-a4b-alpha0.75-Q8_0.gguf,25/25,5/5,6/9,3/3,39/42,0,39,2.2,230168,5480.2,29.1,0.9286
|
| 83 |
-
wk-gemma4-26b-stock-default-r1,wk-gemma4-26b-stock-default,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.83,191275,4554.2,29.05,1.0
|
| 84 |
-
wk-gemma4-26b-stock-default-r2,wk-gemma4-26b-stock-default,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.63,167638,3991.4,28.57,1.0
|
| 85 |
-
wk-gemma4-26b-stock-reasonoff-r1,wk-gemma4-26b-stock-reasonoff,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,0.7,85011,2024.1,33.78,1.0
|
| 86 |
-
wk-gemma4-26b-stock-reasonoff-r2,wk-gemma4-26b-stock-reasonoff,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.96,118271,2816.0,34.09,0.9911
|
| 87 |
-
wk-gemma4-26b-stock-reasonoff-r3,wk-gemma4-26b-stock-reasonoff,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.96,119002,2833.4,34.59,0.9762
|
|
|
|
| 1 |
+
label,group,box,model_path,general,hard,frontier,tier4a,valid_total,void,passed_total,wall_h,tokens_out,tokens_per_task,tok_s_eff,tok_s_prefill,tok_s_decode,prefill_ms,decode_ms,partial_mean
|
| 2 |
+
bolt-e4b-q4,bolt-e4b-q4,gb10,Bolt-E4B-Q4_0.gguf,19/25,1/5,2/9,3/3,25/42,0,25,0.8,133003,3166.7,46.12,3786.1,65.8,124302.3,5560984.5,0.6797
|
| 3 |
+
bonsai2-pq2-oc2-r1,bonsai2-pq2-oc2,gb10,Ternary-Bonsai-2-27B-PQ2_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,6.12,474702,11578.1,21.56,797.5,24.9,653659.2,34407505.5,1.0
|
| 4 |
+
flint-5bpw,flint-5bpw,gb10,Recursive-Flint-APEX-5bpw.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.28,184694,4397.5,40.08,1564.3,53.8,324250.2,8707084.2,1.0
|
| 5 |
+
gemma4-12b-qat-google-r1,gemma4-12b-qat-google,gb10,gemma-4-12b-it-qat-q4_0.gguf,24/24,5/5,7/7,3/3,39/39,3,39,7.86,379608,9733.5,13.41,517.4,16.3,3567807.7,51237639.0,1.0
|
| 6 |
+
gemma4-12b-qat-google-r2,gemma4-12b-qat-google,gb10,gemma-4-12b-it-qat-q4_0.gguf,24/25,5/5,8/8,3/3,40/41,1,40,6.92,312468,7621.2,12.54,519.7,16.4,3378014.9,43725678.2,0.9756
|
| 7 |
+
gemma4-12b-qat-grid-r1,gemma4-12b-qat-grid,gb10,gemma-4-12b-it-qat-grid-q4_0-all.gguf,24/25,5/5,9/9,3/3,41/42,0,41,5.12,249114,5931.3,13.52,520.2,17.3,2773717.9,33117325.2,0.9762
|
| 8 |
+
glimmer-heretic-q4km,glimmer-heretic-q4km,gb10,glimmer-heretic-Q4_K_M.gguf,25/25,5/5,9/9,3/3,42/42,0,42,6.47,205580,4894.8,8.82,704.0,11.9,680903.8,45231122.4,1.0
|
| 9 |
+
glimmer-udq4kxl-oc2-r1,glimmer-udq4kxl-oc2,gb10,Muse-Glimmer-30B-UD-Q4_K_XL.gguf,25/25,5/5,9/9,3/3,42/42,0,42,6.27,210880,5021.0,9.35,723.3,12.9,710177.0,50889483.3,1.0
|
| 10 |
+
glm47-flash-apex-iquality,glm47-flash-apex-iquality,gb10,GLM-4.7-Flash-APEX-i-quality.gguf,24/25,3/5,6/9,3/3,36/42,0,36,4.23,429484,10225.8,28.2,656.1,40.8,1003021.5,28330438.8,0.9391
|
| 11 |
+
k2horizon-v2,k2horizon-v2,gb10,,25/25,5/5,9/9,3/3,42/42,0,42,4.91,109090,2597.4,6.18,,,,,1.0
|
| 12 |
+
kolibri1-apex-2.4bpw,kolibri1-apex-2.4bpw,gb10,Kolibri-1-APEX-2.4bpw.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.21,355575,8466.1,44.67,1585.9,57.4,602249.2,7778697.9,1.0
|
| 13 |
+
kolibri1-uniform-q4k,kolibri1-uniform-q4k,gb10,Kolibri-1-uniform-Q4_K.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.53,375157,8932.3,41.15,1540.5,51.0,3462670.5,39380520.3,1.0
|
| 14 |
+
kolibri1-uniform-q4k-low,kolibri1-uniform-q4k-low,gb10,Kolibri-1-uniform-Q4_K.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.13,303509,7226.4,39.5,1515.9,50.2,1168429.0,13283148.4,1.0
|
| 15 |
+
lagunas21-iquality-v2-oc2-r1,lagunas21-iquality-v2-oc2,gb10,Laguna-S-2.1-APEX-i-quality-v2.gguf,25/25,5/5,7/7,3/3,40/40,2,40,3.15,222316,5557.9,19.63,631.0,27.0,763819.8,30168740.5,1.0
|
| 16 |
+
lagunaxs21-apex-dynamic-v2-oc2-r1,lagunaxs21-apex-dynamic-v2-oc2,gb10,Laguna-XS-2.1-APEX-dynamic-v2.gguf,25/25,5/5,8/9,3/3,41/42,0,41,2.09,301294,7173.7,40.05,1182.5,44.9,728769.3,14050715.4,0.9762
|
| 17 |
+
lagunaxs21-apex-dynamic-v2-oc2-r2,lagunaxs21-apex-dynamic-v2-oc2,gb10,Laguna-XS-2.1-APEX-dynamic-v2.gguf,25/25,5/5,8/9,3/3,41/42,0,41,2.69,379902,9045.3,39.27,1220.8,44.0,717708.9,18281280.1,0.9762
|
| 18 |
+
lagunaxs21-apex-iquality-v2-oc2-r1,lagunaxs21-apex-iquality-v2-oc2,gb10,Laguna-XS-2.1-APEX-i-quality-v2.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.03,175549,4179.7,47.3,1510.7,60.3,423315.3,6796901.1,1.0
|
| 19 |
+
lagunaxs21-apex-iquality-v2-oc2-r2,lagunaxs21-apex-iquality-v2-oc2,gb10,Laguna-XS-2.1-APEX-i-quality-v2.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.89,303993,7237.9,44.56,1276.4,51.6,526452.2,12885832.6,1.0
|
| 20 |
+
lfm25-24b-a2b-bf16src-iq3_xxs-v2-r1,lfm25-24b-a2b-bf16src-iq3_xxs-v2,gb10,LFM2.5-24B-A2B-bf16-IQ3_XXS.gguf,25/25,3/4,5/9,3/3,36/41,1,36,2.69,682432,16644.7,70.55,2112.8,95.7,392539.4,17888749.8,0.9555
|
| 21 |
+
lfm25-24b-a2b-bf16src-iq3_xxs-v2-r2,lfm25-24b-a2b-bf16src-iq3_xxs-v2,gb10,LFM2.5-24B-A2B-bf16-IQ3_XXS.gguf,25/25,4/4,7/9,3/3,39/41,1,39,1.73,489402,11936.6,78.44,2344.9,112.0,269956.4,11872224.8,0.9804
|
| 22 |
+
lfm25-24b-a2b-q8_0-v2-r1,lfm25-24b-a2b-q8_0-v2,gb10,LFM2.5-24B-A2B-Q8_0.gguf,25/25,3/3,8/8,3/3,39/39,3,39,1.65,229133,5875.2,38.68,1964.4,69.0,224856.8,11463036.8,1.0
|
| 23 |
+
lfm25-24b-a2b-qad-q4_0-v2-r1,lfm25-24b-a2b-qad-q4_0-v2,gb10,LFM2.5-24B-A2B-QAD-Q4_0.gguf,25/25,5/5,7/9,3/3,40/42,0,40,1.96,522574,12442.2,73.9,2863.3,90.3,310426.5,13605099.3,0.969
|
| 24 |
+
lfm25-24b-a2b-qad-q4_0-v2-r2,lfm25-24b-a2b-qad-q4_0-v2,gb10,LFM2.5-24B-A2B-QAD-Q4_0.gguf,25/25,3/3,8/8,3/3,39/39,3,39,1.48,347283,8904.7,65.16,2924.2,97.7,202699.3,10264764.4,1.0
|
| 25 |
+
lfm25-24b-a2b-qad-q4_0-v2-r3,lfm25-24b-a2b-qad-q4_0-v2,gb10,LFM2.5-24B-A2B-QAD-Q4_0.gguf,25/25,5/5,7/9,3/3,40/42,0,40,1.59,427171,10170.7,74.64,2908.0,95.9,257484.9,10992361.6,0.9746
|
| 26 |
+
lfm25-24b-a2b-qad-q4_0-v2-r4,lfm25-24b-a2b-qad-q4_0-v2,gb10,LFM2.5-24B-A2B-QAD-Q4_0.gguf,25/25,3/4,6/8,3/3,37/40,2,37,2.03,479900,11997.5,65.77,2900.8,89.4,265816.5,14107931.3,0.9703
|
| 27 |
+
lfm3-1.7b-q4,lfm3-1.7b-q4,gb10,LFM3-1.7B-Q4_0.gguf,23/24,3/4,4/7,2/3,32/38,4,32,2.28,700365,18430.7,85.31,8632.1,135.3,324721.7,15375078.5,0.8949
|
| 28 |
+
mellum21-12b-a2.5b-thinking-q8_0-v2-r1,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,1.11,206172,5028.6,51.79,2536.5,77.0,262515.2,7481445.3,1.0
|
| 29 |
+
mellum21-12b-a2.5b-thinking-q8_0-v2-r2,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf,25/25,4/4,8/9,3/3,40/41,1,40,1.53,255666,6235.8,46.4,2571.0,71.9,289662.0,10551359.2,0.9934
|
| 30 |
+
minicpm5-2b-q8,minicpm5-2b-q8,gb10,MiniCPM5-2B-Q8_0.gguf,23/23,3/3,6/6,3/3,35/35,7,35,1.9,390175,11147.9,57.09,6076.0,64.9,306246.2,16048294.3,1.0
|
| 31 |
+
nemotron35-bolt-74b-nvidia24gb-oc2-r1,nemotron35-bolt-74b-nvidia24gb-oc2,gb10,NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf,24/24,5/5,7/7,3/3,39/39,3,39,2.74,164349,4214.1,16.68,1059.2,38.5,688093.7,13972951.4,1.0
|
| 32 |
+
nemotron35-super-vl-iquality-oc2-r1,nemotron35-super-vl-iquality-oc2,gb10,NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.07,119938,2855.7,10.85,456.8,17.2,893428.2,21066982.9,1.0
|
| 33 |
+
nemotron3super-iquality-oc2-r1,nemotron3super-iquality-oc2,gb10,Nemotron-3-Super-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.4,165619,3943.3,13.52,490.5,17.8,1791178.0,24957317.8,1.0
|
| 34 |
+
ornith122-icompact-r1,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.79,132233,3148.4,20.56,521.4,26.2,607192.2,12091745.9,1.0
|
| 35 |
+
ornith122-icompact-r2,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.83,146077,3478.0,22.14,542.7,26.1,629024.9,12407008.6,1.0
|
| 36 |
+
ornith122-iquality-r1,ornith122-iquality,gb10,Ornith-1.5-122B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.86,134646,3205.9,20.07,469.0,24.4,648353.8,12609150.7,1.0
|
| 37 |
+
ornith122-iquality-r2,ornith122-iquality,gb10,Ornith-1.5-122B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.75,120711,2874.1,19.17,446.9,24.4,582766.9,11858646.5,1.0
|
| 38 |
+
ornith35-apex-iquality,ornith35-apex-iquality,gb10,Ornith-1.5-35B-A3B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.15,211193,5028.4,50.84,1680.6,64.5,262948.1,7500255.1,1.0
|
| 39 |
+
qwen38-27b-q4km,qwen38-27b-q4km,gb10,Qwen3.8-27B-Q4_K_M.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.71,93914,2236.0,9.64,612.1,11.6,380881.8,18969895.6,1.0
|
| 40 |
+
qwen38-flash-next-middle,qwen38-flash-next-middle,gb10,qwen3.8-flash-next-MIDDLE.gguf,24/24,5/5,9/9,3/3,41/41,1,41,4.24,244781,5970.3,16.06,439.1,25.9,1324779.8,26351504.1,1.0
|
| 41 |
+
ripple65b-dflash16,ripple65b-dflash16,gb10,Ripple-65B-Preview.Q4_K_M.gguf,25/25,5/5,9/9,3/3,42/42,0,42,2.68,91901,2188.1,9.53,281.5,12.7,1175589.6,13181868.8,1.0
|
| 42 |
+
scout-iquality-proxy-oc2-r4,scout-iquality-proxy-oc2,gb10,Llama-4-Scout-17B-16E-Instruct-APEX-i-quality.gguf,0/0,0/0,0/1,0/0,0/1,41,0,1.69,90,90.0,0.01,764.3,14.0,388168.5,11674687.8,0.5004
|
| 43 |
+
spark-x2.5-4b-bf16,spark-x2.5-4b-bf16,gb10,spark-x2.5-4b-BF16.gguf,22/22,3/3,4/4,3/3,32/32,10,32,14.54,574489,17952.8,10.97,2193.1,18.5,2569813.3,206451517.5,1.0
|
| 44 |
+
spark-x2.5-4b-q6_k,spark-x2.5-4b-q6_k,gb10,,22/22,3/3,2/2,3/3,30/30,12,30,6.64,387490,12916.3,16.21,,,,,1.0
|
| 45 |
+
spark-x2.6-4b-q6_k,spark-x2.6-4b-q6_k,gb10,,25/25,4/4,7/7,3/3,39/39,3,39,2.5,224657,5760.4,24.98,,,,,1.0
|
| 46 |
+
thor-gemma4-26b-r1,thor-gemma4-26b,thor,,25/25,5/5,9/9,3/3,42/42,0,42,2.31,252002,6000.0,30.35,,,,,1.0
|
| 47 |
+
thor-gemma4-26b-r2,thor-gemma4-26b,thor,,25/25,5/5,8/8,3/3,41/41,1,41,3.58,437605,10673.3,33.96,,,,,1.0
|
| 48 |
+
thor-gptoss20b-r1,thor-gptoss20b,thor,,25/25,4/5,8/9,3/3,40/42,0,40,1.53,277208,6600.2,50.26,,,,,0.9524
|
| 49 |
+
thor-gptoss20b-r2,thor-gptoss20b,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.77,321411,7652.6,50.34,,,,,0.9887
|
| 50 |
+
thor-kolibri-iq-high-r1,thor-kolibri-iq-high,thor,,25/25,5/5,9/9,3/3,42/42,0,42,6.12,347312,8269.3,15.75,,,,,1.0
|
| 51 |
+
thor-kolibri-iq-low-r1,thor-kolibri-iq-low,thor,,25/25,5/5,9/9,3/3,42/42,0,42,4.15,235190,5599.8,15.75,,,,,1.0
|
| 52 |
+
thor-lightning-r1,thor-lightning,thor,,25/25,5/5,9/9,3/3,42/42,0,42,4.42,554855,13210.8,34.88,,,,,1.0
|
| 53 |
+
thor-lightning-r2,thor-lightning,thor,,24/24,5/5,8/9,3/3,40/41,1,40,3.71,433310,10568.5,32.42,,,,,0.9756
|
| 54 |
+
thor-mellum21-apex-3.1bpw-v2-r1,thor-mellum21-apex-3.1bpw-v2,thor,,24/25,1/5,8/9,3/3,36/42,0,36,1.16,410727,9779.2,98.32,,,,,0.8777
|
| 55 |
+
thor-mellum21-apex-3.1bpw-v2-r2,thor-mellum21-apex-3.1bpw-v2,thor,,25/25,5/5,7/9,3/3,40/42,0,40,1.33,450630,10729.3,93.8,,,,,0.979
|
| 56 |
+
thor-mellum21-apex-3.7bpw-v2-r1,thor-mellum21-apex-3.7bpw-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,0.83,233584,5561.5,77.93,,,,,0.9806
|
| 57 |
+
thor-mellum21-apex-3.7bpw-v2-r2,thor-mellum21-apex-3.7bpw-v2,thor,,25/25,5/5,9/9,3/3,42/42,0,42,0.84,251673,5992.2,83.0,,,,,1.0
|
| 58 |
+
thor-mellum21-apex-4.3bpw-v2-r1,thor-mellum21-apex-4.3bpw-v2,thor,,25/25,5/5,7/9,3/3,40/42,0,40,1.92,293682,6992.4,42.52,,,,,0.9669
|
| 59 |
+
thor-mellum21-apex-4.3bpw-v2-r2,thor-mellum21-apex-4.3bpw-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.0,266940,6355.7,73.8,,,,,0.9915
|
| 60 |
+
thor-mellum21-apex-4.9bpw-v2-r1,thor-mellum21-apex-4.9bpw-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.1,284856,6782.3,71.74,,,,,0.9956
|
| 61 |
+
thor-mellum21-apex-4.9bpw-v2-r2,thor-mellum21-apex-4.9bpw-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,0.88,231263,5506.3,72.87,,,,,0.9952
|
| 62 |
+
thor-mellum21-q4_k_m-stock-v2-r1,thor-mellum21-q4_k_m-stock-v2,thor,,25/25,5/5,7/9,3/3,40/42,0,40,1.05,246107,5859.7,65.35,,,,,0.9955
|
| 63 |
+
thor-mellum21-q4_k_m-stock-v2-r2,thor-mellum21-q4_k_m-stock-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.0,248624,5919.6,68.89,,,,,0.9956
|
| 64 |
+
thor-mellum21-q6_k_l-bartowski-v2-r1,thor-mellum21-q6_k_l-bartowski-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.09,226675,5397.0,57.61,,,,,0.9826
|
| 65 |
+
thor-mellum21-q6_k_l-bartowski-v2-r2,thor-mellum21-q6_k_l-bartowski-v2,thor,,25/25,5/5,8/9,3/3,41/42,0,41,1.01,208741,4970.0,57.34,,,,,0.9762
|
| 66 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.15,wk-gemma4-26b-lora-run2-tok5M-s0.15,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,0.89,76153,1857.4,23.77,1795.1,39.8,1522941.0,16782982.9,1.0
|
| 67 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.15-think,wk-gemma4-26b-lora-run2-tok5M-s0.15-think,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.94,188312,4483.6,27.02,1754.8,40.0,785097.0,12876881.0,1.0
|
| 68 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.2,wk-gemma4-26b-lora-run2-tok5M-s0.2,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.95,100577,2394.7,29.56,1749.2,39.4,3869627.6,27693176.0,0.998
|
| 69 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.25,wk-gemma4-26b-lora-run2-tok5M-s0.25,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,0.6,52265,1244.4,24.17,1772.1,39.5,3020867.2,22479051.5,1.0
|
| 70 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.25-r2,wk-gemma4-26b-lora-run2-tok5M-s0.25,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.65,71879,1711.4,30.85,1808.5,39.1,711669.8,3693755.6,0.9968
|
| 71 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.25-r3,wk-gemma4-26b-lora-run2-tok5M-s0.25,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.63,65175,1551.8,28.83,1673.8,39.6,741021.2,3309901.0,0.9881
|
| 72 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.25-think,wk-gemma4-26b-lora-run2-tok5M-s0.25-think,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,1.84,190589,4537.8,28.78,1776.6,39.7,823970.8,12144481.4,0.9762
|
| 73 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.3,wk-gemma4-26b-lora-run2-tok5M-s0.3,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/8,2/3,40/41,1,40,1.17,104072,2538.3,24.64,1655.9,39.2,892457.7,5573081.8,0.9919
|
| 74 |
+
wk-gemma4-26b-lora-run2-tok5M-s0.5,wk-gemma4-26b-lora-run2-tok5M-s0.5,gb10,gemma4-26b-a4b-it-Q8_0.gguf,24/25,5/5,8/9,2/3,39/42,0,39,0.89,101100,2407.1,31.63,1699.4,38.7,793048.5,5243169.5,0.9616
|
| 75 |
+
wk-gemma4-26b-lora-run4-tok5M-s0.1,wk-gemma4-26b-lora-run4-tok5M-s0.1,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,1.1,129632,3086.5,32.84,1819.5,39.0,3064273.5,29611642.6,0.9911
|
| 76 |
+
wk-gemma4-26b-lora-run4-tok5M-s0.15,wk-gemma4-26b-lora-run4-tok5M-s0.15,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.54,58164,1384.9,30.08,1831.4,39.1,2259537.9,22835449.4,0.9802
|
| 77 |
+
wk-gemma4-26b-lora-run4-tok5M-s0.15-r2,wk-gemma4-26b-lora-run4-tok5M-s0.15,gb10,gemma4-26b-a4b-it-Q8_0.gguf,24/25,5/5,9/9,3/3,41/42,0,41,0.95,110012,2619.3,32.33,1848.2,38.4,784115.6,5756261.0,0.9762
|
| 78 |
+
wk-gemma4-26b-lora-run4-tok5M-s0.15-think,wk-gemma4-26b-lora-run4-tok5M-s0.15-think,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,2.73,220567,5379.7,22.48,1796.3,39.3,807926.9,14120288.4,1.0
|
| 79 |
+
wk-gemma4-26b-lora-run4-tok5M-s0.25,wk-gemma4-26b-lora-run4-tok5M-s0.25,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,0.89,103233,2457.9,32.08,1856.1,38.4,786105.8,5391252.3,1.0
|
| 80 |
+
wk-gemma4-26b-lora-run4-tok5M-s0.5,wk-gemma4-26b-lora-run4-tok5M-s0.5,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,7/9,3/3,40/42,0,40,1.14,130602,3109.6,31.88,1709.2,38.2,900464.8,6859446.8,0.9915
|
| 81 |
+
wk-gemma4-26b-lora-run4-tok5M-s0.75,wk-gemma4-26b-lora-run4-tok5M-s0.75,gb10,gemma4-26b-a4b-it-Q8_0.gguf,15/25,3/5,6/8,2/3,26/41,1,26,1.39,168131,4100.8,33.62,1810.5,38.1,885536.2,8838411.5,0.7092
|
| 82 |
+
wk-gemma4-26b-posttrain-alpha0.75,wk-gemma4-26b-posttrain-alpha0.75,gb10,gemma4-26b-a4b-alpha0.75-Q8_0.gguf,25/25,5/5,6/9,3/3,39/42,0,39,2.2,230168,5480.2,29.1,1917.2,40.1,942246.7,11493812.6,0.9286
|
| 83 |
+
wk-gemma4-26b-stock-default-r1,wk-gemma4-26b-stock-default,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.83,191275,4554.2,29.05,2019.1,42.5,657656.9,12216754.1,1.0
|
| 84 |
+
wk-gemma4-26b-stock-default-r2,wk-gemma4-26b-stock-default,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.63,167638,3991.4,28.57,2002.3,42.6,632225.8,10843449.4,1.0
|
| 85 |
+
wk-gemma4-26b-stock-reasonoff-r1,wk-gemma4-26b-stock-reasonoff,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,9/9,3/3,42/42,0,42,0.7,85011,2024.1,33.78,1940.6,41.9,679052.9,4077583.4,1.0
|
| 86 |
+
wk-gemma4-26b-stock-reasonoff-r2,wk-gemma4-26b-stock-reasonoff,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.96,118271,2816.0,34.09,1963.4,40.4,743852.8,5882207.2,0.9911
|
| 87 |
+
wk-gemma4-26b-stock-reasonoff-r3,wk-gemma4-26b-stock-reasonoff,gb10,gemma4-26b-a4b-it-Q8_0.gguf,25/25,5/5,8/9,3/3,41/42,0,41,0.96,119002,2833.4,34.59,2032.6,40.7,725411.0,5874504.7,0.9762
|
scripts/aggregate_results.py
CHANGED
|
@@ -42,6 +42,37 @@ SUITES = [
|
|
| 42 |
]
|
| 43 |
|
| 44 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
def model_path(lab: str, label: str) -> str:
|
| 46 |
"""Best-effort: the served GGUF basename from a local server log, if retained."""
|
| 47 |
for path in sorted(glob.glob(os.path.join(lab, "benchmarks", "results", f"{label}*server.log"))):
|
|
@@ -92,6 +123,7 @@ def main() -> None:
|
|
| 92 |
|
| 93 |
runs, partial_runs = [], []
|
| 94 |
for label, suites in sorted(per_label.items()):
|
|
|
|
| 95 |
record = {
|
| 96 |
"label": label,
|
| 97 |
"group": re.sub(r"-(r|rep)\d+$", "", label),
|
|
@@ -111,6 +143,7 @@ def main() -> None:
|
|
| 111 |
"total_void": sum(s["void"] for s in suites.values()),
|
| 112 |
"total_wall_s": round(sum(s["wall_s"] for s in suites.values()), 1),
|
| 113 |
"total_tokens_out": sum(s["tokens_out"] for s in suites.values()),
|
|
|
|
| 114 |
}
|
| 115 |
(runs if len(suites) == len(SUITES) else partial_runs).append(record)
|
| 116 |
|
|
@@ -131,7 +164,9 @@ def main() -> None:
|
|
| 131 |
writer.writerow(["label", "group", "box", "model_path",
|
| 132 |
"general", "hard", "frontier", "tier4a",
|
| 133 |
"valid_total", "void", "passed_total",
|
| 134 |
-
"wall_h", "tokens_out", "tokens_per_task",
|
|
|
|
|
|
|
| 135 |
for r in sorted(runs, key=lambda x: (x["group"], x["label"])):
|
| 136 |
s = r["suites"]
|
| 137 |
writer.writerow([
|
|
@@ -145,6 +180,8 @@ def main() -> None:
|
|
| 145 |
r["total_tokens_out"],
|
| 146 |
round(r["total_tokens_out"] / max(1, r["total_valid"]), 1),
|
| 147 |
round(r["total_tokens_out"] / max(1, r["total_wall_s"]), 2),
|
|
|
|
|
|
|
| 148 |
round(sum(s[k]["partial"] * s[k]["tasks"] for k in s) / max(1, r["total_valid"]), 4),
|
| 149 |
])
|
| 150 |
|
|
|
|
| 42 |
]
|
| 43 |
|
| 44 |
|
| 45 |
+
def server_timings(lab: str, label: str) -> dict:
|
| 46 |
+
"""Parse llama.cpp server logs for raw prefill/decode throughput.
|
| 47 |
+
|
| 48 |
+
Sums ``prompt eval time`` (prefill) and bare ``eval time`` (decode)
|
| 49 |
+
across every server log matching *label* (resumed runs produce 2-4
|
| 50 |
+
files). Returns ``{prefill_tok, prefill_ms, decode_tok, decode_ms}``
|
| 51 |
+
or an empty dict when no log yields timings.
|
| 52 |
+
"""
|
| 53 |
+
prompt_re = re.compile(r"prompt eval time =\s*([0-9.]+) ms /\s*(\d+) tokens")
|
| 54 |
+
decode_re = re.compile(r"(?<!prompt )eval time =\s*([0-9.]+) ms /\s*(\d+) tokens")
|
| 55 |
+
p_tok = p_ms = d_tok = d_ms = 0
|
| 56 |
+
found = False
|
| 57 |
+
for path in sorted(glob.glob(os.path.join(lab, "benchmarks", "results", f"{label}*server.log"))):
|
| 58 |
+
try:
|
| 59 |
+
text = open(path, errors="ignore").read()
|
| 60 |
+
except OSError:
|
| 61 |
+
continue
|
| 62 |
+
for m in prompt_re.finditer(text):
|
| 63 |
+
p_ms += float(m.group(1)); p_tok += int(m.group(2)); found = True
|
| 64 |
+
for m in decode_re.finditer(text):
|
| 65 |
+
d_ms += float(m.group(1)); d_tok += int(m.group(2)); found = True
|
| 66 |
+
if not found:
|
| 67 |
+
return {}
|
| 68 |
+
return {
|
| 69 |
+
"prefill_tok": p_tok, "prefill_ms": round(p_ms, 1),
|
| 70 |
+
"decode_tok": d_tok, "decode_ms": round(d_ms, 1),
|
| 71 |
+
"prefill_tok_s": round(p_tok / (p_ms / 1000), 1) if p_ms else None,
|
| 72 |
+
"decode_tok_s": round(d_tok / (d_ms / 1000), 1) if d_ms else None,
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
def model_path(lab: str, label: str) -> str:
|
| 77 |
"""Best-effort: the served GGUF basename from a local server log, if retained."""
|
| 78 |
for path in sorted(glob.glob(os.path.join(lab, "benchmarks", "results", f"{label}*server.log"))):
|
|
|
|
| 123 |
|
| 124 |
runs, partial_runs = [], []
|
| 125 |
for label, suites in sorted(per_label.items()):
|
| 126 |
+
timings = server_timings(args.lab, label)
|
| 127 |
record = {
|
| 128 |
"label": label,
|
| 129 |
"group": re.sub(r"-(r|rep)\d+$", "", label),
|
|
|
|
| 143 |
"total_void": sum(s["void"] for s in suites.values()),
|
| 144 |
"total_wall_s": round(sum(s["wall_s"] for s in suites.values()), 1),
|
| 145 |
"total_tokens_out": sum(s["tokens_out"] for s in suites.values()),
|
| 146 |
+
**timings,
|
| 147 |
}
|
| 148 |
(runs if len(suites) == len(SUITES) else partial_runs).append(record)
|
| 149 |
|
|
|
|
| 164 |
writer.writerow(["label", "group", "box", "model_path",
|
| 165 |
"general", "hard", "frontier", "tier4a",
|
| 166 |
"valid_total", "void", "passed_total",
|
| 167 |
+
"wall_h", "tokens_out", "tokens_per_task",
|
| 168 |
+
"tok_s_eff", "tok_s_prefill", "tok_s_decode",
|
| 169 |
+
"prefill_ms", "decode_ms", "partial_mean"])
|
| 170 |
for r in sorted(runs, key=lambda x: (x["group"], x["label"])):
|
| 171 |
s = r["suites"]
|
| 172 |
writer.writerow([
|
|
|
|
| 180 |
r["total_tokens_out"],
|
| 181 |
round(r["total_tokens_out"] / max(1, r["total_valid"]), 1),
|
| 182 |
round(r["total_tokens_out"] / max(1, r["total_wall_s"]), 2),
|
| 183 |
+
r.get("prefill_tok_s"), r.get("decode_tok_s"),
|
| 184 |
+
r.get("prefill_ms"), r.get("decode_ms"),
|
| 185 |
round(sum(s[k]["partial"] * s[k]["tasks"] for k in s) / max(1, r["total_valid"]), 4),
|
| 186 |
])
|
| 187 |
|
scripts/render_table.py
CHANGED
|
@@ -41,9 +41,11 @@ def reps_text(model: dict, runs: list[dict]) -> str:
|
|
| 41 |
|
| 42 |
|
| 43 |
def render(models: list[dict], runs: list[dict]) -> str:
|
|
|
|
|
|
|
| 44 |
lines = [
|
| 45 |
-
"| Model | Quant | G (25) | H (5) | F (9) | T4A (3) | Valid | Void | Tok/task |
|
| 46 |
-
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|",
|
| 47 |
]
|
| 48 |
for model in models:
|
| 49 |
run = best_run(model, runs)
|
|
@@ -63,6 +65,8 @@ def render(models: list[dict], runs: list[dict]) -> str:
|
|
| 63 |
f"| **{run['total_passed']}/{run['total_valid']}** "
|
| 64 |
f"| {run['total_void']} "
|
| 65 |
f"| {run['total_tokens_out']/valid:,.0f} "
|
|
|
|
|
|
|
| 66 |
f"| {run['total_tokens_out']/wall:.1f} "
|
| 67 |
f"| {reps_text(model, runs)} |"
|
| 68 |
)
|
|
|
|
| 41 |
|
| 42 |
|
| 43 |
def render(models: list[dict], runs: list[dict]) -> str:
|
| 44 |
+
def fmt(v: float | None) -> str:
|
| 45 |
+
return f"{v:.1f}" if isinstance(v, (int, float)) else "—"
|
| 46 |
lines = [
|
| 47 |
+
"| Model | Quant | G (25) | H (5) | F (9) | T4A (3) | Valid | Void | Tok/task | Prefill | Decode | Eff. | Reps |",
|
| 48 |
+
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|",
|
| 49 |
]
|
| 50 |
for model in models:
|
| 51 |
run = best_run(model, runs)
|
|
|
|
| 65 |
f"| **{run['total_passed']}/{run['total_valid']}** "
|
| 66 |
f"| {run['total_void']} "
|
| 67 |
f"| {run['total_tokens_out']/valid:,.0f} "
|
| 68 |
+
f"| {fmt(run.get('prefill_tok_s'))} "
|
| 69 |
+
f"| {fmt(run.get('decode_tok_s'))} "
|
| 70 |
f"| {run['total_tokens_out']/wall:.1f} "
|
| 71 |
f"| {reps_text(model, runs)} |"
|
| 72 |
)
|