| {"id": "C1", "capability": "German retrieval for RAG (embedder, optional reranker)", "best_local_model": "+bge-rerank", "best_by_value": "+bge-rerank", "licence_class": "open source", "metric": "nDCG@10 on MIRACL de dev", "value": 0.6404, "ci95_low": 0.6047, "ci95_high": 0.6773, "baseline": "BM25", "baseline_value": 0.1991, "sanity_reference": "MTEB bge-m3 0.5759 (sanity gate)", "sanity_reference_ours": 0.5767, "speed_unit": "ms per query (batch 1, encode plus exact search over 71,277 passages, plus reranking the top 100)", "speed_value": 209.445, "vram_peak_gb": 21.83, "energy_unit": "GPU joules per 1,000 passages indexed", "energy_value": 10090.5953, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown (equal peak VRAM, so the better value is shown)", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c1.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c1/findings.md"} | |
| {"id": "C2", "capability": "German speech to text", "best_local_model": "whisper-large-v3", "best_by_value": "canary-1b-v2", "licence_class": "open source", "metric": "WER percent, FLEURS de_de test (lower is better)", "value": 4.7, "ci95_low": 3.95, "ci95_high": 5.5, "baseline": "whisper-large-v3 (anchor)", "baseline_value": 4.7, "sanity_reference": "canary-1b-v2 card 4.40 (sanity gate)", "sanity_reference_ours": 4.52, "speed_unit": "real-time factor, median at batch 1 (seconds of compute per second of audio)", "speed_value": 0.0467, "vram_peak_gb": 8.88, "energy_unit": "GPU joules per audio hour (batch 1)", "energy_value": 22235, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c2.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c2/findings.md"} | |
| {"id": "C3", "capability": "Invoice fields from page images", "best_local_model": "gemma4-12b", "best_by_value": "gemma4-26b", "licence_class": "open source", "metric": "field accuracy over 408 cells (34 invoices x 12 fields)", "value": 0.9779, "ci95_low": 0.9632, "ci95_high": 0.9902, "baseline": "regex over the PDF text layer", "baseline_value": 0.2696, "sanity_reference": "none published; internal render check 391 of 391 (sanity gate)", "sanity_reference_ours": null, "speed_unit": "seconds per invoice, median (2 to 9 pages)", "speed_value": 2.0979, "vram_peak_gb": 9.79, "energy_unit": "GPU joules per invoice", "energy_value": 614, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown", "unfinished_arms": ["qwen3-vl-32b"], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c3.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c3/findings.md"} | |
| {"id": "C4", "capability": "Structured output and tool calling", "best_local_model": "qwen3-14b", "best_by_value": "qwen3-14b", "licence_class": "open source", "metric": "AST accuracy, BFCL v3 non-live, 1,000 items, native tool calling", "value": 0.919, "ci95_low": 0.901, "ci95_high": 0.935, "baseline": "prompt mode of the same model", "baseline_value": 0.946, "sanity_reference": "BFCL leaderboard Qwen3-14B (FC) simple 95.25 % (sanity gate)", "sanity_reference_ours": 0.935, "speed_unit": "ms per tool-calling item, median (one call at a time)", "speed_value": 606.5, "vram_peak_gb": 10.31, "energy_unit": "GPU joules per 1,000 tool-calling items", "energy_value": 288452, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c4.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c4/findings.md"} | |
| {"id": "C5", "capability": "translation German to English and English to German", "best_local_model": "gemma4-12b", "best_by_value": "gemma4-26b", "licence_class": "open source", "metric": "chrF++ on FLORES+ devtest (mean of de-en and en-de, 0 to 100)", "value": 66.5, "ci95_low": 65.69, "ci95_high": 67.31, "baseline": "source copied as the translation", "baseline_value": 19.83, "sanity_reference": "OPUS-MT leaderboard opus-2020-02-26 on flores101-devtest, plain chrF 66.70 de-en and 63.63 en-de (sanity gate, tolerance 1.5)", "sanity_reference_ours": 65.13, "speed_unit": "segments per second at batch 1", "speed_value": 4.19, "vram_peak_gb": 8.72, "energy_unit": "GPU joules per 1,000 segments", "energy_value": 73847, "fits_24gb": true, "tie_note": "the paired test does not separate the two in either direction; the one with less peak VRAM is shown", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c5.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c5/findings.md"} | |
| {"id": "C6", "capability": "German reading comprehension (answer from a passage)", "best_local_model": "gelectra-large", "best_by_value": "gelectra-large", "licence_class": "open source", "metric": "F1 on 1,000 GermanQuAD test questions", "value": 0.8721, "ci95_low": 0.8551, "ci95_high": 0.8883, "baseline": "best-overlap sentence of the passage", "baseline_value": 0.2213, "sanity_reference": "GELECTRA-large card F1 88.1 on the full test set (sanity gate)", "sanity_reference_ours": 0.871, "speed_unit": "ms per question, median", "speed_value": 27.5, "vram_peak_gb": 1.73, "energy_unit": "GPU joules per 1,000 questions", "energy_value": 10638, "fits_24gb": true, "tie_note": null, "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c6.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c6/findings.md"} | |
| {"id": "C7", "capability": "Long context, 4k to 32k tokens", "best_local_model": "qwen3-14b", "best_by_value": "qwen3-14b", "licence_class": "open source", "metric": "needle accuracy at 32k tokens (180 prompts)", "value": 1.0, "ci95_low": 1.0, "ci95_high": 1.0, "baseline": "the same model at 4k", "baseline_value": 1.0, "sanity_reference": "none published; internal check: 95 % on single at 4k (sanity gate)", "sanity_reference_ours": null, "speed_unit": "seconds of prompt processing at 32k tokens, median", "speed_value": 8.0632, "vram_peak_gb": 13.97, "energy_unit": "GPU joules per 32k-token prompt", "energy_value": 6412, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c7.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c7/findings.md"} | |
| {"id": "C8", "capability": "Throughput and energy per model size", "best_local_model": "granite4-3b", "best_by_value": "granite4-3b", "licence_class": "open source", "metric": "none (throughput test); headline: most one-token decisions per hour at 16 parallel clients among arms that stay on the card", "value": null, "ci95_low": null, "ci95_high": null, "baseline": null, "baseline_value": null, "sanity_reference": null, "sanity_reference_ours": null, "speed_unit": "decisions per hour at 16 parallel clients (route1 one-token method)", "speed_value": 260731, "vram_peak_gb": 8.88, "energy_unit": "GPU joules per 1,000 decisions at 16 parallel clients", "energy_value": 3169, "fits_24gb": true, "tie_note": null, "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c8.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c8/findings.md"} | |
| {"id": "C9", "capability": "personal data detection in German text", "best_local_model": "gemma4-12b", "best_by_value": "gemma4-12b", "licence_class": "open source", "metric": "span F1 on the audited gold of 40 documents (exact boundaries, type-agnostic); gold noisy, descriptive only", "value": 0.83871, "ci95_low": 0.705858, "ci95_high": 0.928965, "baseline": "regex (e-mail, IBAN, phone)", "baseline_value": 0.15, "sanity_reference": "none published for this German subset; internal sanity gate regex e-mail recall >= 0.95", "sanity_reference_ours": 0.965116, "speed_unit": "median ms per document", "speed_value": 516.9, "vram_peak_gb": 9.19, "energy_unit": "GPU joules per 1,000 documents", "energy_value": 158520, "fits_24gb": true, "tie_note": "the paired bootstrap on the 40 audited documents does not separate the two; the one with less peak VRAM is shown. On all 476 documents (original gold) the order is the other way round and equally not separated", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/plan/c9.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/c9/findings.md"} | |