lab-bench 1cb5dab2b8ed1ad228a67a3d36f6ec0363f56a39

data/capabilities.jsonl

@@ -1,3 +1,3 @@
-{"id": "C1", "capability": "German retrieval for RAG (embedder, optional reranker)", "best_local_model": "+bge-rerank", "best_by_value": "+bge-rerank", "licence_class": "open source", "metric": "nDCG@10 on MIRACL de dev", "value": 0.6404, "ci95_low": 0.6047, "ci95_high": 0.6773, "baseline": "BM25", "baseline_value": 0.1991, "sanity_reference": "MTEB bge-m3 0.5759 (sanity gate)", "sanity_reference_ours": 0.5767, "speed_unit": "ms per query (batch 1, encode plus exact search over 71,277 passages, plus reranking the top 100)", "speed_value": 209.445, "vram_peak_gb": 21.83, "energy_unit": "GPU joules per 1,000 passages indexed", "energy_value": 10090.5953, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown (equal peak VRAM, so the better value is shown)", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/99200d408dd455f123c4345d65d71b193182fb26/plan/c1.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/99200d408dd455f123c4345d65d71b193182fb26/results/c1/findings.md"}
-{"id": "C2", "capability": "German speech to text", "best_local_model": "whisper-large-v3", "best_by_value": "canary-1b-v2", "licence_class": "open source", "metric": "WER percent, FLEURS de_de test (lower is better)", "value": 4.7, "ci95_low": 3.95, "ci95_high": 5.5, "baseline": "whisper-large-v3 (anchor)", "baseline_value": 4.7, "sanity_reference": "canary-1b-v2 card 4.40 (sanity gate)", "sanity_reference_ours": 4.52, "speed_unit": "real-time factor, median at batch 1 (seconds of compute per second of audio)", "speed_value": 0.0467, "vram_peak_gb": 8.88, "energy_unit": "GPU joules per audio hour (batch 1)", "energy_value": 22235, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/99200d408dd455f123c4345d65d71b193182fb26/plan/c2.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/99200d408dd455f123c4345d65d71b193182fb26/results/c2/findings.md"}
-{"id": "C3", "capability": "Invoice fields from page images", "best_local_model": "gemma4-12b", "best_by_value": "gemma4-26b", "licence_class": "open source", "metric": "field accuracy over 408 cells (34 invoices x 12 fields)", "value": 0.9779, "ci95_low": 0.9632, "ci95_high": 0.9902, "baseline": "regex over the PDF text layer", "baseline_value": 0.2696, "sanity_reference": "none published; internal render check 391 of 391 (sanity gate)", "sanity_reference_ours": null, "speed_unit": "seconds per invoice, median (2 to 9 pages)", "speed_value": 2.0979, "vram_peak_gb": 9.79, "energy_unit": "GPU joules per invoice", "energy_value": 614, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown", "unfinished_arms": ["qwen3-vl-32b"], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/99200d408dd455f123c4345d65d71b193182fb26/plan/c3.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/99200d408dd455f123c4345d65d71b193182fb26/results/c3/findings.md"}
+{"id": "C1", "capability": "German retrieval for RAG (embedder, optional reranker)", "best_local_model": "+bge-rerank", "best_by_value": "+bge-rerank", "licence_class": "open source", "metric": "nDCG@10 on MIRACL de dev", "value": 0.6404, "ci95_low": 0.6047, "ci95_high": 0.6773, "baseline": "BM25", "baseline_value": 0.1991, "sanity_reference": "MTEB bge-m3 0.5759 (sanity gate)", "sanity_reference_ours": 0.5767, "speed_unit": "ms per query (batch 1, encode plus exact search over 71,277 passages, plus reranking the top 100)", "speed_value": 209.445, "vram_peak_gb": 21.83, "energy_unit": "GPU joules per 1,000 passages indexed", "energy_value": 10090.5953, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown (equal peak VRAM, so the better value is shown)", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/1cb5dab2b8ed1ad228a67a3d36f6ec0363f56a39/plan/c1.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/1cb5dab2b8ed1ad228a67a3d36f6ec0363f56a39/results/c1/findings.md"}
+{"id": "C2", "capability": "German speech to text", "best_local_model": "whisper-large-v3", "best_by_value": "canary-1b-v2", "licence_class": "open source", "metric": "WER percent, FLEURS de_de test (lower is better)", "value": 4.7, "ci95_low": 3.95, "ci95_high": 5.5, "baseline": "whisper-large-v3 (anchor)", "baseline_value": 4.7, "sanity_reference": "canary-1b-v2 card 4.40 (sanity gate)", "sanity_reference_ours": 4.52, "speed_unit": "real-time factor, median at batch 1 (seconds of compute per second of audio)", "speed_value": 0.0467, "vram_peak_gb": 8.88, "energy_unit": "GPU joules per audio hour (batch 1)", "energy_value": 22235, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown", "unfinished_arms": [], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/1cb5dab2b8ed1ad228a67a3d36f6ec0363f56a39/plan/c2.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/1cb5dab2b8ed1ad228a67a3d36f6ec0363f56a39/results/c2/findings.md"}
+{"id": "C3", "capability": "Invoice fields from page images", "best_local_model": "gemma4-12b", "best_by_value": "gemma4-26b", "licence_class": "open source", "metric": "field accuracy over 408 cells (34 invoices x 12 fields)", "value": 0.9779, "ci95_low": 0.9632, "ci95_high": 0.9902, "baseline": "regex over the PDF text layer", "baseline_value": 0.2696, "sanity_reference": "none published; internal render check 391 of 391 (sanity gate)", "sanity_reference_ours": null, "speed_unit": "seconds per invoice, median (2 to 9 pages)", "speed_value": 2.0979, "vram_peak_gb": 9.79, "energy_unit": "GPU joules per invoice", "energy_value": 614, "fits_24gb": true, "tie_note": "intervals overlap and the paired test does not separate the two; the one with less peak VRAM is shown", "unfinished_arms": ["qwen3-vl-32b"], "machine": "local-rtx4090: one RTX 4090 24 GB, Windows 11, Ryzen 9 7900, 128 GB RAM (results/<cN>/<arm>/queue-status.json)", "design_url": "https://github.com/bks-lab/lab-bench/blob/1cb5dab2b8ed1ad228a67a3d36f6ec0363f56a39/plan/c3.md", "report_url": "https://github.com/bks-lab/lab-bench/blob/1cb5dab2b8ed1ad228a67a3d36f6ec0363f56a39/results/c3/findings.md"}