Spaces:

bks-lab
/

lab-bench

Running

File size: 89,876 Bytes

{"task_id": "route1-scenario-de", "kind": "route", "task": "route: MASSIVE scenario (18 classes)", "lang": "de", "metric": "accuracy", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 72.8, "baseline": 13.5, "baseline_name": "most frequent answer", "skill": 0.686, "n": 2974, "answered": 2974, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": 232, "measured": "2026-10-09", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"cost_note\": \"median input tokens per call x USD 0.042 per million input tokens, output free (docs.typesafe.ai/models, read 2026-10-02); per source and language, not per question group\", \"macro_f1\": 73.3, \"median_input_tokens_per_decision\": 510, \"tied_with\": [], \"usd_per_1000_decisions\": 0.0214}"}
{"task_id": "route1-scenario-de", "kind": "route", "task": "route: MASSIVE scenario (18 classes)", "lang": "de", "metric": "accuracy", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide", "arms_compared": 2, "value": 48.5, "baseline": 13.5, "baseline_name": "most frequent answer", "skill": 0.404, "n": 2974, "answered": 2974, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": 14, "measured": "2026-10-09", "model_version": "fastino/GLiNER2.5-multi-Decide (revision e173bca1f0c4)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"english_model_extra\": {\"arm\": \"gliner2.5-decide\", \"note\": \"English-only model on German rows, labelled extra in the report, not eligible as best arm\", \"skill\": 0.41, \"tied_with\": [], \"value\": 49}, \"macro_f1\": 46.4, \"tied_with\": [], \"vs_top_only_arm_right\": 188, \"vs_top_only_top_right\": 912, \"vs_top_paired\": 2974}"}
{"task_id": "route1-scenario-de", "kind": "route", "task": "route: MASSIVE scenario (18 classes)", "lang": "de", "metric": "accuracy", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 71.8, "baseline": 13.5, "baseline_name": "most frequent answer", "skill": 0.674, "n": 2974, "answered": 2974, "top_family": "jev", "mcnemar_vs_top_p": "0.11", "mcnemar_vs_jev_p": "0.11", "p50_ms_per_decision": 89, "measured": "2026-10-09", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0 (Ollama digest b5bedb246945, ollama 0.35.0)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 72.3, \"tied_with\": [], \"vs_top_only_arm_right\": 161, \"vs_top_only_top_right\": 192, \"vs_top_paired\": 2974}"}
{"task_id": "route1-scenario-de", "kind": "route", "task": "route: MASSIVE scenario (18 classes)", "lang": "de", "metric": "accuracy", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "measured", "best_arm": "classic-e5-lr", "arms_compared": 2, "value": 86.9, "baseline": 13.5, "baseline_name": "most frequent answer", "skill": 0.848, "n": 2974, "answered": 2974, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": 25, "measured": "2026-10-09", "model_version": "intfloat/multilingual-e5-base + sklearn LogisticRegression(C=1.0, multinomial) (revision d12875059715)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 86.1, \"tied_with\": [], \"vs_top_only_arm_right\": 559, \"vs_top_only_top_right\": 141, \"vs_top_paired\": 2974}"}
{"task_id": "route1-scenario-en", "kind": "route", "task": "route: MASSIVE scenario (18 classes)", "lang": "en", "metric": "accuracy", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 74.1, "baseline": 13.5, "baseline_name": "most frequent answer", "skill": 0.701, "n": 2974, "answered": 2974, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": 231, "measured": "2026-10-09", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"cost_note\": \"median input tokens per call x USD 0.042 per million input tokens, output free (docs.typesafe.ai/models, read 2026-10-02); per source and language, not per question group\", \"macro_f1\": 74.2, \"median_input_tokens_per_decision\": 504, \"tied_with\": [], \"usd_per_1000_decisions\": 0.0212}"}
{"task_id": "route1-scenario-en", "kind": "route", "task": "route: MASSIVE scenario (18 classes)", "lang": "en", "metric": "accuracy", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide", "arms_compared": 6, "value": 60.9, "baseline": 13.5, "baseline_name": "most frequent answer", "skill": 0.547, "n": 2974, "answered": 2974, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": 26, "measured": "2026-10-09", "model_version": "fastino/GLiNER2.5-Decide (revision 9d1bfb848cd1)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 60.2, \"tied_with\": [], \"vs_top_only_arm_right\": 149, \"vs_top_only_top_right\": 543, \"vs_top_paired\": 2974}"}
{"task_id": "route1-scenario-en", "kind": "route", "task": "route: MASSIVE scenario (18 classes)", "lang": "en", "metric": "accuracy", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 70.6, "baseline": 13.5, "baseline_name": "most frequent answer", "skill": 0.661, "n": 2974, "answered": 2974, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": 92, "measured": "2026-10-09", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0 (Ollama digest b5bedb246945, ollama 0.35.0)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 71.8, \"tied_with\": [], \"vs_top_only_arm_right\": 134, \"vs_top_only_top_right\": 237, \"vs_top_paired\": 2974}"}
{"task_id": "route1-scenario-en", "kind": "route", "task": "route: MASSIVE scenario (18 classes)", "lang": "en", "metric": "accuracy", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "measured", "best_arm": "classic-e5-lr", "arms_compared": 2, "value": 89.8, "baseline": 13.5, "baseline_name": "most frequent answer", "skill": 0.883, "n": 2974, "answered": 2974, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": 23, "measured": "2026-10-09", "model_version": "intfloat/multilingual-e5-base + sklearn LogisticRegression(C=1.0, multinomial) (revision d12875059715)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 89.6, \"tied_with\": [], \"vs_top_only_arm_right\": 560, \"vs_top_only_top_right\": 92, \"vs_top_paired\": 2974}"}
{"task_id": "route1-intent2-de", "kind": "route", "task": "route: MASSIVE intent given the gold scenario, 2+ options", "lang": "de", "metric": "accuracy", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 90.7, "baseline": 50.2, "baseline_name": "most frequent answer", "skill": 0.812, "n": 2622, "answered": 2622, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": 234, "measured": "2026-10-09", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"cost_note\": \"median input tokens per call x USD 0.042 per million input tokens, output free (docs.typesafe.ai/models, read 2026-10-02); per source and language, not per question group\", \"macro_f1\": 86.5, \"median_input_tokens_per_decision\": 510, \"tied_with\": [], \"usd_per_1000_decisions\": 0.0214}"}
{"task_id": "route1-intent2-de", "kind": "route", "task": "route: MASSIVE intent given the gold scenario, 2+ options", "lang": "de", "metric": "accuracy", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide", "arms_compared": 2, "value": 58.9, "baseline": 50.2, "baseline_name": "most frequent answer", "skill": 0.175, "n": 2622, "answered": 2622, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": 13, "measured": "2026-10-09", "model_version": "fastino/GLiNER2.5-multi-Decide (revision e173bca1f0c4)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"english_model_extra\": {\"arm\": \"gliner2.5-decide\", \"note\": \"English-only model on German rows, labelled extra in the report, not eligible as best arm\", \"skill\": 0.316, \"tied_with\": [], \"value\": 65.9}, \"macro_f1\": 54.9, \"tied_with\": [], \"vs_top_only_arm_right\": 67, \"vs_top_only_top_right\": 899, \"vs_top_paired\": 2622}"}
{"task_id": "route1-intent2-de", "kind": "route", "task": "route: MASSIVE intent given the gold scenario, 2+ options", "lang": "de", "metric": "accuracy", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 89, "baseline": 50.2, "baseline_name": "most frequent answer", "skill": 0.779, "n": 2622, "answered": 2621, "top_family": "jev", "mcnemar_vs_top_p": "0.002", "mcnemar_vs_jev_p": "0.002", "p50_ms_per_decision": 84, "measured": "2026-10-09", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0 (Ollama digest b5bedb246945, ollama 0.35.0)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 82.5, \"tied_with\": [], \"vs_top_only_arm_right\": 71, \"vs_top_only_top_right\": 115, \"vs_top_paired\": 2622}"}
{"task_id": "route1-intent2-de", "kind": "route", "task": "route: MASSIVE intent given the gold scenario, 2+ options", "lang": "de", "metric": "accuracy", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "measured", "best_arm": "classic-e5-lr", "arms_compared": 2, "value": 86.9, "baseline": 50.2, "baseline_name": "most frequent answer", "skill": 0.737, "n": 2622, "answered": 2622, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": 24, "measured": "2026-10-09", "model_version": "intfloat/multilingual-e5-base + sklearn LogisticRegression(C=1.0, multinomial) (revision d12875059715)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 72.5, \"tied_with\": [], \"vs_top_only_arm_right\": 144, \"vs_top_only_top_right\": 243, \"vs_top_paired\": 2622}"}
{"task_id": "route1-intent2-en", "kind": "route", "task": "route: MASSIVE intent given the gold scenario, 2+ options", "lang": "en", "metric": "accuracy", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 92, "baseline": 50.2, "baseline_name": "most frequent answer", "skill": 0.838, "n": 2622, "answered": 2622, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": 230, "measured": "2026-10-09", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"cost_note\": \"median input tokens per call x USD 0.042 per million input tokens, output free (docs.typesafe.ai/models, read 2026-10-02); per source and language, not per question group\", \"macro_f1\": 88.4, \"median_input_tokens_per_decision\": 504, \"tied_with\": [], \"usd_per_1000_decisions\": 0.0212}"}
{"task_id": "route1-intent2-en", "kind": "route", "task": "route: MASSIVE intent given the gold scenario, 2+ options", "lang": "en", "metric": "accuracy", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide", "arms_compared": 6, "value": 78.3, "baseline": 50.2, "baseline_name": "most frequent answer", "skill": 0.565, "n": 2622, "answered": 2622, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": 26, "measured": "2026-10-09", "model_version": "fastino/GLiNER2.5-Decide (revision 9d1bfb848cd1)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 74.3, \"tied_with\": [], \"vs_top_only_arm_right\": 71, \"vs_top_only_top_right\": 428, \"vs_top_paired\": 2622}"}
{"task_id": "route1-intent2-en", "kind": "route", "task": "route: MASSIVE intent given the gold scenario, 2+ options", "lang": "en", "metric": "accuracy", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 91.6, "baseline": 50.2, "baseline_name": "most frequent answer", "skill": 0.832, "n": 2622, "answered": 2622, "top_family": "jev", "mcnemar_vs_top_p": "0.575", "mcnemar_vs_jev_p": "0.575", "p50_ms_per_decision": 88, "measured": "2026-10-09", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0 (Ollama digest b5bedb246945, ollama 0.35.0)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 86.4, \"tied_with\": [], \"vs_top_only_arm_right\": 74, \"vs_top_only_top_right\": 82, \"vs_top_paired\": 2622}"}
{"task_id": "route1-intent2-en", "kind": "route", "task": "route: MASSIVE intent given the gold scenario, 2+ options", "lang": "en", "metric": "accuracy", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "measured", "best_arm": "classic-e5-lr", "arms_compared": 2, "value": 91.2, "baseline": 50.2, "baseline_name": "most frequent answer", "skill": 0.824, "n": 2622, "answered": 2622, "top_family": "jev", "mcnemar_vs_top_p": "0.298", "mcnemar_vs_jev_p": "0.298", "p50_ms_per_decision": 23, "measured": "2026-10-09", "model_version": "intfloat/multilingual-e5-base + sklearn LogisticRegression(C=1.0, multinomial) (revision d12875059715)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 79.6, \"tied_with\": [], \"vs_top_only_arm_right\": 140, \"vs_top_only_top_right\": 159, \"vs_top_paired\": 2622}"}
{"task_id": "route1-fastdec26-en", "kind": "route", "task": "route: fast-decisions dev, single-label tasks with 26 labels or fewer", "lang": "en", "metric": "accuracy", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 63, "baseline": 31.2, "baseline_name": "most frequent answer", "skill": 0.461, "n": 2500, "answered": 2500, "top_family": "gliner", "mcnemar_vs_top_p": "0.168", "mcnemar_vs_jev_p": null, "p50_ms_per_decision": 240, "measured": "2026-10-09", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"cost_note\": \"median input tokens per call x USD 0.042 per million input tokens, output free (docs.typesafe.ai/models, read 2026-10-02); per source and language, not per question group\", \"macro_f1\": 55.9, \"median_input_tokens_per_decision\": 544, \"tied_with\": [], \"usd_per_1000_decisions\": 0.0228, \"vs_top_only_arm_right\": 305, \"vs_top_only_top_right\": 341, \"vs_top_paired\": 2500}"}
{"task_id": "route1-fastdec26-en", "kind": "route", "task": "route: fast-decisions dev, single-label tasks with 26 labels or fewer", "lang": "en", "metric": "accuracy", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide-1b-intext", "arms_compared": 6, "value": 64.4, "baseline": 31.2, "baseline_name": "most frequent answer", "skill": 0.482, "n": 2500, "answered": 2500, "top_family": "gliner", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": "0.168", "p50_ms_per_decision": 26, "measured": "2026-10-09", "model_version": "fastino/GLiNER2.5-Decide-1B (revision 67af949e17be)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 60.4, \"tied_with\": []}"}
{"task_id": "route1-fastdec26-en", "kind": "route", "task": "route: fast-decisions dev, single-label tasks with 26 labels or fewer", "lang": "en", "metric": "accuracy", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 63.6, "baseline": 31.2, "baseline_name": "most frequent answer", "skill": 0.471, "n": 2500, "answered": 2500, "top_family": "gliner", "mcnemar_vs_top_p": "0.483", "mcnemar_vs_jev_p": "0.309", "p50_ms_per_decision": 120, "measured": "2026-10-09", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0 (Ollama digest b5bedb246945, ollama 0.35.0)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"macro_f1\": 55.3, \"tied_with\": [], \"vs_top_only_arm_right\": 319, \"vs_top_only_top_right\": 338, \"vs_top_paired\": 2500}"}
{"task_id": "route1-fastdec26-en", "kind": "route", "task": "route: fast-decisions dev, single-label tasks with 26 labels or fewer", "lang": "en", "metric": "accuracy", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "fast-decisions publishes only the dev split, no train split, so the classic classifier has nothing to train on (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/score.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "ex1-slots-de", "kind": "extract", "task": "slot extraction (MASSIVE slots)", "lang": "de", "metric": "micro F1, threshold 0.5 for GLiNER", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "Jev has no extraction primitive (choice, score, noul only), docs.typesafe.ai/introduction", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/ex1/method.md", "extra_json": "{}"}
{"task_id": "ex1-slots-de", "kind": "extract", "task": "slot extraction (MASSIVE slots)", "lang": "de", "metric": "micro F1, threshold 0.5 for GLiNER", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi", "arms_compared": 1, "value": 22.5, "baseline": 0, "baseline_name": "no extraction", "skill": 0.225, "n": 1000, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-09", "model_version": "fastino/gliner2.5-multi-v1 (revision cf5593a5d45e)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/ex1/score.md", "extra_json": "{\"exact_frame\": 27.5, \"exact_frame_baseline\": 33.5, \"exact_frame_skill\": -0.09, \"gold_slots\": 941, \"p50_ms_per_utterance\": 19, \"precision\": 28.8, \"recall\": 18.5, \"skill_note\": \"baseline F1 is 0, so skill equals F1\", \"tied_with\": []}"}
{"task_id": "ex1-slots-de", "kind": "extract", "task": "slot extraction (MASSIVE slots)", "lang": "de", "metric": "micro F1, threshold 0.5 for GLiNER", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 45.1, "baseline": 0, "baseline_name": "no extraction", "skill": 0.451, "n": 1000, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-09", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0 (Ollama digest b5bedb246945, ollama 0.35.0)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/ex1/score.md", "extra_json": "{\"exact_frame\": 45.1, \"exact_frame_baseline\": 33.5, \"exact_frame_skill\": 0.174, \"gold_slots\": 941, \"p50_ms_per_utterance\": 389, \"precision\": 52.2, \"recall\": 39.6, \"skill_note\": \"baseline F1 is 0, so skill equals F1\", \"tied_with\": []}"}
{"task_id": "ex1-slots-de", "kind": "extract", "task": "slot extraction (MASSIVE slots)", "lang": "de", "metric": "micro F1, threshold 0.5 for GLiNER", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "open", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "a trained slot tagger is possible (MASSIVE train has slot annotations) but is a different model; T3 covers classification only", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/ex1/score.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "ex1-slots-en", "kind": "extract", "task": "slot extraction (MASSIVE slots)", "lang": "en", "metric": "micro F1, threshold 0.5 for GLiNER", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "Jev has no extraction primitive (choice, score, noul only), docs.typesafe.ai/introduction", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/ex1/method.md", "extra_json": "{}"}
{"task_id": "ex1-slots-en", "kind": "extract", "task": "slot extraction (MASSIVE slots)", "lang": "en", "metric": "micro F1, threshold 0.5 for GLiNER", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2-large", "arms_compared": 2, "value": 38.1, "baseline": 0, "baseline_name": "no extraction", "skill": 0.381, "n": 1000, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-09", "model_version": "fastino/gliner2-large-v1 (revision d7aa8a2850d1)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/ex1/score.md", "extra_json": "{\"exact_frame\": 22.4, \"exact_frame_baseline\": 33.5, \"exact_frame_skill\": -0.167, \"gold_slots\": 937, \"p50_ms_per_utterance\": 30, \"precision\": 31, \"recall\": 49.3, \"skill_note\": \"baseline F1 is 0, so skill equals F1\", \"tied_with\": []}"}
{"task_id": "ex1-slots-en", "kind": "extract", "task": "slot extraction (MASSIVE slots)", "lang": "en", "metric": "micro F1, threshold 0.5 for GLiNER", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 48.9, "baseline": 0, "baseline_name": "no extraction", "skill": 0.489, "n": 1000, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-09", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0 (Ollama digest b5bedb246945, ollama 0.35.0)", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/ex1/score.md", "extra_json": "{\"exact_frame\": 44.9, \"exact_frame_baseline\": 33.5, \"exact_frame_skill\": 0.171, \"gold_slots\": 937, \"p50_ms_per_utterance\": 401, \"precision\": 51.2, \"recall\": 46.9, \"skill_note\": \"baseline F1 is 0, so skill equals F1\", \"tied_with\": []}"}
{"task_id": "ex1-slots-en", "kind": "extract", "task": "slot extraction (MASSIVE slots)", "lang": "en", "metric": "micro F1, threshold 0.5 for GLiNER", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "open", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "a trained slot tagger is possible (MASSIVE train has slot annotations) but is a different model; T3 covers classification only", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/ex1/score.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-is_req-de", "kind": "match", "task": "pv1 is_req", "lang": "de", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 88.2, "baseline": 85.3, "baseline_name": "most frequent answer (yes)", "skill": 0.2, "n": 136, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": []}"}
{"task_id": "pv1-is_req-de", "kind": "match", "task": "pv1 is_req", "lang": "de", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide", "arms_compared": 3, "value": 85.3, "baseline": 85.3, "baseline_name": "most frequent answer (yes)", "skill": 0, "n": 136, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "0.125", "mcnemar_vs_jev_p": "0.125", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-multi-Decide", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"english_model_extra\": {\"arm\": \"gliner2.5-decide\", \"note\": \"English-only model on German rows, not eligible as best arm\", \"skill\": -0.05, \"tied_with\": [], \"value\": 84.6}, \"tied_with\": [], \"vs_jev_only_arm_right\": 0, \"vs_jev_only_jev_right\": 4, \"vs_top_only_arm_right\": 0, \"vs_top_only_top_right\": 4, \"vs_top_paired\": 136}"}
{"task_id": "pv1-is_req-de", "kind": "match", "task": "pv1 is_req", "lang": "de", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 4, "value": 86.8, "baseline": 85.3, "baseline_name": "most frequent answer (yes)", "skill": 0.1, "n": 136, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "0.5", "mcnemar_vs_jev_p": "0.5", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 0, \"vs_jev_only_jev_right\": 2, \"vs_top_only_arm_right\": 0, \"vs_top_only_top_right\": 2, \"vs_top_paired\": 136}"}
{"task_id": "pv1-is_req-de", "kind": "match", "task": "pv1 is_req", "lang": "de", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-must-de", "kind": "match", "task": "pv1 must", "lang": "de", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 93.1, "baseline": 94, "baseline_name": "most frequent answer (yes)", "skill": -0.143, "n": 116, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "0.008", "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_top_only_arm_right\": 0, \"vs_top_only_top_right\": 8, \"vs_top_paired\": 116}"}
{"task_id": "pv1-must-de", "kind": "match", "task": "pv1 must", "lang": "de", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide", "arms_compared": 3, "value": 94, "baseline": 94, "baseline_name": "most frequent answer (yes)", "skill": 0, "n": 116, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "0.016", "mcnemar_vs_jev_p": "1", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-multi-Decide", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"english_model_extra\": {\"arm\": \"gliner2.5-decide-1b-focus\", \"note\": \"English-only model on German rows, not eligible as best arm\", \"skill\": 0, \"tied_with\": [], \"value\": 94}, \"tied_with\": [\"gliner2.5-multi-decide-focus\", \"gliner2.5-multi-decide-intext\"], \"vs_jev_only_arm_right\": 8, \"vs_jev_only_jev_right\": 7, \"vs_top_only_arm_right\": 0, \"vs_top_only_top_right\": 7, \"vs_top_paired\": 116}"}
{"task_id": "pv1-must-de", "kind": "match", "task": "pv1 must", "lang": "de", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "qwen3-32b", "arms_compared": 4, "value": 100, "baseline": 94, "baseline_name": "most frequent answer (yes)", "skill": 1, "n": 116, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": "0.008", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "qwen3:32b", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 8, \"vs_jev_only_jev_right\": 0}"}
{"task_id": "pv1-must-de", "kind": "match", "task": "pv1 must", "lang": "de", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-axis-de", "kind": "match", "task": "pv1 axis", "lang": "de", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 93.1, "baseline": 16.4, "baseline_name": "most frequent answer (testauto)", "skill": 0.918, "n": 116, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": []}"}
{"task_id": "pv1-axis-de", "kind": "match", "task": "pv1 axis", "lang": "de", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide-focus", "arms_compared": 3, "value": 32.8, "baseline": 16.4, "baseline_name": "most frequent answer (testauto)", "skill": 0.196, "n": 116, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-multi-Decide", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"english_model_extra\": {\"arm\": \"gliner2.5-decide-focus\", \"note\": \"English-only model on German rows, not eligible as best arm\", \"skill\": 0.474, \"tied_with\": [], \"value\": 56}, \"tied_with\": [], \"vs_jev_only_arm_right\": 1, \"vs_jev_only_jev_right\": 71, \"vs_top_only_arm_right\": 1, \"vs_top_only_top_right\": 71, \"vs_top_paired\": 116}"}
{"task_id": "pv1-axis-de", "kind": "match", "task": "pv1 axis", "lang": "de", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 4, "value": 87.1, "baseline": 16.4, "baseline_name": "most frequent answer (testauto)", "skill": 0.845, "n": 116, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "0.065", "mcnemar_vs_jev_p": "0.065", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 2, \"vs_jev_only_jev_right\": 9, \"vs_top_only_arm_right\": 2, \"vs_top_only_top_right\": 9, \"vs_top_paired\": 116}"}
{"task_id": "pv1-axis-de", "kind": "match", "task": "pv1 axis", "lang": "de", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-evidence-de", "kind": "match", "task": "pv1 evidence", "lang": "de", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 81, "baseline": 41.4, "baseline_name": "most frequent answer (none)", "skill": 0.676, "n": 116, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "0.077", "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_top_only_arm_right\": 4, \"vs_top_only_top_right\": 12, \"vs_top_paired\": 116}"}
{"task_id": "pv1-evidence-de", "kind": "match", "task": "pv1 evidence", "lang": "de", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide", "arms_compared": 3, "value": 19, "baseline": 41.4, "baseline_name": "most frequent answer (none)", "skill": -0.382, "n": 116, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-multi-Decide", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"english_model_extra\": {\"arm\": \"gliner2.5-decide\", \"note\": \"English-only model on German rows, not eligible as best arm\", \"skill\": -0.441, \"tied_with\": [\"gliner2.5-decide-focus\", \"gliner2.5-decide-intext\"], \"value\": 15.5}, \"tied_with\": [\"gliner2.5-multi-decide-focus\", \"gliner2.5-multi-decide-intext\"], \"vs_jev_only_arm_right\": 3, \"vs_jev_only_jev_right\": 75, \"vs_top_only_arm_right\": 4, \"vs_top_only_top_right\": 84, \"vs_top_paired\": 116}"}
{"task_id": "pv1-evidence-de", "kind": "match", "task": "pv1 evidence", "lang": "de", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "qwen3.8-27b", "arms_compared": 4, "value": 87.9, "baseline": 41.4, "baseline_name": "most frequent answer (none)", "skill": 0.794, "n": 116, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": "0.077", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "qwen3.8:27b", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 12, \"vs_jev_only_jev_right\": 4}"}
{"task_id": "pv1-evidence-de", "kind": "match", "task": "pv1 evidence", "lang": "de", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-level-de", "kind": "match", "task": "pv1 level", "lang": "de", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 79.3, "baseline": 34.5, "baseline_name": "most frequent answer (3)", "skill": 0.684, "n": 116, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": []}"}
{"task_id": "pv1-level-de", "kind": "match", "task": "pv1 level", "lang": "de", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide-intext", "arms_compared": 3, "value": 24.1, "baseline": 34.5, "baseline_name": "most frequent answer (3)", "skill": -0.158, "n": 116, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-multi-Decide", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"english_model_extra\": {\"arm\": \"gliner2.5-decide\", \"note\": \"English-only model on German rows, not eligible as best arm\", \"skill\": 0.066, \"tied_with\": [\"gliner2.5-decide-focus\"], \"value\": 38.8}, \"tied_with\": [], \"vs_jev_only_arm_right\": 1, \"vs_jev_only_jev_right\": 65, \"vs_top_only_arm_right\": 1, \"vs_top_only_top_right\": 65, \"vs_top_paired\": 116}"}
{"task_id": "pv1-level-de", "kind": "match", "task": "pv1 level", "lang": "de", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 4, "value": 77.6, "baseline": 34.5, "baseline_name": "most frequent answer (3)", "skill": 0.658, "n": 116, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "0.824", "mcnemar_vs_jev_p": "0.824", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 9, \"vs_jev_only_jev_right\": 11, \"vs_top_only_arm_right\": 9, \"vs_top_only_top_right\": 11, \"vs_top_paired\": 116}"}
{"task_id": "pv1-level-de", "kind": "match", "task": "pv1 level", "lang": "de", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-de-adjudicated.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-is_req-en", "kind": "match", "task": "pv1 is_req", "lang": "en", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 88.5, "baseline": 85.6, "baseline_name": "most frequent answer (yes)", "skill": 0.2, "n": 139, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "1", "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_top_only_arm_right\": 0, \"vs_top_only_top_right\": 1, \"vs_top_paired\": 139}"}
{"task_id": "pv1-is_req-en", "kind": "match", "task": "pv1 is_req", "lang": "en", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide", "arms_compared": 9, "value": 85.6, "baseline": 85.6, "baseline_name": "most frequent answer (yes)", "skill": 0, "n": 139, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "0.063", "mcnemar_vs_jev_p": "0.125", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-Decide", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [\"gliner2.5-multi-decide\"], \"vs_jev_only_arm_right\": 0, \"vs_jev_only_jev_right\": 4, \"vs_top_only_arm_right\": 0, \"vs_top_only_top_right\": 5, \"vs_top_paired\": 139}"}
{"task_id": "pv1-is_req-en", "kind": "match", "task": "pv1 is_req", "lang": "en", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 4, "value": 89.2, "baseline": 85.6, "baseline_name": "most frequent answer (yes)", "skill": 0.25, "n": 139, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": "1", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 1, \"vs_jev_only_jev_right\": 0}"}
{"task_id": "pv1-is_req-en", "kind": "match", "task": "pv1 is_req", "lang": "en", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-must-en", "kind": "match", "task": "pv1 must", "lang": "en", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 89.1, "baseline": 94.1, "baseline_name": "most frequent answer (yes)", "skill": -0.857, "n": 119, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_top_only_arm_right\": 0, \"vs_top_only_top_right\": 13, \"vs_top_paired\": 119}"}
{"task_id": "pv1-must-en", "kind": "match", "task": "pv1 must", "lang": "en", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide-1b", "arms_compared": 9, "value": 94.1, "baseline": 94.1, "baseline_name": "most frequent answer (yes)", "skill": 0, "n": 119, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "0.016", "mcnemar_vs_jev_p": "0.263", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-Decide-1B", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [\"gliner2.5-decide-1b-focus\", \"gliner2.5-multi-decide\", \"gliner2.5-multi-decide-focus\", \"gliner2.5-multi-decide-intext\"], \"vs_jev_only_arm_right\": 13, \"vs_jev_only_jev_right\": 7, \"vs_top_only_arm_right\": 0, \"vs_top_only_top_right\": 7, \"vs_top_paired\": 119}"}
{"task_id": "pv1-must-en", "kind": "match", "task": "pv1 must", "lang": "en", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "qwen3-32b", "arms_compared": 4, "value": 100, "baseline": 94.1, "baseline_name": "most frequent answer (yes)", "skill": 1, "n": 119, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "qwen3:32b", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 13, \"vs_jev_only_jev_right\": 0}"}
{"task_id": "pv1-must-en", "kind": "match", "task": "pv1 must", "lang": "en", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-axis-en", "kind": "match", "task": "pv1 axis", "lang": "en", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 91.6, "baseline": 16, "baseline_name": "most frequent answer (ai)", "skill": 0.9, "n": 119, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": []}"}
{"task_id": "pv1-axis-en", "kind": "match", "task": "pv1 axis", "lang": "en", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide-1b-focus", "arms_compared": 9, "value": 62.2, "baseline": 16, "baseline_name": "most frequent answer (ai)", "skill": 0.55, "n": 119, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-Decide-1B", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 3, \"vs_jev_only_jev_right\": 38, \"vs_top_only_arm_right\": 3, \"vs_top_only_top_right\": 38, \"vs_top_paired\": 119}"}
{"task_id": "pv1-axis-en", "kind": "match", "task": "pv1 axis", "lang": "en", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 4, "value": 88.2, "baseline": 16, "baseline_name": "most frequent answer (ai)", "skill": 0.86, "n": 119, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "0.289", "mcnemar_vs_jev_p": "0.289", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 2, \"vs_jev_only_jev_right\": 6, \"vs_top_only_arm_right\": 2, \"vs_top_only_top_right\": 6, \"vs_top_paired\": 119}"}
{"task_id": "pv1-axis-en", "kind": "match", "task": "pv1 axis", "lang": "en", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-evidence-en", "kind": "match", "task": "pv1 evidence", "lang": "en", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 79, "baseline": 37, "baseline_name": "most frequent answer (none)", "skill": 0.667, "n": 119, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "0.503", "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_top_only_arm_right\": 8, \"vs_top_only_top_right\": 12, \"vs_top_paired\": 119}"}
{"task_id": "pv1-evidence-en", "kind": "match", "task": "pv1 evidence", "lang": "en", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide", "arms_compared": 9, "value": 14.3, "baseline": 37, "baseline_name": "most frequent answer (none)", "skill": -0.36, "n": 119, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-Decide", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [\"gliner2.5-decide-focus\", \"gliner2.5-decide-intext\"], \"vs_jev_only_arm_right\": 1, \"vs_jev_only_jev_right\": 78, \"vs_top_only_arm_right\": 4, \"vs_top_only_top_right\": 85, \"vs_top_paired\": 119}"}
{"task_id": "pv1-evidence-en", "kind": "match", "task": "pv1 evidence", "lang": "en", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "qwen3.8-27b", "arms_compared": 4, "value": 82.4, "baseline": 37, "baseline_name": "most frequent answer (none)", "skill": 0.72, "n": 119, "answered": null, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": "0.503", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "qwen3.8:27b", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 12, \"vs_jev_only_jev_right\": 8}"}
{"task_id": "pv1-evidence-en", "kind": "match", "task": "pv1 evidence", "lang": "en", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "pv1-level-en", "kind": "match", "task": "pv1 level", "lang": "en", "metric": "accuracy against the blind reference", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 74.8, "baseline": 34.5, "baseline_name": "most frequent answer (3)", "skill": 0.615, "n": 119, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "jev-1.13.0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": []}"}
{"task_id": "pv1-level-en", "kind": "match", "task": "pv1 level", "lang": "en", "metric": "accuracy against the blind reference", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide", "arms_compared": 9, "value": 38.7, "baseline": 34.5, "baseline_name": "most frequent answer (3)", "skill": 0.064, "n": 119, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "<0.001", "mcnemar_vs_jev_p": "<0.001", "p50_ms_per_decision": null, "measured": "2026-10-08", "model_version": "fastino/GLiNER2.5-Decide", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [\"gliner2.5-decide-focus\"], \"vs_jev_only_arm_right\": 8, \"vs_jev_only_jev_right\": 51, \"vs_top_only_arm_right\": 8, \"vs_top_only_top_right\": 51, \"vs_top_paired\": 119}"}
{"task_id": "pv1-level-en", "kind": "match", "task": "pv1 level", "lang": "en", "metric": "accuracy against the blind reference", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 4, "value": 70.6, "baseline": 34.5, "baseline_name": "most frequent answer (3)", "skill": 0.551, "n": 119, "answered": null, "top_family": "jev", "mcnemar_vs_top_p": "0.442", "mcnemar_vs_jev_p": "0.442", "p50_ms_per_decision": null, "measured": "2026-10-01", "model_version": "hf.co/EldanRing/Winnow-12B:Q8_0", "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"baseline_source\": \"computed from reference/pv1 (same rule as results/pv1/gliner.md)\", \"tied_with\": [], \"vs_jev_only_arm_right\": 11, \"vs_jev_only_jev_right\": 16, \"vs_top_only_arm_right\": 11, \"vs_top_only_top_right\": 16, \"vs_top_paired\": 119}"}
{"task_id": "pv1-level-en", "kind": "match", "task": "pv1 level", "lang": "en", "metric": "accuracy against the blind reference", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data for this task; the only labels are the blind reference of the test rows themselves (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/score-en.md", "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "einv1-yesno-de", "kind": "yesno", "task": "yes/no question over the free text of an e-invoice (KoSIT XRechnung test suite, 34 invoices, ten questions)", "lang": "de", "metric": "accuracy against a blind reference, pooled over 340 cells and per question", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "planned", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": null, "extra_json": "{\"plan\": \"plan/einv1.md\"}"}
{"task_id": "einv1-yesno-de", "kind": "yesno", "task": "yes/no question over the free text of an e-invoice (KoSIT XRechnung test suite, 34 invoices, ten questions)", "lang": "de", "metric": "accuracy against a blind reference, pooled over 340 cells and per question", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "planned", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": null, "extra_json": "{\"plan\": \"plan/einv1.md\"}"}
{"task_id": "einv1-yesno-de", "kind": "yesno", "task": "yes/no question over the free text of an e-invoice (KoSIT XRechnung test suite, 34 invoices, ten questions)", "lang": "de", "metric": "accuracy against a blind reference, pooled over 340 cells and per question", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "planned", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": null, "extra_json": "{\"plan\": \"plan/einv1.md\"}"}
{"task_id": "einv1-yesno-de", "kind": "yesno", "task": "yes/no question over the free text of an e-invoice (KoSIT XRechnung test suite, 34 invoices, ten questions)", "lang": "de", "metric": "accuracy against a blind reference, pooled over 340 cells and per question", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no labelled training data; the 34 invoices are the test set (plan/t3.md)", "report_url": null, "extra_json": "{\"plan\": \"plan/t3.md\"}"}
{"task_id": "route1-automation95-de", "kind": "automation", "task": "automation: route1 (MASSIVE scenario, intent 2+ options; en also fast-decisions)", "lang": "de", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 60.3, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.603, "n": 5596, "answered": 5596, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/automation.md", "extra_json": "{\"accuracy_all\": 81.2, \"accuracy_automated\": 95.2, \"automation90\": 78.2, \"automation90_baseline\": 0, \"brier\": 0.2853, \"crossfit90\": 78.3, \"crossfit90_accuracy\": 90, \"crossfit95\": 60.3, \"crossfit95_accuracy\": 95.2, \"ece\": 0.0817, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.96, \"tied_with\": []}"}
{"task_id": "route1-automation95-de", "kind": "automation", "task": "automation: route1 (MASSIVE scenario, intent 2+ options; en also fast-decisions)", "lang": "de", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide", "arms_compared": 2, "value": 4.4, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.044, "n": 5596, "answered": 5596, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/automation.md", "extra_json": "{\"accuracy_all\": 53.4, \"accuracy_automated\": 95.1, \"automation90\": 12.7, \"automation90_baseline\": 0, \"brier\": 0.6012, \"crossfit90\": 12.6, \"crossfit90_accuracy\": 90.2, \"crossfit95\": 4.8, \"crossfit95_accuracy\": 94.8, \"ece\": 0.0668, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.8968, \"tied_with\": []}"}
{"task_id": "route1-automation95-de", "kind": "automation", "task": "automation: route1 (MASSIVE scenario, intent 2+ options; en also fast-decisions)", "lang": "de", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 58, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.58, "n": 5596, "answered": 5595, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/automation.md", "extra_json": "{\"accuracy_all\": 79.8, \"accuracy_automated\": 95, \"automation90\": 75.4, \"automation90_baseline\": 0, \"brier\": 0.3528, \"crossfit90\": 75.4, \"crossfit90_accuracy\": 89.9, \"crossfit95\": 57.9, \"crossfit95_accuracy\": 95.1, \"ece\": 0.1576, \"letter_mass_median\": 0.9998, \"letter_mass_p10\": 0.9906, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.998, \"tied_with\": []}"}
{"task_id": "route1-automation95-de", "kind": "automation", "task": "automation: route1 (MASSIVE scenario, intent 2+ options; en also fast-decisions)", "lang": "de", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "measured", "best_arm": "classic-e5-lr", "arms_compared": 2, "value": 76.4, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.764, "n": 5596, "answered": 5596, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/automation.md", "extra_json": "{\"accuracy_all\": 86.9, \"accuracy_automated\": 95, \"automation90\": 93.3, \"automation90_baseline\": 0, \"brier\": 0.2384, \"crossfit90\": 93.3, \"crossfit90_accuracy\": 90, \"crossfit95\": 76.7, \"crossfit95_accuracy\": 94.8, \"ece\": 0.1744, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.52, \"tied_with\": []}"}
{"task_id": "route1-automation95-en", "kind": "automation", "task": "automation: route1 (MASSIVE scenario, intent 2+ options; en also fast-decisions)", "lang": "en", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 42.6, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.426, "n": 8196, "answered": 8196, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/automation.md", "extra_json": "{\"accuracy_all\": 76.3, \"accuracy_automated\": 95.2, \"automation90\": 64.7, \"automation90_baseline\": 0, \"brier\": 0.3614, \"crossfit90\": 64.7, \"crossfit90_accuracy\": 90.1, \"crossfit95\": 42.6, \"crossfit95_accuracy\": 95.2, \"ece\": 0.1297, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 1, \"tied_with\": []}"}
{"task_id": "route1-automation95-en", "kind": "automation", "task": "automation: route1 (MASSIVE scenario, intent 2+ options; en also fast-decisions)", "lang": "en", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide-1b", "arms_compared": 6, "value": 10.8, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.108, "n": 8196, "answered": 8196, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/automation.md", "extra_json": "{\"accuracy_all\": 62.6, \"accuracy_automated\": 95, \"automation90\": 23.3, \"automation90_baseline\": 0, \"brier\": 0.4964, \"crossfit90\": 23.2, \"crossfit90_accuracy\": 90.1, \"crossfit95\": 9.2, \"crossfit95_accuracy\": 94.7, \"ece\": 0.061, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.8815, \"tied_with\": []}"}
{"task_id": "route1-automation95-en", "kind": "automation", "task": "automation: route1 (MASSIVE scenario, intent 2+ options; en also fast-decisions)", "lang": "en", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "winnow-12b", "arms_compared": 1, "value": 41.3, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.413, "n": 8196, "answered": 8096, "top_family": "jev", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/automation.md", "extra_json": "{\"accuracy_all\": 74.4, \"accuracy_automated\": 95.1, \"automation90\": 58.7, \"automation90_baseline\": 0, \"brier\": 0.4355, \"crossfit90\": 58.7, \"crossfit90_accuracy\": 90, \"crossfit95\": 41.9, \"crossfit95_accuracy\": 94.9, \"ece\": 0.1972, \"letter_mass_median\": 0.9997, \"letter_mass_p10\": 0.9903, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.9996, \"tied_with\": []}"}
{"task_id": "route1-automation95-en", "kind": "automation", "task": "automation: route1 (MASSIVE scenario, intent 2+ options; en also fast-decisions)", "lang": "en", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "measured", "best_arm": "classic-e5-lr", "arms_compared": 2, "value": 87.7, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.877, "n": 5596, "answered": 5596, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/route1/automation.md", "extra_json": "{\"accuracy_all\": 90.5, \"accuracy_automated\": 95, \"automation90\": 100, \"automation90_baseline\": 0, \"brier\": 0.1882, \"crossfit90\": 100, \"crossfit90_accuracy\": 90.5, \"crossfit95\": 87.8, \"crossfit95_accuracy\": 95, \"ece\": 0.1651, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.4581, \"tied_with\": []}"}
{"task_id": "pv1-automation95-de", "kind": "automation", "task": "automation: pv1 line questions (is_req, must, axis, evidence, level)", "lang": "de", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 53.5, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.535, "n": 600, "answered": 600, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/automation.md", "extra_json": "{\"accuracy_all\": 87, \"accuracy_automated\": 95.3, \"automation90\": 90.8, \"automation90_baseline\": 19.3, \"brier\": 0.2215, \"crossfit90\": 91.3, \"crossfit90_accuracy\": 90, \"crossfit95\": 53.4, \"crossfit95_accuracy\": 95, \"ece\": 0.073, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.92, \"tied_with\": []}"}
{"task_id": "pv1-automation95-de", "kind": "automation", "task": "automation: pv1 line questions (is_req, must, axis, evidence, level)", "lang": "de", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-multi-decide-focus", "arms_compared": 3, "value": 28.5, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.285, "n": 600, "answered": 600, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/automation.md", "extra_json": "{\"accuracy_all\": 51.5, \"accuracy_automated\": 95.9, \"automation90\": 38.5, \"automation90_baseline\": 19.3, \"brier\": 0.6362, \"crossfit90\": 37.8, \"crossfit90_accuracy\": 90.8, \"crossfit95\": 24, \"crossfit95_accuracy\": 94.5, \"ece\": 0.1876, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.5652, \"tied_with\": []}"}
{"task_id": "pv1-automation95-de", "kind": "automation", "task": "automation: pv1 line questions (is_req, must, axis, evidence, level)", "lang": "de", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "qwen3.8-27b", "arms_compared": 4, "value": 68.2, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.682, "n": 600, "answered": 600, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/automation.md", "extra_json": "{\"accuracy_all\": 85.2, \"accuracy_automated\": 95.1, \"automation90\": 90.5, \"automation90_baseline\": 19.3, \"brier\": 0.2046, \"crossfit90\": 90.2, \"crossfit90_accuracy\": 89.6, \"crossfit95\": 70.1, \"crossfit95_accuracy\": 94.8, \"ece\": 0.0319, \"letter_mass_median\": 0.998, \"letter_mass_p10\": 0.9963, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.8147, \"tied_with\": []}"}
{"task_id": "pv1-automation95-de", "kind": "automation", "task": "automation: pv1 line questions (is_req, must, axis, evidence, level)", "lang": "de", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no classic arm on pv1: the classic classifier needs a train split, pv1 has none (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/automation.md", "extra_json": "{}"}
{"task_id": "pv1-automation95-en", "kind": "automation", "task": "automation: pv1 line questions (is_req, must, axis, evidence, level)", "lang": "en", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "jev", "family_label": "TypeSafe Jev, hosted API (api.typesafe.ai)", "state": "measured", "best_arm": "jev", "arms_compared": 1, "value": 51.9, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.519, "n": 615, "answered": 615, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/automation.md", "extra_json": "{\"accuracy_all\": 84.7, \"accuracy_automated\": 95.3, \"automation90\": 79.3, \"automation90_baseline\": 19.4, \"brier\": 0.2283, \"crossfit90\": 83.9, \"crossfit90_accuracy\": 89.3, \"crossfit95\": 58, \"crossfit95_accuracy\": 93.8, \"ece\": 0.037, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.92, \"tied_with\": []}"}
{"task_id": "pv1-automation95-en", "kind": "automation", "task": "automation: pv1 line questions (is_req, must, axis, evidence, level)", "lang": "en", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "gliner", "family_label": "Fastino GLiNER2 / GLiNER2.5 encoders, local GPU (RTX 4090)", "state": "measured", "best_arm": "gliner2.5-decide-1b-focus", "arms_compared": 9, "value": 24.6, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.246, "n": 615, "answered": 615, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/automation.md", "extra_json": "{\"accuracy_all\": 52.8, \"accuracy_automated\": 95.4, \"automation90\": 38.9, \"automation90_baseline\": 19.4, \"brier\": 0.5757, \"crossfit90\": 38.8, \"crossfit90_accuracy\": 89.5, \"crossfit95\": 25.9, \"crossfit95_accuracy\": 94.3, \"ece\": 0.1324, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.6556, \"tied_with\": []}"}
{"task_id": "pv1-automation95-en", "kind": "automation", "task": "automation: pv1 line questions (is_req, must, axis, evidence, level)", "lang": "en", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "local-llm", "family_label": "open LLM through Ollama, local GPU (RTX 4090)", "state": "measured", "best_arm": "qwen3.8-27b", "arms_compared": 4, "value": 56.4, "baseline": 0, "baseline_name": "most frequent answer per option set, confidence = its in-sample share", "skill": 0.564, "n": 615, "answered": 615, "top_family": "local-llm", "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": null, "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/automation.md", "extra_json": "{\"accuracy_all\": 82.3, \"accuracy_automated\": 95.1, \"automation90\": 85.7, \"automation90_baseline\": 19.4, \"brier\": 0.2413, \"crossfit90\": 84.3, \"crossfit90_accuracy\": 89.8, \"crossfit95\": 59.2, \"crossfit95_accuracy\": 93.6, \"ece\": 0.0463, \"letter_mass_median\": 0.9977, \"letter_mass_p10\": 0.9949, \"skill_note\": \"baseline is 0, so skill equals value / 100\", \"threshold\": 0.8692, \"tied_with\": []}"}
{"task_id": "pv1-automation95-en", "kind": "automation", "task": "automation: pv1 line questions (is_req, must, axis, evidence, level)", "lang": "en", "metric": "automation rate at 95 % accuracy: largest share of items, ranked by the probability of the arm's own answer, whose accuracy stays >= 95 %; threshold chosen in-sample", "family": "classic", "family_label": "sentence embeddings plus a trained logistic regression, local (Mac, MPS)", "state": "not_applicable", "best_arm": null, "arms_compared": null, "value": null, "baseline": null, "baseline_name": null, "skill": null, "n": null, "answered": null, "top_family": null, "mcnemar_vs_top_p": null, "mcnemar_vs_jev_p": null, "p50_ms_per_decision": null, "measured": null, "model_version": null, "reason": "no classic arm on pv1: the classic classifier needs a train split, pv1 has none (plan/t3.md)", "report_url": "https://github.com/bks-lab/lab-bench/blob/3584dfc8fddeb353916650f6da1f449cf20e2ccb/results/pv1/automation.md", "extra_json": "{}"}