{ "variant": "v4_vocab32k-mlx-8bit", "backend": "mlx", "size_bytes": 181239101, "validation_all": { "unweighted": { "rows": 4914, "weighted": false, "complexity": { "rows": 4850, "mae": 0.07598711761813803, "spearman": 0.7325308792002598, "qwk_rounded": 0.7541114941160277, "confusion_true_by_pred": [ [ 144.0, 158.0, 6.0, 0.0, 0.0 ], [ 38.0, 2572.0, 267.0, 1.0, 0.0 ], [ 4.0, 395.0, 890.0, 56.0, 0.0 ], [ 1.0, 10.0, 74.0, 226.0, 0.0 ], [ 0.0, 0.0, 2.0, 2.0, 4.0 ] ], "support": { "trivial": 308, "low": 2878, "medium": 1345, "high": 311, "expert": 8 } }, "task": { "accuracy": 0.8376068376068376, "macro_f1": 0.7475081932159393, "macro_f1_present": 0.7475081932159393, "per_class": { "information_seeking": { "f1": 0.8682170542635659, "support": 1528 }, "reasoning": { "f1": 0.5679012345679012, "support": 92 }, "planning": { "f1": 0.7132867132867133, "support": 81 }, "editing": { "f1": 0.8702290076335878, "support": 193 }, "summarization": { "f1": 0.8571428571428571, "support": 128 }, "extraction": { "f1": 0.7965738758029979, "support": 229 }, "classification": { "f1": 0.8901098901098901, "support": 89 }, "translation": { "f1": 0.9166666666666666, "support": 73 }, "coding_debugging": { "f1": 0.8972375690607735, "support": 453 }, "math": { "f1": 0.8704453441295547, "support": 236 }, "data_analysis": { "f1": 0.72, "support": 22 }, "writing": { "f1": 0.9018218623481782, "support": 991 }, "role_playing": { "f1": 0.8771929824561403, "support": 83 }, "advice_seeking": { "f1": 0.7197452229299363, "support": 157 }, "brainstorming": { "f1": 0.5863453815261044, "support": 134 }, "completion": { "f1": 0.7564575645756457, "support": 267 }, "action_execution": { "f1": 0.5263157894736842, "support": 11 }, "chat": { "f1": 0.5679012345679012, "support": 91 }, "other": { "f1": 0.29906542056074764, "support": 56 } } }, "needs_tools": { "f1": 0.6178489702517163, "auroc": 0.9482045454545455, "positives": 250 } }, "weighted": { "rows": 4914, "weighted": true, "complexity": { "rows": 4850, "mae": 0.07375405856631606, "spearman": 0.6926722884622424, "qwk_rounded": 0.7033805871701464, "confusion_true_by_pred": [ [ 138.1, 155.3, 5.0, 0.0, 0.0 ], [ 40.3, 2764.5, 288.3, 1.1, 0.0 ], [ 4.1, 397.4, 893.8, 55.9, 0.0 ], [ 0.4, 4.1, 31.4, 95.4, 0.0 ], [ 0.0, 0.0, 0.3, 0.3, 0.6 ] ], "support": { "trivial": 308, "low": 2878, "medium": 1345, "high": 311, "expert": 8 } }, "task": { "accuracy": 0.8394332602849471, "macro_f1": 0.7340867004427265, "macro_f1_present": 0.7340867004427265, "per_class": { "information_seeking": { "f1": 0.873880050094105, "support": 1528 }, "reasoning": { "f1": 0.5754507335128349, "support": 92 }, "planning": { "f1": 0.6923419444052038, "support": 81 }, "editing": { "f1": 0.8750917385680073, "support": 193 }, "summarization": { "f1": 0.8582092786862726, "support": 128 }, "extraction": { "f1": 0.8006995649406149, "support": 229 }, "classification": { "f1": 0.8889433148815616, "support": 89 }, "translation": { "f1": 0.9166481424971284, "support": 73 }, "coding_debugging": { "f1": 0.898547879675547, "support": 453 }, "math": { "f1": 0.8655105453546306, "support": 236 }, "data_analysis": { "f1": 0.6220404272481576, "support": 22 }, "writing": { "f1": 0.8966525522027537, "support": 991 }, "role_playing": { "f1": 0.856782491891807, "support": 83 }, "advice_seeking": { "f1": 0.7325427527778512, "support": 157 }, "brainstorming": { "f1": 0.5877632803140802, "support": 134 }, "completion": { "f1": 0.7654770750187783, "support": 267 }, "action_execution": { "f1": 0.4050722792506393, "support": 11 }, "chat": { "f1": 0.5710913302828049, "support": 91 }, "other": { "f1": 0.2649019268090265, "support": 56 } } }, "needs_tools": { "f1": 0.6202797007576061, "auroc": 0.9485384523441318, "positives": 250 } } }, "validation_four_sources": { "unweighted": { "rows": 3130, "weighted": false, "complexity": { "rows": 3101, "mae": 0.07344912939439163, "spearman": 0.7432828892500157, "qwk_rounded": 0.7605925596882378, "confusion_true_by_pred": [ [ 98.0, 99.0, 4.0, 0.0, 0.0 ], [ 25.0, 1652.0, 161.0, 0.0, 0.0 ], [ 3.0, 245.0, 586.0, 38.0, 0.0 ], [ 1.0, 4.0, 45.0, 139.0, 0.0 ], [ 0.0, 0.0, 0.0, 0.0, 1.0 ] ], "support": { "trivial": 201, "low": 1838, "medium": 872, "high": 189, "expert": 1 } }, "task": { "accuracy": 0.8460063897763578, "macro_f1": 0.7465511584840749, "macro_f1_present": 0.7465511584840749, "per_class": { "information_seeking": { "f1": 0.8811835413777162, "support": 1068 }, "reasoning": { "f1": 0.5747126436781609, "support": 48 }, "planning": { "f1": 0.651685393258427, "support": 53 }, "editing": { "f1": 0.8857142857142857, "support": 139 }, "summarization": { "f1": 0.8631578947368421, "support": 98 }, "extraction": { "f1": 0.8174386920980926, "support": 180 }, "classification": { "f1": 0.875, "support": 47 }, "translation": { "f1": 0.9243697478991597, "support": 61 }, "coding_debugging": { "f1": 0.9079754601226994, "support": 246 }, "math": { "f1": 0.7272727272727273, "support": 41 }, "data_analysis": { "f1": 0.6666666666666666, "support": 5 }, "writing": { "f1": 0.9019054340155258, "support": 705 }, "role_playing": { "f1": 0.9152542372881356, "support": 30 }, "advice_seeking": { "f1": 0.6901408450704225, "support": 75 }, "brainstorming": { "f1": 0.55, "support": 83 }, "completion": { "f1": 0.7710843373493976, "support": 162 }, "action_execution": { "f1": 0.625, "support": 8 }, "chat": { "f1": 0.5977011494252874, "support": 47 }, "other": { "f1": 0.3582089552238806, "support": 34 } } }, "needs_tools": { "f1": 0.5808823529411765, "auroc": 0.9604978034209988, "positives": 153 } }, "weighted": { "rows": 3130, "weighted": true, "complexity": { "rows": 3101, "mae": 0.07160471797212513, "spearman": 0.704867234170456, "qwk_rounded": 0.7130554560684663, "confusion_true_by_pred": [ [ 93.8, 98.4, 3.5, 0.0, 0.0 ], [ 26.1, 1789.3, 176.0, 0.0, 0.0 ], [ 3.1, 248.3, 595.1, 37.6, 0.0 ], [ 0.4, 1.6, 19.2, 58.5, 0.0 ], [ 0.0, 0.0, 0.0, 0.0, 0.1 ] ], "support": { "trivial": 201, "low": 1838, "medium": 872, "high": 189, "expert": 1 } }, "task": { "accuracy": 0.8481247463011802, "macro_f1": 0.7286895651247581, "macro_f1_present": 0.7286895651247581, "per_class": { "information_seeking": { "f1": 0.8851440898353958, "support": 1068 }, "reasoning": { "f1": 0.5783684198451182, "support": 48 }, "planning": { "f1": 0.6322924335863169, "support": 53 }, "editing": { "f1": 0.8913461323150959, "support": 139 }, "summarization": { "f1": 0.8655264865753218, "support": 98 }, "extraction": { "f1": 0.8194825792197448, "support": 180 }, "classification": { "f1": 0.8744675021779321, "support": 47 }, "translation": { "f1": 0.9243253837334996, "support": 61 }, "coding_debugging": { "f1": 0.9078078299006189, "support": 246 }, "math": { "f1": 0.7056980579142964, "support": 41 }, "data_analysis": { "f1": 0.5512676749948888, "support": 5 }, "writing": { "f1": 0.8987480780377212, "support": 705 }, "role_playing": { "f1": 0.9018366831956603, "support": 30 }, "advice_seeking": { "f1": 0.7056692072538584, "support": 75 }, "brainstorming": { "f1": 0.5509812080931199, "support": 83 }, "completion": { "f1": 0.7813712009865946, "support": 162 }, "action_execution": { "f1": 0.46184553221044683, "support": 8 }, "chat": { "f1": 0.5921133541119249, "support": 47 }, "other": { "f1": 0.31680988338284877, "support": 34 } } }, "needs_tools": { "f1": 0.5785238379679463, "auroc": 0.9630877898213843, "positives": 153 } } }, "agreement_with_reference": { "max_abs_logit_difference": 16.63292694091797, "mean_abs_logit_difference": 0.7649034857749939, "same_task": 0.9177859177859178, "same_needs_tools": 0.9820919820919821, "same_complexity_tier": 0.9092389092389093, "same_policy_tier": 0.9151404151404151 }, "agreement_with_parent": { "max_abs_logit_difference": 1.028977870941162, "mean_abs_logit_difference": 0.06225224584341049, "same_task": 0.9947089947089947, "same_needs_tools": 0.9991859991859992, "same_complexity_tier": 0.99002849002849, "same_policy_tier": 0.9922669922669922 }, "latency_ms_batch1": { "p50": 5.329958999936935, "p95": 34.000291998381726, "mean": 17.72976168012974, "prompts": 200, "device": "cpu" } }