scombat's picture
Add v4_vocab32k-mlx-8bit (six-source development model)
048925f verified
Raw History Blame Contribute Delete
11.1 kB
{
"variant": "v4_vocab32k-mlx-8bit",
"backend": "mlx",
"size_bytes": 181239101,
"validation_all": {
"unweighted": {
"rows": 4914,
"weighted": false,
"complexity": {
"rows": 4850,
"mae": 0.07598711761813803,
"spearman": 0.7325308792002598,
"qwk_rounded": 0.7541114941160277,
"confusion_true_by_pred": [
[
144.0,
158.0,
6.0,
0.0,
0.0
],
[
38.0,
2572.0,
267.0,
1.0,
0.0
],
[
4.0,
395.0,
890.0,
56.0,
0.0
],
[
1.0,
10.0,
74.0,
226.0,
0.0
],
[
0.0,
0.0,
2.0,
2.0,
4.0
]
],
"support": {
"trivial": 308,
"low": 2878,
"medium": 1345,
"high": 311,
"expert": 8
}
},
"task": {
"accuracy": 0.8376068376068376,
"macro_f1": 0.7475081932159393,
"macro_f1_present": 0.7475081932159393,
"per_class": {
"information_seeking": {
"f1": 0.8682170542635659,
"support": 1528
},
"reasoning": {
"f1": 0.5679012345679012,
"support": 92
},
"planning": {
"f1": 0.7132867132867133,
"support": 81
},
"editing": {
"f1": 0.8702290076335878,
"support": 193
},
"summarization": {
"f1": 0.8571428571428571,
"support": 128
},
"extraction": {
"f1": 0.7965738758029979,
"support": 229
},
"classification": {
"f1": 0.8901098901098901,
"support": 89
},
"translation": {
"f1": 0.9166666666666666,
"support": 73
},
"coding_debugging": {
"f1": 0.8972375690607735,
"support": 453
},
"math": {
"f1": 0.8704453441295547,
"support": 236
},
"data_analysis": {
"f1": 0.72,
"support": 22
},
"writing": {
"f1": 0.9018218623481782,
"support": 991
},
"role_playing": {
"f1": 0.8771929824561403,
"support": 83
},
"advice_seeking": {
"f1": 0.7197452229299363,
"support": 157
},
"brainstorming": {
"f1": 0.5863453815261044,
"support": 134
},
"completion": {
"f1": 0.7564575645756457,
"support": 267
},
"action_execution": {
"f1": 0.5263157894736842,
"support": 11
},
"chat": {
"f1": 0.5679012345679012,
"support": 91
},
"other": {
"f1": 0.29906542056074764,
"support": 56
}
}
},
"needs_tools": {
"f1": 0.6178489702517163,
"auroc": 0.9482045454545455,
"positives": 250
}
},
"weighted": {
"rows": 4914,
"weighted": true,
"complexity": {
"rows": 4850,
"mae": 0.07375405856631606,
"spearman": 0.6926722884622424,
"qwk_rounded": 0.7033805871701464,
"confusion_true_by_pred": [
[
138.1,
155.3,
5.0,
0.0,
0.0
],
[
40.3,
2764.5,
288.3,
1.1,
0.0
],
[
4.1,
397.4,
893.8,
55.9,
0.0
],
[
0.4,
4.1,
31.4,
95.4,
0.0
],
[
0.0,
0.0,
0.3,
0.3,
0.6
]
],
"support": {
"trivial": 308,
"low": 2878,
"medium": 1345,
"high": 311,
"expert": 8
}
},
"task": {
"accuracy": 0.8394332602849471,
"macro_f1": 0.7340867004427265,
"macro_f1_present": 0.7340867004427265,
"per_class": {
"information_seeking": {
"f1": 0.873880050094105,
"support": 1528
},
"reasoning": {
"f1": 0.5754507335128349,
"support": 92
},
"planning": {
"f1": 0.6923419444052038,
"support": 81
},
"editing": {
"f1": 0.8750917385680073,
"support": 193
},
"summarization": {
"f1": 0.8582092786862726,
"support": 128
},
"extraction": {
"f1": 0.8006995649406149,
"support": 229
},
"classification": {
"f1": 0.8889433148815616,
"support": 89
},
"translation": {
"f1": 0.9166481424971284,
"support": 73
},
"coding_debugging": {
"f1": 0.898547879675547,
"support": 453
},
"math": {
"f1": 0.8655105453546306,
"support": 236
},
"data_analysis": {
"f1": 0.6220404272481576,
"support": 22
},
"writing": {
"f1": 0.8966525522027537,
"support": 991
},
"role_playing": {
"f1": 0.856782491891807,
"support": 83
},
"advice_seeking": {
"f1": 0.7325427527778512,
"support": 157
},
"brainstorming": {
"f1": 0.5877632803140802,
"support": 134
},
"completion": {
"f1": 0.7654770750187783,
"support": 267
},
"action_execution": {
"f1": 0.4050722792506393,
"support": 11
},
"chat": {
"f1": 0.5710913302828049,
"support": 91
},
"other": {
"f1": 0.2649019268090265,
"support": 56
}
}
},
"needs_tools": {
"f1": 0.6202797007576061,
"auroc": 0.9485384523441318,
"positives": 250
}
}
},
"validation_four_sources": {
"unweighted": {
"rows": 3130,
"weighted": false,
"complexity": {
"rows": 3101,
"mae": 0.07344912939439163,
"spearman": 0.7432828892500157,
"qwk_rounded": 0.7605925596882378,
"confusion_true_by_pred": [
[
98.0,
99.0,
4.0,
0.0,
0.0
],
[
25.0,
1652.0,
161.0,
0.0,
0.0
],
[
3.0,
245.0,
586.0,
38.0,
0.0
],
[
1.0,
4.0,
45.0,
139.0,
0.0
],
[
0.0,
0.0,
0.0,
0.0,
1.0
]
],
"support": {
"trivial": 201,
"low": 1838,
"medium": 872,
"high": 189,
"expert": 1
}
},
"task": {
"accuracy": 0.8460063897763578,
"macro_f1": 0.7465511584840749,
"macro_f1_present": 0.7465511584840749,
"per_class": {
"information_seeking": {
"f1": 0.8811835413777162,
"support": 1068
},
"reasoning": {
"f1": 0.5747126436781609,
"support": 48
},
"planning": {
"f1": 0.651685393258427,
"support": 53
},
"editing": {
"f1": 0.8857142857142857,
"support": 139
},
"summarization": {
"f1": 0.8631578947368421,
"support": 98
},
"extraction": {
"f1": 0.8174386920980926,
"support": 180
},
"classification": {
"f1": 0.875,
"support": 47
},
"translation": {
"f1": 0.9243697478991597,
"support": 61
},
"coding_debugging": {
"f1": 0.9079754601226994,
"support": 246
},
"math": {
"f1": 0.7272727272727273,
"support": 41
},
"data_analysis": {
"f1": 0.6666666666666666,
"support": 5
},
"writing": {
"f1": 0.9019054340155258,
"support": 705
},
"role_playing": {
"f1": 0.9152542372881356,
"support": 30
},
"advice_seeking": {
"f1": 0.6901408450704225,
"support": 75
},
"brainstorming": {
"f1": 0.55,
"support": 83
},
"completion": {
"f1": 0.7710843373493976,
"support": 162
},
"action_execution": {
"f1": 0.625,
"support": 8
},
"chat": {
"f1": 0.5977011494252874,
"support": 47
},
"other": {
"f1": 0.3582089552238806,
"support": 34
}
}
},
"needs_tools": {
"f1": 0.5808823529411765,
"auroc": 0.9604978034209988,
"positives": 153
}
},
"weighted": {
"rows": 3130,
"weighted": true,
"complexity": {
"rows": 3101,
"mae": 0.07160471797212513,
"spearman": 0.704867234170456,
"qwk_rounded": 0.7130554560684663,
"confusion_true_by_pred": [
[
93.8,
98.4,
3.5,
0.0,
0.0
],
[
26.1,
1789.3,
176.0,
0.0,
0.0
],
[
3.1,
248.3,
595.1,
37.6,
0.0
],
[
0.4,
1.6,
19.2,
58.5,
0.0
],
[
0.0,
0.0,
0.0,
0.0,
0.1
]
],
"support": {
"trivial": 201,
"low": 1838,
"medium": 872,
"high": 189,
"expert": 1
}
},
"task": {
"accuracy": 0.8481247463011802,
"macro_f1": 0.7286895651247581,
"macro_f1_present": 0.7286895651247581,
"per_class": {
"information_seeking": {
"f1": 0.8851440898353958,
"support": 1068
},
"reasoning": {
"f1": 0.5783684198451182,
"support": 48
},
"planning": {
"f1": 0.6322924335863169,
"support": 53
},
"editing": {
"f1": 0.8913461323150959,
"support": 139
},
"summarization": {
"f1": 0.8655264865753218,
"support": 98
},
"extraction": {
"f1": 0.8194825792197448,
"support": 180
},
"classification": {
"f1": 0.8744675021779321,
"support": 47
},
"translation": {
"f1": 0.9243253837334996,
"support": 61
},
"coding_debugging": {
"f1": 0.9078078299006189,
"support": 246
},
"math": {
"f1": 0.7056980579142964,
"support": 41
},
"data_analysis": {
"f1": 0.5512676749948888,
"support": 5
},
"writing": {
"f1": 0.8987480780377212,
"support": 705
},
"role_playing": {
"f1": 0.9018366831956603,
"support": 30
},
"advice_seeking": {
"f1": 0.7056692072538584,
"support": 75
},
"brainstorming": {
"f1": 0.5509812080931199,
"support": 83
},
"completion": {
"f1": 0.7813712009865946,
"support": 162
},
"action_execution": {
"f1": 0.46184553221044683,
"support": 8
},
"chat": {
"f1": 0.5921133541119249,
"support": 47
},
"other": {
"f1": 0.31680988338284877,
"support": 34
}
}
},
"needs_tools": {
"f1": 0.5785238379679463,
"auroc": 0.9630877898213843,
"positives": 153
}
}
},
"agreement_with_reference": {
"max_abs_logit_difference": 16.63292694091797,
"mean_abs_logit_difference": 0.7649034857749939,
"same_task": 0.9177859177859178,
"same_needs_tools": 0.9820919820919821,
"same_complexity_tier": 0.9092389092389093,
"same_policy_tier": 0.9151404151404151
},
"agreement_with_parent": {
"max_abs_logit_difference": 1.028977870941162,
"mean_abs_logit_difference": 0.06225224584341049,
"same_task": 0.9947089947089947,
"same_needs_tools": 0.9991859991859992,
"same_complexity_tier": 0.99002849002849,
"same_policy_tier": 0.9922669922669922
},
"latency_ms_batch1": {
"p50": 5.329958999936935,
"p95": 34.000291998381726,
"mean": 17.72976168012974,
"prompts": 200,
"device": "cpu"
}
}