fev-4b / evaluation /benchmarks /kev /report.json
murabcd's picture
feat: add compiled serving shared training and benchmarks
5adf069 verified
Raw History Blame Contribute Delete
18.5 kB
{
"overall": {
"questions": 908,
"answered": 908,
"errors": 0,
"correct": 756,
"accuracyAllQuestions": 0.8325991189427313,
"brierAnswered": 0.23049057438945586,
"nllAnswered": 0.42905102574468273,
"ece10Answered": 0.051600521203983195,
"confidentWrongAt09": 6,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 828,
"coverage": 0.9118942731277533,
"errorRate": 0.12560386473429952
},
{
"threshold": 0.7,
"accepted": 667,
"coverage": 0.7345814977973568,
"errorRate": 0.05997001499250375
},
{
"threshold": 0.8,
"accepted": 531,
"coverage": 0.5848017621145375,
"errorRate": 0.04331450094161959
},
{
"threshold": 0.9,
"accepted": 368,
"coverage": 0.4052863436123348,
"errorRate": 0.016304347826086956
},
{
"threshold": 0.95,
"accepted": 276,
"coverage": 0.3039647577092511,
"errorRate": 0.007246376811594203
},
{
"threshold": 0.99,
"accepted": 88,
"coverage": 0.09691629955947137,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"byType": {
"choice": {
"questions": 492,
"answered": 492,
"errors": 0,
"correct": 400,
"accuracyAllQuestions": 0.8130081300813008,
"brierAnswered": 0.24971593029461644,
"nllAnswered": 0.5001996781390384,
"ece10Answered": 0.05950644744126973,
"confidentWrongAt09": 2,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 421,
"coverage": 0.8556910569105691,
"errorRate": 0.11163895486935867
},
{
"threshold": 0.7,
"accepted": 345,
"coverage": 0.7012195121951219,
"errorRate": 0.04927536231884058
},
{
"threshold": 0.8,
"accepted": 276,
"coverage": 0.5609756097560976,
"errorRate": 0.036231884057971016
},
{
"threshold": 0.9,
"accepted": 193,
"coverage": 0.39227642276422764,
"errorRate": 0.010362694300518135
},
{
"threshold": 0.95,
"accepted": 144,
"coverage": 0.2926829268292683,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 36,
"coverage": 0.07317073170731707,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"noul": {
"questions": 328,
"answered": 328,
"errors": 0,
"correct": 284,
"accuracyAllQuestions": 0.8658536585365854,
"brierAnswered": 0.19400091681333753,
"nllAnswered": 0.3162898738046143,
"ece10Answered": 0.03553192036633847,
"confidentWrongAt09": 3,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 328,
"coverage": 1.0,
"errorRate": 0.13414634146341464
},
{
"threshold": 0.7,
"accepted": 264,
"coverage": 0.8048780487804879,
"errorRate": 0.07196969696969698
},
{
"threshold": 0.8,
"accepted": 216,
"coverage": 0.6585365853658537,
"errorRate": 0.05092592592592592
},
{
"threshold": 0.9,
"accepted": 154,
"coverage": 0.4695121951219512,
"errorRate": 0.01948051948051948
},
{
"threshold": 0.95,
"accepted": 119,
"coverage": 0.3628048780487805,
"errorRate": 0.01680672268907563
},
{
"threshold": 0.99,
"accepted": 50,
"coverage": 0.1524390243902439,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"score": {
"questions": 88,
"answered": 88,
"errors": 0,
"correct": 72,
"accuracyAllQuestions": 0.8181818181818182,
"brierAnswered": 0.2590102627943176,
"nllAnswered": 0.451556944589222,
"ece10Answered": 0.08293436635062447,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 79,
"coverage": 0.8977272727272727,
"errorRate": 0.16455696202531644
},
{
"threshold": 0.7,
"accepted": 58,
"coverage": 0.6590909090909091,
"errorRate": 0.06896551724137931
},
{
"threshold": 0.8,
"accepted": 39,
"coverage": 0.4431818181818182,
"errorRate": 0.05128205128205128
},
{
"threshold": 0.9,
"accepted": 21,
"coverage": 0.23863636363636365,
"errorRate": 0.047619047619047616
},
{
"threshold": 0.95,
"accepted": 13,
"coverage": 0.14772727272727273,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 2,
"coverage": 0.022727272727272728,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
}
},
"bySource": {
"authored-russian-typed": {
"questions": 144,
"answered": 144,
"errors": 0,
"correct": 143,
"accuracyAllQuestions": 0.9930555555555556,
"brierAnswered": 0.10416501037063129,
"nllAnswered": 0.2628194880491901,
"ece10Answered": 0.21249311017124553,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 137,
"coverage": 0.9513888888888888,
"errorRate": 0.0
},
{
"threshold": 0.7,
"accepted": 125,
"coverage": 0.8680555555555556,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 63,
"coverage": 0.4375,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 18,
"coverage": 0.125,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 14,
"coverage": 0.09722222222222222,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 1,
"coverage": 0.006944444444444444,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"composition_holdout": {
"questions": 96,
"answered": 96,
"errors": 0,
"correct": 94,
"accuracyAllQuestions": 0.9791666666666666,
"brierAnswered": 0.07265750882301934,
"nllAnswered": 0.14970972837197358,
"ece10Answered": 0.07865301631331481,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 96,
"coverage": 1.0,
"errorRate": 0.020833333333333332
},
{
"threshold": 0.7,
"accepted": 85,
"coverage": 0.8854166666666666,
"errorRate": 0.011764705882352941
},
{
"threshold": 0.8,
"accepted": 80,
"coverage": 0.8333333333333334,
"errorRate": 0.0125
},
{
"threshold": 0.9,
"accepted": 64,
"coverage": 0.6666666666666666,
"errorRate": 0.015625
},
{
"threshold": 0.95,
"accepted": 52,
"coverage": 0.5416666666666666,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 13,
"coverage": 0.13541666666666666,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"emotion": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 65,
"accuracyAllQuestions": 0.5603448275862069,
"brierAnswered": 0.5283768574785999,
"nllAnswered": 1.0464486013956598,
"ece10Answered": 0.10797638647123994,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 89,
"coverage": 0.7672413793103449,
"errorRate": 0.33707865168539325
},
{
"threshold": 0.7,
"accepted": 48,
"coverage": 0.41379310344827586,
"errorRate": 0.20833333333333334
},
{
"threshold": 0.8,
"accepted": 34,
"coverage": 0.29310344827586204,
"errorRate": 0.17647058823529413
},
{
"threshold": 0.9,
"accepted": 12,
"coverage": 0.10344827586206896,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 3,
"coverage": 0.02586206896551724,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"legacy_holdout": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 65,
"accuracyAllQuestions": 0.8125,
"brierAnswered": 0.18524784583046267,
"nllAnswered": 0.28116834855200057,
"ece10Answered": 0.08460090352211151,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 78,
"coverage": 0.975,
"errorRate": 0.16666666666666666
},
{
"threshold": 0.7,
"accepted": 66,
"coverage": 0.825,
"errorRate": 0.06060606060606061
},
{
"threshold": 0.8,
"accepted": 63,
"coverage": 0.7875,
"errorRate": 0.031746031746031744
},
{
"threshold": 0.9,
"accepted": 58,
"coverage": 0.725,
"errorRate": 0.017241379310344827
},
{
"threshold": 0.95,
"accepted": 53,
"coverage": 0.6625,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 39,
"coverage": 0.4875,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"mmlu": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 79,
"accuracyAllQuestions": 0.6810344827586207,
"brierAnswered": 0.3964835619347099,
"nllAnswered": 0.740201823820986,
"ece10Answered": 0.06660618296076969,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 74,
"coverage": 0.6379310344827587,
"errorRate": 0.17567567567567569
},
{
"threshold": 0.7,
"accepted": 59,
"coverage": 0.5086206896551724,
"errorRate": 0.1016949152542373
},
{
"threshold": 0.8,
"accepted": 44,
"coverage": 0.3793103448275862,
"errorRate": 0.06818181818181818
},
{
"threshold": 0.9,
"accepted": 28,
"coverage": 0.2413793103448276,
"errorRate": 0.03571428571428571
},
{
"threshold": 0.95,
"accepted": 15,
"coverage": 0.12931034482758622,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 1,
"coverage": 0.008620689655172414,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"paws": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 64,
"accuracyAllQuestions": 0.8,
"brierAnswered": 0.30918811576472116,
"nllAnswered": 0.48629027291665167,
"ece10Answered": 0.0669704500818625,
"confidentWrongAt09": 2,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.2
},
{
"threshold": 0.7,
"accepted": 65,
"coverage": 0.8125,
"errorRate": 0.16923076923076924
},
{
"threshold": 0.8,
"accepted": 55,
"coverage": 0.6875,
"errorRate": 0.12727272727272726
},
{
"threshold": 0.9,
"accepted": 37,
"coverage": 0.4625,
"errorRate": 0.05405405405405406
},
{
"threshold": 0.95,
"accepted": 23,
"coverage": 0.2875,
"errorRate": 0.08695652173913043
},
{
"threshold": 0.99,
"accepted": 2,
"coverage": 0.025,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"qnli": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 73,
"accuracyAllQuestions": 0.9125,
"brierAnswered": 0.14017388395851296,
"nllAnswered": 0.2368256396206597,
"ece10Answered": 0.028304750133247607,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.0875
},
{
"threshold": 0.7,
"accepted": 70,
"coverage": 0.875,
"errorRate": 0.05714285714285714
},
{
"threshold": 0.8,
"accepted": 63,
"coverage": 0.7875,
"errorRate": 0.031746031746031744
},
{
"threshold": 0.9,
"accepted": 50,
"coverage": 0.625,
"errorRate": 0.02
},
{
"threshold": 0.95,
"accepted": 38,
"coverage": 0.475,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 10,
"coverage": 0.125,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"sciq": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 114,
"accuracyAllQuestions": 0.9827586206896551,
"brierAnswered": 0.04432013428162752,
"nllAnswered": 0.11124410753498573,
"ece10Answered": 0.07043185982245576,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 114,
"coverage": 0.9827586206896551,
"errorRate": 0.017543859649122806
},
{
"threshold": 0.7,
"accepted": 106,
"coverage": 0.9137931034482759,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 98,
"coverage": 0.8448275862068966,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 89,
"coverage": 0.7672413793103449,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 74,
"coverage": 0.6379310344827587,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 22,
"coverage": 0.1896551724137931,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"tweet_offensive": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 59,
"accuracyAllQuestions": 0.7375,
"brierAnswered": 0.3014603416542088,
"nllAnswered": 0.4607710560880617,
"ece10Answered": 0.0959448419045657,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.2625
},
{
"threshold": 0.7,
"accepted": 43,
"coverage": 0.5375,
"errorRate": 0.09302325581395349
},
{
"threshold": 0.8,
"accepted": 31,
"coverage": 0.3875,
"errorRate": 0.06451612903225806
},
{
"threshold": 0.9,
"accepted": 12,
"coverage": 0.15,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 4,
"coverage": 0.05,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
}
},
"provenance": {
"revision": "6cfce5c2fa4b4bd64026336ab649c5ca78857d52",
"codeRevision": "2074cc44256200bb37540103fa4572901faa10f3",
"dataSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
"precision": "BF16 merged, official eager SDPA; no graphs/fused",
"gpu": "NVIDIA A10G",
"torch": "2.8.0+cu128",
"temperature": 2.406050072164233
},
"servedModels": [
"kev-4b@6cfce5c2fa4b4bd64026336ab649c5ca78857d52"
],
"records": 812,
"startedUtc": null,
"wallLatencyMs": {
"median": 67.72997199999509,
"p95": 83.42849299998534
},
"latencyMeaning": "Sequential wall time including client overhead/cold calls; not model-only time or throughput"
}