Download evaluation/benchmarks/results.json from murabcd/fev-4b: direct link, hf CLI and curl.
- Browser
- Download file 72.1 kB
-
https://huggingface.co/murabcd/fev-4b/resolve/main/evaluation/benchmarks/results.json
- Command line
-
hf download hf://murabcd/fev-4b/evaluation/benchmarks/results.json
-
curl -L -o results.json https://huggingface.co/murabcd/fev-4b/resolve/main/evaluation/benchmarks/results.json
72.1 kB
{
"models": {
"Fev": {
"all": {
"questions": 908,
"answered": 908,
"errors": 0,
"correct": 783,
"accuracyAllQuestions": 0.8623348017621145,
"brierAnswered": 0.19616500539754675,
"nllAnswered": 0.374025080065555,
"ece10Answered": 0.04207503699664424,
"confidentWrongAt09": 26,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 889,
"coverage": 0.9790748898678414,
"errorRate": 0.12373453318335208
},
{
"threshold": 0.7,
"accepted": 819,
"coverage": 0.9019823788546255,
"errorRate": 0.08424908424908426
},
{
"threshold": 0.8,
"accepted": 767,
"coverage": 0.8447136563876652,
"errorRate": 0.06649282920469361
},
{
"threshold": 0.9,
"accepted": 676,
"coverage": 0.7444933920704846,
"errorRate": 0.038461538461538464
},
{
"threshold": 0.95,
"accepted": 567,
"coverage": 0.6244493392070485,
"errorRate": 0.012345679012345678
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.060723818086750196
},
"publicTransfer": {
"questions": 764,
"answered": 764,
"errors": 0,
"correct": 639,
"accuracyAllQuestions": 0.8363874345549738,
"brierAnswered": 0.2329500652363337,
"nllAnswered": 0.43991052107250256,
"ece10Answered": 0.054555210421981894,
"confidentWrongAt09": 26,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 745,
"coverage": 0.975130890052356,
"errorRate": 0.1476510067114094
},
{
"threshold": 0.7,
"accepted": 675,
"coverage": 0.8835078534031413,
"errorRate": 0.10222222222222223
},
{
"threshold": 0.8,
"accepted": 623,
"coverage": 0.8154450261780105,
"errorRate": 0.08186195826645265
},
{
"threshold": 0.9,
"accepted": 532,
"coverage": 0.6963350785340314,
"errorRate": 0.04887218045112782
},
{
"threshold": 0.95,
"accepted": 423,
"coverage": 0.5536649214659686,
"errorRate": 0.016548463356973995
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.10620271886956209
},
"authoredRussian": {
"questions": 144,
"answered": 144,
"errors": 0,
"correct": 144,
"accuracyAllQuestions": 1.0,
"brierAnswered": 0.0009998268084272204,
"nllAnswered": 0.024466212500916604,
"ece10Answered": 0.024139216454452272,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.7,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.02282473410107361
},
"byType": {
"choice": {
"questions": 492,
"answered": 492,
"errors": 0,
"correct": 400,
"accuracyAllQuestions": 0.8130081300813008,
"brierAnswered": 0.2615263089500927,
"nllAnswered": 0.5247360934156637,
"ece10Answered": 0.06533461933177653,
"confidentWrongAt09": 20,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 473,
"coverage": 0.9613821138211383,
"errorRate": 0.16279069767441862
},
{
"threshold": 0.7,
"accepted": 424,
"coverage": 0.8617886178861789,
"errorRate": 0.10377358490566038
},
{
"threshold": 0.8,
"accepted": 386,
"coverage": 0.7845528455284553,
"errorRate": 0.08031088082901554
},
{
"threshold": 0.9,
"accepted": 331,
"coverage": 0.6727642276422764,
"errorRate": 0.06042296072507553
},
{
"threshold": 0.95,
"accepted": 267,
"coverage": 0.5426829268292683,
"errorRate": 0.026217228464419477
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"noul": {
"questions": 328,
"answered": 328,
"errors": 0,
"correct": 298,
"accuracyAllQuestions": 0.9085365853658537,
"brierAnswered": 0.13871514597249474,
"nllAnswered": 0.22139737153892355,
"ece10Answered": 0.04994271263817423,
"confidentWrongAt09": 5,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 328,
"coverage": 1.0,
"errorRate": 0.09146341463414634
},
{
"threshold": 0.7,
"accepted": 309,
"coverage": 0.9420731707317073,
"errorRate": 0.07766990291262135
},
{
"threshold": 0.8,
"accepted": 296,
"coverage": 0.9024390243902439,
"errorRate": 0.06418918918918919
},
{
"threshold": 0.9,
"accepted": 261,
"coverage": 0.7957317073170732,
"errorRate": 0.019157088122605363
},
{
"threshold": 0.95,
"accepted": 220,
"coverage": 0.6707317073170732,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"score": {
"questions": 88,
"answered": 88,
"errors": 0,
"correct": 85,
"accuracyAllQuestions": 0.9659090909090909,
"brierAnswered": 0.04486719339259727,
"nllAnswered": 0.10029860084375529,
"ece10Answered": 0.03799778408573792,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 88,
"coverage": 1.0,
"errorRate": 0.03409090909090909
},
{
"threshold": 0.7,
"accepted": 86,
"coverage": 0.9772727272727273,
"errorRate": 0.011627906976744186
},
{
"threshold": 0.8,
"accepted": 85,
"coverage": 0.9659090909090909,
"errorRate": 0.011764705882352941
},
{
"threshold": 0.9,
"accepted": 84,
"coverage": 0.9545454545454546,
"errorRate": 0.011904761904761904
},
{
"threshold": 0.95,
"accepted": 80,
"coverage": 0.9090909090909091,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
}
},
"bySource": {
"authored-russian-typed": {
"questions": 144,
"answered": 144,
"errors": 0,
"correct": 144,
"accuracyAllQuestions": 1.0,
"brierAnswered": 0.0009998268084272204,
"nllAnswered": 0.024466212500916604,
"ece10Answered": 0.024139216454452272,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.7,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"composition_holdout": {
"questions": 96,
"answered": 96,
"errors": 0,
"correct": 84,
"accuracyAllQuestions": 0.875,
"brierAnswered": 0.18821988619559113,
"nllAnswered": 0.31506340814867745,
"ece10Answered": 0.08715274769631735,
"confidentWrongAt09": 9,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 96,
"coverage": 1.0,
"errorRate": 0.125
},
{
"threshold": 0.7,
"accepted": 92,
"coverage": 0.9583333333333334,
"errorRate": 0.09782608695652174
},
{
"threshold": 0.8,
"accepted": 92,
"coverage": 0.9583333333333334,
"errorRate": 0.09782608695652174
},
{
"threshold": 0.9,
"accepted": 92,
"coverage": 0.9583333333333334,
"errorRate": 0.09782608695652174
},
{
"threshold": 0.95,
"accepted": 86,
"coverage": 0.8958333333333334,
"errorRate": 0.03488372093023256
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"emotion": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 69,
"accuracyAllQuestions": 0.5948275862068966,
"brierAnswered": 0.5245559470753833,
"nllAnswered": 1.0208149336604613,
"ece10Answered": 0.12016921458494689,
"confidentWrongAt09": 2,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 101,
"coverage": 0.8706896551724138,
"errorRate": 0.3564356435643564
},
{
"threshold": 0.7,
"accepted": 70,
"coverage": 0.603448275862069,
"errorRate": 0.22857142857142856
},
{
"threshold": 0.8,
"accepted": 44,
"coverage": 0.3793103448275862,
"errorRate": 0.18181818181818182
},
{
"threshold": 0.9,
"accepted": 8,
"coverage": 0.06896551724137931,
"errorRate": 0.25
},
{
"threshold": 0.95,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"legacy_holdout": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 77,
"accuracyAllQuestions": 0.9625,
"brierAnswered": 0.04910351545555701,
"nllAnswered": 0.10416551178163372,
"ece10Answered": 0.03574970703743059,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.0375
},
{
"threshold": 0.7,
"accepted": 78,
"coverage": 0.975,
"errorRate": 0.01282051282051282
},
{
"threshold": 0.8,
"accepted": 77,
"coverage": 0.9625,
"errorRate": 0.012987012987012988
},
{
"threshold": 0.9,
"accepted": 76,
"coverage": 0.95,
"errorRate": 0.013157894736842105
},
{
"threshold": 0.95,
"accepted": 72,
"coverage": 0.9,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"mmlu": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 86,
"accuracyAllQuestions": 0.7413793103448276,
"brierAnswered": 0.39416899029285185,
"nllAnswered": 0.8394104030580065,
"ece10Answered": 0.12528513793721022,
"confidentWrongAt09": 9,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 112,
"coverage": 0.9655172413793104,
"errorRate": 0.23214285714285715
},
{
"threshold": 0.7,
"accepted": 100,
"coverage": 0.8620689655172413,
"errorRate": 0.18
},
{
"threshold": 0.8,
"accepted": 91,
"coverage": 0.7844827586206896,
"errorRate": 0.14285714285714285
},
{
"threshold": 0.9,
"accepted": 77,
"coverage": 0.6637931034482759,
"errorRate": 0.11688311688311688
},
{
"threshold": 0.95,
"accepted": 42,
"coverage": 0.3620689655172414,
"errorRate": 0.09523809523809523
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"paws": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 68,
"accuracyAllQuestions": 0.85,
"brierAnswered": 0.20695251414059662,
"nllAnswered": 0.3140944708626177,
"ece10Answered": 0.08749262200436604,
"confidentWrongAt09": 2,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.15
},
{
"threshold": 0.7,
"accepted": 75,
"coverage": 0.9375,
"errorRate": 0.12
},
{
"threshold": 0.8,
"accepted": 73,
"coverage": 0.9125,
"errorRate": 0.0958904109589041
},
{
"threshold": 0.9,
"accepted": 65,
"coverage": 0.8125,
"errorRate": 0.03076923076923077
},
{
"threshold": 0.95,
"accepted": 58,
"coverage": 0.725,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"qnli": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 74,
"accuracyAllQuestions": 0.925,
"brierAnswered": 0.1185264487594759,
"nllAnswered": 0.19157792439378024,
"ece10Answered": 0.07130364771118254,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.075
},
{
"threshold": 0.7,
"accepted": 79,
"coverage": 0.9875,
"errorRate": 0.0759493670886076
},
{
"threshold": 0.8,
"accepted": 77,
"coverage": 0.9625,
"errorRate": 0.06493506493506493
},
{
"threshold": 0.9,
"accepted": 71,
"coverage": 0.8875,
"errorRate": 0.014084507042253521
},
{
"threshold": 0.95,
"accepted": 60,
"coverage": 0.75,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"sciq": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 113,
"accuracyAllQuestions": 0.9741379310344828,
"brierAnswered": 0.03409531064541125,
"nllAnswered": 0.09114362656929587,
"ece10Answered": 0.05454464085684041,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 116,
"coverage": 1.0,
"errorRate": 0.02586206896551724
},
{
"threshold": 0.7,
"accepted": 114,
"coverage": 0.9827586206896551,
"errorRate": 0.008771929824561403
},
{
"threshold": 0.8,
"accepted": 111,
"coverage": 0.9568965517241379,
"errorRate": 0.009009009009009009
},
{
"threshold": 0.9,
"accepted": 106,
"coverage": 0.9137931034482759,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 91,
"coverage": 0.7844827586206896,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"tweet_offensive": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 68,
"accuracyAllQuestions": 0.85,
"brierAnswered": 0.24263742159686036,
"nllAnswered": 0.38374648265869776,
"ece10Answered": 0.05576197181938556,
"confidentWrongAt09": 2,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.15
},
{
"threshold": 0.7,
"accepted": 67,
"coverage": 0.8375,
"errorRate": 0.13432835820895522
},
{
"threshold": 0.8,
"accepted": 58,
"coverage": 0.725,
"errorRate": 0.1206896551724138
},
{
"threshold": 0.9,
"accepted": 37,
"coverage": 0.4625,
"errorRate": 0.05405405405405406
},
{
"threshold": 0.95,
"accepted": 14,
"coverage": 0.175,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
}
}
},
"Kev": {
"all": {
"questions": 908,
"answered": 908,
"errors": 0,
"correct": 756,
"accuracyAllQuestions": 0.8325991189427313,
"brierAnswered": 0.23049057438945586,
"nllAnswered": 0.42905102574468273,
"ece10Answered": 0.051600521203983195,
"confidentWrongAt09": 6,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 828,
"coverage": 0.9118942731277533,
"errorRate": 0.12560386473429952
},
{
"threshold": 0.7,
"accepted": 667,
"coverage": 0.7345814977973568,
"errorRate": 0.05997001499250375
},
{
"threshold": 0.8,
"accepted": 531,
"coverage": 0.5848017621145375,
"errorRate": 0.04331450094161959
},
{
"threshold": 0.9,
"accepted": 368,
"coverage": 0.4052863436123348,
"errorRate": 0.016304347826086956
},
{
"threshold": 0.95,
"accepted": 276,
"coverage": 0.3039647577092511,
"errorRate": 0.007246376811594203
},
{
"threshold": 0.99,
"accepted": 88,
"coverage": 0.09691629955947137,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.3190549633819242
},
"publicTransfer": {
"questions": 764,
"answered": 764,
"errors": 0,
"correct": 613,
"accuracyAllQuestions": 0.8023560209424084,
"brierAnswered": 0.2543006283406479,
"nllAnswered": 0.460382624472629,
"ece10Answered": 0.033847807241890064,
"confidentWrongAt09": 6,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 691,
"coverage": 0.9044502617801047,
"errorRate": 0.15050651230101303
},
{
"threshold": 0.7,
"accepted": 542,
"coverage": 0.7094240837696335,
"errorRate": 0.07380073800738007
},
{
"threshold": 0.8,
"accepted": 468,
"coverage": 0.612565445026178,
"errorRate": 0.049145299145299144
},
{
"threshold": 0.9,
"accepted": 350,
"coverage": 0.4581151832460733,
"errorRate": 0.017142857142857144
},
{
"threshold": 0.95,
"accepted": 262,
"coverage": 0.34293193717277487,
"errorRate": 0.007633587786259542
},
{
"threshold": 0.99,
"accepted": 87,
"coverage": 0.11387434554973822,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.34230038267246843
},
"authoredRussian": {
"questions": 144,
"answered": 144,
"errors": 0,
"correct": 143,
"accuracyAllQuestions": 0.9930555555555556,
"brierAnswered": 0.10416501037063129,
"nllAnswered": 0.2628194880491901,
"ece10Answered": 0.21249311017124553,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 137,
"coverage": 0.9513888888888888,
"errorRate": 0.0
},
{
"threshold": 0.7,
"accepted": 125,
"coverage": 0.8680555555555556,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 63,
"coverage": 0.4375,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 18,
"coverage": 0.125,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 14,
"coverage": 0.09722222222222222,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 1,
"coverage": 0.006944444444444444,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.29968378063980394
},
"byType": {
"choice": {
"questions": 492,
"answered": 492,
"errors": 0,
"correct": 400,
"accuracyAllQuestions": 0.8130081300813008,
"brierAnswered": 0.24971593029461644,
"nllAnswered": 0.5001996781390384,
"ece10Answered": 0.05950644744126973,
"confidentWrongAt09": 2,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 421,
"coverage": 0.8556910569105691,
"errorRate": 0.11163895486935867
},
{
"threshold": 0.7,
"accepted": 345,
"coverage": 0.7012195121951219,
"errorRate": 0.04927536231884058
},
{
"threshold": 0.8,
"accepted": 276,
"coverage": 0.5609756097560976,
"errorRate": 0.036231884057971016
},
{
"threshold": 0.9,
"accepted": 193,
"coverage": 0.39227642276422764,
"errorRate": 0.010362694300518135
},
{
"threshold": 0.95,
"accepted": 144,
"coverage": 0.2926829268292683,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 36,
"coverage": 0.07317073170731707,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"noul": {
"questions": 328,
"answered": 328,
"errors": 0,
"correct": 284,
"accuracyAllQuestions": 0.8658536585365854,
"brierAnswered": 0.19400091681333753,
"nllAnswered": 0.3162898738046143,
"ece10Answered": 0.03553192036633847,
"confidentWrongAt09": 3,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 328,
"coverage": 1.0,
"errorRate": 0.13414634146341464
},
{
"threshold": 0.7,
"accepted": 264,
"coverage": 0.8048780487804879,
"errorRate": 0.07196969696969698
},
{
"threshold": 0.8,
"accepted": 216,
"coverage": 0.6585365853658537,
"errorRate": 0.05092592592592592
},
{
"threshold": 0.9,
"accepted": 154,
"coverage": 0.4695121951219512,
"errorRate": 0.01948051948051948
},
{
"threshold": 0.95,
"accepted": 119,
"coverage": 0.3628048780487805,
"errorRate": 0.01680672268907563
},
{
"threshold": 0.99,
"accepted": 50,
"coverage": 0.1524390243902439,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"score": {
"questions": 88,
"answered": 88,
"errors": 0,
"correct": 72,
"accuracyAllQuestions": 0.8181818181818182,
"brierAnswered": 0.2590102627943176,
"nllAnswered": 0.451556944589222,
"ece10Answered": 0.08293436635062447,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 79,
"coverage": 0.8977272727272727,
"errorRate": 0.16455696202531644
},
{
"threshold": 0.7,
"accepted": 58,
"coverage": 0.6590909090909091,
"errorRate": 0.06896551724137931
},
{
"threshold": 0.8,
"accepted": 39,
"coverage": 0.4431818181818182,
"errorRate": 0.05128205128205128
},
{
"threshold": 0.9,
"accepted": 21,
"coverage": 0.23863636363636365,
"errorRate": 0.047619047619047616
},
{
"threshold": 0.95,
"accepted": 13,
"coverage": 0.14772727272727273,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 2,
"coverage": 0.022727272727272728,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
}
},
"bySource": {
"authored-russian-typed": {
"questions": 144,
"answered": 144,
"errors": 0,
"correct": 143,
"accuracyAllQuestions": 0.9930555555555556,
"brierAnswered": 0.10416501037063129,
"nllAnswered": 0.2628194880491901,
"ece10Answered": 0.21249311017124553,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 137,
"coverage": 0.9513888888888888,
"errorRate": 0.0
},
{
"threshold": 0.7,
"accepted": 125,
"coverage": 0.8680555555555556,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 63,
"coverage": 0.4375,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 18,
"coverage": 0.125,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 14,
"coverage": 0.09722222222222222,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 1,
"coverage": 0.006944444444444444,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"composition_holdout": {
"questions": 96,
"answered": 96,
"errors": 0,
"correct": 94,
"accuracyAllQuestions": 0.9791666666666666,
"brierAnswered": 0.07265750882301934,
"nllAnswered": 0.14970972837197358,
"ece10Answered": 0.07865301631331481,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 96,
"coverage": 1.0,
"errorRate": 0.020833333333333332
},
{
"threshold": 0.7,
"accepted": 85,
"coverage": 0.8854166666666666,
"errorRate": 0.011764705882352941
},
{
"threshold": 0.8,
"accepted": 80,
"coverage": 0.8333333333333334,
"errorRate": 0.0125
},
{
"threshold": 0.9,
"accepted": 64,
"coverage": 0.6666666666666666,
"errorRate": 0.015625
},
{
"threshold": 0.95,
"accepted": 52,
"coverage": 0.5416666666666666,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 13,
"coverage": 0.13541666666666666,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"emotion": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 65,
"accuracyAllQuestions": 0.5603448275862069,
"brierAnswered": 0.5283768574785999,
"nllAnswered": 1.0464486013956598,
"ece10Answered": 0.10797638647123994,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 89,
"coverage": 0.7672413793103449,
"errorRate": 0.33707865168539325
},
{
"threshold": 0.7,
"accepted": 48,
"coverage": 0.41379310344827586,
"errorRate": 0.20833333333333334
},
{
"threshold": 0.8,
"accepted": 34,
"coverage": 0.29310344827586204,
"errorRate": 0.17647058823529413
},
{
"threshold": 0.9,
"accepted": 12,
"coverage": 0.10344827586206896,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 3,
"coverage": 0.02586206896551724,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"legacy_holdout": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 65,
"accuracyAllQuestions": 0.8125,
"brierAnswered": 0.18524784583046267,
"nllAnswered": 0.28116834855200057,
"ece10Answered": 0.08460090352211151,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 78,
"coverage": 0.975,
"errorRate": 0.16666666666666666
},
{
"threshold": 0.7,
"accepted": 66,
"coverage": 0.825,
"errorRate": 0.06060606060606061
},
{
"threshold": 0.8,
"accepted": 63,
"coverage": 0.7875,
"errorRate": 0.031746031746031744
},
{
"threshold": 0.9,
"accepted": 58,
"coverage": 0.725,
"errorRate": 0.017241379310344827
},
{
"threshold": 0.95,
"accepted": 53,
"coverage": 0.6625,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 39,
"coverage": 0.4875,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"mmlu": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 79,
"accuracyAllQuestions": 0.6810344827586207,
"brierAnswered": 0.3964835619347099,
"nllAnswered": 0.740201823820986,
"ece10Answered": 0.06660618296076969,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 74,
"coverage": 0.6379310344827587,
"errorRate": 0.17567567567567569
},
{
"threshold": 0.7,
"accepted": 59,
"coverage": 0.5086206896551724,
"errorRate": 0.1016949152542373
},
{
"threshold": 0.8,
"accepted": 44,
"coverage": 0.3793103448275862,
"errorRate": 0.06818181818181818
},
{
"threshold": 0.9,
"accepted": 28,
"coverage": 0.2413793103448276,
"errorRate": 0.03571428571428571
},
{
"threshold": 0.95,
"accepted": 15,
"coverage": 0.12931034482758622,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 1,
"coverage": 0.008620689655172414,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"paws": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 64,
"accuracyAllQuestions": 0.8,
"brierAnswered": 0.30918811576472116,
"nllAnswered": 0.48629027291665167,
"ece10Answered": 0.0669704500818625,
"confidentWrongAt09": 2,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.2
},
{
"threshold": 0.7,
"accepted": 65,
"coverage": 0.8125,
"errorRate": 0.16923076923076924
},
{
"threshold": 0.8,
"accepted": 55,
"coverage": 0.6875,
"errorRate": 0.12727272727272726
},
{
"threshold": 0.9,
"accepted": 37,
"coverage": 0.4625,
"errorRate": 0.05405405405405406
},
{
"threshold": 0.95,
"accepted": 23,
"coverage": 0.2875,
"errorRate": 0.08695652173913043
},
{
"threshold": 0.99,
"accepted": 2,
"coverage": 0.025,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"qnli": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 73,
"accuracyAllQuestions": 0.9125,
"brierAnswered": 0.14017388395851296,
"nllAnswered": 0.2368256396206597,
"ece10Answered": 0.028304750133247607,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.0875
},
{
"threshold": 0.7,
"accepted": 70,
"coverage": 0.875,
"errorRate": 0.05714285714285714
},
{
"threshold": 0.8,
"accepted": 63,
"coverage": 0.7875,
"errorRate": 0.031746031746031744
},
{
"threshold": 0.9,
"accepted": 50,
"coverage": 0.625,
"errorRate": 0.02
},
{
"threshold": 0.95,
"accepted": 38,
"coverage": 0.475,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 10,
"coverage": 0.125,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"sciq": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 114,
"accuracyAllQuestions": 0.9827586206896551,
"brierAnswered": 0.04432013428162752,
"nllAnswered": 0.11124410753498573,
"ece10Answered": 0.07043185982245576,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 114,
"coverage": 0.9827586206896551,
"errorRate": 0.017543859649122806
},
{
"threshold": 0.7,
"accepted": 106,
"coverage": 0.9137931034482759,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 98,
"coverage": 0.8448275862068966,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 89,
"coverage": 0.7672413793103449,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 74,
"coverage": 0.6379310344827587,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 22,
"coverage": 0.1896551724137931,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"tweet_offensive": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 59,
"accuracyAllQuestions": 0.7375,
"brierAnswered": 0.3014603416542088,
"nllAnswered": 0.4607710560880617,
"ece10Answered": 0.0959448419045657,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.2625
},
{
"threshold": 0.7,
"accepted": 43,
"coverage": 0.5375,
"errorRate": 0.09302325581395349
},
{
"threshold": 0.8,
"accepted": 31,
"coverage": 0.3875,
"errorRate": 0.06451612903225806
},
{
"threshold": 0.9,
"accepted": 12,
"coverage": 0.15,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 4,
"coverage": 0.05,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
}
}
},
"Jev": {
"all": {
"questions": 908,
"answered": 908,
"errors": 0,
"correct": 790,
"accuracyAllQuestions": 0.8700440528634361,
"brierAnswered": 0.18933414586842034,
"nllAnswered": 0.8838910343827637,
"ece10Answered": 0.04661137809816233,
"confidentWrongAt09": 36,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 900,
"coverage": 0.9911894273127754,
"errorRate": 0.12444444444444444
},
{
"threshold": 0.7,
"accepted": 807,
"coverage": 0.8887665198237885,
"errorRate": 0.08178438661710037
},
{
"threshold": 0.8,
"accepted": 757,
"coverage": 0.8337004405286343,
"errorRate": 0.06340819022457067
},
{
"threshold": 0.9,
"accepted": 695,
"coverage": 0.7654185022026432,
"errorRate": 0.051798561151079135
},
{
"threshold": 0.95,
"accepted": 629,
"coverage": 0.6927312775330396,
"errorRate": 0.04292527821939587
},
{
"threshold": 0.99,
"accepted": 443,
"coverage": 0.4878854625550661,
"errorRate": 0.040632054176072234
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.0325
},
"publicTransfer": {
"questions": 764,
"answered": 764,
"errors": 0,
"correct": 646,
"accuracyAllQuestions": 0.8455497382198953,
"brierAnswered": 0.22499791158183988,
"nllAnswered": 1.049527189121179,
"ece10Answered": 0.056352266116663954,
"confidentWrongAt09": 36,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 756,
"coverage": 0.9895287958115183,
"errorRate": 0.14814814814814814
},
{
"threshold": 0.7,
"accepted": 663,
"coverage": 0.8678010471204188,
"errorRate": 0.09954751131221719
},
{
"threshold": 0.8,
"accepted": 613,
"coverage": 0.8023560209424084,
"errorRate": 0.07830342577487764
},
{
"threshold": 0.9,
"accepted": 551,
"coverage": 0.7212041884816754,
"errorRate": 0.06533575317604355
},
{
"threshold": 0.95,
"accepted": 485,
"coverage": 0.6348167539267016,
"errorRate": 0.05567010309278351
},
{
"threshold": 0.99,
"accepted": 305,
"coverage": 0.3992146596858639,
"errorRate": 0.05901639344262295
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.07075000000000001
},
"authoredRussian": {
"questions": 144,
"answered": 144,
"errors": 0,
"correct": 144,
"accuracyAllQuestions": 1.0,
"brierAnswered": 0.00011805555555555571,
"nllAnswered": 0.005099213409504898,
"ece10Answered": 0.005069444444444349,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.7,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 138,
"coverage": 0.9583333333333334,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.0006250000000000006
},
"byType": {
"choice": {
"questions": 492,
"answered": 492,
"errors": 0,
"correct": 414,
"accuracyAllQuestions": 0.8414634146341463,
"brierAnswered": 0.2359069196108245,
"nllAnswered": 1.4490444419682396,
"ece10Answered": 0.08126652705921,
"confidentWrongAt09": 28,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 484,
"coverage": 0.983739837398374,
"errorRate": 0.1487603305785124
},
{
"threshold": 0.7,
"accepted": 437,
"coverage": 0.8882113821138211,
"errorRate": 0.10526315789473684
},
{
"threshold": 0.8,
"accepted": 411,
"coverage": 0.8353658536585366,
"errorRate": 0.0827250608272506
},
{
"threshold": 0.9,
"accepted": 384,
"coverage": 0.7804878048780488,
"errorRate": 0.07291666666666667
},
{
"threshold": 0.95,
"accepted": 362,
"coverage": 0.7357723577235772,
"errorRate": 0.06906077348066299
},
{
"threshold": 0.99,
"accepted": 320,
"coverage": 0.6504065040650406,
"errorRate": 0.05625
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"noul": {
"questions": 328,
"answered": 328,
"errors": 0,
"correct": 289,
"accuracyAllQuestions": 0.8810975609756098,
"brierAnswered": 0.16377439024390245,
"nllAnswered": 0.26158678411952796,
"ece10Answered": 0.015304878048780484,
"confidentWrongAt09": 8,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 328,
"coverage": 1.0,
"errorRate": 0.11890243902439024
},
{
"threshold": 0.7,
"accepted": 285,
"coverage": 0.8689024390243902,
"errorRate": 0.06666666666666667
},
{
"threshold": 0.8,
"accepted": 262,
"coverage": 0.7987804878048781,
"errorRate": 0.05343511450381679
},
{
"threshold": 0.9,
"accepted": 229,
"coverage": 0.698170731707317,
"errorRate": 0.034934497816593885
},
{
"threshold": 0.95,
"accepted": 186,
"coverage": 0.5670731707317073,
"errorRate": 0.010752688172043012
},
{
"threshold": 0.99,
"accepted": 55,
"coverage": 0.1676829268292683,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"score": {
"questions": 88,
"answered": 88,
"errors": 0,
"correct": 87,
"accuracyAllQuestions": 0.9886363636363636,
"brierAnswered": 0.02421818181818182,
"nllAnswered": 0.043667370226935225,
"ece10Answered": 0.03250000000000011,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 88,
"coverage": 1.0,
"errorRate": 0.011363636363636364
},
{
"threshold": 0.7,
"accepted": 85,
"coverage": 0.9659090909090909,
"errorRate": 0.011764705882352941
},
{
"threshold": 0.8,
"accepted": 84,
"coverage": 0.9545454545454546,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 82,
"coverage": 0.9318181818181818,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 81,
"coverage": 0.9204545454545454,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 68,
"coverage": 0.7727272727272727,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
}
},
"bySource": {
"authored-russian-typed": {
"questions": 144,
"answered": 144,
"errors": 0,
"correct": 144,
"accuracyAllQuestions": 1.0,
"brierAnswered": 0.00011805555555555571,
"nllAnswered": 0.005099213409504898,
"ece10Answered": 0.005069444444444349,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.7,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.8,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 144,
"coverage": 1.0,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 138,
"coverage": 0.9583333333333334,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"composition_holdout": {
"questions": 96,
"answered": 96,
"errors": 0,
"correct": 86,
"accuracyAllQuestions": 0.8958333333333334,
"brierAnswered": 0.11480833333333335,
"nllAnswered": 0.16559070638097498,
"ece10Answered": 0.041458333333333354,
"confidentWrongAt09": 1,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 96,
"coverage": 1.0,
"errorRate": 0.10416666666666667
},
{
"threshold": 0.7,
"accepted": 81,
"coverage": 0.84375,
"errorRate": 0.012345679012345678
},
{
"threshold": 0.8,
"accepted": 80,
"coverage": 0.8333333333333334,
"errorRate": 0.0125
},
{
"threshold": 0.9,
"accepted": 77,
"coverage": 0.8020833333333334,
"errorRate": 0.012987012987012988
},
{
"threshold": 0.95,
"accepted": 73,
"coverage": 0.7604166666666666,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 65,
"coverage": 0.6770833333333334,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"emotion": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 70,
"accuracyAllQuestions": 0.603448275862069,
"brierAnswered": 0.5788285340341767,
"nllAnswered": 3.146503884340211,
"ece10Answered": 0.23599616858237546,
"confidentWrongAt09": 14,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 110,
"coverage": 0.9482758620689655,
"errorRate": 0.37272727272727274
},
{
"threshold": 0.7,
"accepted": 91,
"coverage": 0.7844827586206896,
"errorRate": 0.31868131868131866
},
{
"threshold": 0.8,
"accepted": 75,
"coverage": 0.646551724137931,
"errorRate": 0.25333333333333335
},
{
"threshold": 0.9,
"accepted": 59,
"coverage": 0.5086206896551724,
"errorRate": 0.23728813559322035
},
{
"threshold": 0.95,
"accepted": 52,
"coverage": 0.4482758620689655,
"errorRate": 0.25
},
{
"threshold": 0.99,
"accepted": 39,
"coverage": 0.33620689655172414,
"errorRate": 0.1794871794871795
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"legacy_holdout": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 79,
"accuracyAllQuestions": 0.9875,
"brierAnswered": 0.02695,
"nllAnswered": 0.05636262223757969,
"ece10Answered": 0.043999999999999984,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.0125
},
{
"threshold": 0.7,
"accepted": 77,
"coverage": 0.9625,
"errorRate": 0.012987012987012988
},
{
"threshold": 0.8,
"accepted": 76,
"coverage": 0.95,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 74,
"coverage": 0.925,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 73,
"coverage": 0.9125,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 31,
"coverage": 0.3875,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"mmlu": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 97,
"accuracyAllQuestions": 0.8362068965517241,
"brierAnswered": 0.28328529741863073,
"nllAnswered": 2.795124001333949,
"ece10Answered": 0.12087251828631135,
"confidentWrongAt09": 13,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 114,
"coverage": 0.9827586206896551,
"errorRate": 0.15789473684210525
},
{
"threshold": 0.7,
"accepted": 105,
"coverage": 0.9051724137931034,
"errorRate": 0.13333333333333333
},
{
"threshold": 0.8,
"accepted": 99,
"coverage": 0.853448275862069,
"errorRate": 0.13131313131313133
},
{
"threshold": 0.9,
"accepted": 93,
"coverage": 0.8017241379310345,
"errorRate": 0.13978494623655913
},
{
"threshold": 0.95,
"accepted": 82,
"coverage": 0.7068965517241379,
"errorRate": 0.14634146341463414
},
{
"threshold": 0.99,
"accepted": 67,
"coverage": 0.5775862068965517,
"errorRate": 0.16417910447761194
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"paws": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 63,
"accuracyAllQuestions": 0.7875,
"brierAnswered": 0.3104425,
"nllAnswered": 0.4745888407794543,
"ece10Answered": 0.0953749999999999,
"confidentWrongAt09": 6,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.2125
},
{
"threshold": 0.7,
"accepted": 70,
"coverage": 0.875,
"errorRate": 0.17142857142857143
},
{
"threshold": 0.8,
"accepted": 67,
"coverage": 0.8375,
"errorRate": 0.16417910447761194
},
{
"threshold": 0.9,
"accepted": 56,
"coverage": 0.7,
"errorRate": 0.10714285714285714
},
{
"threshold": 0.95,
"accepted": 39,
"coverage": 0.4875,
"errorRate": 0.05128205128205128
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"qnli": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 73,
"accuracyAllQuestions": 0.9125,
"brierAnswered": 0.094585,
"nllAnswered": 0.1699941228795627,
"ece10Answered": 0.08575000000000002,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.0875
},
{
"threshold": 0.7,
"accepted": 71,
"coverage": 0.8875,
"errorRate": 0.014084507042253521
},
{
"threshold": 0.8,
"accepted": 63,
"coverage": 0.7875,
"errorRate": 0.0
},
{
"threshold": 0.9,
"accepted": 55,
"coverage": 0.6875,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 42,
"coverage": 0.525,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 1,
"coverage": 0.0125,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"sciq": {
"questions": 116,
"answered": 116,
"errors": 0,
"correct": 113,
"accuracyAllQuestions": 0.9741379310344828,
"brierAnswered": 0.04341551724137931,
"nllAnswered": 0.0658923924477742,
"ece10Answered": 0.01767241379310347,
"confidentWrongAt09": 0,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 116,
"coverage": 1.0,
"errorRate": 0.02586206896551724
},
{
"threshold": 0.7,
"accepted": 112,
"coverage": 0.9655172413793104,
"errorRate": 0.017857142857142856
},
{
"threshold": 0.8,
"accepted": 109,
"coverage": 0.9396551724137931,
"errorRate": 0.009174311926605505
},
{
"threshold": 0.9,
"accepted": 107,
"coverage": 0.9224137931034483,
"errorRate": 0.0
},
{
"threshold": 0.95,
"accepted": 107,
"coverage": 0.9224137931034483,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 101,
"coverage": 0.8706896551724138,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
},
"tweet_offensive": {
"questions": 80,
"answered": 80,
"errors": 0,
"correct": 65,
"accuracyAllQuestions": 0.8125,
"brierAnswered": 0.265965,
"nllAnswered": 0.4124258192766881,
"ece10Answered": 0.02999999999999997,
"confidentWrongAt09": 2,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 80,
"coverage": 1.0,
"errorRate": 0.1875
},
{
"threshold": 0.7,
"accepted": 56,
"coverage": 0.7,
"errorRate": 0.10714285714285714
},
{
"threshold": 0.8,
"accepted": 44,
"coverage": 0.55,
"errorRate": 0.06818181818181818
},
{
"threshold": 0.9,
"accepted": 30,
"coverage": 0.375,
"errorRate": 0.06666666666666667
},
{
"threshold": 0.95,
"accepted": 17,
"coverage": 0.2125,
"errorRate": 0.0
},
{
"threshold": 0.99,
"accepted": 1,
"coverage": 0.0125,
"errorRate": 0.0
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
}
}
}
},
"pairedOutcomes": {
"Fev vs Kev": {
"aOnlyCorrect": 63,
"bOnlyCorrect": 36,
"bothCorrect": 720,
"neitherCorrect": 89
},
"Fev vs Jev": {
"aOnlyCorrect": 28,
"bOnlyCorrect": 35,
"bothCorrect": 755,
"neitherCorrect": 90
},
"Kev vs Jev": {
"aOnlyCorrect": 35,
"bOnlyCorrect": 69,
"bothCorrect": 721,
"neitherCorrect": 83
}
},
"interpretation": "Descriptive comparison on fixed public development and repetitive authored inputs; no significance or unseen customer accuracy claim",
"datasetSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
"provenance": {
"Fev": {
"revision": "3ef077cd7cc3c97d696c18150358ab4de3a6884e",
"codeZipSha256": "b3b7808d7c2cbe2ce26d4b31ee0dde2b3e894c6776537931b61cd173ef526f8e",
"execution": "reference BF16 backbone, FP32 shared adapter unmerged",
"dataSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
"gpu": "NVIDIA A10G",
"torch": "2.14.0+cu130",
"questionIsolation": false
},
"Kev": {
"revision": "6cfce5c2fa4b4bd64026336ab649c5ca78857d52",
"codeRevision": "2074cc44256200bb37540103fa4572901faa10f3",
"dataSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
"precision": "BF16 merged, official eager SDPA; no graphs/fused",
"gpu": "NVIDIA A10G",
"torch": "2.8.0+cu128",
"temperature": 2.406050072164233
},
"Jev": {
"model": "jev-1.14.0",
"endpoint": "https://api.typesafe.ai/v1/systemone",
"records": 812,
"datasetSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
"estimatedCapUsd": 0.15,
"conservativeByteTokenUpperEstimateUsd": 0.018575508,
"priceUsdPerMillionInputTokens": 0.042,
"noAutomaticRetries": true,
"publicAuthoredInputsOnly": true
}
}
}
{
"models": {
"Fev": {
"all": {
"questions": 908,
"answered": 908,
"errors": 0,
"correct": 783,
"accuracyAllQuestions": 0.8623348017621145,
"brierAnswered": 0.19616500539754675,
"nllAnswered": 0.374025080065555,
"ece10Answered": 0.04207503699664424,
"confidentWrongAt09": 26,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 889,
"coverage": 0.9790748898678414,
"errorRate": 0.12373453318335208
},
{
"threshold": 0.7,
"accepted": 819,
"coverage": 0.9019823788546255,
"errorRate": 0.08424908424908426
},
{
"threshold": 0.8,
"accepted": 767,
"coverage": 0.8447136563876652,
"errorRate": 0.06649282920469361
},
{
"threshold": 0.9,
"accepted": 676,
"coverage": 0.7444933920704846,
"errorRate": 0.038461538461538464
},
{
"threshold": 0.95,
"accepted": 567,
"coverage": 0.6244493392070485,
"errorRate": 0.012345679012345678
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null
}
],
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
"ordinalMeanAbsoluteError": 0.060723818086750196
},
"publicTransfer": {
"questions": 764,
"answered": 764,
"errors": 0,
"correct": 639,
"accuracyAllQuestions": 0.8363874345549738,
"brierAnswered": 0.2329500652363337,
"nllAnswered": 0.43991052107250256,
"ece10Answered": 0.054555210421981894,
"confidentWrongAt09": 26,
"coverageByFixedThreshold": [
{
"threshold": 0.5,
"accepted": 745,
"coverage": 0.975130890052356,
"errorRate": 0.1476510067114094
},
{
"threshold": 0.7,
"accepted": 675,
"coverage": 0.8835078534031413,
"errorRate": 0.10222222222222223
},
{
"threshold": 0.8,
"accepted": 623,
"coverage": 0.8154450261780105,
"errorRate": 0.08186195826645265
},
{
"threshold": 0.9,
"accepted": 532,
"coverage": 0.6963350785340314,
"errorRate": 0.04887218045112782
},
{
"threshold": 0.95,
"accepted": 423,
"coverage": 0.5536649214659686,
"errorRate": 0.016548463356973995
},
{
"threshold": 0.99,
"accepted": 0,
"coverage": 0.0,
"errorRate": null