Download evaluation/benchmarks/jev/report.json from murabcd/fev-4b: direct link, hf CLI and curl.
- Browser
- Download file 18.4 kB
-
https://huggingface.co/murabcd/fev-4b/resolve/main/evaluation/benchmarks/jev/report.json
- Command line
-
hf download hf://murabcd/fev-4b/evaluation/benchmarks/jev/report.json
-
curl -L -o report.json https://huggingface.co/murabcd/fev-4b/resolve/main/evaluation/benchmarks/jev/report.json
18.4 kB
| { | |
| "overall": { | |
| "questions": 908, | |
| "answered": 908, | |
| "errors": 0, | |
| "correct": 790, | |
| "accuracyAllQuestions": 0.8700440528634361, | |
| "brierAnswered": 0.18933414586842034, | |
| "nllAnswered": 0.8838910343827637, | |
| "ece10Answered": 0.04661137809816233, | |
| "confidentWrongAt09": 36, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 900, | |
| "coverage": 0.9911894273127754, | |
| "errorRate": 0.12444444444444444 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 807, | |
| "coverage": 0.8887665198237885, | |
| "errorRate": 0.08178438661710037 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 757, | |
| "coverage": 0.8337004405286343, | |
| "errorRate": 0.06340819022457067 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 695, | |
| "coverage": 0.7654185022026432, | |
| "errorRate": 0.051798561151079135 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 629, | |
| "coverage": 0.6927312775330396, | |
| "errorRate": 0.04292527821939587 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 443, | |
| "coverage": 0.4878854625550661, | |
| "errorRate": 0.040632054176072234 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "byType": { | |
| "choice": { | |
| "questions": 492, | |
| "answered": 492, | |
| "errors": 0, | |
| "correct": 414, | |
| "accuracyAllQuestions": 0.8414634146341463, | |
| "brierAnswered": 0.2359069196108245, | |
| "nllAnswered": 1.4490444419682396, | |
| "ece10Answered": 0.08126652705921, | |
| "confidentWrongAt09": 28, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 484, | |
| "coverage": 0.983739837398374, | |
| "errorRate": 0.1487603305785124 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 437, | |
| "coverage": 0.8882113821138211, | |
| "errorRate": 0.10526315789473684 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 411, | |
| "coverage": 0.8353658536585366, | |
| "errorRate": 0.0827250608272506 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 384, | |
| "coverage": 0.7804878048780488, | |
| "errorRate": 0.07291666666666667 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 362, | |
| "coverage": 0.7357723577235772, | |
| "errorRate": 0.06906077348066299 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 320, | |
| "coverage": 0.6504065040650406, | |
| "errorRate": 0.05625 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "noul": { | |
| "questions": 328, | |
| "answered": 328, | |
| "errors": 0, | |
| "correct": 289, | |
| "accuracyAllQuestions": 0.8810975609756098, | |
| "brierAnswered": 0.16377439024390245, | |
| "nllAnswered": 0.26158678411952796, | |
| "ece10Answered": 0.015304878048780484, | |
| "confidentWrongAt09": 8, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 328, | |
| "coverage": 1.0, | |
| "errorRate": 0.11890243902439024 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 285, | |
| "coverage": 0.8689024390243902, | |
| "errorRate": 0.06666666666666667 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 262, | |
| "coverage": 0.7987804878048781, | |
| "errorRate": 0.05343511450381679 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 229, | |
| "coverage": 0.698170731707317, | |
| "errorRate": 0.034934497816593885 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 186, | |
| "coverage": 0.5670731707317073, | |
| "errorRate": 0.010752688172043012 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 55, | |
| "coverage": 0.1676829268292683, | |
| "errorRate": 0.0 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "score": { | |
| "questions": 88, | |
| "answered": 88, | |
| "errors": 0, | |
| "correct": 87, | |
| "accuracyAllQuestions": 0.9886363636363636, | |
| "brierAnswered": 0.02421818181818182, | |
| "nllAnswered": 0.043667370226935225, | |
| "ece10Answered": 0.03250000000000011, | |
| "confidentWrongAt09": 0, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 88, | |
| "coverage": 1.0, | |
| "errorRate": 0.011363636363636364 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 85, | |
| "coverage": 0.9659090909090909, | |
| "errorRate": 0.011764705882352941 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 84, | |
| "coverage": 0.9545454545454546, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 82, | |
| "coverage": 0.9318181818181818, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 81, | |
| "coverage": 0.9204545454545454, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 68, | |
| "coverage": 0.7727272727272727, | |
| "errorRate": 0.0 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| } | |
| }, | |
| "bySource": { | |
| "authored-russian-typed": { | |
| "questions": 144, | |
| "answered": 144, | |
| "errors": 0, | |
| "correct": 144, | |
| "accuracyAllQuestions": 1.0, | |
| "brierAnswered": 0.00011805555555555571, | |
| "nllAnswered": 0.005099213409504898, | |
| "ece10Answered": 0.005069444444444349, | |
| "confidentWrongAt09": 0, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 144, | |
| "coverage": 1.0, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 144, | |
| "coverage": 1.0, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 144, | |
| "coverage": 1.0, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 144, | |
| "coverage": 1.0, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 144, | |
| "coverage": 1.0, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 138, | |
| "coverage": 0.9583333333333334, | |
| "errorRate": 0.0 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "composition_holdout": { | |
| "questions": 96, | |
| "answered": 96, | |
| "errors": 0, | |
| "correct": 86, | |
| "accuracyAllQuestions": 0.8958333333333334, | |
| "brierAnswered": 0.11480833333333335, | |
| "nllAnswered": 0.16559070638097498, | |
| "ece10Answered": 0.041458333333333354, | |
| "confidentWrongAt09": 1, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 96, | |
| "coverage": 1.0, | |
| "errorRate": 0.10416666666666667 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 81, | |
| "coverage": 0.84375, | |
| "errorRate": 0.012345679012345678 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 80, | |
| "coverage": 0.8333333333333334, | |
| "errorRate": 0.0125 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 77, | |
| "coverage": 0.8020833333333334, | |
| "errorRate": 0.012987012987012988 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 73, | |
| "coverage": 0.7604166666666666, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 65, | |
| "coverage": 0.6770833333333334, | |
| "errorRate": 0.0 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "emotion": { | |
| "questions": 116, | |
| "answered": 116, | |
| "errors": 0, | |
| "correct": 70, | |
| "accuracyAllQuestions": 0.603448275862069, | |
| "brierAnswered": 0.5788285340341767, | |
| "nllAnswered": 3.146503884340211, | |
| "ece10Answered": 0.23599616858237546, | |
| "confidentWrongAt09": 14, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 110, | |
| "coverage": 0.9482758620689655, | |
| "errorRate": 0.37272727272727274 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 91, | |
| "coverage": 0.7844827586206896, | |
| "errorRate": 0.31868131868131866 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 75, | |
| "coverage": 0.646551724137931, | |
| "errorRate": 0.25333333333333335 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 59, | |
| "coverage": 0.5086206896551724, | |
| "errorRate": 0.23728813559322035 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 52, | |
| "coverage": 0.4482758620689655, | |
| "errorRate": 0.25 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 39, | |
| "coverage": 0.33620689655172414, | |
| "errorRate": 0.1794871794871795 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "legacy_holdout": { | |
| "questions": 80, | |
| "answered": 80, | |
| "errors": 0, | |
| "correct": 79, | |
| "accuracyAllQuestions": 0.9875, | |
| "brierAnswered": 0.02695, | |
| "nllAnswered": 0.05636262223757969, | |
| "ece10Answered": 0.043999999999999984, | |
| "confidentWrongAt09": 0, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 80, | |
| "coverage": 1.0, | |
| "errorRate": 0.0125 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 77, | |
| "coverage": 0.9625, | |
| "errorRate": 0.012987012987012988 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 76, | |
| "coverage": 0.95, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 74, | |
| "coverage": 0.925, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 73, | |
| "coverage": 0.9125, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 31, | |
| "coverage": 0.3875, | |
| "errorRate": 0.0 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "mmlu": { | |
| "questions": 116, | |
| "answered": 116, | |
| "errors": 0, | |
| "correct": 97, | |
| "accuracyAllQuestions": 0.8362068965517241, | |
| "brierAnswered": 0.28328529741863073, | |
| "nllAnswered": 2.795124001333949, | |
| "ece10Answered": 0.12087251828631135, | |
| "confidentWrongAt09": 13, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 114, | |
| "coverage": 0.9827586206896551, | |
| "errorRate": 0.15789473684210525 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 105, | |
| "coverage": 0.9051724137931034, | |
| "errorRate": 0.13333333333333333 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 99, | |
| "coverage": 0.853448275862069, | |
| "errorRate": 0.13131313131313133 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 93, | |
| "coverage": 0.8017241379310345, | |
| "errorRate": 0.13978494623655913 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 82, | |
| "coverage": 0.7068965517241379, | |
| "errorRate": 0.14634146341463414 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 67, | |
| "coverage": 0.5775862068965517, | |
| "errorRate": 0.16417910447761194 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "paws": { | |
| "questions": 80, | |
| "answered": 80, | |
| "errors": 0, | |
| "correct": 63, | |
| "accuracyAllQuestions": 0.7875, | |
| "brierAnswered": 0.3104425, | |
| "nllAnswered": 0.4745888407794543, | |
| "ece10Answered": 0.0953749999999999, | |
| "confidentWrongAt09": 6, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 80, | |
| "coverage": 1.0, | |
| "errorRate": 0.2125 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 70, | |
| "coverage": 0.875, | |
| "errorRate": 0.17142857142857143 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 67, | |
| "coverage": 0.8375, | |
| "errorRate": 0.16417910447761194 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 56, | |
| "coverage": 0.7, | |
| "errorRate": 0.10714285714285714 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 39, | |
| "coverage": 0.4875, | |
| "errorRate": 0.05128205128205128 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 0, | |
| "coverage": 0.0, | |
| "errorRate": null | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "qnli": { | |
| "questions": 80, | |
| "answered": 80, | |
| "errors": 0, | |
| "correct": 73, | |
| "accuracyAllQuestions": 0.9125, | |
| "brierAnswered": 0.094585, | |
| "nllAnswered": 0.1699941228795627, | |
| "ece10Answered": 0.08575000000000002, | |
| "confidentWrongAt09": 0, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 80, | |
| "coverage": 1.0, | |
| "errorRate": 0.0875 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 71, | |
| "coverage": 0.8875, | |
| "errorRate": 0.014084507042253521 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 63, | |
| "coverage": 0.7875, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 55, | |
| "coverage": 0.6875, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 42, | |
| "coverage": 0.525, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 1, | |
| "coverage": 0.0125, | |
| "errorRate": 0.0 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "sciq": { | |
| "questions": 116, | |
| "answered": 116, | |
| "errors": 0, | |
| "correct": 113, | |
| "accuracyAllQuestions": 0.9741379310344828, | |
| "brierAnswered": 0.04341551724137931, | |
| "nllAnswered": 0.0658923924477742, | |
| "ece10Answered": 0.01767241379310347, | |
| "confidentWrongAt09": 0, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 116, | |
| "coverage": 1.0, | |
| "errorRate": 0.02586206896551724 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 112, | |
| "coverage": 0.9655172413793104, | |
| "errorRate": 0.017857142857142856 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 109, | |
| "coverage": 0.9396551724137931, | |
| "errorRate": 0.009174311926605505 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 107, | |
| "coverage": 0.9224137931034483, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 107, | |
| "coverage": 0.9224137931034483, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 101, | |
| "coverage": 0.8706896551724138, | |
| "errorRate": 0.0 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| }, | |
| "tweet_offensive": { | |
| "questions": 80, | |
| "answered": 80, | |
| "errors": 0, | |
| "correct": 65, | |
| "accuracyAllQuestions": 0.8125, | |
| "brierAnswered": 0.265965, | |
| "nllAnswered": 0.4124258192766881, | |
| "ece10Answered": 0.02999999999999997, | |
| "confidentWrongAt09": 2, | |
| "coverageByFixedThreshold": [ | |
| { | |
| "threshold": 0.5, | |
| "accepted": 80, | |
| "coverage": 1.0, | |
| "errorRate": 0.1875 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "accepted": 56, | |
| "coverage": 0.7, | |
| "errorRate": 0.10714285714285714 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "accepted": 44, | |
| "coverage": 0.55, | |
| "errorRate": 0.06818181818181818 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "accepted": 30, | |
| "coverage": 0.375, | |
| "errorRate": 0.06666666666666667 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "accepted": 17, | |
| "coverage": 0.2125, | |
| "errorRate": 0.0 | |
| }, | |
| { | |
| "threshold": 0.99, | |
| "accepted": 1, | |
| "coverage": 0.0125, | |
| "errorRate": 0.0 | |
| } | |
| ], | |
| "coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here." | |
| } | |
| }, | |
| "provenance": { | |
| "model": "jev-1.14.0", | |
| "endpoint": "https://api.typesafe.ai/v1/systemone", | |
| "records": 812, | |
| "datasetSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91", | |
| "estimatedCapUsd": 0.15, | |
| "conservativeByteTokenUpperEstimateUsd": 0.018575508, | |
| "priceUsdPerMillionInputTokens": 0.042, | |
| "noAutomaticRetries": true, | |
| "publicAuthoredInputsOnly": true | |
| }, | |
| "servedModels": [ | |
| "jev-1.14.0" | |
| ], | |
| "records": 812, | |
| "usageInputTokens": 323757, | |
| "estimatedApiCostUsd": 0.013597794000000002, | |
| "seconds": 289.84684014320374, | |
| "wallLatencyMs": { | |
| "median": 311.6002500173636 | |
| }, | |
| "latencyMeaning": "Remote network/client wall time; excludes reused preflight; not comparable GPU kernels or throughput" | |
| } | |