| { | |
| "arc_easy": { | |
| "n": 2376, | |
| "accuracy": 0.9558080808080808, | |
| "nll": 0.22179265867232187, | |
| "brier": 0.07866352502747752, | |
| "ece": 0.03341212654329345, | |
| "latency_p50_ms": 153.7982515001204, | |
| "latency_p95_ms": 216.1515892476018, | |
| "mean_confidence": 0.9868527784304538 | |
| }, | |
| "arc_challenge": { | |
| "n": 1172, | |
| "accuracy": 0.8737201365187713, | |
| "nll": 0.6831067400476731, | |
| "brier": 0.21717192716282274, | |
| "ece": 0.0939787680494793, | |
| "latency_p50_ms": 158.1707515015296, | |
| "latency_p95_ms": 239.32236594882852, | |
| "mean_confidence": 0.9676989045682511 | |
| }, | |
| "winogrande": { | |
| "n": 1267, | |
| "accuracy": 0.601420678768745, | |
| "nll": 1.2788768116978686, | |
| "brier": 0.6614413444986983, | |
| "ece": 0.3052572014015491, | |
| "latency_p50_ms": 154.72152499933145, | |
| "latency_p95_ms": 223.75241669906248, | |
| "mean_confidence": 0.9024290672675535 | |
| }, | |
| "hellaswag": { | |
| "n": 10042, | |
| "accuracy": 0.7650866361282613, | |
| "nll": 1.1006902005839516, | |
| "brier": 0.3895498661609883, | |
| "ece": 0.16545367857507678, | |
| "latency_p50_ms": 259.9183570000605, | |
| "latency_p95_ms": 484.0487622521322, | |
| "mean_confidence": 0.9305298846809178 | |
| }, | |
| "mmlu": { | |
| "n": 14042, | |
| "accuracy": 0.6618715282723259, | |
| "nll": 1.6791403682676391, | |
| "brier": 0.5577819683925032, | |
| "ece": 0.24181974065906128, | |
| "latency_p50_ms": 176.33997150187497, | |
| "latency_p95_ms": 324.31165310117643, | |
| "mean_confidence": 0.9036912689313833 | |
| } | |
| } | |