| { |
| "arc_challenge": { |
| "n": 1172, |
| "accuracy": 0.878839590443686, |
| "nll": 0.3434354264088279, |
| "brier": 0.18004018282065612, |
| "ece": 0.018640702693919233, |
| "latency_p50_ms": 173.45179200128769, |
| "latency_p95_ms": 241.31601209974178, |
| "mean_confidence": 0.8609673274686053 |
| }, |
| "winogrande": { |
| "n": 1267, |
| "accuracy": 0.664561957379637, |
| "nll": 0.8732282598352692, |
| "brier": 0.5079363220206212, |
| "ece": 0.1986676781891346, |
| "latency_p50_ms": 168.07529400102794, |
| "latency_p95_ms": 241.4132092009823, |
| "mean_confidence": 0.8493462892926714 |
| }, |
| "arc_easy": { |
| "n": 2376, |
| "accuracy": 0.9562289562289562, |
| "nll": 0.15331558248276225, |
| "brier": 0.07269179989034444, |
| "ece": 0.029571857402955024, |
| "latency_p50_ms": 172.40582649901626, |
| "latency_p95_ms": 231.58336125015921, |
| "mean_confidence": 0.9269831239375699 |
| }, |
| "hellaswag": { |
| "n": 10042, |
| "accuracy": 0.8977295359490142, |
| "nll": 0.2736215284007136, |
| "brier": 0.14571887516017584, |
| "ece": 0.012820075397336044, |
| "latency_p50_ms": 230.12878749796073, |
| "latency_p95_ms": 276.0439652007335, |
| "mean_confidence": 0.8883940220236488 |
| }, |
| "mmlu": { |
| "n": 14042, |
| "accuracy": 0.6441390115368181, |
| "nll": 0.8705675106095903, |
| "brier": 0.45396635565340543, |
| "ece": 0.03314363695835977, |
| "latency_p50_ms": 196.04511249963252, |
| "latency_p95_ms": 303.28189720094076, |
| "mean_confidence": 0.6772229938346321 |
| } |
| } |
|
|
