| { |
| "model": "runs/v3b", |
| "quant": null, |
| "agreement": { |
| "rows": 5125, |
| "agree": 0.6874146341463415, |
| "kl": 0.6401720955939848, |
| "brier": 0.28186191172584546, |
| "per_file": { |
| "data/eval_mmlu_pro_think512.jsonl": { |
| "rows": 1999, |
| "agree": 0.49024512256128067, |
| "kl": 1.345718805884049, |
| "brier": 0.5798118903688055 |
| }, |
| "data/eval_wide.jsonl": { |
| "rows": 920, |
| "agree": 0.8521739130434782, |
| "kl": 0.24507102074775225, |
| "brier": 0.09771515623017962 |
| }, |
| "data/eval_xnli.jsonl": { |
| "rows": 740, |
| "agree": 0.8405405405405405, |
| "kl": 0.12840145847645495, |
| "brier": 0.07536055317105994 |
| }, |
| "data/eval_multi.jsonl": { |
| "rows": 1466, |
| "agree": 0.7755798090040927, |
| "kl": 0.18438450108898327, |
| "brier": 0.09538443087951312 |
| } |
| } |
| }, |
| "mmlu_pro": { |
| "rows": 2000, |
| "accuracy": 0.4195 |
| }, |
| "latency": [ |
| { |
| "questions": 1, |
| "p50_ms": 27.914823964238167, |
| "p99_ms": 28.93548645079136 |
| }, |
| { |
| "questions": 10, |
| "p50_ms": 47.78458643704653, |
| "p99_ms": 48.7666130065918 |
| } |
| ] |
| } |
