{
  "model": "runs/v3b",
  "quant": null,
  "agreement": {
    "rows": 5125,
    "agree": 0.6874146341463415,
    "kl": 0.6401720955939848,
    "brier": 0.28186191172584546,
    "per_file": {
      "data/eval_mmlu_pro_think512.jsonl": {
        "rows": 1999,
        "agree": 0.49024512256128067,
        "kl": 1.345718805884049,
        "brier": 0.5798118903688055
      },
      "data/eval_wide.jsonl": {
        "rows": 920,
        "agree": 0.8521739130434782,
        "kl": 0.24507102074775225,
        "brier": 0.09771515623017962
      },
      "data/eval_xnli.jsonl": {
        "rows": 740,
        "agree": 0.8405405405405405,
        "kl": 0.12840145847645495,
        "brier": 0.07536055317105994
      },
      "data/eval_multi.jsonl": {
        "rows": 1466,
        "agree": 0.7755798090040927,
        "kl": 0.18438450108898327,
        "brier": 0.09538443087951312
      }
    }
  },
  "mmlu_pro": {
    "rows": 2000,
    "accuracy": 0.4195
  },
  "latency": [
    {
      "questions": 1,
      "p50_ms": 27.914823964238167,
      "p99_ms": 28.93548645079136
    },
    {
      "questions": 10,
      "p50_ms": 47.78458643704653,
      "p99_ms": 48.7666130065918
    }
  ]
}