{
  "model": "C:\\Users\\henry\\new-omida-project\\pluto-2\\artifacts\\pluto-2",
  "benchmark": "Hanno-Labs/decision-bench",
  "split": "eval",
  "requested_rows": 1992,
  "included_rows": 1565,
  "skipped_options": 403,
  "skipped_prompt": 24,
  "max_len": 2048,
  "max_options": 16,
  "batch_size": 8,
  "n": 1565,
  "accuracy": 0.5789137380191693,
  "nll": 1.086787512493374,
  "brier": 0.5476896818997071,
  "ece": 0.03040878511978975,
  "by_primitive": {
    "ordinal_scoring": {
      "n": 142,
      "accuracy": 0.31690140845070425,
      "nll": 1.702245548212995,
      "brier": 0.7919354885271652,
      "ece": 0.23922968191870858
    },
    "candidate_selection": {
      "n": 893,
      "accuracy": 0.5139977603583427,
      "nll": 1.3175748448567437,
      "brier": 0.6268954549540386,
      "ece": 0.05335862384965334
    },
    "binary_classification": {
      "n": 530,
      "accuracy": 0.7584905660377359,
      "nll": 0.5330363259411564,
      "brier": 0.34879598401552414,
      "ece": 0.06172943167517076
    }
  },
  "by_candidate_count_bucket": {
    "2": {
      "n": 571,
      "accuracy": 0.7758318739054291,
      "nll": 0.5050864657848015,
      "brier": 0.327164851226164,
      "ece": 0.06657381248932939
    },
    "3": {
      "n": 52,
      "accuracy": 0.7307692307692307,
      "nll": 0.8230841673369436,
      "brier": 0.4656287957639696,
      "ece": 0.1768807057572377
    },
    "4": {
      "n": 417,
      "accuracy": 0.38369304556354916,
      "nll": 1.3917934761391375,
      "brier": 0.7077339030970196,
      "ece": 0.1366008560847453
    },
    "5": {
      "n": 53,
      "accuracy": 0.41509433962264153,
      "nll": 1.3906180619397066,
      "brier": 0.6965652674513569,
      "ece": 0.12229648836302624
    },
    "6": {
      "n": 62,
      "accuracy": 0.5161290322580645,
      "nll": 1.4950352941490053,
      "brier": 0.7103141629730844,
      "ece": 0.20879640991101237
    },
    "7": {
      "n": 45,
      "accuracy": 0.4444444444444444,
      "nll": 1.6899349194297915,
      "brier": 0.7423954014990986,
      "ece": 0.12905421665120007
    },
    "8": {
      "n": 84,
      "accuracy": 0.7738095238095238,
      "nll": 0.9060157136010754,
      "brier": 0.3785204070880192,
      "ece": 0.24301153029444017
    },
    "9": {
      "n": 204,
      "accuracy": 0.36764705882352944,
      "nll": 1.8435587409953924,
      "brier": 0.812923101907946,
      "ece": 0.22095318662488653
    },
    "10": {
      "n": 20,
      "accuracy": 0.55,
      "nll": 1.3466658370996674,
      "brier": 0.581430754465156,
      "ece": 0.19840851353183617
    },
    "11": {
      "n": 11,
      "accuracy": 0.45454545454545453,
      "nll": 1.742375978409565,
      "brier": 0.7057783122595841,
      "ece": 0.14119367689162138
    },
    "12": {
      "n": 5,
      "accuracy": 0.6,
      "nll": 1.60027353448161,
      "brier": 0.7113295784136134,
      "ece": 0.3569272335501148
    },
    "13": {
      "n": 2,
      "accuracy": 0.0,
      "nll": 2.7291098611081326,
      "brier": 1.0117958954704067,
      "ece": 0.26010035168011614
    },
    "14": {
      "n": 5,
      "accuracy": 0.2,
      "nll": 2.0200694744604966,
      "brier": 0.8290373434160234,
      "ece": 0.08302967212398918
    },
    "15": {
      "n": 4,
      "accuracy": 0.75,
      "nll": 1.4792958719146407,
      "brier": 0.6844986526248836,
      "ece": 0.4951474916901011
    },
    "16": {
      "n": 30,
      "accuracy": 0.9333333333333333,
      "nll": 0.6343312692816507,
      "brier": 0.23775440952147306,
      "ece": 0.2986224605056543
    }
  },
  "prompt_state_truncated": 48,
  "latency_s": 21.23680189999868,
  "latency_ms_per_row": 13.56984146964772,
  "length_tokens": {
    "p50": 250,
    "p90": 662,
    "p95": 1086,
    "p99": 2048,
    "max": 2048
  }
}