{
  "model": "artifacts/pluto-typed-ensemble/config.json",
  "benchmark": "LocalLLaMA/typed-decisions",
  "split": "test",
  "mode": "fitted",
  "n_cases": 400,
  "n_decisions": 2000,
  "accuracy": 0.7515,
  "kl_from_gold": 0.11825331703553803,
  "brier": 0.06491284081552073,
  "ece": 0.14000920995022426,
  "by_kind": {
    "choice": {
      "n_decisions": 600,
      "accuracy": 0.7166666666666667,
      "kl_from_gold": 0.1750736015750848,
      "brier": 0.09019795780750824,
      "ece": 0.14079999397409826
    },
    "noul": {
      "n_decisions": 600,
      "accuracy": 0.8416666666666667,
      "kl_from_gold": 0.04831293153979232,
      "brier": 0.03659910507461351,
      "ece": 0.08494916795437828
    },
    "score": {
      "n_decisions": 800,
      "accuracy": 0.71,
      "kl_from_gold": 0.12809339275268725,
      "brier": 0.06718430487721054,
      "ece": 0.1819052732641962
    }
  },
  "by_workflow": {
    "agent_trace_observability": {
      "n_decisions": 500,
      "accuracy": 0.7,
      "kl_from_gold": 0.09569835841938794,
      "brier": 0.05901824673184943,
      "ece": 0.12393117519401484
    },
    "customer_service": {
      "n_decisions": 500,
      "accuracy": 0.754,
      "kl_from_gold": 0.17659215574016174,
      "brier": 0.09179313928678065,
      "ece": 0.1660653832562083
    },
    "invoice_processing": {
      "n_decisions": 500,
      "accuracy": 0.834,
      "kl_from_gold": 0.11278324721085041,
      "brier": 0.06002676292418177,
      "ece": 0.12675089177637838
    },
    "security_incidents": {
      "n_decisions": 500,
      "accuracy": 0.718,
      "kl_from_gold": 0.0879395067717521,
      "brier": 0.04881321431927111,
      "ece": 0.14328938957429524
    }
  },
  "latency_ms_per_case_amortized": 66.0180790000004,
  "latency_ms_single_p50": 54.230400000051304,
  "latency_ms_single_p95": 66.72152500023005,
  "params_m": 332.84
}