{
  "model": "artifacts/pluto-2-typed",
  "benchmark": "LocalLLaMA/typed-decisions",
  "split": "test",
  "mode": "fitted",
  "n_cases": 400,
  "n_decisions": 2000,
  "accuracy": 0.803,
  "kl_from_gold": 0.11923040849736305,
  "brier": 0.0672323907787711,
  "ece": 0.07477404832751905,
  "by_kind": {
    "choice": {
      "n_decisions": 600,
      "accuracy": 0.78,
      "kl_from_gold": 0.15241561932803213,
      "brier": 0.081281756228142,
      "ece": 0.07988290969168543
    },
    "noul": {
      "n_decisions": 600,
      "accuracy": 0.8516666666666667,
      "kl_from_gold": 0.06329251245082283,
      "brier": 0.04614648872056481,
      "ece": 0.04405483458049834
    },
    "score": {
      "n_decisions": 800,
      "accuracy": 0.78375,
      "kl_from_gold": 0.13629492240926638,
      "brier": 0.07250979323539765,
      "ece": 0.10546664498729107
    }
  },
  "by_workflow": {
    "agent_trace_observability": {
      "n_decisions": 500,
      "accuracy": 0.764,
      "kl_from_gold": 0.1149915606592059,
      "brier": 0.06838644564523914,
      "ece": 0.09413051276996476
    },
    "customer_service": {
      "n_decisions": 500,
      "accuracy": 0.83,
      "kl_from_gold": 0.12424451907243046,
      "brier": 0.0661457304954968,
      "ece": 0.07617299813519858
    },
    "invoice_processing": {
      "n_decisions": 500,
      "accuracy": 0.846,
      "kl_from_gold": 0.1235060928782072,
      "brier": 0.06848787497824028,
      "ece": 0.05575175429481842
    },
    "security_incidents": {
      "n_decisions": 500,
      "accuracy": 0.772,
      "kl_from_gold": 0.11417946137960859,
      "brier": 0.06590951199610816,
      "ece": 0.08430837784308685
    }
  },
  "latency_ms_per_case_amortized": 50.90324725002574,
  "latency_ms_single_p50": 42.323649977333844,
  "latency_ms_single_p95": 43.27478500053985,
  "params_m": 395.85
}