| { | |
| "model": "artifacts/pluto-2-typed", | |
| "benchmark": "LocalLLaMA/typed-decisions", | |
| "split": "test", | |
| "mode": "fitted", | |
| "n_cases": 400, | |
| "n_decisions": 2000, | |
| "accuracy": 0.803, | |
| "kl_from_gold": 0.11923040849736305, | |
| "brier": 0.0672323907787711, | |
| "ece": 0.07477404832751905, | |
| "by_kind": { | |
| "choice": { | |
| "n_decisions": 600, | |
| "accuracy": 0.78, | |
| "kl_from_gold": 0.15241561932803213, | |
| "brier": 0.081281756228142, | |
| "ece": 0.07988290969168543 | |
| }, | |
| "noul": { | |
| "n_decisions": 600, | |
| "accuracy": 0.8516666666666667, | |
| "kl_from_gold": 0.06329251245082283, | |
| "brier": 0.04614648872056481, | |
| "ece": 0.04405483458049834 | |
| }, | |
| "score": { | |
| "n_decisions": 800, | |
| "accuracy": 0.78375, | |
| "kl_from_gold": 0.13629492240926638, | |
| "brier": 0.07250979323539765, | |
| "ece": 0.10546664498729107 | |
| } | |
| }, | |
| "by_workflow": { | |
| "agent_trace_observability": { | |
| "n_decisions": 500, | |
| "accuracy": 0.764, | |
| "kl_from_gold": 0.1149915606592059, | |
| "brier": 0.06838644564523914, | |
| "ece": 0.09413051276996476 | |
| }, | |
| "customer_service": { | |
| "n_decisions": 500, | |
| "accuracy": 0.83, | |
| "kl_from_gold": 0.12424451907243046, | |
| "brier": 0.0661457304954968, | |
| "ece": 0.07617299813519858 | |
| }, | |
| "invoice_processing": { | |
| "n_decisions": 500, | |
| "accuracy": 0.846, | |
| "kl_from_gold": 0.1235060928782072, | |
| "brier": 0.06848787497824028, | |
| "ece": 0.05575175429481842 | |
| }, | |
| "security_incidents": { | |
| "n_decisions": 500, | |
| "accuracy": 0.772, | |
| "kl_from_gold": 0.11417946137960859, | |
| "brier": 0.06590951199610816, | |
| "ece": 0.08430837784308685 | |
| } | |
| }, | |
| "latency_ms_per_case_amortized": 50.90324725002574, | |
| "latency_ms_single_p50": 42.323649977333844, | |
| "latency_ms_single_p95": 43.27478500053985, | |
| "params_m": 395.85 | |
| } |