Pluto-1 private release candidate: dual-encoder ensemble, calibrated Typed Decisions specialist
2778af7 verified | { | |
| "model": "artifacts/pluto-typed-ensemble/config.json", | |
| "benchmark": "LocalLLaMA/typed-decisions", | |
| "split": "test", | |
| "mode": "fitted", | |
| "n_cases": 400, | |
| "n_decisions": 2000, | |
| "accuracy": 0.7515, | |
| "kl_from_gold": 0.11825331703553803, | |
| "brier": 0.06491284081552073, | |
| "ece": 0.14000920995022426, | |
| "by_kind": { | |
| "choice": { | |
| "n_decisions": 600, | |
| "accuracy": 0.7166666666666667, | |
| "kl_from_gold": 0.1750736015750848, | |
| "brier": 0.09019795780750824, | |
| "ece": 0.14079999397409826 | |
| }, | |
| "noul": { | |
| "n_decisions": 600, | |
| "accuracy": 0.8416666666666667, | |
| "kl_from_gold": 0.04831293153979232, | |
| "brier": 0.03659910507461351, | |
| "ece": 0.08494916795437828 | |
| }, | |
| "score": { | |
| "n_decisions": 800, | |
| "accuracy": 0.71, | |
| "kl_from_gold": 0.12809339275268725, | |
| "brier": 0.06718430487721054, | |
| "ece": 0.1819052732641962 | |
| } | |
| }, | |
| "by_workflow": { | |
| "agent_trace_observability": { | |
| "n_decisions": 500, | |
| "accuracy": 0.7, | |
| "kl_from_gold": 0.09569835841938794, | |
| "brier": 0.05901824673184943, | |
| "ece": 0.12393117519401484 | |
| }, | |
| "customer_service": { | |
| "n_decisions": 500, | |
| "accuracy": 0.754, | |
| "kl_from_gold": 0.17659215574016174, | |
| "brier": 0.09179313928678065, | |
| "ece": 0.1660653832562083 | |
| }, | |
| "invoice_processing": { | |
| "n_decisions": 500, | |
| "accuracy": 0.834, | |
| "kl_from_gold": 0.11278324721085041, | |
| "brier": 0.06002676292418177, | |
| "ece": 0.12675089177637838 | |
| }, | |
| "security_incidents": { | |
| "n_decisions": 500, | |
| "accuracy": 0.718, | |
| "kl_from_gold": 0.0879395067717521, | |
| "brier": 0.04881321431927111, | |
| "ece": 0.14328938957429524 | |
| } | |
| }, | |
| "latency_ms_per_case_amortized": 66.0180790000004, | |
| "latency_ms_single_p50": 54.230400000051304, | |
| "latency_ms_single_p95": 66.72152500023005, | |
| "params_m": 332.84 | |
| } |