File size: 1,925 Bytes
4649798 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | {
"model": "artifacts/pluto-2-typed",
"benchmark": "LocalLLaMA/typed-decisions",
"split": "test",
"mode": "fitted",
"n_cases": 400,
"n_decisions": 2000,
"accuracy": 0.803,
"kl_from_gold": 0.11923040849736305,
"brier": 0.0672323907787711,
"ece": 0.07477404832751905,
"by_kind": {
"choice": {
"n_decisions": 600,
"accuracy": 0.78,
"kl_from_gold": 0.15241561932803213,
"brier": 0.081281756228142,
"ece": 0.07988290969168543
},
"noul": {
"n_decisions": 600,
"accuracy": 0.8516666666666667,
"kl_from_gold": 0.06329251245082283,
"brier": 0.04614648872056481,
"ece": 0.04405483458049834
},
"score": {
"n_decisions": 800,
"accuracy": 0.78375,
"kl_from_gold": 0.13629492240926638,
"brier": 0.07250979323539765,
"ece": 0.10546664498729107
}
},
"by_workflow": {
"agent_trace_observability": {
"n_decisions": 500,
"accuracy": 0.764,
"kl_from_gold": 0.1149915606592059,
"brier": 0.06838644564523914,
"ece": 0.09413051276996476
},
"customer_service": {
"n_decisions": 500,
"accuracy": 0.83,
"kl_from_gold": 0.12424451907243046,
"brier": 0.0661457304954968,
"ece": 0.07617299813519858
},
"invoice_processing": {
"n_decisions": 500,
"accuracy": 0.846,
"kl_from_gold": 0.1235060928782072,
"brier": 0.06848787497824028,
"ece": 0.05575175429481842
},
"security_incidents": {
"n_decisions": 500,
"accuracy": 0.772,
"kl_from_gold": 0.11417946137960859,
"brier": 0.06590951199610816,
"ece": 0.08430837784308685
}
},
"latency_ms_per_case_amortized": 50.90324725002574,
"latency_ms_single_p50": 42.323649977333844,
"latency_ms_single_p95": 43.27478500053985,
"params_m": 395.85
} |