| { | |
| "model": "C:\\Users\\henry\\new-omida-project\\pluto-2\\artifacts\\pluto-2", | |
| "benchmark": "Hanno-Labs/decision-bench", | |
| "split": "eval", | |
| "requested_rows": 1992, | |
| "included_rows": 1565, | |
| "skipped_options": 403, | |
| "skipped_prompt": 24, | |
| "max_len": 2048, | |
| "max_options": 16, | |
| "batch_size": 8, | |
| "n": 1565, | |
| "accuracy": 0.5789137380191693, | |
| "nll": 1.086787512493374, | |
| "brier": 0.5476896818997071, | |
| "ece": 0.03040878511978975, | |
| "by_primitive": { | |
| "ordinal_scoring": { | |
| "n": 142, | |
| "accuracy": 0.31690140845070425, | |
| "nll": 1.702245548212995, | |
| "brier": 0.7919354885271652, | |
| "ece": 0.23922968191870858 | |
| }, | |
| "candidate_selection": { | |
| "n": 893, | |
| "accuracy": 0.5139977603583427, | |
| "nll": 1.3175748448567437, | |
| "brier": 0.6268954549540386, | |
| "ece": 0.05335862384965334 | |
| }, | |
| "binary_classification": { | |
| "n": 530, | |
| "accuracy": 0.7584905660377359, | |
| "nll": 0.5330363259411564, | |
| "brier": 0.34879598401552414, | |
| "ece": 0.06172943167517076 | |
| } | |
| }, | |
| "by_candidate_count_bucket": { | |
| "2": { | |
| "n": 571, | |
| "accuracy": 0.7758318739054291, | |
| "nll": 0.5050864657848015, | |
| "brier": 0.327164851226164, | |
| "ece": 0.06657381248932939 | |
| }, | |
| "3": { | |
| "n": 52, | |
| "accuracy": 0.7307692307692307, | |
| "nll": 0.8230841673369436, | |
| "brier": 0.4656287957639696, | |
| "ece": 0.1768807057572377 | |
| }, | |
| "4": { | |
| "n": 417, | |
| "accuracy": 0.38369304556354916, | |
| "nll": 1.3917934761391375, | |
| "brier": 0.7077339030970196, | |
| "ece": 0.1366008560847453 | |
| }, | |
| "5": { | |
| "n": 53, | |
| "accuracy": 0.41509433962264153, | |
| "nll": 1.3906180619397066, | |
| "brier": 0.6965652674513569, | |
| "ece": 0.12229648836302624 | |
| }, | |
| "6": { | |
| "n": 62, | |
| "accuracy": 0.5161290322580645, | |
| "nll": 1.4950352941490053, | |
| "brier": 0.7103141629730844, | |
| "ece": 0.20879640991101237 | |
| }, | |
| "7": { | |
| "n": 45, | |
| "accuracy": 0.4444444444444444, | |
| "nll": 1.6899349194297915, | |
| "brier": 0.7423954014990986, | |
| "ece": 0.12905421665120007 | |
| }, | |
| "8": { | |
| "n": 84, | |
| "accuracy": 0.7738095238095238, | |
| "nll": 0.9060157136010754, | |
| "brier": 0.3785204070880192, | |
| "ece": 0.24301153029444017 | |
| }, | |
| "9": { | |
| "n": 204, | |
| "accuracy": 0.36764705882352944, | |
| "nll": 1.8435587409953924, | |
| "brier": 0.812923101907946, | |
| "ece": 0.22095318662488653 | |
| }, | |
| "10": { | |
| "n": 20, | |
| "accuracy": 0.55, | |
| "nll": 1.3466658370996674, | |
| "brier": 0.581430754465156, | |
| "ece": 0.19840851353183617 | |
| }, | |
| "11": { | |
| "n": 11, | |
| "accuracy": 0.45454545454545453, | |
| "nll": 1.742375978409565, | |
| "brier": 0.7057783122595841, | |
| "ece": 0.14119367689162138 | |
| }, | |
| "12": { | |
| "n": 5, | |
| "accuracy": 0.6, | |
| "nll": 1.60027353448161, | |
| "brier": 0.7113295784136134, | |
| "ece": 0.3569272335501148 | |
| }, | |
| "13": { | |
| "n": 2, | |
| "accuracy": 0.0, | |
| "nll": 2.7291098611081326, | |
| "brier": 1.0117958954704067, | |
| "ece": 0.26010035168011614 | |
| }, | |
| "14": { | |
| "n": 5, | |
| "accuracy": 0.2, | |
| "nll": 2.0200694744604966, | |
| "brier": 0.8290373434160234, | |
| "ece": 0.08302967212398918 | |
| }, | |
| "15": { | |
| "n": 4, | |
| "accuracy": 0.75, | |
| "nll": 1.4792958719146407, | |
| "brier": 0.6844986526248836, | |
| "ece": 0.4951474916901011 | |
| }, | |
| "16": { | |
| "n": 30, | |
| "accuracy": 0.9333333333333333, | |
| "nll": 0.6343312692816507, | |
| "brier": 0.23775440952147306, | |
| "ece": 0.2986224605056543 | |
| } | |
| }, | |
| "prompt_state_truncated": 48, | |
| "latency_s": 21.23680189999868, | |
| "latency_ms_per_row": 13.56984146964772, | |
| "length_tokens": { | |
| "p50": 250, | |
| "p90": 662, | |
| "p95": 1086, | |
| "p99": 2048, | |
| "max": 2048 | |
| } | |
| } |