| { |
| "run": "runs/tm1b", |
| "mode": "native", |
| "max_state": 4096, |
| "limit": 0, |
| "eval": { |
| "u_chaosnli": { |
| "n": 1594, |
| "n_total": 1594, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 1.014103315474928, |
| "brier": 0.17091631474795344, |
| "jsd": 0.06646093306121999, |
| "acc": 0.5131744040150564, |
| "acc_k": 0.5131744040150564, |
| "ece": 0.10623594570713328, |
| "sel_acc80": 0.5313479623824452, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "": { |
| "nll": 1.014103315474928, |
| "brier": 0.17091631474795344, |
| "jsd": 0.06646093306121999, |
| "acc": 0.5131744040150564, |
| "acc_k": 0.5131744040150564, |
| "ece": 0.10623594570713328, |
| "sel_acc80": 0.5313479623824452, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1594 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "answer": { |
| "nll": 1.014103315474928, |
| "brier": 0.17091631474795344, |
| "jsd": 0.06646093306121999, |
| "acc": 0.5131744040150564, |
| "acc_k": 0.5131744040150564, |
| "ece": 0.10623594570713328, |
| "sel_acc80": 0.5313479623824452, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1594 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.46361355081555833 |
| } |
| }, |
| "u_real_heldout": { |
| "n": 562, |
| "n_total": 562, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.6395687143934158, |
| "brier": 0.11684180639496176, |
| "jsd": 0.05530236546295147, |
| "acc": 0.6583629893238434, |
| "acc_k": 0.6583629893238434, |
| "ece": 0.04802694863696116, |
| "sel_acc80": 0.7, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "": { |
| "nll": 0.6395687143934158, |
| "brier": 0.11684180639496176, |
| "jsd": 0.05530236546295147, |
| "acc": 0.6583629893238434, |
| "acc_k": 0.6583629893238434, |
| "ece": 0.04802694863696116, |
| "sel_acc80": 0.7, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 562 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.6395687143934158, |
| "brier": 0.11684180639496176, |
| "jsd": 0.05530236546295147, |
| "acc": 0.6583629893238434, |
| "acc_k": 0.6583629893238434, |
| "ece": 0.04802694863696116, |
| "sel_acc80": 0.7, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 562 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.5071174377224199 |
| } |
| }, |
| "u_synthetic_heldout": { |
| "n": 475, |
| "n_total": 475, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.6619388394682302, |
| "brier": 0.029335292543417674, |
| "jsd": 0.01347952047234738, |
| "acc": 0.9073684210526316, |
| "acc_k": 0.9073684210526316, |
| "ece": 0.17639653883482279, |
| "sel_acc80": 0.9815789473684211, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "": { |
| "nll": 0.6619388394682302, |
| "brier": 0.029335292543417674, |
| "jsd": 0.01347952047234738, |
| "acc": 0.9073684210526316, |
| "acc_k": 0.9073684210526316, |
| "ece": 0.17639653883482279, |
| "sel_acc80": 0.9815789473684211, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 475 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.6619388394682302, |
| "brier": 0.029335292543417674, |
| "jsd": 0.01347952047234738, |
| "acc": 0.9073684210526316, |
| "acc_k": 0.9073684210526316, |
| "ece": 0.17639653883482279, |
| "sel_acc80": 0.9815789473684211, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 475 |
| } |
| }, |
| "baselines": { |
| "majority_class": 1, |
| "majority_acc": 0.5747368421052632 |
| } |
| }, |
| "wf_heldout_choice": { |
| "n": 1500, |
| "n_total": 1500, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.08894784373441748, |
| "brier": 0.03804468692396801, |
| "jsd": 0.017131709690636845, |
| "acc": 0.978, |
| "acc_k": 0.9907997169143666, |
| "ece": 0.011830694854259511, |
| "sel_acc80": 0.9958333333333333, |
| "conf_wrong": 0.0, |
| "coverage": 0.9273333333333333, |
| "false_abstain": 0.01556970983722576, |
| "null_recall": 1.0, |
| "auroc_null": 0.9994834500654839, |
| "auroc_conf": 0.9995119213217171 |
| }, |
| "by_family": { |
| "tool_select": { |
| "nll": 0.08894784373441748, |
| "brier": 0.03804468692396801, |
| "jsd": 0.017131709690636845, |
| "acc": 0.978, |
| "acc_k": 0.9907997169143666, |
| "ece": 0.011830694854259511, |
| "sel_acc80": 0.9958333333333333, |
| "conf_wrong": 0.0, |
| "coverage": 0.9273333333333333, |
| "false_abstain": 0.01556970983722576, |
| "null_recall": 1.0, |
| "auroc_null": 0.9994834500654839, |
| "auroc_conf": 0.9995119213217171, |
| "n": 1500 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 0.08894784373441748, |
| "brier": 0.03804468692396801, |
| "jsd": 0.017131709690636845, |
| "acc": 0.978, |
| "acc_k": 0.9907997169143666, |
| "ece": 0.011830694854259511, |
| "sel_acc80": 0.9958333333333333, |
| "conf_wrong": 0.0, |
| "coverage": 0.9273333333333333, |
| "false_abstain": 0.01556970983722576, |
| "null_recall": 1.0, |
| "auroc_null": 0.9994834500654839, |
| "auroc_conf": 0.9995119213217171, |
| "n": 1500 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.09437487539006816, |
| "brier": 0.04038581321521866, |
| "jsd": 0.018169384833798215, |
| "acc": 0.9766454352441614, |
| "acc_k": 0.9907997169143666, |
| "ece": 0.012608277211354707, |
| "sel_acc80": 0.9946949602122016, |
| "coverage": 0.9844302901627743, |
| "false_abstain": 0.01556970983722576, |
| "null_recall": "n/a", |
| "n": 1413 |
| }, |
| "null": { |
| "nll": 0.0008053640857458489, |
| "brier": 2.1566814345354523e-05, |
| "jsd": 0.0002784340896366467, |
| "acc": 1.0, |
| "ece": 0.0007983151523546228, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.0, |
| "coverage": 0.0, |
| "false_abstain": "n/a", |
| "null_recall": 1.0, |
| "n": 87 |
| } |
| }, |
| "baselines": { |
| "majority_class": 2, |
| "majority_acc": 0.21333333333333335 |
| } |
| }, |
| "wf_heldout_noul": { |
| "n": 1500, |
| "n_total": 1500, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.5731820141841979, |
| "brier": 0.30167717441890574, |
| "jsd": 0.10894462183976376, |
| "acc": 0.8106666666666666, |
| "acc_k": 0.8106666666666666, |
| "ece": 0.13052070760726933, |
| "sel_acc80": 0.88, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "eligibility": { |
| "nll": 0.5731820141841979, |
| "brier": 0.30167717441890574, |
| "jsd": 0.10894462183976376, |
| "acc": 0.8106666666666666, |
| "acc_k": 0.8106666666666666, |
| "ece": 0.13052070760726933, |
| "sel_acc80": 0.88, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1500 |
| } |
| }, |
| "by_qtype": { |
| "noul": { |
| "nll": 0.5731820141841979, |
| "brier": 0.30167717441890574, |
| "jsd": 0.10894462183976376, |
| "acc": 0.8106666666666666, |
| "acc_k": 0.8106666666666666, |
| "ece": 0.13052070760726933, |
| "sel_acc80": 0.88, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1500 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.5731820141841979, |
| "brier": 0.30167717441890574, |
| "jsd": 0.10894462183976376, |
| "acc": 0.8106666666666666, |
| "acc_k": 0.8106666666666666, |
| "ece": 0.13052070760726933, |
| "sel_acc80": 0.88, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1500 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.5833333333333334, |
| "always_positive_acc": 0.4166666666666667 |
| } |
| }, |
| "wf_heldout_score": { |
| "n": 1502, |
| "n_total": 1502, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 1.0185536074805717, |
| "brier": 0.5644523897741041, |
| "jsd": 0.2801056596145076, |
| "acc": 0.5279627163781625, |
| "acc_k": 0.5279627163781625, |
| "ece": 0.05101245270667476, |
| "sel_acc80": 0.5806988352745425, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "urgency": { |
| "nll": 1.0185536074805717, |
| "brier": 0.5644523897741041, |
| "jsd": 0.2801056596145076, |
| "acc": 0.5279627163781625, |
| "acc_k": 0.5279627163781625, |
| "ece": 0.05101245270667476, |
| "sel_acc80": 0.5806988352745425, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1502 |
| } |
| }, |
| "by_qtype": { |
| "score": { |
| "nll": 1.0185536074805717, |
| "brier": 0.5644523897741041, |
| "jsd": 0.2801056596145076, |
| "acc": 0.5279627163781625, |
| "acc_k": 0.5279627163781625, |
| "ece": 0.05101245270667476, |
| "sel_acc80": 0.5806988352745425, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1502 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 1.0185536074805717, |
| "brier": 0.5644523897741041, |
| "jsd": 0.2801056596145076, |
| "acc": 0.5279627163781625, |
| "acc_k": 0.5279627163781625, |
| "ece": 0.05101245270667476, |
| "sel_acc80": 0.5806988352745425, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1502 |
| } |
| }, |
| "baselines": { |
| "majority_class": 2, |
| "majority_acc": 0.3548601864181092 |
| } |
| }, |
| "wf_heldout_style": { |
| "n": 1627, |
| "n_total": 1627, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.35412793927257424, |
| "brier": 0.2060959847410378, |
| "jsd": 0.10287023165278557, |
| "acc": 0.8586355255070682, |
| "acc_k": 0.8564294631710362, |
| "ece": 0.05414908969592812, |
| "sel_acc80": 0.9308755760368663, |
| "conf_wrong": 0.0, |
| "coverage": 0.9846342962507683, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0 |
| }, |
| "by_family": { |
| "relevance": { |
| "nll": 0.6980276300431268, |
| "brier": 0.42102710856000347, |
| "jsd": 0.21389350686536507, |
| "acc": 0.72, |
| "acc_k": 0.72, |
| "ece": 0.17414700448513035, |
| "sel_acc80": 0.775, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 150 |
| }, |
| "support": { |
| "nll": 0.20406603972307857, |
| "brier": 0.1248183124169987, |
| "jsd": 0.055781303712672675, |
| "acc": 0.912751677852349, |
| "acc_k": 0.912751677852349, |
| "ece": 0.043371342572589816, |
| "sel_acc80": 0.975, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 149 |
| }, |
| "adequacy": { |
| "nll": 0.17072481903574357, |
| "brier": 0.10599008454380157, |
| "jsd": 0.047014351937127816, |
| "acc": 0.9266666666666666, |
| "acc_k": 0.9266666666666666, |
| "ece": 0.04129962166150417, |
| "sel_acc80": 0.9916666666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 150 |
| }, |
| "action_select": { |
| "nll": 0.231281707526007, |
| "brier": 0.13311344023592248, |
| "jsd": 0.058601355529894074, |
| "acc": 0.9133333333333333, |
| "acc_k": 0.9064748201438849, |
| "ece": 0.05428580959637955, |
| "sel_acc80": 0.9833333333333333, |
| "conf_wrong": 0.0, |
| "coverage": 0.9266666666666666, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 150 |
| }, |
| "completeness": { |
| "nll": 0.7433181718309183, |
| "brier": 0.4258628433595575, |
| "jsd": 0.22361098694674933, |
| "acc": 0.7248322147651006, |
| "acc_k": 0.7248322147651006, |
| "ece": 0.19188565155803758, |
| "sel_acc80": 0.75, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 149 |
| }, |
| "quality": { |
| "nll": 0.8425104976168987, |
| "brier": 0.4753130219919474, |
| "jsd": 0.24646774262839125, |
| "acc": 0.6259541984732825, |
| "acc_k": 0.6259541984732825, |
| "ece": 0.1351061500893294, |
| "sel_acc80": 0.6857142857142857, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 131 |
| }, |
| "enum_extract": { |
| "nll": 0.0033332703444646995, |
| "brier": 0.00024124939750805901, |
| "jsd": 0.0011463143230046637, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.003245462179183957, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.0, |
| "coverage": 0.9666666666666667, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 150 |
| }, |
| "categorical": { |
| "nll": 0.24690092856485552, |
| "brier": 0.14982400101731513, |
| "jsd": 0.07302109473186505, |
| "acc": 0.889763779527559, |
| "acc_k": 0.8870967741935484, |
| "ece": 0.053563406148294794, |
| "sel_acc80": 0.9607843137254902, |
| "conf_wrong": 0.0, |
| "coverage": 0.9763779527559056, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 127 |
| }, |
| "routing": { |
| "nll": 0.21774816429206942, |
| "brier": 0.13021437367644959, |
| "jsd": 0.055718942276041364, |
| "acc": 0.9090909090909091, |
| "acc_k": 0.9047619047619048, |
| "ece": 0.05593117032990313, |
| "sel_acc80": 0.9905660377358491, |
| "conf_wrong": 0.0, |
| "coverage": 0.9545454545454546, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 132 |
| }, |
| "severity": { |
| "nll": 0.6434271258602455, |
| "brier": 0.347468069379254, |
| "jsd": 0.19560876673722952, |
| "acc": 0.7974683544303798, |
| "acc_k": 0.7974683544303798, |
| "ece": 0.22888638286650936, |
| "sel_acc80": 0.875, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 79 |
| }, |
| "policy_permit": { |
| "nll": 0.08175127275497936, |
| "brier": 0.03988201939839468, |
| "jsd": 0.02053868180510852, |
| "acc": 0.9798657718120806, |
| "acc_k": 0.9798657718120806, |
| "ece": 0.031195846179987764, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 149 |
| }, |
| "fact": { |
| "nll": 0.3245140742970864, |
| "brier": 0.201230115726297, |
| "jsd": 0.09200789581633895, |
| "acc": 0.8558558558558559, |
| "acc_k": 0.8558558558558559, |
| "ece": 0.0778693476238766, |
| "sel_acc80": 0.9213483146067416, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 111 |
| } |
| }, |
| "by_qtype": { |
| "score": { |
| "nll": 0.7399964051867366, |
| "brier": 0.4249972756512392, |
| "jsd": 0.2222837523407285, |
| "acc": 0.7092337917485265, |
| "acc_k": 0.7092337917485265, |
| "ece": 0.1720800453354193, |
| "sel_acc80": 0.7549019607843137, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 509 |
| }, |
| "noul": { |
| "nll": 0.186433926049318, |
| "brier": 0.11229947221410162, |
| "jsd": 0.0512284562939753, |
| "acc": 0.9230769230769231, |
| "acc_k": 0.9230769230769231, |
| "ece": 0.024164358063972435, |
| "sel_acc80": 0.9754464285714286, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 559 |
| }, |
| "choice": { |
| "nll": 0.1704676606348132, |
| "brier": 0.10057092826387289, |
| "jsd": 0.04577948101845999, |
| "acc": 0.9302325581395349, |
| "acc_k": 0.9269662921348315, |
| "ece": 0.0223422101657164, |
| "sel_acc80": 0.9933035714285714, |
| "conf_wrong": 0.0, |
| "coverage": 0.9552772808586762, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 559 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.35959165383560376, |
| "brier": 0.2093046663699308, |
| "jsd": 0.10445406727566603, |
| "acc": 0.8564294631710362, |
| "acc_k": 0.8564294631710362, |
| "ece": 0.05493348601568356, |
| "sel_acc80": 0.9297971918876755, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1602 |
| }, |
| "null": { |
| "nll": 0.004013110073643866, |
| "brier": 0.0004836659615737134, |
| "jsd": 0.0013780449386066767, |
| "acc": 1.0, |
| "ece": 0.003884973526001012, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.0, |
| "coverage": 0.0, |
| "false_abstain": "n/a", |
| "null_recall": 1.0, |
| "n": 25 |
| } |
| }, |
| "baselines": { |
| "majority_class": 1, |
| "majority_acc": 0.36447449293177625 |
| } |
| }, |
| "wf_rubric_flip": { |
| "n": 2062, |
| "n_total": 2062, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.5140816127009797, |
| "brier": 0.27501173451171385, |
| "jsd": 0.1274834860045779, |
| "acc": 0.7856450048496605, |
| "acc_k": 0.7914645974781765, |
| "ece": 0.04033603424533379, |
| "sel_acc80": 0.8878787878787879, |
| "coverage": 0.9932104752667313, |
| "false_abstain": 0.006789524733268671, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "urgency": { |
| "nll": 1.0503140999721947, |
| "brier": 0.5826411232348119, |
| "jsd": 0.2870499222013987, |
| "acc": 0.5142857142857142, |
| "acc_k": 0.5142857142857142, |
| "ece": 0.061800167475427895, |
| "sel_acc80": 0.5464285714285714, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 700 |
| }, |
| "eligibility": { |
| "nll": 0.3789669930673108, |
| "brier": 0.19723455708945753, |
| "jsd": 0.07565788618043696, |
| "acc": 0.8704268292682927, |
| "acc_k": 0.8704268292682927, |
| "ece": 0.0704190259481349, |
| "sel_acc80": 0.9314285714285714, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 656 |
| }, |
| "tool_select": { |
| "nll": 0.10795193761576238, |
| "brier": 0.042265638595044515, |
| "jsd": 0.017428370065288763, |
| "acc": 0.9759206798866855, |
| "acc_k": 0.9929178470254958, |
| "ece": 0.015375537465381884, |
| "sel_acc80": 0.9946902654867257, |
| "coverage": 0.9801699716713881, |
| "false_abstain": 0.019830028328611898, |
| "null_recall": "n/a", |
| "n": 706 |
| } |
| }, |
| "by_qtype": { |
| "score": { |
| "nll": 1.0503140999721947, |
| "brier": 0.5826411232348119, |
| "jsd": 0.2870499222013987, |
| "acc": 0.5142857142857142, |
| "acc_k": 0.5142857142857142, |
| "ece": 0.061800167475427895, |
| "sel_acc80": 0.5464285714285714, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 700 |
| }, |
| "noul": { |
| "nll": 0.3789669930673108, |
| "brier": 0.19723455708945753, |
| "jsd": 0.07565788618043696, |
| "acc": 0.8704268292682927, |
| "acc_k": 0.8704268292682927, |
| "ece": 0.0704190259481349, |
| "sel_acc80": 0.9314285714285714, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 656 |
| }, |
| "choice": { |
| "nll": 0.10795193761576238, |
| "brier": 0.042265638595044515, |
| "jsd": 0.017428370065288763, |
| "acc": 0.9759206798866855, |
| "acc_k": 0.9929178470254958, |
| "ece": 0.015375537465381884, |
| "sel_acc80": 0.9946902654867257, |
| "coverage": 0.9801699716713881, |
| "false_abstain": 0.019830028328611898, |
| "null_recall": "n/a", |
| "n": 706 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.5140816127009797, |
| "brier": 0.27501173451171385, |
| "jsd": 0.1274834860045779, |
| "acc": 0.7856450048496605, |
| "acc_k": 0.7914645974781765, |
| "ece": 0.04033603424533379, |
| "sel_acc80": 0.8878787878787879, |
| "coverage": 0.9932104752667313, |
| "false_abstain": 0.006789524733268671, |
| "null_recall": "n/a", |
| "n": 2062 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.32104752667313285 |
| }, |
| "flip": { |
| "n_pairs": 1031, |
| "flip_rate": 0.8205625606207565, |
| "both_correct": 0.6702230843840931, |
| "acc": 0.7914645974781765 |
| }, |
| "flip_by_family": { |
| "eligibility": { |
| "n_pairs": 328, |
| "flip_rate": 0.7591463414634146, |
| "both_correct": 0.75, |
| "acc": 0.8704268292682927 |
| }, |
| "tool_select": { |
| "n_pairs": 353, |
| "flip_rate": 0.9915014164305949, |
| "both_correct": 0.9858356940509915, |
| "acc": 0.9929178470254958 |
| }, |
| "urgency": { |
| "n_pairs": 350, |
| "flip_rate": 0.7057142857142857, |
| "both_correct": 0.27714285714285714, |
| "acc": 0.5142857142857142 |
| } |
| } |
| }, |
| "wf_rubric_shuffled": { |
| "n": 3775, |
| "n_total": 3775, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 3.3019875953051896, |
| "brier": 0.9847791419883737, |
| "jsd": 0.38205552797027176, |
| "acc": 0.3801324503311258, |
| "acc_k": 0.3811920529801324, |
| "ece": 0.39123494437198764, |
| "sel_acc80": 0.39470198675496687, |
| "coverage": 0.9925827814569537, |
| "false_abstain": 0.007417218543046358, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "tool_select": { |
| "nll": 8.01855352489357, |
| "brier": 1.585383500778014, |
| "jsd": 0.5686605163566438, |
| "acc": 0.15359116022099448, |
| "acc_k": 0.1580110497237569, |
| "ece": 0.7718642068172686, |
| "sel_acc80": 0.13535911602209943, |
| "coverage": 0.969060773480663, |
| "false_abstain": 0.030939226519337018, |
| "null_recall": "n/a", |
| "n": 905 |
| }, |
| "eligibility": { |
| "nll": 2.0992631849983714, |
| "brier": 0.8251410346340735, |
| "jsd": 0.28859514090273525, |
| "acc": 0.5234758234057463, |
| "acc_k": 0.5234758234057463, |
| "ece": 0.39013672221684337, |
| "sel_acc80": 0.5332749562171629, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1427 |
| }, |
| "urgency": { |
| "nll": 1.5333081547163776, |
| "brier": 0.7659687708795461, |
| "jsd": 0.35744739065613984, |
| "acc": 0.3804573804573805, |
| "acc_k": 0.3804573804573805, |
| "ece": 0.15649522523952702, |
| "sel_acc80": 0.38961038961038963, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1443 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 8.01855352489357, |
| "brier": 1.585383500778014, |
| "jsd": 0.5686605163566438, |
| "acc": 0.15359116022099448, |
| "acc_k": 0.1580110497237569, |
| "ece": 0.7718642068172686, |
| "sel_acc80": 0.13535911602209943, |
| "coverage": 0.969060773480663, |
| "false_abstain": 0.030939226519337018, |
| "null_recall": "n/a", |
| "n": 905 |
| }, |
| "noul": { |
| "nll": 2.0992631849983714, |
| "brier": 0.8251410346340735, |
| "jsd": 0.28859514090273525, |
| "acc": 0.5234758234057463, |
| "acc_k": 0.5234758234057463, |
| "ece": 0.39013672221684337, |
| "sel_acc80": 0.5332749562171629, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1427 |
| }, |
| "score": { |
| "nll": 1.5333081547163776, |
| "brier": 0.7659687708795461, |
| "jsd": 0.35744739065613984, |
| "acc": 0.3804573804573805, |
| "acc_k": 0.3804573804573805, |
| "ece": 0.15649522523952702, |
| "sel_acc80": 0.38961038961038963, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1443 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 3.3019875953051896, |
| "brier": 0.9847791419883737, |
| "jsd": 0.38205552797027176, |
| "acc": 0.3801324503311258, |
| "acc_k": 0.3811920529801324, |
| "ece": 0.39123494437198764, |
| "sel_acc80": 0.39470198675496687, |
| "coverage": 0.9925827814569537, |
| "false_abstain": 0.007417218543046358, |
| "null_recall": "n/a", |
| "n": 3775 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.36 |
| } |
| }, |
| "cua_s1_forms_demo": { |
| "n": 196, |
| "n_total": 196, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.0006697119946596961, |
| "brier": 2.5804204528327477e-06, |
| "jsd": 0.00023202367765799874, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.0006688650773496452, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "cua_s1_forms": { |
| "nll": 0.0006697119946596961, |
| "brier": 2.5804204528327477e-06, |
| "jsd": 0.00023202367765799874, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.0006688650773496452, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 196 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 0.0006697119946596961, |
| "brier": 2.5804204528327477e-06, |
| "jsd": 0.00023202367765799874, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.0006688650773496452, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 196 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.0006697119946596961, |
| "brier": 2.5804204528327477e-06, |
| "jsd": 0.00023202367765799874, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.0006688650773496452, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 196 |
| } |
| }, |
| "baselines": { |
| "majority_class": 18, |
| "majority_acc": 0.27040816326530615 |
| } |
| }, |
| "cua_s1_forms_test": { |
| "n": 24370, |
| "n_total": 24370, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.032467243628837364, |
| "brier": 0.01572971627781683, |
| "jsd": 0.006402877388207127, |
| "acc": 0.9902749281903981, |
| "acc_k": 0.9902749281903981, |
| "ece": 0.004090282016471624, |
| "sel_acc80": 0.9998974148543291, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "cua_s1_forms": { |
| "nll": 0.032467243628837364, |
| "brier": 0.01572971627781683, |
| "jsd": 0.006402877388207127, |
| "acc": 0.9902749281903981, |
| "acc_k": 0.9902749281903981, |
| "ece": 0.004090282016471624, |
| "sel_acc80": 0.9998974148543291, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 24370 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 0.032467243628837364, |
| "brier": 0.01572971627781683, |
| "jsd": 0.006402877388207127, |
| "acc": 0.9902749281903981, |
| "acc_k": 0.9902749281903981, |
| "ece": 0.004090282016471624, |
| "sel_acc80": 0.9998974148543291, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 24370 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.032467243628837364, |
| "brier": 0.01572971627781683, |
| "jsd": 0.006402877388207127, |
| "acc": 0.9902749281903981, |
| "acc_k": 0.9902749281903981, |
| "ece": 0.004090282016471624, |
| "sel_acc80": 0.9998974148543291, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 24370 |
| } |
| }, |
| "baselines": { |
| "majority_class": 16, |
| "majority_acc": 0.07033237587197373 |
| } |
| }, |
| "jev4b_adversarial": { |
| "n": 600, |
| "n_total": 600, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.21596318335730846, |
| "brier": 0.11449747234656535, |
| "jsd": 0.0566693510735706, |
| "acc": 0.9266666666666666, |
| "acc_k": 0.9266666666666666, |
| "ece": 0.033218660900990224, |
| "sel_acc80": 0.98125, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "jev4b_distill": { |
| "nll": 0.21596318335730846, |
| "brier": 0.11449747234656535, |
| "jsd": 0.0566693510735706, |
| "acc": 0.9266666666666666, |
| "acc_k": 0.9266666666666666, |
| "ece": 0.033218660900990224, |
| "sel_acc80": 0.98125, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 600 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 0.15764157777699858, |
| "brier": 0.07730203604858563, |
| "jsd": 0.03353047215992909, |
| "acc": 0.9527777777777777, |
| "acc_k": 0.9527777777777777, |
| "ece": 0.027492753830220998, |
| "sel_acc80": 0.9895833333333334, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 360 |
| }, |
| "score": { |
| "nll": 0.588537222505538, |
| "brier": 0.3360254493815728, |
| "jsd": 0.17654749465911643, |
| "acc": 0.775, |
| "acc_k": 0.775, |
| "ece": 0.11445025528470672, |
| "sel_acc80": 0.8125, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 120 |
| }, |
| "noul": { |
| "nll": 0.018353960950008512, |
| "brier": 0.004555804205497079, |
| "jsd": 0.006207844228949319, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.01696336468060807, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 120 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.21596318335730846, |
| "brier": 0.11449747234656535, |
| "jsd": 0.0566693510735706, |
| "acc": 0.9266666666666666, |
| "acc_k": 0.9266666666666666, |
| "ece": 0.033218660900990224, |
| "sel_acc80": 0.98125, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 600 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.42833333333333334 |
| } |
| }, |
| "jevlogs_triage": { |
| "n": 5080, |
| "n_total": 5080, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.8200663549047869, |
| "brier": 0.5245085358820047, |
| "jsd": 0.19936215850961117, |
| "acc": 0.6734251968503937, |
| "acc_k": 0.6734251968503937, |
| "ece": 0.19179721678805164, |
| "sel_acc80": 0.6801181102362205, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "log_hdfs": { |
| "nll": 0.8322974851760632, |
| "brier": 0.5211627531136993, |
| "jsd": 0.1981452827621227, |
| "acc": 0.6664, |
| "acc_k": 0.6664, |
| "ece": 0.17615253834724426, |
| "sel_acc80": 0.6785, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 2500 |
| }, |
| "log_bgl": { |
| "nll": 0.7998629474253376, |
| "brier": 0.521462450412092, |
| "jsd": 0.1985896785858705, |
| "acc": 0.6856, |
| "acc_k": 0.6856, |
| "ece": 0.24997946472167967, |
| "sel_acc80": 0.659, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 2500 |
| }, |
| "log_adversarial": { |
| "nll": 1.0692000176602048, |
| "brier": 0.7242544183263233, |
| "jsd": 0.26152952323552203, |
| "acc": 0.5125, |
| "acc_k": 0.5125, |
| "ece": 0.304364737868309, |
| "sel_acc80": 0.515625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 80 |
| } |
| }, |
| "by_qtype": { |
| "noul": { |
| "nll": 0.8200663549047869, |
| "brier": 0.5245085358820047, |
| "jsd": 0.19936215850961117, |
| "acc": 0.6734251968503937, |
| "acc_k": 0.6734251968503937, |
| "ece": 0.19179721678805164, |
| "sel_acc80": 0.6801181102362205, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 5080 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.8200663549047869, |
| "brier": 0.5245085358820047, |
| "jsd": 0.19936215850961117, |
| "acc": 0.6734251968503937, |
| "acc_k": 0.6734251968503937, |
| "ece": 0.19179721678805164, |
| "sel_acc80": 0.6801181102362205, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 5080 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.6968503937007874, |
| "always_positive_acc": 0.3031496062992126 |
| } |
| }, |
| "mind2web_choice": { |
| "n": 1600, |
| "n_total": 1600, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 1.2406711424343253, |
| "brier": 0.5732404237664536, |
| "jsd": 0.2788736075063562, |
| "acc": 0.544375, |
| "acc_k": 0.544375, |
| "ece": 0.08040638350881635, |
| "sel_acc80": 0.615625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "mind2web": { |
| "nll": 1.2406711424343253, |
| "brier": 0.5732404237664536, |
| "jsd": 0.2788736075063562, |
| "acc": 0.544375, |
| "acc_k": 0.544375, |
| "ece": 0.08040638350881635, |
| "sel_acc80": 0.615625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1600 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 1.8521192870863956, |
| "brier": 0.7087342212760166, |
| "jsd": 0.3756742847617898, |
| "acc": 0.44, |
| "acc_k": 0.44, |
| "ece": 0.08787944102659824, |
| "sel_acc80": 0.49375, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 800 |
| }, |
| "noul": { |
| "nll": 0.6292229977822548, |
| "brier": 0.4377466262568903, |
| "jsd": 0.18207293025092264, |
| "acc": 0.64875, |
| "acc_k": 0.64875, |
| "ece": 0.07293332599103454, |
| "sel_acc80": 0.6875, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 800 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 1.2406711424343253, |
| "brier": 0.5732404237664536, |
| "jsd": 0.2788736075063562, |
| "acc": 0.544375, |
| "acc_k": 0.544375, |
| "ece": 0.08040638350881635, |
| "sel_acc80": 0.615625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1600 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.426875 |
| } |
| }, |
| "pagerduty_trigger": { |
| "n": 6000, |
| "n_total": 6000, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.396707458891549, |
| "brier": 0.23002239391086174, |
| "jsd": 0.11322283340814882, |
| "acc": 0.838, |
| "acc_k": 0.838, |
| "ece": 0.07345882829030351, |
| "sel_acc80": 0.8979166666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "replica_lag_data_risk": { |
| "nll": 0.29740572278535377, |
| "brier": 0.14798980251485705, |
| "jsd": 0.09449495177807844, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.2269586962566041, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 114 |
| }, |
| "tls_hours_left_serving": { |
| "nll": 2.462503098332096, |
| "brier": 1.4718128845820069, |
| "jsd": 0.5430561946798185, |
| "acc": 0.0, |
| "acc_k": 0.0, |
| "ece": 0.7213651537895203, |
| "sel_acc80": 0.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 98 |
| }, |
| "disk_full_writes_failing": { |
| "nll": 0.6112621743776719, |
| "brier": 0.43961606766122513, |
| "jsd": 0.1828291039570283, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.40561820756305345, |
| "sel_acc80": 0.625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 110 |
| }, |
| "revoked_key_used": { |
| "nll": 2.25962306280277, |
| "brier": 1.3607013127715883, |
| "jsd": 0.49743537591946874, |
| "acc": 0.0, |
| "acc_k": 0.0, |
| "ece": 0.7095300555229187, |
| "sel_acc80": 0.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 102 |
| }, |
| "checksum_corruption": { |
| "nll": 0.16655108974517663, |
| "brier": 0.05001836994744468, |
| "jsd": 0.05564429813812224, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.14739903873867463, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 90 |
| }, |
| "checkout_slo_burn": { |
| "nll": 0.8234049439981885, |
| "brier": 0.5886945952552634, |
| "jsd": 0.24262392305224867, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.5280300080776215, |
| "sel_acc80": 0.375, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 100 |
| }, |
| "payments_processor_down": { |
| "nll": 0.7458985031769221, |
| "brier": 0.5397173510738811, |
| "jsd": 0.21208411419957937, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.3847963486580139, |
| "sel_acc80": 0.618421052631579, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 94 |
| }, |
| "crash_loop_user_impact": { |
| "nll": 0.45723675404949216, |
| "brier": 0.28838964871857825, |
| "jsd": 0.14411361556383404, |
| "acc": 0.7916666666666666, |
| "acc_k": 0.7916666666666666, |
| "ece": 0.3439861076573531, |
| "sel_acc80": 0.7402597402597403, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 96 |
| }, |
| "credential_stuffing": { |
| "nll": 0.6855976842756064, |
| "brier": 0.5141043463970237, |
| "jsd": 0.19799073218670762, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.4205113059530656, |
| "sel_acc80": 0.6233766233766234, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 96 |
| }, |
| "queue_backup_checkout": { |
| "nll": 0.4943918081315674, |
| "brier": 0.3165144497680829, |
| "jsd": 0.1554112283610238, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.3690876066684723, |
| "sel_acc80": 0.625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 100 |
| }, |
| "queue_slow_growth": { |
| "nll": 0.8056600958296468, |
| "brier": 0.46479321367422693, |
| "jsd": 0.2233220382938683, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.2907683104276657, |
| "sel_acc80": 0.6222222222222222, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 56 |
| }, |
| "cert_30d": { |
| "nll": 0.5765844556293318, |
| "brier": 0.459360806409668, |
| "jsd": 0.1663273119345, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.34605565667152405, |
| "sel_acc80": 0.6170212765957447, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 58 |
| }, |
| "disk_80pct_48h": { |
| "nll": 0.6364577616833813, |
| "brier": 0.3603717509503686, |
| "jsd": 0.1950357110171284, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.44600894767791033, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 64 |
| }, |
| "cpu_high_no_impact": { |
| "nll": 0.7085339514792242, |
| "brier": 0.5101478528535615, |
| "jsd": 0.19822264049400234, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.37314837455749506, |
| "sel_acc80": 0.625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 50 |
| }, |
| "single_pod_oom": { |
| "nll": 0.597222631375, |
| "brier": 0.3511328330652838, |
| "jsd": 0.1808235790015383, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.32174784059707934, |
| "sel_acc80": 0.6190476190476191, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 52 |
| }, |
| "deprecation": { |
| "nll": 1.2131927809820504, |
| "brier": 0.8289228409108279, |
| "jsd": 0.2701922670787307, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.45578983426094055, |
| "sel_acc80": 0.625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 60 |
| }, |
| "canary_within_budget": { |
| "nll": 0.68097187510421, |
| "brier": 0.46432714017336996, |
| "jsd": 0.19212515668071925, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.33959993720054626, |
| "sel_acc80": 0.6153846153846154, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 48 |
| }, |
| "replica_lag_seconds": { |
| "nll": 0.48507738285690005, |
| "brier": 0.2914945314683177, |
| "jsd": 0.14510412619084756, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.21687236428260803, |
| "sel_acc80": 0.625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 50 |
| }, |
| "tls_expires_7d": { |
| "nll": 0.8200619861584703, |
| "brier": 0.6263737845037869, |
| "jsd": 0.21875945370810285, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.41936272382736206, |
| "sel_acc80": 0.6190476190476191, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 26 |
| }, |
| "slow_slo_burn": { |
| "nll": 0.4646857777411605, |
| "brier": 0.2633606501114195, |
| "jsd": 0.15017855189982032, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.3715018928050995, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 36 |
| }, |
| "injection_noise": { |
| "nll": 0.3191649156767602, |
| "brier": 0.15667065379041875, |
| "jsd": 0.10271966665831857, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.2534384096822431, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 310 |
| }, |
| "health_ok": { |
| "nll": 0.11813930759934398, |
| "brier": 0.039244944056013954, |
| "jsd": 0.03965546739519377, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.10602798960567297, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 306 |
| }, |
| "kafka_caught_up": { |
| "nll": 0.1239690909443528, |
| "brier": 0.03284989912051308, |
| "jsd": 0.04179599987809453, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.11298148923640626, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 278 |
| }, |
| "invalid_coupon_error": { |
| "nll": 0.4095562707396914, |
| "brier": 0.2070045846973819, |
| "jsd": 0.12855307030768603, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.3042865756505765, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 322 |
| }, |
| "stack_trace_handled": { |
| "nll": 0.1369772675048073, |
| "brier": 0.04242081045781862, |
| "jsd": 0.04599428397435028, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.12313097069062383, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 332 |
| }, |
| "retry_then_ok": { |
| "nll": 0.23577570062147413, |
| "brier": 0.09354840300021036, |
| "jsd": 0.07807766271472766, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.20314176128783692, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 284 |
| }, |
| "feature_flag": { |
| "nll": 0.02957982981648557, |
| "brier": 0.0025294010832958157, |
| "jsd": 0.010185593857561997, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.028911458685042968, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 282 |
| }, |
| "cache_hit": { |
| "nll": 0.05577471799801635, |
| "brier": 0.010092673936511587, |
| "jsd": 0.019045345808299963, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.05296187488349169, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 286 |
| }, |
| "metrics_scrape": { |
| "nll": 0.19512496383591874, |
| "brier": 0.06495891768903242, |
| "jsd": 0.06532458959189952, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.17378984264484268, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 328 |
| }, |
| "cron_ok": { |
| "nll": 0.8260068202421647, |
| "brier": 0.5908376052070468, |
| "jsd": 0.24215122771741, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.3292480845605173, |
| "sel_acc80": 0.375, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 310 |
| }, |
| "rate_limit_bot": { |
| "nll": 0.2224008145703419, |
| "brier": 0.10465719261617362, |
| "jsd": 0.0723341471248896, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.18123274389654398, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 256 |
| }, |
| "http_404": { |
| "nll": 0.0272498202190122, |
| "brier": 0.0025453067235740637, |
| "jsd": 0.009377000999055841, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.026570967720739525, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 318 |
| }, |
| "deploy_success": { |
| "nll": 0.11437534991524133, |
| "brier": 0.022894189185427016, |
| "jsd": 0.03898171724942419, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.10790608627874346, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 244 |
| }, |
| "user_not_found": { |
| "nll": 0.7467332928805962, |
| "brier": 0.4869266134970554, |
| "jsd": 0.20417162215696066, |
| "acc": 0.5, |
| "acc_k": 0.5, |
| "ece": 0.3224844210812835, |
| "sel_acc80": 0.6228813559322034, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 294 |
| }, |
| "request_ok": { |
| "nll": 0.15962773605536434, |
| "brier": 0.045111188981216126, |
| "jsd": 0.05396733304793474, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.14663683669907707, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 350 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 0.48022365288831015, |
| "brier": 0.25716945445373307, |
| "jsd": 0.14152253563996733, |
| "acc": 0.851, |
| "acc_k": 0.851, |
| "ece": 0.13448732795317966, |
| "sel_acc80": 0.905, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 3000 |
| }, |
| "noul": { |
| "nll": 0.31319126489478805, |
| "brier": 0.20287533336799038, |
| "jsd": 0.0849231311763303, |
| "acc": 0.825, |
| "acc_k": 0.825, |
| "ece": 0.10489587610960006, |
| "sel_acc80": 0.9566666666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 3000 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.396707458891549, |
| "brier": 0.23002239391086174, |
| "jsd": 0.11322283340814882, |
| "acc": 0.838, |
| "acc_k": 0.838, |
| "ece": 0.07345882829030351, |
| "sel_acc80": 0.8979166666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 6000 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.7916666666666666 |
| } |
| }, |
| "systemone_lite_hard": { |
| "n": 5400, |
| "n_total": 5400, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.43450476654815856, |
| "brier": 0.23583658134544866, |
| "jsd": 0.11106552302444929, |
| "acc": 0.850925925925926, |
| "acc_k": 0.850925925925926, |
| "ece": 0.05852483691440687, |
| "sel_acc80": 0.8976851851851851, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "systemone_lite": { |
| "nll": 0.43450476654815856, |
| "brier": 0.23583658134544866, |
| "jsd": 0.11106552302444929, |
| "acc": 0.850925925925926, |
| "acc_k": 0.850925925925926, |
| "ece": 0.05852483691440687, |
| "sel_acc80": 0.8976851851851851, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 5400 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 0.47634740362930744, |
| "brier": 0.22078883830669857, |
| "jsd": 0.0890289134343277, |
| "acc": 0.8522222222222222, |
| "acc_k": 0.8522222222222222, |
| "ece": 0.06805240516861279, |
| "sel_acc80": 0.9319444444444445, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1800 |
| }, |
| "noul": { |
| "nll": 0.23199631304256532, |
| "brier": 0.14232818861742158, |
| "jsd": 0.06226400940582674, |
| "acc": 0.8994444444444445, |
| "acc_k": 0.8994444444444445, |
| "ece": 0.01822677158647116, |
| "sel_acc80": 0.9597222222222223, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1800 |
| }, |
| "score": { |
| "nll": 0.595170582972603, |
| "brier": 0.3443927171122257, |
| "jsd": 0.18190364623319336, |
| "acc": 0.8011111111111111, |
| "acc_k": 0.8011111111111111, |
| "ece": 0.17747255620029237, |
| "sel_acc80": 0.8513888888888889, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1800 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.43450476654815856, |
| "brier": 0.23583658134544866, |
| "jsd": 0.11106552302444929, |
| "acc": 0.850925925925926, |
| "acc_k": 0.850925925925926, |
| "ece": 0.05852483691440687, |
| "sel_acc80": 0.8976851851851851, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 5400 |
| } |
| }, |
| "baselines": { |
| "majority_class": 1, |
| "majority_acc": 0.44166666666666665 |
| } |
| }, |
| "tree_choice_cap": { |
| "n": 720, |
| "n_total": 720, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 1.0937767414927673, |
| "brier": 0.4118990134350543, |
| "jsd": 0.2282905942697906, |
| "acc": 0.6902777777777778, |
| "acc_k": 0.7319444444444444, |
| "ece": 0.07905487779838341, |
| "sel_acc80": 0.7934027777777778, |
| "coverage": 0.8611111111111112, |
| "false_abstain": 0.1388888888888889, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "incident_catalog": { |
| "nll": 1.0937767414927673, |
| "brier": 0.4118990134350543, |
| "jsd": 0.2282905942697906, |
| "acc": 0.6902777777777778, |
| "acc_k": 0.7319444444444444, |
| "ece": 0.07905487779838341, |
| "sel_acc80": 0.7934027777777778, |
| "coverage": 0.8611111111111112, |
| "false_abstain": 0.1388888888888889, |
| "null_recall": "n/a", |
| "n": 720 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 1.0937767414927673, |
| "brier": 0.4118990134350543, |
| "jsd": 0.2282905942697906, |
| "acc": 0.6902777777777778, |
| "acc_k": 0.7319444444444444, |
| "ece": 0.07905487779838341, |
| "sel_acc80": 0.7934027777777778, |
| "coverage": 0.8611111111111112, |
| "false_abstain": 0.1388888888888889, |
| "null_recall": "n/a", |
| "n": 720 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 1.0937767414927673, |
| "brier": 0.4118990134350543, |
| "jsd": 0.2282905942697906, |
| "acc": 0.6902777777777778, |
| "acc_k": 0.7319444444444444, |
| "ece": 0.07905487779838341, |
| "sel_acc80": 0.7934027777777778, |
| "coverage": 0.8611111111111112, |
| "false_abstain": 0.1388888888888889, |
| "null_recall": "n/a", |
| "n": 720 |
| } |
| }, |
| "baselines": { |
| "majority_class": 3, |
| "majority_acc": 0.11388888888888889 |
| } |
| }, |
| "typed_decisions_test": { |
| "n": 2000, |
| "n_total": 2000, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 1.183971535602263, |
| "brier": 0.21124234378656148, |
| "jsd": 0.08978436233492346, |
| "acc": 0.5325, |
| "acc_k": 0.5325, |
| "ece": 0.0960818936675787, |
| "sel_acc80": 0.55125, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "agent_trace_observability": { |
| "nll": 1.4513733308753711, |
| "brier": 0.27746615896441984, |
| "jsd": 0.11437914671823042, |
| "acc": 0.4, |
| "acc_k": 0.4, |
| "ece": 0.21149230748414996, |
| "sel_acc80": 0.4125, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 500 |
| }, |
| "customer_service": { |
| "nll": 1.1394922824048543, |
| "brier": 0.1685314240724029, |
| "jsd": 0.07635420781877246, |
| "acc": 0.638, |
| "acc_k": 0.638, |
| "ece": 0.09256491619348527, |
| "sel_acc80": 0.6825, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 500 |
| }, |
| "invoice_processing": { |
| "nll": 1.0241192434455133, |
| "brier": 0.25477993975703184, |
| "jsd": 0.11173192033283712, |
| "acc": 0.582, |
| "acc_k": 0.582, |
| "ece": 0.08850071275234224, |
| "sel_acc80": 0.64, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 500 |
| }, |
| "security_incidents": { |
| "nll": 1.1209012856833143, |
| "brier": 0.14419185235239126, |
| "jsd": 0.056672174469853814, |
| "acc": 0.51, |
| "acc_k": 0.51, |
| "ece": 0.1298270362019539, |
| "sel_acc80": 0.505, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 500 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 1.4624093197198917, |
| "brier": 0.22882638324834792, |
| "jsd": 0.10929266556165232, |
| "acc": 0.49166666666666664, |
| "acc_k": 0.49166666666666664, |
| "ece": 0.12719455897808077, |
| "sel_acc80": 0.5333333333333333, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 600 |
| }, |
| "noul": { |
| "nll": 0.8045478477317457, |
| "brier": 0.25816601068728884, |
| "jsd": 0.08310478332934974, |
| "acc": 0.5783333333333334, |
| "acc_k": 0.5783333333333334, |
| "ece": 0.15133020063241323, |
| "sel_acc80": 0.5895833333333333, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 600 |
| }, |
| "score": { |
| "nll": 1.2597109634169303, |
| "brier": 0.16286156401467608, |
| "jsd": 0.08016281916905708, |
| "acc": 0.52875, |
| "acc_k": 0.52875, |
| "ece": 0.07651440937072038, |
| "sel_acc80": 0.55, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 800 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 1.183971535602263, |
| "brier": 0.21124234378656148, |
| "jsd": 0.08978436233492346, |
| "acc": 0.5325, |
| "acc_k": 0.5325, |
| "ece": 0.0960818936675787, |
| "sel_acc80": 0.55125, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 2000 |
| } |
| }, |
| "baselines": { |
| "majority_class": 1, |
| "majority_acc": 0.2905 |
| } |
| }, |
| "typed_decisions_train": { |
| "n": 6000, |
| "n_total": 6000, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 1.1985372981046378, |
| "brier": 0.2329991959274657, |
| "jsd": 0.09837575798408288, |
| "acc": 0.5223333333333333, |
| "acc_k": 0.5223333333333333, |
| "ece": 0.10562264278034368, |
| "sel_acc80": 0.5375, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "agent_trace_observability": { |
| "nll": 1.416481477970894, |
| "brier": 0.25422688167917284, |
| "jsd": 0.10777658676024122, |
| "acc": 0.4266666666666667, |
| "acc_k": 0.4266666666666667, |
| "ece": 0.18357536943753558, |
| "sel_acc80": 0.42916666666666664, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1500 |
| }, |
| "customer_service": { |
| "nll": 1.1214988914219903, |
| "brier": 0.19128662847914574, |
| "jsd": 0.08370220742725006, |
| "acc": 0.6106666666666667, |
| "acc_k": 0.6106666666666667, |
| "ece": 0.08198627370595929, |
| "sel_acc80": 0.6433333333333333, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1500 |
| }, |
| "invoice_processing": { |
| "nll": 1.1031419902595583, |
| "brier": 0.3050021381875509, |
| "jsd": 0.13027031117221108, |
| "acc": 0.5333333333333333, |
| "acc_k": 0.5333333333333333, |
| "ece": 0.10525429819027585, |
| "sel_acc80": 0.575, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1500 |
| }, |
| "security_incidents": { |
| "nll": 1.1530268327661088, |
| "brier": 0.1814811353639933, |
| "jsd": 0.07175392657662916, |
| "acc": 0.5186666666666667, |
| "acc_k": 0.5186666666666667, |
| "ece": 0.12120020204782486, |
| "sel_acc80": 0.5258333333333334, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1500 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "nll": 1.4635702511121844, |
| "brier": 0.2532055828202569, |
| "jsd": 0.11942735571638688, |
| "acc": 0.4961111111111111, |
| "acc_k": 0.4961111111111111, |
| "ece": 0.12604168651832473, |
| "sel_acc80": 0.5368055555555555, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1800 |
| }, |
| "noul": { |
| "nll": 0.8394133296564307, |
| "brier": 0.2816899204625882, |
| "jsd": 0.09059050796156125, |
| "acc": 0.5633333333333334, |
| "acc_k": 0.5633333333333334, |
| "ece": 0.1775682842400339, |
| "sel_acc80": 0.5701388888888889, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 1800 |
| }, |
| "score": { |
| "nll": 1.2691055596851333, |
| "brier": 0.18132636235653043, |
| "jsd": 0.08842599720174607, |
| "acc": 0.51125, |
| "acc_k": 0.51125, |
| "ece": 0.08015046964089076, |
| "sel_acc80": 0.5317708333333333, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 2400 |
| } |
| }, |
| "by_label_class": { |
| "answer": { |
| "nll": 1.1985372981046378, |
| "brier": 0.2329991959274657, |
| "jsd": 0.09837575798408288, |
| "acc": 0.5223333333333333, |
| "acc_k": 0.5223333333333333, |
| "ece": 0.10562264278034368, |
| "sel_acc80": 0.5375, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 6000 |
| } |
| }, |
| "baselines": { |
| "majority_class": 1, |
| "majority_acc": 0.3105 |
| } |
| }, |
| "wh_heldout_family": { |
| "n": 422, |
| "n_total": 422, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.3240292072876919, |
| "brier": 0.17772530751240193, |
| "jsd": 0.07779958816769972, |
| "acc": 0.8744075829383886, |
| "acc_k": 0.9045092838196287, |
| "ece": 0.04873869755256795, |
| "sel_acc80": 0.9437869822485208, |
| "conf_wrong": 0.0, |
| "coverage": 0.8625592417061612, |
| "false_abstain": 0.04774535809018567, |
| "null_recall": 0.8888888888888888, |
| "auroc_null": 0.9826112584733274, |
| "auroc_conf": 0.9737105806071323 |
| }, |
| "by_family": { |
| "procurement_request": { |
| "nll": 0.31726594171575667, |
| "brier": 0.15641665156788495, |
| "jsd": 0.06340224526164076, |
| "acc": 0.9014084507042254, |
| "acc_k": 0.9375, |
| "ece": 0.07553887535148941, |
| "sel_acc80": 0.956140350877193, |
| "conf_wrong": 0.0, |
| "coverage": 0.8802816901408451, |
| "false_abstain": 0.0390625, |
| "null_recall": 0.8571428571428571, |
| "auroc_null": 0.9827008928571429, |
| "auroc_conf": 0.9782366071428572, |
| "n": 142 |
| }, |
| "research_ethics_review": { |
| "nll": 0.41234907161332957, |
| "brier": 0.23254687981548977, |
| "jsd": 0.10529413759841766, |
| "acc": 0.8396946564885496, |
| "acc_k": 0.8376068376068376, |
| "ece": 0.07992691961863568, |
| "sel_acc80": 0.9238095238095239, |
| "conf_wrong": 0.0, |
| "coverage": 0.8549618320610687, |
| "false_abstain": 0.042735042735042736, |
| "null_recall": 1.0, |
| "auroc_null": 0.9865689865689865, |
| "auroc_conf": 0.9847374847374847, |
| "n": 131 |
| }, |
| "subscription_cancellation": { |
| "nll": 0.25282438503639165, |
| "brier": 0.14983405363600533, |
| "jsd": 0.06734748559881597, |
| "acc": 0.8791946308724832, |
| "acc_k": 0.9318181818181818, |
| "ece": 0.04965120893996835, |
| "sel_acc80": 0.9666666666666667, |
| "conf_wrong": 0.0, |
| "coverage": 0.8523489932885906, |
| "false_abstain": 0.06060606060606061, |
| "null_recall": 0.8235294117647058, |
| "auroc_null": 0.9795008912655971, |
| "auroc_conf": 0.9714795008912657, |
| "n": 149 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.3270902509793986, |
| "brier": 0.1794731921212136, |
| "jsd": 0.07638232337116445, |
| "acc": 0.8726790450928382, |
| "acc_k": 0.9045092838196287, |
| "ece": 0.04732005681536247, |
| "sel_acc80": 0.9403973509933775, |
| "coverage": 0.9522546419098143, |
| "false_abstain": 0.04774535809018567, |
| "null_recall": "n/a", |
| "n": 377 |
| }, |
| "null": { |
| "nll": 0.2983844634705046, |
| "brier": 0.16308191867857966, |
| "jsd": 0.08967311768533967, |
| "acc": 0.8888888888888888, |
| "ece": 0.13434556325276692, |
| "sel_acc80": 0.9722222222222222, |
| "conf_wrong": 0.0, |
| "coverage": 0.11111111111111116, |
| "false_abstain": "n/a", |
| "null_recall": 0.8888888888888888, |
| "n": 45 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.38388625592417064 |
| } |
| }, |
| "wh_heldout_grammar": { |
| "n": 816, |
| "n_total": 816, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.2472013947656661, |
| "brier": 0.14714674629576405, |
| "jsd": 0.0655881649228148, |
| "acc": 0.8933823529411765, |
| "acc_k": 0.925414364640884, |
| "ece": 0.02050217598968861, |
| "sel_acc80": 0.9647779479326187, |
| "conf_wrong": 0.0, |
| "coverage": 0.8455882352941176, |
| "false_abstain": 0.05386740331491713, |
| "null_recall": 0.9456521739130435, |
| "auroc_null": 0.9844012731203459, |
| "auroc_conf": 0.9806629834254144 |
| }, |
| "by_family": { |
| "visa_extension": { |
| "nll": 0.3876584563899138, |
| "brier": 0.23389678729229724, |
| "jsd": 0.10288573687884103, |
| "acc": 0.851063829787234, |
| "acc_k": 0.8782608695652174, |
| "ece": 0.0661268077843578, |
| "sel_acc80": 0.9203539823008849, |
| "conf_wrong": 0.0, |
| "coverage": 0.7588652482269503, |
| "false_abstain": 0.0782608695652174, |
| "null_recall": 0.9615384615384616, |
| "auroc_null": 0.9709030100334448, |
| "auroc_conf": 0.9692307692307692, |
| "n": 141 |
| }, |
| "loan_approval": { |
| "nll": 7.454925248028152e-05, |
| "brier": 1.315695971451629e-07, |
| "jsd": 2.5833865575709257e-05, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 7.45158125884382e-05, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.0, |
| "coverage": 0.9708029197080292, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 137 |
| }, |
| "content_moderation": { |
| "nll": 0.5935040969149519, |
| "brier": 0.3596948371934755, |
| "jsd": 0.16132759921807785, |
| "acc": 0.703125, |
| "acc_k": 0.8247422680412371, |
| "ece": 0.10629645804874599, |
| "sel_acc80": 0.7864077669902912, |
| "conf_wrong": 0.0, |
| "coverage": 0.5703125, |
| "false_abstain": 0.26804123711340205, |
| "null_recall": 0.9354838709677419, |
| "auroc_null": 0.9002327901563019, |
| "auroc_conf": 0.9068839374792153, |
| "n": 128 |
| }, |
| "warranty_claim": { |
| "nll": 0.04521754519140885, |
| "brier": 0.030164132379325935, |
| "jsd": 0.012534975904067302, |
| "acc": 0.971830985915493, |
| "acc_k": 0.9710144927536232, |
| "ece": 0.025068008563887925, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.0, |
| "coverage": 0.971830985915493, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 142 |
| }, |
| "shift_swap": { |
| "nll": 0.31995094781288197, |
| "brier": 0.1809786294088722, |
| "jsd": 0.07359853995759635, |
| "acc": 0.8615384615384616, |
| "acc_k": 0.8907563025210085, |
| "ece": 0.054814853805762016, |
| "sel_acc80": 0.9423076923076923, |
| "conf_wrong": 0.0, |
| "coverage": 0.9, |
| "false_abstain": 0.03361344537815126, |
| "null_recall": 0.8181818181818182, |
| "auroc_null": 0.985485103132162, |
| "auroc_conf": 0.9572192513368983, |
| "n": 130 |
| }, |
| "vendor_onboarding": { |
| "nll": 0.16712495284237217, |
| "brier": 0.09594791421965362, |
| "jsd": 0.05081018090389403, |
| "acc": 0.9565217391304348, |
| "acc_k": 0.9508196721311475, |
| "ece": 0.07468755521636082, |
| "sel_acc80": 0.990990990990991, |
| "conf_wrong": 0.0, |
| "coverage": 0.8840579710144928, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 138 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "null": { |
| "nll": 0.277453074058921, |
| "brier": 0.14561172018866156, |
| "jsd": 0.08687484324735249, |
| "acc": 0.9456521739130435, |
| "ece": 0.1358135566115379, |
| "sel_acc80": 0.972972972972973, |
| "conf_wrong": 0.0, |
| "coverage": 0.05434782608695654, |
| "false_abstain": "n/a", |
| "null_recall": 0.9456521739130435, |
| "n": 92 |
| }, |
| "answer": { |
| "nll": 0.24335725872287678, |
| "brier": 0.14734180486185994, |
| "jsd": 0.06288322789814979, |
| "acc": 0.8867403314917127, |
| "acc_k": 0.925414364640884, |
| "ece": 0.03288542010013568, |
| "sel_acc80": 0.9706896551724138, |
| "coverage": 0.9461325966850829, |
| "false_abstain": 0.05386740331491713, |
| "null_recall": "n/a", |
| "n": 724 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.38480392156862747 |
| } |
| }, |
| "wh_heldout_style": { |
| "n": 403, |
| "n_total": 403, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.26691198757004164, |
| "brier": 0.1464047042686628, |
| "jsd": 0.0662673268053048, |
| "acc": 0.890818858560794, |
| "acc_k": 0.890818858560794, |
| "ece": 0.031059944674039948, |
| "sel_acc80": 0.9845201238390093, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "feature_flag_rollout": { |
| "nll": 0.19010573836138, |
| "brier": 0.11432481201229953, |
| "jsd": 0.05277888022237791, |
| "acc": 0.9090909090909091, |
| "acc_k": 0.9090909090909091, |
| "ece": 0.039632322571494384, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 55 |
| }, |
| "shift_swap": { |
| "nll": 0.3043634096913596, |
| "brier": 0.1503282508374881, |
| "jsd": 0.06654764978820293, |
| "acc": 0.8939393939393939, |
| "acc_k": 0.8939393939393939, |
| "ece": 0.062232637044155215, |
| "sel_acc80": 0.9811320754716981, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 66 |
| }, |
| "content_moderation": { |
| "nll": 0.4789173250257905, |
| "brier": 0.26149167112429633, |
| "jsd": 0.1279300537119714, |
| "acc": 0.813953488372093, |
| "acc_k": 0.813953488372093, |
| "ece": 0.06183511365291682, |
| "sel_acc80": 0.8571428571428571, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 43 |
| }, |
| "insurance_claim": { |
| "nll": 0.29568089754215104, |
| "brier": 0.16360065632472767, |
| "jsd": 0.06646606171635495, |
| "acc": 0.8529411764705882, |
| "acc_k": 0.8529411764705882, |
| "ece": 0.1020367794177111, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 34 |
| }, |
| "visa_extension": { |
| "nll": 0.09558577342528488, |
| "brier": 0.04933176843684583, |
| "jsd": 0.02922906947317498, |
| "acc": 0.972972972972973, |
| "acc_k": 0.972972972972973, |
| "ece": 0.03434973310779885, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 37 |
| }, |
| "warranty_claim": { |
| "nll": 0.31847504141119937, |
| "brier": 0.1722281656536195, |
| "jsd": 0.07974897957552128, |
| "acc": 0.8813559322033898, |
| "acc_k": 0.8813559322033898, |
| "ece": 0.03718661453764316, |
| "sel_acc80": 0.9791666666666666, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 59 |
| }, |
| "loan_approval": { |
| "nll": 0.12007633533318733, |
| "brier": 0.06839648073371568, |
| "jsd": 0.027442892286818, |
| "acc": 0.9583333333333334, |
| "acc_k": 0.9583333333333334, |
| "ece": 0.04480526472131414, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 48 |
| }, |
| "vendor_onboarding": { |
| "nll": 0.299750322830979, |
| "brier": 0.17565957622934011, |
| "jsd": 0.07452437378826761, |
| "acc": 0.8524590163934426, |
| "acc_k": 0.8524590163934426, |
| "ece": 0.08769017262536971, |
| "sel_acc80": 0.9591836734693877, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 61 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.26691198757004164, |
| "brier": 0.1464047042686628, |
| "jsd": 0.0662673268053048, |
| "acc": 0.890818858560794, |
| "acc_k": 0.890818858560794, |
| "ece": 0.031059944674039948, |
| "sel_acc80": 0.9845201238390093, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 403 |
| } |
| }, |
| "baselines": { |
| "majority_class": 1, |
| "majority_acc": 0.4267990074441687 |
| } |
| }, |
| "wh_level7": { |
| "n": 880, |
| "n_total": 880, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.8425315491747045, |
| "brier": 0.566066637960518, |
| "jsd": 0.22709053783966102, |
| "acc": 0.5443181818181818, |
| "acc_k": 0.5443181818181818, |
| "ece": 0.17712931768460705, |
| "sel_acc80": 0.5738636363636364, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "by_family": { |
| "temporal": { |
| "nll": 0.7525894214290174, |
| "brier": 0.5283106465099253, |
| "jsd": 0.20930622376758312, |
| "acc": 0.5727272727272728, |
| "acc_k": 0.5727272727272728, |
| "ece": 0.18158261044458907, |
| "sel_acc80": 0.6193181818181818, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "probability": { |
| "nll": 0.804850408692942, |
| "brier": 0.5061907772166189, |
| "jsd": 0.22027018987749325, |
| "acc": 0.5727272727272728, |
| "acc_k": 0.5727272727272728, |
| "ece": 0.12663529461080378, |
| "sel_acc80": 0.6136363636363636, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "numeric": { |
| "nll": 0.854408628287633, |
| "brier": 0.6255809344308126, |
| "jsd": 0.24148170952539819, |
| "acc": 0.4909090909090909, |
| "acc_k": 0.4909090909090909, |
| "ece": 0.24755196327512913, |
| "sel_acc80": 0.4772727272727273, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "tradeoff": { |
| "nll": 0.9582777382892255, |
| "brier": 0.6041841936847155, |
| "jsd": 0.23730402818816942, |
| "acc": 0.5409090909090909, |
| "acc_k": 0.5409090909090909, |
| "ece": 0.21417698643424293, |
| "sel_acc80": 0.5965909090909091, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.8425315491747045, |
| "brier": 0.566066637960518, |
| "jsd": 0.22709053783966102, |
| "acc": 0.5443181818181818, |
| "acc_k": 0.5443181818181818, |
| "ece": 0.17712931768460705, |
| "sel_acc80": 0.5738636363636364, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 880 |
| } |
| }, |
| "baselines": { |
| "majority_class": 1, |
| "majority_acc": 0.4602272727272727 |
| } |
| }, |
| "wh_rubric_flip": { |
| "n": 2206, |
| "n_total": 2206, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 0.5215484658596726, |
| "brier": 0.33112145645101976, |
| "jsd": 0.13730397267005034, |
| "acc": 0.742973708068903, |
| "acc_k": 0.7501196744853997, |
| "ece": 0.08118663341860714, |
| "sel_acc80": 0.8164305949008499, |
| "conf_wrong": 0.0, |
| "coverage": 0.9301903898458749, |
| "false_abstain": 0.022498803255146003, |
| "null_recall": 0.9145299145299145, |
| "auroc_null": 0.9924267530777824, |
| "auroc_conf": 0.9735570530209114 |
| }, |
| "by_family": { |
| "loan_approval": { |
| "nll": 0.03986105987120578, |
| "brier": 0.022708606037172275, |
| "jsd": 0.00911794257520729, |
| "acc": 0.9861111111111112, |
| "acc_k": 0.9858156028368794, |
| "ece": 0.014913849118683096, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.0, |
| "coverage": 0.9791666666666666, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 144 |
| }, |
| "shift_swap": { |
| "nll": 0.3556272740222601, |
| "brier": 0.19030986550930082, |
| "jsd": 0.07994201505668534, |
| "acc": 0.8554216867469879, |
| "acc_k": 0.8782051282051282, |
| "ece": 0.05131704243550819, |
| "sel_acc80": 0.9398496240601504, |
| "conf_wrong": 0.0, |
| "coverage": 0.927710843373494, |
| "false_abstain": 0.02564102564102564, |
| "null_recall": 0.8, |
| "auroc_null": 0.9878205128205129, |
| "auroc_conf": 0.948076923076923, |
| "n": 166 |
| }, |
| "vendor_onboarding": { |
| "nll": 0.2460892508917783, |
| "brier": 0.14163160753660067, |
| "jsd": 0.06893476279267717, |
| "acc": 0.9102564102564102, |
| "acc_k": 0.9014084507042254, |
| "ece": 0.054101008826341355, |
| "sel_acc80": 0.976, |
| "conf_wrong": 0.0, |
| "coverage": 0.9102564102564102, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 156 |
| }, |
| "content_moderation": { |
| "nll": 0.5707868104784541, |
| "brier": 0.3399881864130089, |
| "jsd": 0.1565919344337606, |
| "acc": 0.7297297297297297, |
| "acc_k": 0.8166666666666667, |
| "ece": 0.07247401008734834, |
| "sel_acc80": 0.7983193277310925, |
| "conf_wrong": 0.0, |
| "coverage": 0.6891891891891893, |
| "false_abstain": 0.16666666666666666, |
| "null_recall": 0.9285714285714286, |
| "auroc_null": 0.9428571428571428, |
| "auroc_conf": 0.9383928571428571, |
| "n": 148 |
| }, |
| "warranty_claim": { |
| "nll": 0.1533494565143874, |
| "brier": 0.08736313763136712, |
| "jsd": 0.038794858019505045, |
| "acc": 0.935064935064935, |
| "acc_k": 0.9342105263157895, |
| "ece": 0.03250992956099568, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.0, |
| "coverage": 0.987012987012987, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0, |
| "n": 154 |
| }, |
| "visa_extension": { |
| "nll": 0.3792518490833633, |
| "brier": 0.22995354363423323, |
| "jsd": 0.10051831683881524, |
| "acc": 0.8493150684931506, |
| "acc_k": 0.8790322580645161, |
| "ece": 0.06351610569104751, |
| "sel_acc80": 0.9230769230769231, |
| "conf_wrong": 0.0, |
| "coverage": 0.7945205479452055, |
| "false_abstain": 0.07258064516129033, |
| "null_recall": 0.9545454545454546, |
| "auroc_null": 0.9736070381231671, |
| "auroc_conf": 0.9721407624633431, |
| "n": 146 |
| }, |
| "procurement_request": { |
| "nll": 0.3881801381792412, |
| "brier": 0.18757470893100156, |
| "jsd": 0.07515705849986501, |
| "acc": 0.8909090909090909, |
| "acc_k": 0.9292929292929293, |
| "ece": 0.08982028690251438, |
| "sel_acc80": 0.9204545454545454, |
| "conf_wrong": 0.0, |
| "coverage": 0.8818181818181818, |
| "false_abstain": 0.04040404040404041, |
| "null_recall": 0.8181818181818182, |
| "auroc_null": 0.977961432506887, |
| "auroc_conf": 0.9687786960514233, |
| "n": 110 |
| }, |
| "research_ethics_review": { |
| "nll": 0.4421237581343384, |
| "brier": 0.24773885698642706, |
| "jsd": 0.11269835014231615, |
| "acc": 0.8333333333333334, |
| "acc_k": 0.8315789473684211, |
| "ece": 0.103033944412514, |
| "sel_acc80": 0.9080459770114943, |
| "conf_wrong": 0.0, |
| "coverage": 0.8425925925925926, |
| "false_abstain": 0.042105263157894736, |
| "null_recall": 1.0, |
| "auroc_null": 0.9854251012145749, |
| "auroc_conf": 0.9838056680161944, |
| "n": 108 |
| }, |
| "subscription_cancellation": { |
| "nll": 0.2644555372286351, |
| "brier": 0.15680793083414146, |
| "jsd": 0.06982178903755307, |
| "acc": 0.8728813559322034, |
| "acc_k": 0.9326923076923077, |
| "ece": 0.061388740095041576, |
| "sel_acc80": 0.9578947368421052, |
| "conf_wrong": 0.0, |
| "coverage": 0.8559322033898304, |
| "false_abstain": 0.057692307692307696, |
| "null_recall": 0.7857142857142857, |
| "auroc_null": 0.9745879120879122, |
| "auroc_conf": 0.9684065934065934, |
| "n": 118 |
| }, |
| "feature_flag_rollout": { |
| "nll": 0.2267314556395052, |
| "brier": 0.1367964508738713, |
| "jsd": 0.06290056031954905, |
| "acc": 0.8913043478260869, |
| "acc_k": 0.8913043478260869, |
| "ece": 0.04663619204707767, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 46 |
| }, |
| "insurance_claim": { |
| "nll": 0.33329503028518, |
| "brier": 0.185439438184308, |
| "jsd": 0.07467994362635967, |
| "acc": 0.8333333333333334, |
| "acc_k": 0.8333333333333334, |
| "ece": 0.11369339625040695, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 30 |
| }, |
| "temporal": { |
| "nll": 0.7525550379081508, |
| "brier": 0.5281081163658832, |
| "jsd": 0.20925588567068154, |
| "acc": 0.5727272727272728, |
| "acc_k": 0.5727272727272728, |
| "ece": 0.1753265808929097, |
| "sel_acc80": 0.625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "numeric": { |
| "nll": 0.8554757856422319, |
| "brier": 0.6267431100109008, |
| "jsd": 0.2417873556438029, |
| "acc": 0.4909090909090909, |
| "acc_k": 0.4909090909090909, |
| "ece": 0.2482705327597531, |
| "sel_acc80": 0.4659090909090909, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "probability": { |
| "nll": 0.8047299060130065, |
| "brier": 0.5061678304732362, |
| "jsd": 0.220257933543653, |
| "acc": 0.5727272727272728, |
| "acc_k": 0.5727272727272728, |
| "ece": 0.11572440903295172, |
| "sel_acc80": 0.6136363636363636, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "tradeoff": { |
| "nll": 0.9591729335220512, |
| "brier": 0.6044553194922022, |
| "jsd": 0.2374179176613618, |
| "acc": 0.5454545454545454, |
| "acc_k": 0.5454545454545454, |
| "ece": 0.20948043235323643, |
| "sel_acc80": 0.5909090909090909, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "answer": { |
| "nll": 0.5344578349380356, |
| "brier": 0.340925948758752, |
| "jsd": 0.13998262548140297, |
| "acc": 0.7333652465294399, |
| "acc_k": 0.7501196744853997, |
| "ece": 0.09119710230610484, |
| "sel_acc80": 0.8068181818181818, |
| "coverage": 0.977501196744854, |
| "false_abstain": 0.022498803255146003, |
| "null_recall": "n/a", |
| "n": 2089 |
| }, |
| "null": { |
| "nll": 0.2910555427425767, |
| "brier": 0.15606517926424532, |
| "jsd": 0.08947742802974543, |
| "acc": 0.9145299145299145, |
| "ece": 0.1208790940606696, |
| "sel_acc80": 0.9680851063829787, |
| "conf_wrong": 0.0, |
| "coverage": 0.0854700854700855, |
| "false_abstain": "n/a", |
| "null_recall": 0.9145299145299145, |
| "n": 117 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.39981867633726204 |
| } |
| }, |
| "wh_rubric_shuffled": { |
| "n": 2437, |
| "n_total": 2437, |
| "suffix_overflow": 0, |
| "raw": { |
| "nll": 3.3731109702781983, |
| "brier": 1.0166531838701214, |
| "jsd": 0.3668337750954112, |
| "acc": 0.37792367665162085, |
| "acc_k": 0.4059104737070839, |
| "ece": 0.46684057980616855, |
| "sel_acc80": 0.3553846153846154, |
| "conf_wrong": 0.17647058823529413, |
| "coverage": 0.926138695116947, |
| "false_abstain": 0.049109083007388095, |
| "null_recall": 0.49264705882352944, |
| "auroc_null": 0.9573203466523506, |
| "auroc_conf": 0.8061951964618963 |
| }, |
| "by_family": { |
| "temporal": { |
| "nll": 1.0978670096057097, |
| "brier": 0.7736457181852393, |
| "jsd": 0.2832470099926861, |
| "acc": 0.4090909090909091, |
| "acc_k": 0.4090909090909091, |
| "ece": 0.34149774312973025, |
| "sel_acc80": 0.3693181818181818, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "shift_swap": { |
| "nll": 4.813958214372273, |
| "brier": 1.1627913595629051, |
| "jsd": 0.41357994903252204, |
| "acc": 0.35602094240837695, |
| "acc_k": 0.37222222222222223, |
| "ece": 0.5618253638919112, |
| "sel_acc80": 0.35294117647058826, |
| "conf_wrong": 0.2727272727272727, |
| "coverage": 0.9319371727748691, |
| "false_abstain": 0.044444444444444446, |
| "null_recall": 0.45454545454545453, |
| "auroc_null": 0.961111111111111, |
| "auroc_conf": 0.8565656565656565, |
| "n": 191 |
| }, |
| "vendor_onboarding": { |
| "nll": 4.468025309710248, |
| "brier": 1.1531663283624074, |
| "jsd": 0.4069967513654248, |
| "acc": 0.31958762886597936, |
| "acc_k": 0.39325842696629215, |
| "ece": 0.5820424177290239, |
| "sel_acc80": 0.34615384615384615, |
| "conf_wrong": 0.1875, |
| "coverage": 0.9175257731958762, |
| "false_abstain": 0.07303370786516854, |
| "null_recall": 0.1875, |
| "auroc_null": 0.9371488764044944, |
| "auroc_conf": 0.8433988764044944, |
| "n": 194 |
| }, |
| "loan_approval": { |
| "nll": 7.430180710009384, |
| "brier": 1.3288401062960586, |
| "jsd": 0.4609968152472018, |
| "acc": 0.33146067415730335, |
| "acc_k": 0.3505747126436782, |
| "ece": 0.6675498291347803, |
| "sel_acc80": 0.3356643356643357, |
| "conf_wrong": 0.5, |
| "coverage": 0.9662921348314607, |
| "false_abstain": 0.022988505747126436, |
| "null_recall": 0.5, |
| "auroc_null": 0.978448275862069, |
| "auroc_conf": 0.9396551724137931, |
| "n": 178 |
| }, |
| "probability": { |
| "nll": 1.5095582926439517, |
| "brier": 0.9146099048716316, |
| "jsd": 0.35344105098573364, |
| "acc": 0.35, |
| "acc_k": 0.35, |
| "ece": 0.3669138718735088, |
| "sel_acc80": 0.3181818181818182, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "feature_flag_rollout": { |
| "nll": 4.963032018178103, |
| "brier": 1.250659962667729, |
| "jsd": 0.4386572342122217, |
| "acc": 0.34782608695652173, |
| "acc_k": 0.34782608695652173, |
| "ece": 0.6342515031928602, |
| "sel_acc80": 0.3783783783783784, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 46 |
| }, |
| "research_ethics_review": { |
| "nll": 3.379301129427671, |
| "brier": 1.1293730274385836, |
| "jsd": 0.4101792599802621, |
| "acc": 0.31746031746031744, |
| "acc_k": 0.38392857142857145, |
| "ece": 0.5505591124769241, |
| "sel_acc80": 0.3069306930693069, |
| "conf_wrong": 0.07142857142857142, |
| "coverage": 0.8492063492063492, |
| "false_abstain": 0.125, |
| "null_recall": 0.35714285714285715, |
| "auroc_null": 0.8852040816326531, |
| "auroc_conf": 0.7633928571428572, |
| "n": 126 |
| }, |
| "warranty_claim": { |
| "nll": 5.425085813770316, |
| "brier": 1.222989340354703, |
| "jsd": 0.42426978503284835, |
| "acc": 0.35978835978835977, |
| "acc_k": 0.3675675675675676, |
| "ece": 0.6112382035091439, |
| "sel_acc80": 0.3684210526315789, |
| "conf_wrong": 0.5, |
| "coverage": 0.9788359788359788, |
| "false_abstain": 0.010810810810810811, |
| "null_recall": 0.5, |
| "auroc_null": 0.9905405405405405, |
| "auroc_conf": 0.8783783783783783, |
| "n": 189 |
| }, |
| "numeric": { |
| "nll": 0.7009633054040385, |
| "brier": 0.5083317633419103, |
| "jsd": 0.19623566233246967, |
| "acc": 0.5636363636363636, |
| "acc_k": 0.5636363636363636, |
| "ece": 0.20822090967134996, |
| "sel_acc80": 0.5795454545454546, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "content_moderation": { |
| "nll": 2.532750401458349, |
| "brier": 0.9723702132508161, |
| "jsd": 0.3676846987407566, |
| "acc": 0.3670886075949367, |
| "acc_k": 0.4094488188976378, |
| "ece": 0.4052060521856139, |
| "sel_acc80": 0.3464566929133858, |
| "conf_wrong": 0.06451612903225806, |
| "coverage": 0.6645569620253164, |
| "false_abstain": 0.23622047244094488, |
| "null_recall": 0.7419354838709677, |
| "auroc_null": 0.8270256540513081, |
| "auroc_conf": 0.7680975361950725, |
| "n": 158 |
| }, |
| "visa_extension": { |
| "nll": 3.4319388280589163, |
| "brier": 1.185777007774496, |
| "jsd": 0.4211514399505007, |
| "acc": 0.3023255813953488, |
| "acc_k": 0.3698630136986301, |
| "ece": 0.5790469986061717, |
| "sel_acc80": 0.2971014492753623, |
| "conf_wrong": 0.11538461538461539, |
| "coverage": 0.8023255813953488, |
| "false_abstain": 0.14383561643835616, |
| "null_recall": 0.5, |
| "auroc_null": 0.8730242360379347, |
| "auroc_conf": 0.8132244467860906, |
| "n": 172 |
| }, |
| "subscription_cancellation": { |
| "nll": 4.124055456988184, |
| "brier": 1.142628609921072, |
| "jsd": 0.40527967431266604, |
| "acc": 0.3546099290780142, |
| "acc_k": 0.424, |
| "ece": 0.5671253876483187, |
| "sel_acc80": 0.35398230088495575, |
| "conf_wrong": 0.3125, |
| "coverage": 0.8652482269503546, |
| "false_abstain": 0.096, |
| "null_recall": 0.4375, |
| "auroc_null": 0.9035, |
| "auroc_conf": 0.801, |
| "n": 141 |
| }, |
| "procurement_request": { |
| "nll": 5.064798743828172, |
| "brier": 1.1244811567056938, |
| "jsd": 0.3945059149254654, |
| "acc": 0.37593984962406013, |
| "acc_k": 0.4369747899159664, |
| "ece": 0.5752717774165303, |
| "sel_acc80": 0.3925233644859813, |
| "conf_wrong": 0.21428571428571427, |
| "coverage": 0.8796992481203008, |
| "false_abstain": 0.07563025210084033, |
| "null_recall": 0.5, |
| "auroc_null": 0.9357743097238895, |
| "auroc_conf": 0.8715486194477791, |
| "n": 133 |
| }, |
| "tradeoff": { |
| "nll": 1.295216810789425, |
| "brier": 0.8154554789002251, |
| "jsd": 0.30319613299194453, |
| "acc": 0.4318181818181818, |
| "acc_k": 0.4318181818181818, |
| "ece": 0.35886814662001354, |
| "sel_acc80": 0.3806818181818182, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 220 |
| }, |
| "insurance_claim": { |
| "nll": 5.987324790766054, |
| "brier": 1.150424499572334, |
| "jsd": 0.407765815124327, |
| "acc": 0.41379310344827586, |
| "acc_k": 0.41379310344827586, |
| "ece": 0.5872780434016523, |
| "sel_acc80": 0.4166666666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a", |
| "n": 29 |
| } |
| }, |
| "by_qtype": {}, |
| "by_label_class": { |
| "answer": { |
| "nll": 3.4790294568471882, |
| "brier": 1.0318484361922742, |
| "jsd": 0.3710045446095289, |
| "acc": 0.37114298131247286, |
| "acc_k": 0.4059104737070839, |
| "ece": 0.47994829282352375, |
| "sel_acc80": 0.3508962520369365, |
| "coverage": 0.9508909169926119, |
| "false_abstain": 0.049109083007388095, |
| "null_recall": "n/a", |
| "n": 2301 |
| }, |
| "null": { |
| "nll": 1.5810636350190332, |
| "brier": 0.7595629221548795, |
| "jsd": 0.2962680350072858, |
| "acc": 0.49264705882352944, |
| "ece": 0.3134835373391124, |
| "sel_acc80": 0.45871559633027525, |
| "conf_wrong": 0.17647058823529413, |
| "coverage": 0.5073529411764706, |
| "false_abstain": "n/a", |
| "null_recall": 0.49264705882352944, |
| "n": 136 |
| } |
| }, |
| "baselines": { |
| "majority_class": 0, |
| "majority_acc": 0.4095199015182602 |
| } |
| } |
| } |
| } |
