{ "run": "runs/tm1b", "mode": "native", "max_state": 4096, "limit": 0, "eval": { "u_chaosnli": { "n": 1594, "n_total": 1594, "suffix_overflow": 0, "raw": { "nll": 1.014103315474928, "brier": 0.17091631474795344, "jsd": 0.06646093306121999, "acc": 0.5131744040150564, "acc_k": 0.5131744040150564, "ece": 0.10623594570713328, "sel_acc80": 0.5313479623824452, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "": { "nll": 1.014103315474928, "brier": 0.17091631474795344, "jsd": 0.06646093306121999, "acc": 0.5131744040150564, "acc_k": 0.5131744040150564, "ece": 0.10623594570713328, "sel_acc80": 0.5313479623824452, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1594 } }, "by_qtype": {}, "by_label_class": { "answer": { "nll": 1.014103315474928, "brier": 0.17091631474795344, "jsd": 0.06646093306121999, "acc": 0.5131744040150564, "acc_k": 0.5131744040150564, "ece": 0.10623594570713328, "sel_acc80": 0.5313479623824452, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1594 } }, "baselines": { "majority_class": 0, "majority_acc": 0.46361355081555833 } }, "u_real_heldout": { "n": 562, "n_total": 562, "suffix_overflow": 0, "raw": { "nll": 0.6395687143934158, "brier": 0.11684180639496176, "jsd": 0.05530236546295147, "acc": 0.6583629893238434, "acc_k": 0.6583629893238434, "ece": 0.04802694863696116, "sel_acc80": 0.7, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "": { "nll": 0.6395687143934158, "brier": 0.11684180639496176, "jsd": 0.05530236546295147, "acc": 0.6583629893238434, "acc_k": 0.6583629893238434, "ece": 0.04802694863696116, "sel_acc80": 0.7, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 562 } }, "by_qtype": {}, "by_label_class": { "answer": { "nll": 0.6395687143934158, "brier": 0.11684180639496176, "jsd": 0.05530236546295147, "acc": 0.6583629893238434, "acc_k": 0.6583629893238434, "ece": 0.04802694863696116, "sel_acc80": 0.7, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 562 } }, "baselines": { "majority_class": 0, "majority_acc": 0.5071174377224199 } }, "u_synthetic_heldout": { "n": 475, "n_total": 475, "suffix_overflow": 0, "raw": { "nll": 0.6619388394682302, "brier": 0.029335292543417674, "jsd": 0.01347952047234738, "acc": 0.9073684210526316, "acc_k": 0.9073684210526316, "ece": 0.17639653883482279, "sel_acc80": 0.9815789473684211, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "": { "nll": 0.6619388394682302, "brier": 0.029335292543417674, "jsd": 0.01347952047234738, "acc": 0.9073684210526316, "acc_k": 0.9073684210526316, "ece": 0.17639653883482279, "sel_acc80": 0.9815789473684211, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 475 } }, "by_qtype": {}, "by_label_class": { "answer": { "nll": 0.6619388394682302, "brier": 0.029335292543417674, "jsd": 0.01347952047234738, "acc": 0.9073684210526316, "acc_k": 0.9073684210526316, "ece": 0.17639653883482279, "sel_acc80": 0.9815789473684211, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 475 } }, "baselines": { "majority_class": 1, "majority_acc": 0.5747368421052632 } }, "wf_heldout_choice": { "n": 1500, "n_total": 1500, "suffix_overflow": 0, "raw": { "nll": 0.08894784373441748, "brier": 0.03804468692396801, "jsd": 0.017131709690636845, "acc": 0.978, "acc_k": 0.9907997169143666, "ece": 0.011830694854259511, "sel_acc80": 0.9958333333333333, "conf_wrong": 0.0, "coverage": 0.9273333333333333, "false_abstain": 0.01556970983722576, "null_recall": 1.0, "auroc_null": 0.9994834500654839, "auroc_conf": 0.9995119213217171 }, "by_family": { "tool_select": { "nll": 0.08894784373441748, "brier": 0.03804468692396801, "jsd": 0.017131709690636845, "acc": 0.978, "acc_k": 0.9907997169143666, "ece": 0.011830694854259511, "sel_acc80": 0.9958333333333333, "conf_wrong": 0.0, "coverage": 0.9273333333333333, "false_abstain": 0.01556970983722576, "null_recall": 1.0, "auroc_null": 0.9994834500654839, "auroc_conf": 0.9995119213217171, "n": 1500 } }, "by_qtype": { "choice": { "nll": 0.08894784373441748, "brier": 0.03804468692396801, "jsd": 0.017131709690636845, "acc": 0.978, "acc_k": 0.9907997169143666, "ece": 0.011830694854259511, "sel_acc80": 0.9958333333333333, "conf_wrong": 0.0, "coverage": 0.9273333333333333, "false_abstain": 0.01556970983722576, "null_recall": 1.0, "auroc_null": 0.9994834500654839, "auroc_conf": 0.9995119213217171, "n": 1500 } }, "by_label_class": { "answer": { "nll": 0.09437487539006816, "brier": 0.04038581321521866, "jsd": 0.018169384833798215, "acc": 0.9766454352441614, "acc_k": 0.9907997169143666, "ece": 0.012608277211354707, "sel_acc80": 0.9946949602122016, "coverage": 0.9844302901627743, "false_abstain": 0.01556970983722576, "null_recall": "n/a", "n": 1413 }, "null": { "nll": 0.0008053640857458489, "brier": 2.1566814345354523e-05, "jsd": 0.0002784340896366467, "acc": 1.0, "ece": 0.0007983151523546228, "sel_acc80": 1.0, "conf_wrong": 0.0, "coverage": 0.0, "false_abstain": "n/a", "null_recall": 1.0, "n": 87 } }, "baselines": { "majority_class": 2, "majority_acc": 0.21333333333333335 } }, "wf_heldout_noul": { "n": 1500, "n_total": 1500, "suffix_overflow": 0, "raw": { "nll": 0.5731820141841979, "brier": 0.30167717441890574, "jsd": 0.10894462183976376, "acc": 0.8106666666666666, "acc_k": 0.8106666666666666, "ece": 0.13052070760726933, "sel_acc80": 0.88, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "eligibility": { "nll": 0.5731820141841979, "brier": 0.30167717441890574, "jsd": 0.10894462183976376, "acc": 0.8106666666666666, "acc_k": 0.8106666666666666, "ece": 0.13052070760726933, "sel_acc80": 0.88, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1500 } }, "by_qtype": { "noul": { "nll": 0.5731820141841979, "brier": 0.30167717441890574, "jsd": 0.10894462183976376, "acc": 0.8106666666666666, "acc_k": 0.8106666666666666, "ece": 0.13052070760726933, "sel_acc80": 0.88, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1500 } }, "by_label_class": { "answer": { "nll": 0.5731820141841979, "brier": 0.30167717441890574, "jsd": 0.10894462183976376, "acc": 0.8106666666666666, "acc_k": 0.8106666666666666, "ece": 0.13052070760726933, "sel_acc80": 0.88, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1500 } }, "baselines": { "majority_class": 0, "majority_acc": 0.5833333333333334, "always_positive_acc": 0.4166666666666667 } }, "wf_heldout_score": { "n": 1502, "n_total": 1502, "suffix_overflow": 0, "raw": { "nll": 1.0185536074805717, "brier": 0.5644523897741041, "jsd": 0.2801056596145076, "acc": 0.5279627163781625, "acc_k": 0.5279627163781625, "ece": 0.05101245270667476, "sel_acc80": 0.5806988352745425, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "urgency": { "nll": 1.0185536074805717, "brier": 0.5644523897741041, "jsd": 0.2801056596145076, "acc": 0.5279627163781625, "acc_k": 0.5279627163781625, "ece": 0.05101245270667476, "sel_acc80": 0.5806988352745425, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1502 } }, "by_qtype": { "score": { "nll": 1.0185536074805717, "brier": 0.5644523897741041, "jsd": 0.2801056596145076, "acc": 0.5279627163781625, "acc_k": 0.5279627163781625, "ece": 0.05101245270667476, "sel_acc80": 0.5806988352745425, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1502 } }, "by_label_class": { "answer": { "nll": 1.0185536074805717, "brier": 0.5644523897741041, "jsd": 0.2801056596145076, "acc": 0.5279627163781625, "acc_k": 0.5279627163781625, "ece": 0.05101245270667476, "sel_acc80": 0.5806988352745425, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1502 } }, "baselines": { "majority_class": 2, "majority_acc": 0.3548601864181092 } }, "wf_heldout_style": { "n": 1627, "n_total": 1627, "suffix_overflow": 0, "raw": { "nll": 0.35412793927257424, "brier": 0.2060959847410378, "jsd": 0.10287023165278557, "acc": 0.8586355255070682, "acc_k": 0.8564294631710362, "ece": 0.05414908969592812, "sel_acc80": 0.9308755760368663, "conf_wrong": 0.0, "coverage": 0.9846342962507683, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0 }, "by_family": { "relevance": { "nll": 0.6980276300431268, "brier": 0.42102710856000347, "jsd": 0.21389350686536507, "acc": 0.72, "acc_k": 0.72, "ece": 0.17414700448513035, "sel_acc80": 0.775, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 150 }, "support": { "nll": 0.20406603972307857, "brier": 0.1248183124169987, "jsd": 0.055781303712672675, "acc": 0.912751677852349, "acc_k": 0.912751677852349, "ece": 0.043371342572589816, "sel_acc80": 0.975, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 149 }, "adequacy": { "nll": 0.17072481903574357, "brier": 0.10599008454380157, "jsd": 0.047014351937127816, "acc": 0.9266666666666666, "acc_k": 0.9266666666666666, "ece": 0.04129962166150417, "sel_acc80": 0.9916666666666667, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 150 }, "action_select": { "nll": 0.231281707526007, "brier": 0.13311344023592248, "jsd": 0.058601355529894074, "acc": 0.9133333333333333, "acc_k": 0.9064748201438849, "ece": 0.05428580959637955, "sel_acc80": 0.9833333333333333, "conf_wrong": 0.0, "coverage": 0.9266666666666666, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 150 }, "completeness": { "nll": 0.7433181718309183, "brier": 0.4258628433595575, "jsd": 0.22361098694674933, "acc": 0.7248322147651006, "acc_k": 0.7248322147651006, "ece": 0.19188565155803758, "sel_acc80": 0.75, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 149 }, "quality": { "nll": 0.8425104976168987, "brier": 0.4753130219919474, "jsd": 0.24646774262839125, "acc": 0.6259541984732825, "acc_k": 0.6259541984732825, "ece": 0.1351061500893294, "sel_acc80": 0.6857142857142857, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 131 }, "enum_extract": { "nll": 0.0033332703444646995, "brier": 0.00024124939750805901, "jsd": 0.0011463143230046637, "acc": 1.0, "acc_k": 1.0, "ece": 0.003245462179183957, "sel_acc80": 1.0, "conf_wrong": 0.0, "coverage": 0.9666666666666667, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 150 }, "categorical": { "nll": 0.24690092856485552, "brier": 0.14982400101731513, "jsd": 0.07302109473186505, "acc": 0.889763779527559, "acc_k": 0.8870967741935484, "ece": 0.053563406148294794, "sel_acc80": 0.9607843137254902, "conf_wrong": 0.0, "coverage": 0.9763779527559056, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 127 }, "routing": { "nll": 0.21774816429206942, "brier": 0.13021437367644959, "jsd": 0.055718942276041364, "acc": 0.9090909090909091, "acc_k": 0.9047619047619048, "ece": 0.05593117032990313, "sel_acc80": 0.9905660377358491, "conf_wrong": 0.0, "coverage": 0.9545454545454546, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 132 }, "severity": { "nll": 0.6434271258602455, "brier": 0.347468069379254, "jsd": 0.19560876673722952, "acc": 0.7974683544303798, "acc_k": 0.7974683544303798, "ece": 0.22888638286650936, "sel_acc80": 0.875, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 79 }, "policy_permit": { "nll": 0.08175127275497936, "brier": 0.03988201939839468, "jsd": 0.02053868180510852, "acc": 0.9798657718120806, "acc_k": 0.9798657718120806, "ece": 0.031195846179987764, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 149 }, "fact": { "nll": 0.3245140742970864, "brier": 0.201230115726297, "jsd": 0.09200789581633895, "acc": 0.8558558558558559, "acc_k": 0.8558558558558559, "ece": 0.0778693476238766, "sel_acc80": 0.9213483146067416, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 111 } }, "by_qtype": { "score": { "nll": 0.7399964051867366, "brier": 0.4249972756512392, "jsd": 0.2222837523407285, "acc": 0.7092337917485265, "acc_k": 0.7092337917485265, "ece": 0.1720800453354193, "sel_acc80": 0.7549019607843137, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 509 }, "noul": { "nll": 0.186433926049318, "brier": 0.11229947221410162, "jsd": 0.0512284562939753, "acc": 0.9230769230769231, "acc_k": 0.9230769230769231, "ece": 0.024164358063972435, "sel_acc80": 0.9754464285714286, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 559 }, "choice": { "nll": 0.1704676606348132, "brier": 0.10057092826387289, "jsd": 0.04577948101845999, "acc": 0.9302325581395349, "acc_k": 0.9269662921348315, "ece": 0.0223422101657164, "sel_acc80": 0.9933035714285714, "conf_wrong": 0.0, "coverage": 0.9552772808586762, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 559 } }, "by_label_class": { "answer": { "nll": 0.35959165383560376, "brier": 0.2093046663699308, "jsd": 0.10445406727566603, "acc": 0.8564294631710362, "acc_k": 0.8564294631710362, "ece": 0.05493348601568356, "sel_acc80": 0.9297971918876755, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1602 }, "null": { "nll": 0.004013110073643866, "brier": 0.0004836659615737134, "jsd": 0.0013780449386066767, "acc": 1.0, "ece": 0.003884973526001012, "sel_acc80": 1.0, "conf_wrong": 0.0, "coverage": 0.0, "false_abstain": "n/a", "null_recall": 1.0, "n": 25 } }, "baselines": { "majority_class": 1, "majority_acc": 0.36447449293177625 } }, "wf_rubric_flip": { "n": 2062, "n_total": 2062, "suffix_overflow": 0, "raw": { "nll": 0.5140816127009797, "brier": 0.27501173451171385, "jsd": 0.1274834860045779, "acc": 0.7856450048496605, "acc_k": 0.7914645974781765, "ece": 0.04033603424533379, "sel_acc80": 0.8878787878787879, "coverage": 0.9932104752667313, "false_abstain": 0.006789524733268671, "null_recall": "n/a" }, "by_family": { "urgency": { "nll": 1.0503140999721947, "brier": 0.5826411232348119, "jsd": 0.2870499222013987, "acc": 0.5142857142857142, "acc_k": 0.5142857142857142, "ece": 0.061800167475427895, "sel_acc80": 0.5464285714285714, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 700 }, "eligibility": { "nll": 0.3789669930673108, "brier": 0.19723455708945753, "jsd": 0.07565788618043696, "acc": 0.8704268292682927, "acc_k": 0.8704268292682927, "ece": 0.0704190259481349, "sel_acc80": 0.9314285714285714, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 656 }, "tool_select": { "nll": 0.10795193761576238, "brier": 0.042265638595044515, "jsd": 0.017428370065288763, "acc": 0.9759206798866855, "acc_k": 0.9929178470254958, "ece": 0.015375537465381884, "sel_acc80": 0.9946902654867257, "coverage": 0.9801699716713881, "false_abstain": 0.019830028328611898, "null_recall": "n/a", "n": 706 } }, "by_qtype": { "score": { "nll": 1.0503140999721947, "brier": 0.5826411232348119, "jsd": 0.2870499222013987, "acc": 0.5142857142857142, "acc_k": 0.5142857142857142, "ece": 0.061800167475427895, "sel_acc80": 0.5464285714285714, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 700 }, "noul": { "nll": 0.3789669930673108, "brier": 0.19723455708945753, "jsd": 0.07565788618043696, "acc": 0.8704268292682927, "acc_k": 0.8704268292682927, "ece": 0.0704190259481349, "sel_acc80": 0.9314285714285714, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 656 }, "choice": { "nll": 0.10795193761576238, "brier": 0.042265638595044515, "jsd": 0.017428370065288763, "acc": 0.9759206798866855, "acc_k": 0.9929178470254958, "ece": 0.015375537465381884, "sel_acc80": 0.9946902654867257, "coverage": 0.9801699716713881, "false_abstain": 0.019830028328611898, "null_recall": "n/a", "n": 706 } }, "by_label_class": { "answer": { "nll": 0.5140816127009797, "brier": 0.27501173451171385, "jsd": 0.1274834860045779, "acc": 0.7856450048496605, "acc_k": 0.7914645974781765, "ece": 0.04033603424533379, "sel_acc80": 0.8878787878787879, "coverage": 0.9932104752667313, "false_abstain": 0.006789524733268671, "null_recall": "n/a", "n": 2062 } }, "baselines": { "majority_class": 0, "majority_acc": 0.32104752667313285 }, "flip": { "n_pairs": 1031, "flip_rate": 0.8205625606207565, "both_correct": 0.6702230843840931, "acc": 0.7914645974781765 }, "flip_by_family": { "eligibility": { "n_pairs": 328, "flip_rate": 0.7591463414634146, "both_correct": 0.75, "acc": 0.8704268292682927 }, "tool_select": { "n_pairs": 353, "flip_rate": 0.9915014164305949, "both_correct": 0.9858356940509915, "acc": 0.9929178470254958 }, "urgency": { "n_pairs": 350, "flip_rate": 0.7057142857142857, "both_correct": 0.27714285714285714, "acc": 0.5142857142857142 } } }, "wf_rubric_shuffled": { "n": 3775, "n_total": 3775, "suffix_overflow": 0, "raw": { "nll": 3.3019875953051896, "brier": 0.9847791419883737, "jsd": 0.38205552797027176, "acc": 0.3801324503311258, "acc_k": 0.3811920529801324, "ece": 0.39123494437198764, "sel_acc80": 0.39470198675496687, "coverage": 0.9925827814569537, "false_abstain": 0.007417218543046358, "null_recall": "n/a" }, "by_family": { "tool_select": { "nll": 8.01855352489357, "brier": 1.585383500778014, "jsd": 0.5686605163566438, "acc": 0.15359116022099448, "acc_k": 0.1580110497237569, "ece": 0.7718642068172686, "sel_acc80": 0.13535911602209943, "coverage": 0.969060773480663, "false_abstain": 0.030939226519337018, "null_recall": "n/a", "n": 905 }, "eligibility": { "nll": 2.0992631849983714, "brier": 0.8251410346340735, "jsd": 0.28859514090273525, "acc": 0.5234758234057463, "acc_k": 0.5234758234057463, "ece": 0.39013672221684337, "sel_acc80": 0.5332749562171629, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1427 }, "urgency": { "nll": 1.5333081547163776, "brier": 0.7659687708795461, "jsd": 0.35744739065613984, "acc": 0.3804573804573805, "acc_k": 0.3804573804573805, "ece": 0.15649522523952702, "sel_acc80": 0.38961038961038963, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1443 } }, "by_qtype": { "choice": { "nll": 8.01855352489357, "brier": 1.585383500778014, "jsd": 0.5686605163566438, "acc": 0.15359116022099448, "acc_k": 0.1580110497237569, "ece": 0.7718642068172686, "sel_acc80": 0.13535911602209943, "coverage": 0.969060773480663, "false_abstain": 0.030939226519337018, "null_recall": "n/a", "n": 905 }, "noul": { "nll": 2.0992631849983714, "brier": 0.8251410346340735, "jsd": 0.28859514090273525, "acc": 0.5234758234057463, "acc_k": 0.5234758234057463, "ece": 0.39013672221684337, "sel_acc80": 0.5332749562171629, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1427 }, "score": { "nll": 1.5333081547163776, "brier": 0.7659687708795461, "jsd": 0.35744739065613984, "acc": 0.3804573804573805, "acc_k": 0.3804573804573805, "ece": 0.15649522523952702, "sel_acc80": 0.38961038961038963, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1443 } }, "by_label_class": { "answer": { "nll": 3.3019875953051896, "brier": 0.9847791419883737, "jsd": 0.38205552797027176, "acc": 0.3801324503311258, "acc_k": 0.3811920529801324, "ece": 0.39123494437198764, "sel_acc80": 0.39470198675496687, "coverage": 0.9925827814569537, "false_abstain": 0.007417218543046358, "null_recall": "n/a", "n": 3775 } }, "baselines": { "majority_class": 0, "majority_acc": 0.36 } }, "cua_s1_forms_demo": { "n": 196, "n_total": 196, "suffix_overflow": 0, "raw": { "nll": 0.0006697119946596961, "brier": 2.5804204528327477e-06, "jsd": 0.00023202367765799874, "acc": 1.0, "acc_k": 1.0, "ece": 0.0006688650773496452, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "cua_s1_forms": { "nll": 0.0006697119946596961, "brier": 2.5804204528327477e-06, "jsd": 0.00023202367765799874, "acc": 1.0, "acc_k": 1.0, "ece": 0.0006688650773496452, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 196 } }, "by_qtype": { "choice": { "nll": 0.0006697119946596961, "brier": 2.5804204528327477e-06, "jsd": 0.00023202367765799874, "acc": 1.0, "acc_k": 1.0, "ece": 0.0006688650773496452, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 196 } }, "by_label_class": { "answer": { "nll": 0.0006697119946596961, "brier": 2.5804204528327477e-06, "jsd": 0.00023202367765799874, "acc": 1.0, "acc_k": 1.0, "ece": 0.0006688650773496452, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 196 } }, "baselines": { "majority_class": 18, "majority_acc": 0.27040816326530615 } }, "cua_s1_forms_test": { "n": 24370, "n_total": 24370, "suffix_overflow": 0, "raw": { "nll": 0.032467243628837364, "brier": 0.01572971627781683, "jsd": 0.006402877388207127, "acc": 0.9902749281903981, "acc_k": 0.9902749281903981, "ece": 0.004090282016471624, "sel_acc80": 0.9998974148543291, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "cua_s1_forms": { "nll": 0.032467243628837364, "brier": 0.01572971627781683, "jsd": 0.006402877388207127, "acc": 0.9902749281903981, "acc_k": 0.9902749281903981, "ece": 0.004090282016471624, "sel_acc80": 0.9998974148543291, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 24370 } }, "by_qtype": { "choice": { "nll": 0.032467243628837364, "brier": 0.01572971627781683, "jsd": 0.006402877388207127, "acc": 0.9902749281903981, "acc_k": 0.9902749281903981, "ece": 0.004090282016471624, "sel_acc80": 0.9998974148543291, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 24370 } }, "by_label_class": { "answer": { "nll": 0.032467243628837364, "brier": 0.01572971627781683, "jsd": 0.006402877388207127, "acc": 0.9902749281903981, "acc_k": 0.9902749281903981, "ece": 0.004090282016471624, "sel_acc80": 0.9998974148543291, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 24370 } }, "baselines": { "majority_class": 16, "majority_acc": 0.07033237587197373 } }, "jev4b_adversarial": { "n": 600, "n_total": 600, "suffix_overflow": 0, "raw": { "nll": 0.21596318335730846, "brier": 0.11449747234656535, "jsd": 0.0566693510735706, "acc": 0.9266666666666666, "acc_k": 0.9266666666666666, "ece": 0.033218660900990224, "sel_acc80": 0.98125, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "jev4b_distill": { "nll": 0.21596318335730846, "brier": 0.11449747234656535, "jsd": 0.0566693510735706, "acc": 0.9266666666666666, "acc_k": 0.9266666666666666, "ece": 0.033218660900990224, "sel_acc80": 0.98125, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 600 } }, "by_qtype": { "choice": { "nll": 0.15764157777699858, "brier": 0.07730203604858563, "jsd": 0.03353047215992909, "acc": 0.9527777777777777, "acc_k": 0.9527777777777777, "ece": 0.027492753830220998, "sel_acc80": 0.9895833333333334, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 360 }, "score": { "nll": 0.588537222505538, "brier": 0.3360254493815728, "jsd": 0.17654749465911643, "acc": 0.775, "acc_k": 0.775, "ece": 0.11445025528470672, "sel_acc80": 0.8125, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 120 }, "noul": { "nll": 0.018353960950008512, "brier": 0.004555804205497079, "jsd": 0.006207844228949319, "acc": 1.0, "acc_k": 1.0, "ece": 0.01696336468060807, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 120 } }, "by_label_class": { "answer": { "nll": 0.21596318335730846, "brier": 0.11449747234656535, "jsd": 0.0566693510735706, "acc": 0.9266666666666666, "acc_k": 0.9266666666666666, "ece": 0.033218660900990224, "sel_acc80": 0.98125, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 600 } }, "baselines": { "majority_class": 0, "majority_acc": 0.42833333333333334 } }, "jevlogs_triage": { "n": 5080, "n_total": 5080, "suffix_overflow": 0, "raw": { "nll": 0.8200663549047869, "brier": 0.5245085358820047, "jsd": 0.19936215850961117, "acc": 0.6734251968503937, "acc_k": 0.6734251968503937, "ece": 0.19179721678805164, "sel_acc80": 0.6801181102362205, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "log_hdfs": { "nll": 0.8322974851760632, "brier": 0.5211627531136993, "jsd": 0.1981452827621227, "acc": 0.6664, "acc_k": 0.6664, "ece": 0.17615253834724426, "sel_acc80": 0.6785, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 2500 }, "log_bgl": { "nll": 0.7998629474253376, "brier": 0.521462450412092, "jsd": 0.1985896785858705, "acc": 0.6856, "acc_k": 0.6856, "ece": 0.24997946472167967, "sel_acc80": 0.659, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 2500 }, "log_adversarial": { "nll": 1.0692000176602048, "brier": 0.7242544183263233, "jsd": 0.26152952323552203, "acc": 0.5125, "acc_k": 0.5125, "ece": 0.304364737868309, "sel_acc80": 0.515625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 80 } }, "by_qtype": { "noul": { "nll": 0.8200663549047869, "brier": 0.5245085358820047, "jsd": 0.19936215850961117, "acc": 0.6734251968503937, "acc_k": 0.6734251968503937, "ece": 0.19179721678805164, "sel_acc80": 0.6801181102362205, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 5080 } }, "by_label_class": { "answer": { "nll": 0.8200663549047869, "brier": 0.5245085358820047, "jsd": 0.19936215850961117, "acc": 0.6734251968503937, "acc_k": 0.6734251968503937, "ece": 0.19179721678805164, "sel_acc80": 0.6801181102362205, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 5080 } }, "baselines": { "majority_class": 0, "majority_acc": 0.6968503937007874, "always_positive_acc": 0.3031496062992126 } }, "mind2web_choice": { "n": 1600, "n_total": 1600, "suffix_overflow": 0, "raw": { "nll": 1.2406711424343253, "brier": 0.5732404237664536, "jsd": 0.2788736075063562, "acc": 0.544375, "acc_k": 0.544375, "ece": 0.08040638350881635, "sel_acc80": 0.615625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "mind2web": { "nll": 1.2406711424343253, "brier": 0.5732404237664536, "jsd": 0.2788736075063562, "acc": 0.544375, "acc_k": 0.544375, "ece": 0.08040638350881635, "sel_acc80": 0.615625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1600 } }, "by_qtype": { "choice": { "nll": 1.8521192870863956, "brier": 0.7087342212760166, "jsd": 0.3756742847617898, "acc": 0.44, "acc_k": 0.44, "ece": 0.08787944102659824, "sel_acc80": 0.49375, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 800 }, "noul": { "nll": 0.6292229977822548, "brier": 0.4377466262568903, "jsd": 0.18207293025092264, "acc": 0.64875, "acc_k": 0.64875, "ece": 0.07293332599103454, "sel_acc80": 0.6875, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 800 } }, "by_label_class": { "answer": { "nll": 1.2406711424343253, "brier": 0.5732404237664536, "jsd": 0.2788736075063562, "acc": 0.544375, "acc_k": 0.544375, "ece": 0.08040638350881635, "sel_acc80": 0.615625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1600 } }, "baselines": { "majority_class": 0, "majority_acc": 0.426875 } }, "pagerduty_trigger": { "n": 6000, "n_total": 6000, "suffix_overflow": 0, "raw": { "nll": 0.396707458891549, "brier": 0.23002239391086174, "jsd": 0.11322283340814882, "acc": 0.838, "acc_k": 0.838, "ece": 0.07345882829030351, "sel_acc80": 0.8979166666666667, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "replica_lag_data_risk": { "nll": 0.29740572278535377, "brier": 0.14798980251485705, "jsd": 0.09449495177807844, "acc": 1.0, "acc_k": 1.0, "ece": 0.2269586962566041, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 114 }, "tls_hours_left_serving": { "nll": 2.462503098332096, "brier": 1.4718128845820069, "jsd": 0.5430561946798185, "acc": 0.0, "acc_k": 0.0, "ece": 0.7213651537895203, "sel_acc80": 0.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 98 }, "disk_full_writes_failing": { "nll": 0.6112621743776719, "brier": 0.43961606766122513, "jsd": 0.1828291039570283, "acc": 0.5, "acc_k": 0.5, "ece": 0.40561820756305345, "sel_acc80": 0.625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 110 }, "revoked_key_used": { "nll": 2.25962306280277, "brier": 1.3607013127715883, "jsd": 0.49743537591946874, "acc": 0.0, "acc_k": 0.0, "ece": 0.7095300555229187, "sel_acc80": 0.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 102 }, "checksum_corruption": { "nll": 0.16655108974517663, "brier": 0.05001836994744468, "jsd": 0.05564429813812224, "acc": 1.0, "acc_k": 1.0, "ece": 0.14739903873867463, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 90 }, "checkout_slo_burn": { "nll": 0.8234049439981885, "brier": 0.5886945952552634, "jsd": 0.24262392305224867, "acc": 0.5, "acc_k": 0.5, "ece": 0.5280300080776215, "sel_acc80": 0.375, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 100 }, "payments_processor_down": { "nll": 0.7458985031769221, "brier": 0.5397173510738811, "jsd": 0.21208411419957937, "acc": 0.5, "acc_k": 0.5, "ece": 0.3847963486580139, "sel_acc80": 0.618421052631579, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 94 }, "crash_loop_user_impact": { "nll": 0.45723675404949216, "brier": 0.28838964871857825, "jsd": 0.14411361556383404, "acc": 0.7916666666666666, "acc_k": 0.7916666666666666, "ece": 0.3439861076573531, "sel_acc80": 0.7402597402597403, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 96 }, "credential_stuffing": { "nll": 0.6855976842756064, "brier": 0.5141043463970237, "jsd": 0.19799073218670762, "acc": 0.5, "acc_k": 0.5, "ece": 0.4205113059530656, "sel_acc80": 0.6233766233766234, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 96 }, "queue_backup_checkout": { "nll": 0.4943918081315674, "brier": 0.3165144497680829, "jsd": 0.1554112283610238, "acc": 0.5, "acc_k": 0.5, "ece": 0.3690876066684723, "sel_acc80": 0.625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 100 }, "queue_slow_growth": { "nll": 0.8056600958296468, "brier": 0.46479321367422693, "jsd": 0.2233220382938683, "acc": 0.5, "acc_k": 0.5, "ece": 0.2907683104276657, "sel_acc80": 0.6222222222222222, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 56 }, "cert_30d": { "nll": 0.5765844556293318, "brier": 0.459360806409668, "jsd": 0.1663273119345, "acc": 0.5, "acc_k": 0.5, "ece": 0.34605565667152405, "sel_acc80": 0.6170212765957447, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 58 }, "disk_80pct_48h": { "nll": 0.6364577616833813, "brier": 0.3603717509503686, "jsd": 0.1950357110171284, "acc": 1.0, "acc_k": 1.0, "ece": 0.44600894767791033, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 64 }, "cpu_high_no_impact": { "nll": 0.7085339514792242, "brier": 0.5101478528535615, "jsd": 0.19822264049400234, "acc": 0.5, "acc_k": 0.5, "ece": 0.37314837455749506, "sel_acc80": 0.625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 50 }, "single_pod_oom": { "nll": 0.597222631375, "brier": 0.3511328330652838, "jsd": 0.1808235790015383, "acc": 0.5, "acc_k": 0.5, "ece": 0.32174784059707934, "sel_acc80": 0.6190476190476191, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 52 }, "deprecation": { "nll": 1.2131927809820504, "brier": 0.8289228409108279, "jsd": 0.2701922670787307, "acc": 0.5, "acc_k": 0.5, "ece": 0.45578983426094055, "sel_acc80": 0.625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 60 }, "canary_within_budget": { "nll": 0.68097187510421, "brier": 0.46432714017336996, "jsd": 0.19212515668071925, "acc": 0.5, "acc_k": 0.5, "ece": 0.33959993720054626, "sel_acc80": 0.6153846153846154, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 48 }, "replica_lag_seconds": { "nll": 0.48507738285690005, "brier": 0.2914945314683177, "jsd": 0.14510412619084756, "acc": 0.5, "acc_k": 0.5, "ece": 0.21687236428260803, "sel_acc80": 0.625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 50 }, "tls_expires_7d": { "nll": 0.8200619861584703, "brier": 0.6263737845037869, "jsd": 0.21875945370810285, "acc": 0.5, "acc_k": 0.5, "ece": 0.41936272382736206, "sel_acc80": 0.6190476190476191, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 26 }, "slow_slo_burn": { "nll": 0.4646857777411605, "brier": 0.2633606501114195, "jsd": 0.15017855189982032, "acc": 1.0, "acc_k": 1.0, "ece": 0.3715018928050995, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 36 }, "injection_noise": { "nll": 0.3191649156767602, "brier": 0.15667065379041875, "jsd": 0.10271966665831857, "acc": 1.0, "acc_k": 1.0, "ece": 0.2534384096822431, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 310 }, "health_ok": { "nll": 0.11813930759934398, "brier": 0.039244944056013954, "jsd": 0.03965546739519377, "acc": 1.0, "acc_k": 1.0, "ece": 0.10602798960567297, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 306 }, "kafka_caught_up": { "nll": 0.1239690909443528, "brier": 0.03284989912051308, "jsd": 0.04179599987809453, "acc": 1.0, "acc_k": 1.0, "ece": 0.11298148923640626, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 278 }, "invalid_coupon_error": { "nll": 0.4095562707396914, "brier": 0.2070045846973819, "jsd": 0.12855307030768603, "acc": 1.0, "acc_k": 1.0, "ece": 0.3042865756505765, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 322 }, "stack_trace_handled": { "nll": 0.1369772675048073, "brier": 0.04242081045781862, "jsd": 0.04599428397435028, "acc": 1.0, "acc_k": 1.0, "ece": 0.12313097069062383, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 332 }, "retry_then_ok": { "nll": 0.23577570062147413, "brier": 0.09354840300021036, "jsd": 0.07807766271472766, "acc": 1.0, "acc_k": 1.0, "ece": 0.20314176128783692, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 284 }, "feature_flag": { "nll": 0.02957982981648557, "brier": 0.0025294010832958157, "jsd": 0.010185593857561997, "acc": 1.0, "acc_k": 1.0, "ece": 0.028911458685042968, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 282 }, "cache_hit": { "nll": 0.05577471799801635, "brier": 0.010092673936511587, "jsd": 0.019045345808299963, "acc": 1.0, "acc_k": 1.0, "ece": 0.05296187488349169, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 286 }, "metrics_scrape": { "nll": 0.19512496383591874, "brier": 0.06495891768903242, "jsd": 0.06532458959189952, "acc": 1.0, "acc_k": 1.0, "ece": 0.17378984264484268, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 328 }, "cron_ok": { "nll": 0.8260068202421647, "brier": 0.5908376052070468, "jsd": 0.24215122771741, "acc": 0.5, "acc_k": 0.5, "ece": 0.3292480845605173, "sel_acc80": 0.375, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 310 }, "rate_limit_bot": { "nll": 0.2224008145703419, "brier": 0.10465719261617362, "jsd": 0.0723341471248896, "acc": 1.0, "acc_k": 1.0, "ece": 0.18123274389654398, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 256 }, "http_404": { "nll": 0.0272498202190122, "brier": 0.0025453067235740637, "jsd": 0.009377000999055841, "acc": 1.0, "acc_k": 1.0, "ece": 0.026570967720739525, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 318 }, "deploy_success": { "nll": 0.11437534991524133, "brier": 0.022894189185427016, "jsd": 0.03898171724942419, "acc": 1.0, "acc_k": 1.0, "ece": 0.10790608627874346, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 244 }, "user_not_found": { "nll": 0.7467332928805962, "brier": 0.4869266134970554, "jsd": 0.20417162215696066, "acc": 0.5, "acc_k": 0.5, "ece": 0.3224844210812835, "sel_acc80": 0.6228813559322034, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 294 }, "request_ok": { "nll": 0.15962773605536434, "brier": 0.045111188981216126, "jsd": 0.05396733304793474, "acc": 1.0, "acc_k": 1.0, "ece": 0.14663683669907707, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 350 } }, "by_qtype": { "choice": { "nll": 0.48022365288831015, "brier": 0.25716945445373307, "jsd": 0.14152253563996733, "acc": 0.851, "acc_k": 0.851, "ece": 0.13448732795317966, "sel_acc80": 0.905, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 3000 }, "noul": { "nll": 0.31319126489478805, "brier": 0.20287533336799038, "jsd": 0.0849231311763303, "acc": 0.825, "acc_k": 0.825, "ece": 0.10489587610960006, "sel_acc80": 0.9566666666666667, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 3000 } }, "by_label_class": { "answer": { "nll": 0.396707458891549, "brier": 0.23002239391086174, "jsd": 0.11322283340814882, "acc": 0.838, "acc_k": 0.838, "ece": 0.07345882829030351, "sel_acc80": 0.8979166666666667, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 6000 } }, "baselines": { "majority_class": 0, "majority_acc": 0.7916666666666666 } }, "systemone_lite_hard": { "n": 5400, "n_total": 5400, "suffix_overflow": 0, "raw": { "nll": 0.43450476654815856, "brier": 0.23583658134544866, "jsd": 0.11106552302444929, "acc": 0.850925925925926, "acc_k": 0.850925925925926, "ece": 0.05852483691440687, "sel_acc80": 0.8976851851851851, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "systemone_lite": { "nll": 0.43450476654815856, "brier": 0.23583658134544866, "jsd": 0.11106552302444929, "acc": 0.850925925925926, "acc_k": 0.850925925925926, "ece": 0.05852483691440687, "sel_acc80": 0.8976851851851851, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 5400 } }, "by_qtype": { "choice": { "nll": 0.47634740362930744, "brier": 0.22078883830669857, "jsd": 0.0890289134343277, "acc": 0.8522222222222222, "acc_k": 0.8522222222222222, "ece": 0.06805240516861279, "sel_acc80": 0.9319444444444445, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1800 }, "noul": { "nll": 0.23199631304256532, "brier": 0.14232818861742158, "jsd": 0.06226400940582674, "acc": 0.8994444444444445, "acc_k": 0.8994444444444445, "ece": 0.01822677158647116, "sel_acc80": 0.9597222222222223, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1800 }, "score": { "nll": 0.595170582972603, "brier": 0.3443927171122257, "jsd": 0.18190364623319336, "acc": 0.8011111111111111, "acc_k": 0.8011111111111111, "ece": 0.17747255620029237, "sel_acc80": 0.8513888888888889, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1800 } }, "by_label_class": { "answer": { "nll": 0.43450476654815856, "brier": 0.23583658134544866, "jsd": 0.11106552302444929, "acc": 0.850925925925926, "acc_k": 0.850925925925926, "ece": 0.05852483691440687, "sel_acc80": 0.8976851851851851, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 5400 } }, "baselines": { "majority_class": 1, "majority_acc": 0.44166666666666665 } }, "tree_choice_cap": { "n": 720, "n_total": 720, "suffix_overflow": 0, "raw": { "nll": 1.0937767414927673, "brier": 0.4118990134350543, "jsd": 0.2282905942697906, "acc": 0.6902777777777778, "acc_k": 0.7319444444444444, "ece": 0.07905487779838341, "sel_acc80": 0.7934027777777778, "coverage": 0.8611111111111112, "false_abstain": 0.1388888888888889, "null_recall": "n/a" }, "by_family": { "incident_catalog": { "nll": 1.0937767414927673, "brier": 0.4118990134350543, "jsd": 0.2282905942697906, "acc": 0.6902777777777778, "acc_k": 0.7319444444444444, "ece": 0.07905487779838341, "sel_acc80": 0.7934027777777778, "coverage": 0.8611111111111112, "false_abstain": 0.1388888888888889, "null_recall": "n/a", "n": 720 } }, "by_qtype": { "choice": { "nll": 1.0937767414927673, "brier": 0.4118990134350543, "jsd": 0.2282905942697906, "acc": 0.6902777777777778, "acc_k": 0.7319444444444444, "ece": 0.07905487779838341, "sel_acc80": 0.7934027777777778, "coverage": 0.8611111111111112, "false_abstain": 0.1388888888888889, "null_recall": "n/a", "n": 720 } }, "by_label_class": { "answer": { "nll": 1.0937767414927673, "brier": 0.4118990134350543, "jsd": 0.2282905942697906, "acc": 0.6902777777777778, "acc_k": 0.7319444444444444, "ece": 0.07905487779838341, "sel_acc80": 0.7934027777777778, "coverage": 0.8611111111111112, "false_abstain": 0.1388888888888889, "null_recall": "n/a", "n": 720 } }, "baselines": { "majority_class": 3, "majority_acc": 0.11388888888888889 } }, "typed_decisions_test": { "n": 2000, "n_total": 2000, "suffix_overflow": 0, "raw": { "nll": 1.183971535602263, "brier": 0.21124234378656148, "jsd": 0.08978436233492346, "acc": 0.5325, "acc_k": 0.5325, "ece": 0.0960818936675787, "sel_acc80": 0.55125, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "agent_trace_observability": { "nll": 1.4513733308753711, "brier": 0.27746615896441984, "jsd": 0.11437914671823042, "acc": 0.4, "acc_k": 0.4, "ece": 0.21149230748414996, "sel_acc80": 0.4125, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 500 }, "customer_service": { "nll": 1.1394922824048543, "brier": 0.1685314240724029, "jsd": 0.07635420781877246, "acc": 0.638, "acc_k": 0.638, "ece": 0.09256491619348527, "sel_acc80": 0.6825, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 500 }, "invoice_processing": { "nll": 1.0241192434455133, "brier": 0.25477993975703184, "jsd": 0.11173192033283712, "acc": 0.582, "acc_k": 0.582, "ece": 0.08850071275234224, "sel_acc80": 0.64, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 500 }, "security_incidents": { "nll": 1.1209012856833143, "brier": 0.14419185235239126, "jsd": 0.056672174469853814, "acc": 0.51, "acc_k": 0.51, "ece": 0.1298270362019539, "sel_acc80": 0.505, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 500 } }, "by_qtype": { "choice": { "nll": 1.4624093197198917, "brier": 0.22882638324834792, "jsd": 0.10929266556165232, "acc": 0.49166666666666664, "acc_k": 0.49166666666666664, "ece": 0.12719455897808077, "sel_acc80": 0.5333333333333333, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 600 }, "noul": { "nll": 0.8045478477317457, "brier": 0.25816601068728884, "jsd": 0.08310478332934974, "acc": 0.5783333333333334, "acc_k": 0.5783333333333334, "ece": 0.15133020063241323, "sel_acc80": 0.5895833333333333, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 600 }, "score": { "nll": 1.2597109634169303, "brier": 0.16286156401467608, "jsd": 0.08016281916905708, "acc": 0.52875, "acc_k": 0.52875, "ece": 0.07651440937072038, "sel_acc80": 0.55, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 800 } }, "by_label_class": { "answer": { "nll": 1.183971535602263, "brier": 0.21124234378656148, "jsd": 0.08978436233492346, "acc": 0.5325, "acc_k": 0.5325, "ece": 0.0960818936675787, "sel_acc80": 0.55125, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 2000 } }, "baselines": { "majority_class": 1, "majority_acc": 0.2905 } }, "typed_decisions_train": { "n": 6000, "n_total": 6000, "suffix_overflow": 0, "raw": { "nll": 1.1985372981046378, "brier": 0.2329991959274657, "jsd": 0.09837575798408288, "acc": 0.5223333333333333, "acc_k": 0.5223333333333333, "ece": 0.10562264278034368, "sel_acc80": 0.5375, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "agent_trace_observability": { "nll": 1.416481477970894, "brier": 0.25422688167917284, "jsd": 0.10777658676024122, "acc": 0.4266666666666667, "acc_k": 0.4266666666666667, "ece": 0.18357536943753558, "sel_acc80": 0.42916666666666664, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1500 }, "customer_service": { "nll": 1.1214988914219903, "brier": 0.19128662847914574, "jsd": 0.08370220742725006, "acc": 0.6106666666666667, "acc_k": 0.6106666666666667, "ece": 0.08198627370595929, "sel_acc80": 0.6433333333333333, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1500 }, "invoice_processing": { "nll": 1.1031419902595583, "brier": 0.3050021381875509, "jsd": 0.13027031117221108, "acc": 0.5333333333333333, "acc_k": 0.5333333333333333, "ece": 0.10525429819027585, "sel_acc80": 0.575, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1500 }, "security_incidents": { "nll": 1.1530268327661088, "brier": 0.1814811353639933, "jsd": 0.07175392657662916, "acc": 0.5186666666666667, "acc_k": 0.5186666666666667, "ece": 0.12120020204782486, "sel_acc80": 0.5258333333333334, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1500 } }, "by_qtype": { "choice": { "nll": 1.4635702511121844, "brier": 0.2532055828202569, "jsd": 0.11942735571638688, "acc": 0.4961111111111111, "acc_k": 0.4961111111111111, "ece": 0.12604168651832473, "sel_acc80": 0.5368055555555555, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1800 }, "noul": { "nll": 0.8394133296564307, "brier": 0.2816899204625882, "jsd": 0.09059050796156125, "acc": 0.5633333333333334, "acc_k": 0.5633333333333334, "ece": 0.1775682842400339, "sel_acc80": 0.5701388888888889, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 1800 }, "score": { "nll": 1.2691055596851333, "brier": 0.18132636235653043, "jsd": 0.08842599720174607, "acc": 0.51125, "acc_k": 0.51125, "ece": 0.08015046964089076, "sel_acc80": 0.5317708333333333, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 2400 } }, "by_label_class": { "answer": { "nll": 1.1985372981046378, "brier": 0.2329991959274657, "jsd": 0.09837575798408288, "acc": 0.5223333333333333, "acc_k": 0.5223333333333333, "ece": 0.10562264278034368, "sel_acc80": 0.5375, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 6000 } }, "baselines": { "majority_class": 1, "majority_acc": 0.3105 } }, "wh_heldout_family": { "n": 422, "n_total": 422, "suffix_overflow": 0, "raw": { "nll": 0.3240292072876919, "brier": 0.17772530751240193, "jsd": 0.07779958816769972, "acc": 0.8744075829383886, "acc_k": 0.9045092838196287, "ece": 0.04873869755256795, "sel_acc80": 0.9437869822485208, "conf_wrong": 0.0, "coverage": 0.8625592417061612, "false_abstain": 0.04774535809018567, "null_recall": 0.8888888888888888, "auroc_null": 0.9826112584733274, "auroc_conf": 0.9737105806071323 }, "by_family": { "procurement_request": { "nll": 0.31726594171575667, "brier": 0.15641665156788495, "jsd": 0.06340224526164076, "acc": 0.9014084507042254, "acc_k": 0.9375, "ece": 0.07553887535148941, "sel_acc80": 0.956140350877193, "conf_wrong": 0.0, "coverage": 0.8802816901408451, "false_abstain": 0.0390625, "null_recall": 0.8571428571428571, "auroc_null": 0.9827008928571429, "auroc_conf": 0.9782366071428572, "n": 142 }, "research_ethics_review": { "nll": 0.41234907161332957, "brier": 0.23254687981548977, "jsd": 0.10529413759841766, "acc": 0.8396946564885496, "acc_k": 0.8376068376068376, "ece": 0.07992691961863568, "sel_acc80": 0.9238095238095239, "conf_wrong": 0.0, "coverage": 0.8549618320610687, "false_abstain": 0.042735042735042736, "null_recall": 1.0, "auroc_null": 0.9865689865689865, "auroc_conf": 0.9847374847374847, "n": 131 }, "subscription_cancellation": { "nll": 0.25282438503639165, "brier": 0.14983405363600533, "jsd": 0.06734748559881597, "acc": 0.8791946308724832, "acc_k": 0.9318181818181818, "ece": 0.04965120893996835, "sel_acc80": 0.9666666666666667, "conf_wrong": 0.0, "coverage": 0.8523489932885906, "false_abstain": 0.06060606060606061, "null_recall": 0.8235294117647058, "auroc_null": 0.9795008912655971, "auroc_conf": 0.9714795008912657, "n": 149 } }, "by_qtype": {}, "by_label_class": { "answer": { "nll": 0.3270902509793986, "brier": 0.1794731921212136, "jsd": 0.07638232337116445, "acc": 0.8726790450928382, "acc_k": 0.9045092838196287, "ece": 0.04732005681536247, "sel_acc80": 0.9403973509933775, "coverage": 0.9522546419098143, "false_abstain": 0.04774535809018567, "null_recall": "n/a", "n": 377 }, "null": { "nll": 0.2983844634705046, "brier": 0.16308191867857966, "jsd": 0.08967311768533967, "acc": 0.8888888888888888, "ece": 0.13434556325276692, "sel_acc80": 0.9722222222222222, "conf_wrong": 0.0, "coverage": 0.11111111111111116, "false_abstain": "n/a", "null_recall": 0.8888888888888888, "n": 45 } }, "baselines": { "majority_class": 0, "majority_acc": 0.38388625592417064 } }, "wh_heldout_grammar": { "n": 816, "n_total": 816, "suffix_overflow": 0, "raw": { "nll": 0.2472013947656661, "brier": 0.14714674629576405, "jsd": 0.0655881649228148, "acc": 0.8933823529411765, "acc_k": 0.925414364640884, "ece": 0.02050217598968861, "sel_acc80": 0.9647779479326187, "conf_wrong": 0.0, "coverage": 0.8455882352941176, "false_abstain": 0.05386740331491713, "null_recall": 0.9456521739130435, "auroc_null": 0.9844012731203459, "auroc_conf": 0.9806629834254144 }, "by_family": { "visa_extension": { "nll": 0.3876584563899138, "brier": 0.23389678729229724, "jsd": 0.10288573687884103, "acc": 0.851063829787234, "acc_k": 0.8782608695652174, "ece": 0.0661268077843578, "sel_acc80": 0.9203539823008849, "conf_wrong": 0.0, "coverage": 0.7588652482269503, "false_abstain": 0.0782608695652174, "null_recall": 0.9615384615384616, "auroc_null": 0.9709030100334448, "auroc_conf": 0.9692307692307692, "n": 141 }, "loan_approval": { "nll": 7.454925248028152e-05, "brier": 1.315695971451629e-07, "jsd": 2.5833865575709257e-05, "acc": 1.0, "acc_k": 1.0, "ece": 7.45158125884382e-05, "sel_acc80": 1.0, "conf_wrong": 0.0, "coverage": 0.9708029197080292, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 137 }, "content_moderation": { "nll": 0.5935040969149519, "brier": 0.3596948371934755, "jsd": 0.16132759921807785, "acc": 0.703125, "acc_k": 0.8247422680412371, "ece": 0.10629645804874599, "sel_acc80": 0.7864077669902912, "conf_wrong": 0.0, "coverage": 0.5703125, "false_abstain": 0.26804123711340205, "null_recall": 0.9354838709677419, "auroc_null": 0.9002327901563019, "auroc_conf": 0.9068839374792153, "n": 128 }, "warranty_claim": { "nll": 0.04521754519140885, "brier": 0.030164132379325935, "jsd": 0.012534975904067302, "acc": 0.971830985915493, "acc_k": 0.9710144927536232, "ece": 0.025068008563887925, "sel_acc80": 1.0, "conf_wrong": 0.0, "coverage": 0.971830985915493, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 142 }, "shift_swap": { "nll": 0.31995094781288197, "brier": 0.1809786294088722, "jsd": 0.07359853995759635, "acc": 0.8615384615384616, "acc_k": 0.8907563025210085, "ece": 0.054814853805762016, "sel_acc80": 0.9423076923076923, "conf_wrong": 0.0, "coverage": 0.9, "false_abstain": 0.03361344537815126, "null_recall": 0.8181818181818182, "auroc_null": 0.985485103132162, "auroc_conf": 0.9572192513368983, "n": 130 }, "vendor_onboarding": { "nll": 0.16712495284237217, "brier": 0.09594791421965362, "jsd": 0.05081018090389403, "acc": 0.9565217391304348, "acc_k": 0.9508196721311475, "ece": 0.07468755521636082, "sel_acc80": 0.990990990990991, "conf_wrong": 0.0, "coverage": 0.8840579710144928, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 138 } }, "by_qtype": {}, "by_label_class": { "null": { "nll": 0.277453074058921, "brier": 0.14561172018866156, "jsd": 0.08687484324735249, "acc": 0.9456521739130435, "ece": 0.1358135566115379, "sel_acc80": 0.972972972972973, "conf_wrong": 0.0, "coverage": 0.05434782608695654, "false_abstain": "n/a", "null_recall": 0.9456521739130435, "n": 92 }, "answer": { "nll": 0.24335725872287678, "brier": 0.14734180486185994, "jsd": 0.06288322789814979, "acc": 0.8867403314917127, "acc_k": 0.925414364640884, "ece": 0.03288542010013568, "sel_acc80": 0.9706896551724138, "coverage": 0.9461325966850829, "false_abstain": 0.05386740331491713, "null_recall": "n/a", "n": 724 } }, "baselines": { "majority_class": 0, "majority_acc": 0.38480392156862747 } }, "wh_heldout_style": { "n": 403, "n_total": 403, "suffix_overflow": 0, "raw": { "nll": 0.26691198757004164, "brier": 0.1464047042686628, "jsd": 0.0662673268053048, "acc": 0.890818858560794, "acc_k": 0.890818858560794, "ece": 0.031059944674039948, "sel_acc80": 0.9845201238390093, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "feature_flag_rollout": { "nll": 0.19010573836138, "brier": 0.11432481201229953, "jsd": 0.05277888022237791, "acc": 0.9090909090909091, "acc_k": 0.9090909090909091, "ece": 0.039632322571494384, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 55 }, "shift_swap": { "nll": 0.3043634096913596, "brier": 0.1503282508374881, "jsd": 0.06654764978820293, "acc": 0.8939393939393939, "acc_k": 0.8939393939393939, "ece": 0.062232637044155215, "sel_acc80": 0.9811320754716981, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 66 }, "content_moderation": { "nll": 0.4789173250257905, "brier": 0.26149167112429633, "jsd": 0.1279300537119714, "acc": 0.813953488372093, "acc_k": 0.813953488372093, "ece": 0.06183511365291682, "sel_acc80": 0.8571428571428571, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 43 }, "insurance_claim": { "nll": 0.29568089754215104, "brier": 0.16360065632472767, "jsd": 0.06646606171635495, "acc": 0.8529411764705882, "acc_k": 0.8529411764705882, "ece": 0.1020367794177111, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 34 }, "visa_extension": { "nll": 0.09558577342528488, "brier": 0.04933176843684583, "jsd": 0.02922906947317498, "acc": 0.972972972972973, "acc_k": 0.972972972972973, "ece": 0.03434973310779885, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 37 }, "warranty_claim": { "nll": 0.31847504141119937, "brier": 0.1722281656536195, "jsd": 0.07974897957552128, "acc": 0.8813559322033898, "acc_k": 0.8813559322033898, "ece": 0.03718661453764316, "sel_acc80": 0.9791666666666666, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 59 }, "loan_approval": { "nll": 0.12007633533318733, "brier": 0.06839648073371568, "jsd": 0.027442892286818, "acc": 0.9583333333333334, "acc_k": 0.9583333333333334, "ece": 0.04480526472131414, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 48 }, "vendor_onboarding": { "nll": 0.299750322830979, "brier": 0.17565957622934011, "jsd": 0.07452437378826761, "acc": 0.8524590163934426, "acc_k": 0.8524590163934426, "ece": 0.08769017262536971, "sel_acc80": 0.9591836734693877, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 61 } }, "by_qtype": {}, "by_label_class": { "answer": { "nll": 0.26691198757004164, "brier": 0.1464047042686628, "jsd": 0.0662673268053048, "acc": 0.890818858560794, "acc_k": 0.890818858560794, "ece": 0.031059944674039948, "sel_acc80": 0.9845201238390093, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 403 } }, "baselines": { "majority_class": 1, "majority_acc": 0.4267990074441687 } }, "wh_level7": { "n": 880, "n_total": 880, "suffix_overflow": 0, "raw": { "nll": 0.8425315491747045, "brier": 0.566066637960518, "jsd": 0.22709053783966102, "acc": 0.5443181818181818, "acc_k": 0.5443181818181818, "ece": 0.17712931768460705, "sel_acc80": 0.5738636363636364, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a" }, "by_family": { "temporal": { "nll": 0.7525894214290174, "brier": 0.5283106465099253, "jsd": 0.20930622376758312, "acc": 0.5727272727272728, "acc_k": 0.5727272727272728, "ece": 0.18158261044458907, "sel_acc80": 0.6193181818181818, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "probability": { "nll": 0.804850408692942, "brier": 0.5061907772166189, "jsd": 0.22027018987749325, "acc": 0.5727272727272728, "acc_k": 0.5727272727272728, "ece": 0.12663529461080378, "sel_acc80": 0.6136363636363636, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "numeric": { "nll": 0.854408628287633, "brier": 0.6255809344308126, "jsd": 0.24148170952539819, "acc": 0.4909090909090909, "acc_k": 0.4909090909090909, "ece": 0.24755196327512913, "sel_acc80": 0.4772727272727273, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "tradeoff": { "nll": 0.9582777382892255, "brier": 0.6041841936847155, "jsd": 0.23730402818816942, "acc": 0.5409090909090909, "acc_k": 0.5409090909090909, "ece": 0.21417698643424293, "sel_acc80": 0.5965909090909091, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 } }, "by_qtype": {}, "by_label_class": { "answer": { "nll": 0.8425315491747045, "brier": 0.566066637960518, "jsd": 0.22709053783966102, "acc": 0.5443181818181818, "acc_k": 0.5443181818181818, "ece": 0.17712931768460705, "sel_acc80": 0.5738636363636364, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 880 } }, "baselines": { "majority_class": 1, "majority_acc": 0.4602272727272727 } }, "wh_rubric_flip": { "n": 2206, "n_total": 2206, "suffix_overflow": 0, "raw": { "nll": 0.5215484658596726, "brier": 0.33112145645101976, "jsd": 0.13730397267005034, "acc": 0.742973708068903, "acc_k": 0.7501196744853997, "ece": 0.08118663341860714, "sel_acc80": 0.8164305949008499, "conf_wrong": 0.0, "coverage": 0.9301903898458749, "false_abstain": 0.022498803255146003, "null_recall": 0.9145299145299145, "auroc_null": 0.9924267530777824, "auroc_conf": 0.9735570530209114 }, "by_family": { "loan_approval": { "nll": 0.03986105987120578, "brier": 0.022708606037172275, "jsd": 0.00911794257520729, "acc": 0.9861111111111112, "acc_k": 0.9858156028368794, "ece": 0.014913849118683096, "sel_acc80": 1.0, "conf_wrong": 0.0, "coverage": 0.9791666666666666, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 144 }, "shift_swap": { "nll": 0.3556272740222601, "brier": 0.19030986550930082, "jsd": 0.07994201505668534, "acc": 0.8554216867469879, "acc_k": 0.8782051282051282, "ece": 0.05131704243550819, "sel_acc80": 0.9398496240601504, "conf_wrong": 0.0, "coverage": 0.927710843373494, "false_abstain": 0.02564102564102564, "null_recall": 0.8, "auroc_null": 0.9878205128205129, "auroc_conf": 0.948076923076923, "n": 166 }, "vendor_onboarding": { "nll": 0.2460892508917783, "brier": 0.14163160753660067, "jsd": 0.06893476279267717, "acc": 0.9102564102564102, "acc_k": 0.9014084507042254, "ece": 0.054101008826341355, "sel_acc80": 0.976, "conf_wrong": 0.0, "coverage": 0.9102564102564102, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 156 }, "content_moderation": { "nll": 0.5707868104784541, "brier": 0.3399881864130089, "jsd": 0.1565919344337606, "acc": 0.7297297297297297, "acc_k": 0.8166666666666667, "ece": 0.07247401008734834, "sel_acc80": 0.7983193277310925, "conf_wrong": 0.0, "coverage": 0.6891891891891893, "false_abstain": 0.16666666666666666, "null_recall": 0.9285714285714286, "auroc_null": 0.9428571428571428, "auroc_conf": 0.9383928571428571, "n": 148 }, "warranty_claim": { "nll": 0.1533494565143874, "brier": 0.08736313763136712, "jsd": 0.038794858019505045, "acc": 0.935064935064935, "acc_k": 0.9342105263157895, "ece": 0.03250992956099568, "sel_acc80": 1.0, "conf_wrong": 0.0, "coverage": 0.987012987012987, "false_abstain": 0.0, "null_recall": 1.0, "auroc_null": 1.0, "auroc_conf": 1.0, "n": 154 }, "visa_extension": { "nll": 0.3792518490833633, "brier": 0.22995354363423323, "jsd": 0.10051831683881524, "acc": 0.8493150684931506, "acc_k": 0.8790322580645161, "ece": 0.06351610569104751, "sel_acc80": 0.9230769230769231, "conf_wrong": 0.0, "coverage": 0.7945205479452055, "false_abstain": 0.07258064516129033, "null_recall": 0.9545454545454546, "auroc_null": 0.9736070381231671, "auroc_conf": 0.9721407624633431, "n": 146 }, "procurement_request": { "nll": 0.3881801381792412, "brier": 0.18757470893100156, "jsd": 0.07515705849986501, "acc": 0.8909090909090909, "acc_k": 0.9292929292929293, "ece": 0.08982028690251438, "sel_acc80": 0.9204545454545454, "conf_wrong": 0.0, "coverage": 0.8818181818181818, "false_abstain": 0.04040404040404041, "null_recall": 0.8181818181818182, "auroc_null": 0.977961432506887, "auroc_conf": 0.9687786960514233, "n": 110 }, "research_ethics_review": { "nll": 0.4421237581343384, "brier": 0.24773885698642706, "jsd": 0.11269835014231615, "acc": 0.8333333333333334, "acc_k": 0.8315789473684211, "ece": 0.103033944412514, "sel_acc80": 0.9080459770114943, "conf_wrong": 0.0, "coverage": 0.8425925925925926, "false_abstain": 0.042105263157894736, "null_recall": 1.0, "auroc_null": 0.9854251012145749, "auroc_conf": 0.9838056680161944, "n": 108 }, "subscription_cancellation": { "nll": 0.2644555372286351, "brier": 0.15680793083414146, "jsd": 0.06982178903755307, "acc": 0.8728813559322034, "acc_k": 0.9326923076923077, "ece": 0.061388740095041576, "sel_acc80": 0.9578947368421052, "conf_wrong": 0.0, "coverage": 0.8559322033898304, "false_abstain": 0.057692307692307696, "null_recall": 0.7857142857142857, "auroc_null": 0.9745879120879122, "auroc_conf": 0.9684065934065934, "n": 118 }, "feature_flag_rollout": { "nll": 0.2267314556395052, "brier": 0.1367964508738713, "jsd": 0.06290056031954905, "acc": 0.8913043478260869, "acc_k": 0.8913043478260869, "ece": 0.04663619204707767, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 46 }, "insurance_claim": { "nll": 0.33329503028518, "brier": 0.185439438184308, "jsd": 0.07467994362635967, "acc": 0.8333333333333334, "acc_k": 0.8333333333333334, "ece": 0.11369339625040695, "sel_acc80": 1.0, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 30 }, "temporal": { "nll": 0.7525550379081508, "brier": 0.5281081163658832, "jsd": 0.20925588567068154, "acc": 0.5727272727272728, "acc_k": 0.5727272727272728, "ece": 0.1753265808929097, "sel_acc80": 0.625, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "numeric": { "nll": 0.8554757856422319, "brier": 0.6267431100109008, "jsd": 0.2417873556438029, "acc": 0.4909090909090909, "acc_k": 0.4909090909090909, "ece": 0.2482705327597531, "sel_acc80": 0.4659090909090909, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "probability": { "nll": 0.8047299060130065, "brier": 0.5061678304732362, "jsd": 0.220257933543653, "acc": 0.5727272727272728, "acc_k": 0.5727272727272728, "ece": 0.11572440903295172, "sel_acc80": 0.6136363636363636, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "tradeoff": { "nll": 0.9591729335220512, "brier": 0.6044553194922022, "jsd": 0.2374179176613618, "acc": 0.5454545454545454, "acc_k": 0.5454545454545454, "ece": 0.20948043235323643, "sel_acc80": 0.5909090909090909, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 } }, "by_qtype": {}, "by_label_class": { "answer": { "nll": 0.5344578349380356, "brier": 0.340925948758752, "jsd": 0.13998262548140297, "acc": 0.7333652465294399, "acc_k": 0.7501196744853997, "ece": 0.09119710230610484, "sel_acc80": 0.8068181818181818, "coverage": 0.977501196744854, "false_abstain": 0.022498803255146003, "null_recall": "n/a", "n": 2089 }, "null": { "nll": 0.2910555427425767, "brier": 0.15606517926424532, "jsd": 0.08947742802974543, "acc": 0.9145299145299145, "ece": 0.1208790940606696, "sel_acc80": 0.9680851063829787, "conf_wrong": 0.0, "coverage": 0.0854700854700855, "false_abstain": "n/a", "null_recall": 0.9145299145299145, "n": 117 } }, "baselines": { "majority_class": 0, "majority_acc": 0.39981867633726204 } }, "wh_rubric_shuffled": { "n": 2437, "n_total": 2437, "suffix_overflow": 0, "raw": { "nll": 3.3731109702781983, "brier": 1.0166531838701214, "jsd": 0.3668337750954112, "acc": 0.37792367665162085, "acc_k": 0.4059104737070839, "ece": 0.46684057980616855, "sel_acc80": 0.3553846153846154, "conf_wrong": 0.17647058823529413, "coverage": 0.926138695116947, "false_abstain": 0.049109083007388095, "null_recall": 0.49264705882352944, "auroc_null": 0.9573203466523506, "auroc_conf": 0.8061951964618963 }, "by_family": { "temporal": { "nll": 1.0978670096057097, "brier": 0.7736457181852393, "jsd": 0.2832470099926861, "acc": 0.4090909090909091, "acc_k": 0.4090909090909091, "ece": 0.34149774312973025, "sel_acc80": 0.3693181818181818, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "shift_swap": { "nll": 4.813958214372273, "brier": 1.1627913595629051, "jsd": 0.41357994903252204, "acc": 0.35602094240837695, "acc_k": 0.37222222222222223, "ece": 0.5618253638919112, "sel_acc80": 0.35294117647058826, "conf_wrong": 0.2727272727272727, "coverage": 0.9319371727748691, "false_abstain": 0.044444444444444446, "null_recall": 0.45454545454545453, "auroc_null": 0.961111111111111, "auroc_conf": 0.8565656565656565, "n": 191 }, "vendor_onboarding": { "nll": 4.468025309710248, "brier": 1.1531663283624074, "jsd": 0.4069967513654248, "acc": 0.31958762886597936, "acc_k": 0.39325842696629215, "ece": 0.5820424177290239, "sel_acc80": 0.34615384615384615, "conf_wrong": 0.1875, "coverage": 0.9175257731958762, "false_abstain": 0.07303370786516854, "null_recall": 0.1875, "auroc_null": 0.9371488764044944, "auroc_conf": 0.8433988764044944, "n": 194 }, "loan_approval": { "nll": 7.430180710009384, "brier": 1.3288401062960586, "jsd": 0.4609968152472018, "acc": 0.33146067415730335, "acc_k": 0.3505747126436782, "ece": 0.6675498291347803, "sel_acc80": 0.3356643356643357, "conf_wrong": 0.5, "coverage": 0.9662921348314607, "false_abstain": 0.022988505747126436, "null_recall": 0.5, "auroc_null": 0.978448275862069, "auroc_conf": 0.9396551724137931, "n": 178 }, "probability": { "nll": 1.5095582926439517, "brier": 0.9146099048716316, "jsd": 0.35344105098573364, "acc": 0.35, "acc_k": 0.35, "ece": 0.3669138718735088, "sel_acc80": 0.3181818181818182, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "feature_flag_rollout": { "nll": 4.963032018178103, "brier": 1.250659962667729, "jsd": 0.4386572342122217, "acc": 0.34782608695652173, "acc_k": 0.34782608695652173, "ece": 0.6342515031928602, "sel_acc80": 0.3783783783783784, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 46 }, "research_ethics_review": { "nll": 3.379301129427671, "brier": 1.1293730274385836, "jsd": 0.4101792599802621, "acc": 0.31746031746031744, "acc_k": 0.38392857142857145, "ece": 0.5505591124769241, "sel_acc80": 0.3069306930693069, "conf_wrong": 0.07142857142857142, "coverage": 0.8492063492063492, "false_abstain": 0.125, "null_recall": 0.35714285714285715, "auroc_null": 0.8852040816326531, "auroc_conf": 0.7633928571428572, "n": 126 }, "warranty_claim": { "nll": 5.425085813770316, "brier": 1.222989340354703, "jsd": 0.42426978503284835, "acc": 0.35978835978835977, "acc_k": 0.3675675675675676, "ece": 0.6112382035091439, "sel_acc80": 0.3684210526315789, "conf_wrong": 0.5, "coverage": 0.9788359788359788, "false_abstain": 0.010810810810810811, "null_recall": 0.5, "auroc_null": 0.9905405405405405, "auroc_conf": 0.8783783783783783, "n": 189 }, "numeric": { "nll": 0.7009633054040385, "brier": 0.5083317633419103, "jsd": 0.19623566233246967, "acc": 0.5636363636363636, "acc_k": 0.5636363636363636, "ece": 0.20822090967134996, "sel_acc80": 0.5795454545454546, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "content_moderation": { "nll": 2.532750401458349, "brier": 0.9723702132508161, "jsd": 0.3676846987407566, "acc": 0.3670886075949367, "acc_k": 0.4094488188976378, "ece": 0.4052060521856139, "sel_acc80": 0.3464566929133858, "conf_wrong": 0.06451612903225806, "coverage": 0.6645569620253164, "false_abstain": 0.23622047244094488, "null_recall": 0.7419354838709677, "auroc_null": 0.8270256540513081, "auroc_conf": 0.7680975361950725, "n": 158 }, "visa_extension": { "nll": 3.4319388280589163, "brier": 1.185777007774496, "jsd": 0.4211514399505007, "acc": 0.3023255813953488, "acc_k": 0.3698630136986301, "ece": 0.5790469986061717, "sel_acc80": 0.2971014492753623, "conf_wrong": 0.11538461538461539, "coverage": 0.8023255813953488, "false_abstain": 0.14383561643835616, "null_recall": 0.5, "auroc_null": 0.8730242360379347, "auroc_conf": 0.8132244467860906, "n": 172 }, "subscription_cancellation": { "nll": 4.124055456988184, "brier": 1.142628609921072, "jsd": 0.40527967431266604, "acc": 0.3546099290780142, "acc_k": 0.424, "ece": 0.5671253876483187, "sel_acc80": 0.35398230088495575, "conf_wrong": 0.3125, "coverage": 0.8652482269503546, "false_abstain": 0.096, "null_recall": 0.4375, "auroc_null": 0.9035, "auroc_conf": 0.801, "n": 141 }, "procurement_request": { "nll": 5.064798743828172, "brier": 1.1244811567056938, "jsd": 0.3945059149254654, "acc": 0.37593984962406013, "acc_k": 0.4369747899159664, "ece": 0.5752717774165303, "sel_acc80": 0.3925233644859813, "conf_wrong": 0.21428571428571427, "coverage": 0.8796992481203008, "false_abstain": 0.07563025210084033, "null_recall": 0.5, "auroc_null": 0.9357743097238895, "auroc_conf": 0.8715486194477791, "n": 133 }, "tradeoff": { "nll": 1.295216810789425, "brier": 0.8154554789002251, "jsd": 0.30319613299194453, "acc": 0.4318181818181818, "acc_k": 0.4318181818181818, "ece": 0.35886814662001354, "sel_acc80": 0.3806818181818182, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 220 }, "insurance_claim": { "nll": 5.987324790766054, "brier": 1.150424499572334, "jsd": 0.407765815124327, "acc": 0.41379310344827586, "acc_k": 0.41379310344827586, "ece": 0.5872780434016523, "sel_acc80": 0.4166666666666667, "coverage": 1.0, "false_abstain": 0.0, "null_recall": "n/a", "n": 29 } }, "by_qtype": {}, "by_label_class": { "answer": { "nll": 3.4790294568471882, "brier": 1.0318484361922742, "jsd": 0.3710045446095289, "acc": 0.37114298131247286, "acc_k": 0.4059104737070839, "ece": 0.47994829282352375, "sel_acc80": 0.3508962520369365, "coverage": 0.9508909169926119, "false_abstain": 0.049109083007388095, "null_recall": "n/a", "n": 2301 }, "null": { "nll": 1.5810636350190332, "brier": 0.7595629221548795, "jsd": 0.2962680350072858, "acc": 0.49264705882352944, "ece": 0.3134835373391124, "sel_acc80": 0.45871559633027525, "conf_wrong": 0.17647058823529413, "coverage": 0.5073529411764706, "false_abstain": "n/a", "null_recall": 0.49264705882352944, "n": 136 } }, "baselines": { "majority_class": 0, "majority_acc": 0.4095199015182602 } } } }