| { |
| "objective": -0.5112698517216128, |
| "paired_flip": { |
| "pairs": 80, |
| "flip_rate": 0.675, |
| "both_correct_rate": 0.625, |
| "invariant_pairs": 32, |
| "invariance_rate": 0.96875, |
| "invariant_both_correct_rate": 0.59375 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 1264, |
| "nll": 0.4989297577819544, |
| "acc": 0.8243670886075949, |
| "ece": 0.05704976909178335, |
| "brier": 0.25535485232026645, |
| "mean_conf": 0.8785235750178558, |
| "confident_error_rate": 0.040348101265822785, |
| "coverage_at_0_9": 0.6566455696202531, |
| "accuracy_at_0_9": 0.9385542168674699, |
| "coverage_at_5pct_error": 0.6337025316455697, |
| "coverage_at_1pct_error": 0.18908227848101267, |
| "aurc": 0.057098658886616284, |
| "error_rate_at_0_9": 0.06144578313253012, |
| "confidence_bias": 0.05415648641026094, |
| "top_bins": { |
| "0.9": { |
| "n": 830, |
| "errors": 51, |
| "error_rate": 0.06144578313253012 |
| }, |
| "0.95": { |
| "n": 765, |
| "errors": 32, |
| "error_rate": 0.04183006535947712 |
| }, |
| "0.99": { |
| "n": 616, |
| "errors": 21, |
| "error_rate": 0.03409090909090909 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9651898734177216, |
| "confidence_cutoff": 0.9884021799746481 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.8913043478260869, |
| "confidence_cutoff": 0.7007823799483796 |
| } |
| }, |
| "score_mae": 0.44288336559433733, |
| "ranked_probability_score": 0.07198397780180195 |
| }, |
| "tasks": { |
| "agnews": { |
| "n": 80, |
| "nll": 0.7033810609329125, |
| "acc": 0.85, |
| "ece": 0.12788153266590807, |
| "brier": 0.2690347009363626, |
| "mean_conf": 0.9626452504559462, |
| "confident_error_rate": 0.1125, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 0.875, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.175, |
| "aurc": 0.06388995806168134, |
| "error_rate_at_0_9": 0.125, |
| "confidence_bias": 0.1126452504559462, |
| "top_bins": { |
| "0.9": { |
| "n": 72, |
| "errors": 9, |
| "error_rate": 0.125 |
| }, |
| "0.95": { |
| "n": 66, |
| "errors": 6, |
| "error_rate": 0.09090909090909091 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 5, |
| "error_rate": 0.08196721311475409 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9983029203561053 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.921875, |
| "confidence_cutoff": 0.9797347727687692 |
| } |
| } |
| }, |
| "agnews_yn": { |
| "n": 160, |
| "nll": 0.218266217089547, |
| "acc": 0.925, |
| "ece": 0.053822844561075917, |
| "brier": 0.12359771772019439, |
| "mean_conf": 0.9648416375603937, |
| "confident_error_rate": 0.03125, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 0.9652777777777778, |
| "coverage_at_5pct_error": 0.94375, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.013180196924715198, |
| "error_rate_at_0_9": 0.034722222222222224, |
| "confidence_bias": 0.039841637560393606, |
| "top_bins": { |
| "0.9": { |
| "n": 144, |
| "errors": 5, |
| "error_rate": 0.034722222222222224 |
| }, |
| "0.95": { |
| "n": 132, |
| "errors": 3, |
| "error_rate": 0.022727272727272728 |
| }, |
| "0.99": { |
| "n": 98, |
| "errors": 1, |
| "error_rate": 0.01020408163265306 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9943423697163807 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9765625, |
| "confidence_cutoff": 0.962791422719902 |
| } |
| } |
| }, |
| "amazon": { |
| "n": 80, |
| "nll": 0.9101254335951751, |
| "acc": 0.65, |
| "ece": 0.09852625629411788, |
| "brier": 0.4737080597217712, |
| "mean_conf": 0.7311588013609494, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.2125, |
| "accuracy_at_0_9": 0.9411764705882353, |
| "coverage_at_5pct_error": 0.1875, |
| "coverage_at_1pct_error": 0.1875, |
| "aurc": 0.16015706656418405, |
| "error_rate_at_0_9": 0.058823529411764705, |
| "confidence_bias": 0.08115880136094933, |
| "top_bins": { |
| "0.9": { |
| "n": 17, |
| "errors": 1, |
| "error_rate": 0.058823529411764705 |
| }, |
| "0.95": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.85, |
| "confidence_cutoff": 0.7263951073590981 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.5548622064540625 |
| } |
| }, |
| "score_mae": 0.45330552002269775, |
| "ranked_probability_score": 0.07559174675562505 |
| }, |
| "banking77": { |
| "n": 80, |
| "nll": 0.3979839296889442, |
| "acc": 0.875, |
| "ece": 0.0922611325054158, |
| "brier": 0.1822296579454367, |
| "mean_conf": 0.8923134569904196, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.725, |
| "accuracy_at_0_9": 0.9482758620689655, |
| "coverage_at_5pct_error": 0.825, |
| "coverage_at_1pct_error": 0.675, |
| "aurc": 0.02199412946893899, |
| "error_rate_at_0_9": 0.05172413793103448, |
| "confidence_bias": 0.01731345699041964, |
| "top_bins": { |
| "0.9": { |
| "n": 58, |
| "errors": 3, |
| "error_rate": 0.05172413793103448 |
| }, |
| "0.95": { |
| "n": 52, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 41, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9931638954155307 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.953125, |
| "confidence_cutoff": 0.845837989712747 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 80, |
| "nll": 0.5220096518672049, |
| "acc": 0.8125, |
| "ece": 0.14833387100163806, |
| "brier": 0.2951043064516692, |
| "mean_conf": 0.9369053870550708, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.8, |
| "accuracy_at_0_9": 0.890625, |
| "coverage_at_5pct_error": 0.35, |
| "coverage_at_1pct_error": 0.2625, |
| "aurc": 0.07004308805415525, |
| "error_rate_at_0_9": 0.109375, |
| "confidence_bias": 0.12440538705507076, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 7, |
| "error_rate": 0.109375 |
| }, |
| "0.95": { |
| "n": 57, |
| "errors": 4, |
| "error_rate": 0.07017543859649122 |
| }, |
| "0.99": { |
| "n": 33, |
| "errors": 3, |
| "error_rate": 0.09090909090909091 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.925, |
| "confidence_cutoff": 0.9818334678368117 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.890625, |
| "confidence_cutoff": 0.906916795952237 |
| } |
| } |
| }, |
| "composition_atom": { |
| "n": 16, |
| "nll": 0.52206201525939, |
| "acc": 0.8125, |
| "ece": 0.33102741792619084, |
| "brier": 0.34727627266455574, |
| "mean_conf": 0.678589841408767, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0625, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.1089238886113886, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.133910158591233, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.7019797950393023 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.7692307692307693, |
| "confidence_cutoff": 0.5925620021652579 |
| } |
| } |
| }, |
| "composition_conditional": { |
| "n": 16, |
| "nll": 0.7085893013145462, |
| "acc": 0.5, |
| "ece": 0.0392985537546805, |
| "brier": 0.5153023405509731, |
| "mean_conf": 0.5392985537546805, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.6738095238095239, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.0392985537546805, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.25, |
| "confidence_cutoff": 0.5389107305136105 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.5714285714285714, |
| "confidence_cutoff": 0.5247236050583006 |
| } |
| } |
| }, |
| "composition_conjunction": { |
| "n": 16, |
| "nll": 0.6954926808023254, |
| "acc": 0.6875, |
| "ece": 0.28356235633747884, |
| "brier": 0.5005900249196042, |
| "mean_conf": 0.5960339565951683, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.571116296897547, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.09146604340483167, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.375, |
| "confidence_cutoff": 0.5774681144198214 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.6428571428571429, |
| "confidence_cutoff": 0.5598945450814203 |
| } |
| } |
| }, |
| "composition_disjunction": { |
| "n": 16, |
| "nll": 0.6223727318080233, |
| "acc": 0.5625, |
| "ece": 0.22532601951230213, |
| "brier": 0.4331353693805437, |
| "mean_conf": 0.5895230696045874, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.3125, |
| "coverage_at_1pct_error": 0.3125, |
| "aurc": 0.22883184523809524, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.027023069604587446, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5625, |
| "accuracy": 0.6666666666666666, |
| "confidence_cutoff": 0.5856707096099854 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.6428571428571429, |
| "confidence_cutoff": 0.5193371772766113 |
| } |
| } |
| }, |
| "composition_exception": { |
| "n": 16, |
| "nll": 0.6747223985633183, |
| "acc": 0.6875, |
| "ece": 0.16649871606180544, |
| "brier": 0.4815999757579885, |
| "mean_conf": 0.5210012839381946, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.3167455808080808, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.16649871606180544, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5625, |
| "accuracy": 0.7777777777777778, |
| "confidence_cutoff": 0.518760085105896 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.6428571428571429, |
| "confidence_cutoff": 0.5035156757687234 |
| } |
| } |
| }, |
| "composition_negation": { |
| "n": 16, |
| "nll": 0.7403130749553563, |
| "acc": 0.5, |
| "ece": 0.08891041545292032, |
| "brier": 0.5451303525712351, |
| "mean_conf": 0.5889104154529203, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.6560387529137529, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.08891041545292033, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.375, |
| "confidence_cutoff": 0.5939869638673642 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.46153846153846156, |
| "confidence_cutoff": 0.5687558393928909 |
| } |
| } |
| }, |
| "composition_nested_and": { |
| "n": 16, |
| "nll": 0.6701464137220343, |
| "acc": 0.5, |
| "ece": 0.1743572557807083, |
| "brier": 0.4801171644212976, |
| "mean_conf": 0.5974301225035049, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.25, |
| "aurc": 0.37770632145632144, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.09743012250350491, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5888296365737915 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.46153846153846156, |
| "confidence_cutoff": 0.5511568948971299 |
| } |
| } |
| }, |
| "composition_nested_or": { |
| "n": 16, |
| "nll": 0.5812368508611689, |
| "acc": 0.5625, |
| "ece": 0.232265838198384, |
| "brier": 0.39731880250019436, |
| "mean_conf": 0.596988431257154, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.375, |
| "coverage_at_1pct_error": 0.375, |
| "aurc": 0.1931575369075369, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.03448843125715395, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.6048937439918518 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.6923076923076923, |
| "confidence_cutoff": 0.5087750402458 |
| } |
| } |
| }, |
| "contrastive_age_eligibility": { |
| "n": 24, |
| "nll": 0.00030557725656388685, |
| "acc": 1.0, |
| "ece": 0.0003054692741715215, |
| "brier": 4.3159530440543707e-07, |
| "mean_conf": 0.9996945307258285, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0003054692741715215, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998034213326141 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996426136047368 |
| } |
| } |
| }, |
| "contrastive_quantity_limit": { |
| "n": 24, |
| "nll": 0.0949641863436191, |
| "acc": 0.9583333333333334, |
| "ece": 0.06630730027669939, |
| "brier": 0.061578113407729874, |
| "mean_conf": 0.9503996922670156, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.8333333333333334, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9166666666666666, |
| "aurc": 0.0035477053140096615, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.007933641066317776, |
| "top_bins": { |
| "0.9": { |
| "n": 20, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 20, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999154857901148 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9568661440200333 |
| } |
| } |
| }, |
| "contrastive_return_window": { |
| "n": 24, |
| "nll": 0.5242983013148222, |
| "acc": 0.75, |
| "ece": 0.13728604635195948, |
| "brier": 0.3391539263874357, |
| "mean_conf": 0.8872860463519596, |
| "confident_error_rate": 0.08333333333333333, |
| "coverage_at_0_9": 0.6666666666666666, |
| "accuracy_at_0_9": 0.875, |
| "coverage_at_5pct_error": 0.4583333333333333, |
| "coverage_at_1pct_error": 0.4583333333333333, |
| "aurc": 0.08703314878649715, |
| "error_rate_at_0_9": 0.125, |
| "confidence_bias": 0.13728604635195962, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 2, |
| "error_rate": 0.125 |
| }, |
| "0.95": { |
| "n": 11, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9166666666666666, |
| "confidence_cutoff": 0.9489070242881575 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 0.8, |
| "confidence_cutoff": 0.7711345531891699 |
| } |
| } |
| }, |
| "contrastive_spend_threshold": { |
| "n": 24, |
| "nll": 9.237405628065089e-06, |
| "acc": 1.0, |
| "ece": 9.237352093016682e-06, |
| "brier": 2.021011459531347e-10, |
| "mean_conf": 0.999990762647907, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -9.237352093016682e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999922291820006 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999883904450512 |
| } |
| } |
| }, |
| "dbpedia14": { |
| "n": 80, |
| "nll": 0.3415167120067256, |
| "acc": 0.9625, |
| "ece": 0.043700092546254746, |
| "brier": 0.07457610595560957, |
| "mean_conf": 0.9898401932810058, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.9625, |
| "accuracy_at_0_9": 0.974025974025974, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.1375, |
| "aurc": 0.02543183035490516, |
| "error_rate_at_0_9": 0.025974025974025976, |
| "confidence_bias": 0.027340193281005787, |
| "top_bins": { |
| "0.9": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999976727949713 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.999980934132584 |
| } |
| } |
| }, |
| "imdb": { |
| "n": 80, |
| "nll": 0.24985978507417891, |
| "acc": 0.9375, |
| "ece": 0.05457930309714365, |
| "brier": 0.11189476167690375, |
| "mean_conf": 0.977379170939226, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 0.9473684210526315, |
| "coverage_at_5pct_error": 0.9375, |
| "coverage_at_1pct_error": 0.475, |
| "aurc": 0.013302134857181047, |
| "error_rate_at_0_9": 0.05263157894736842, |
| "confidence_bias": 0.039879170939225994, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 4, |
| "error_rate": 0.05263157894736842 |
| }, |
| "0.95": { |
| "n": 74, |
| "errors": 3, |
| "error_rate": 0.04054054054054054 |
| }, |
| "0.99": { |
| "n": 68, |
| "errors": 1, |
| "error_rate": 0.014705882352941176 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9992378023250749 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9947106490827637 |
| } |
| } |
| }, |
| "mnli": { |
| "n": 80, |
| "nll": 0.8964775304734388, |
| "acc": 0.825, |
| "ece": 0.16162789615757212, |
| "brier": 0.33240517794745345, |
| "mean_conf": 0.9328373683549026, |
| "confident_error_rate": 0.1375, |
| "coverage_at_0_9": 0.8, |
| "accuracy_at_0_9": 0.828125, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.1279801828240891, |
| "error_rate_at_0_9": 0.171875, |
| "confidence_bias": 0.10783736835490265, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 11, |
| "error_rate": 0.171875 |
| }, |
| "0.95": { |
| "n": 62, |
| "errors": 10, |
| "error_rate": 0.16129032258064516 |
| }, |
| "0.99": { |
| "n": 50, |
| "errors": 7, |
| "error_rate": 0.14 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.9951284115930806 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.828125, |
| "confidence_cutoff": 0.9465963914191013 |
| } |
| } |
| }, |
| "sst5": { |
| "n": 80, |
| "nll": 1.0021265818153486, |
| "acc": 0.6125, |
| "ece": 0.11978396394038553, |
| "brier": 0.5446736304135753, |
| "mean_conf": 0.700510761326975, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.075, |
| "accuracy_at_0_9": 0.8333333333333334, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.3072321995862583, |
| "error_rate_at_0_9": 0.16666666666666666, |
| "confidence_bias": 0.08801076132697494, |
| "top_bins": { |
| "0.9": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 1, |
| "error_rate": 1.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.675, |
| "confidence_cutoff": 0.7306530300819783 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.65625, |
| "confidence_cutoff": 0.5438117284883917 |
| } |
| }, |
| "score_mae": 0.5114106925506208, |
| "ranked_probability_score": 0.08295468856515971 |
| }, |
| "trec": { |
| "n": 80, |
| "nll": 0.2925501809265985, |
| "acc": 0.9375, |
| "ece": 0.05267290937483785, |
| "brier": 0.09750309073685226, |
| "mean_conf": 0.9901729093748377, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 0.9615384615384616, |
| "coverage_at_5pct_error": 0.975, |
| "coverage_at_1pct_error": 0.65, |
| "aurc": 0.010066847867815351, |
| "error_rate_at_0_9": 0.038461538461538464, |
| "confidence_bias": 0.052672909374837706, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 3, |
| "error_rate": 0.038461538461538464 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| }, |
| "0.99": { |
| "n": 74, |
| "errors": 2, |
| "error_rate": 0.02702702702702703 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998811671798993 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9989895525699338 |
| } |
| } |
| }, |
| "yelp": { |
| "n": 80, |
| "nll": 0.724927007873615, |
| "acc": 0.7125, |
| "ece": 0.12184401403681792, |
| "brier": 0.43845374961294875, |
| "mean_conf": 0.7841603935118646, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.25, |
| "accuracy_at_0_9": 0.9, |
| "coverage_at_5pct_error": 0.1625, |
| "coverage_at_1pct_error": 0.1625, |
| "aurc": 0.18107778703006866, |
| "error_rate_at_0_9": 0.1, |
| "confidence_bias": 0.0716603935118646, |
| "top_bins": { |
| "0.9": { |
| "n": 20, |
| "errors": 2, |
| "error_rate": 0.1 |
| }, |
| "0.95": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8, |
| "confidence_cutoff": 0.8030074284504592 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.734375, |
| "confidence_cutoff": 0.6472819362414997 |
| } |
| }, |
| "score_mae": 0.36393388420969347, |
| "ranked_probability_score": 0.0574054980846211 |
| }, |
| "yelp_yn": { |
| "n": 80, |
| "nll": 0.1767395803682206, |
| "acc": 0.925, |
| "ece": 0.04299234681271804, |
| "brier": 0.10751418779018915, |
| "mean_conf": 0.9602990730093268, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.8625, |
| "accuracy_at_0_9": 0.9855072463768116, |
| "coverage_at_5pct_error": 0.9125, |
| "coverage_at_1pct_error": 0.6875, |
| "aurc": 0.009561420798710252, |
| "error_rate_at_0_9": 0.014492753623188406, |
| "confidence_bias": 0.035299073009326776, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 1, |
| "error_rate": 0.014492753623188406 |
| }, |
| "0.95": { |
| "n": 64, |
| "errors": 1, |
| "error_rate": 0.015625 |
| }, |
| "0.99": { |
| "n": 49, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9945756192344355 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9567033266284315 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 1264, |
| "nll": 0.4989297577819544, |
| "acc": 0.8243670886075949, |
| "ece": 0.05704976909178335, |
| "brier": 0.25535485232026645, |
| "mean_conf": 0.8785235750178558, |
| "confident_error_rate": 0.040348101265822785, |
| "coverage_at_0_9": 0.6566455696202531, |
| "accuracy_at_0_9": 0.9385542168674699, |
| "coverage_at_5pct_error": 0.6337025316455697, |
| "coverage_at_1pct_error": 0.18908227848101267, |
| "aurc": 0.057098658886616284, |
| "error_rate_at_0_9": 0.06144578313253012, |
| "confidence_bias": 0.05415648641026094, |
| "top_bins": { |
| "0.9": { |
| "n": 830, |
| "errors": 51, |
| "error_rate": 0.06144578313253012 |
| }, |
| "0.95": { |
| "n": 765, |
| "errors": 32, |
| "error_rate": 0.04183006535947712 |
| }, |
| "0.99": { |
| "n": 616, |
| "errors": 21, |
| "error_rate": 0.03409090909090909 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9651898734177216, |
| "confidence_cutoff": 0.9884021799746481 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.8913043478260869, |
| "confidence_cutoff": 0.7007823799483796 |
| } |
| }, |
| "score_mae": 0.44288336559433733, |
| "ranked_probability_score": 0.07198397780180195 |
| }, |
| "none_absent": { |
| "n": 60, |
| "nll": 0.3973810280057748, |
| "acc": 0.8666666666666667, |
| "ece": 0.06408010262843164, |
| "brier": 0.19542789202339048, |
| "mean_conf": 0.9298361792245137, |
| "confident_error_rate": 0.016666666666666666, |
| "coverage_at_0_9": 0.7666666666666667, |
| "accuracy_at_0_9": 0.9782608695652174, |
| "coverage_at_5pct_error": 0.8, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.09434286048325893, |
| "error_rate_at_0_9": 0.021739130434782608, |
| "confidence_bias": 0.06316951255784697, |
| "top_bins": { |
| "0.9": { |
| "n": 46, |
| "errors": 1, |
| "error_rate": 0.021739130434782608 |
| }, |
| "0.95": { |
| "n": 40, |
| "errors": 1, |
| "error_rate": 0.025 |
| }, |
| "0.99": { |
| "n": 23, |
| "errors": 1, |
| "error_rate": 0.043478260869565216 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9666666666666667, |
| "confidence_cutoff": 0.9822965689718635 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9583333333333334, |
| "confidence_cutoff": 0.8703627257812488 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 60, |
| "nll": 0.8331048448864015, |
| "acc": 0.85, |
| "ece": 0.09958344599948121, |
| "brier": 0.25491634205576813, |
| "mean_conf": 0.8976962829199532, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.7, |
| "accuracy_at_0_9": 0.9285714285714286, |
| "coverage_at_5pct_error": 0.6666666666666666, |
| "coverage_at_1pct_error": 0.1, |
| "aurc": 0.0707109577846649, |
| "error_rate_at_0_9": 0.07142857142857142, |
| "confidence_bias": 0.0476962829199532, |
| "top_bins": { |
| "0.9": { |
| "n": 42, |
| "errors": 3, |
| "error_rate": 0.07142857142857142 |
| }, |
| "0.95": { |
| "n": 36, |
| "errors": 2, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.99": { |
| "n": 25, |
| "errors": 1, |
| "error_rate": 0.04 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9333333333333333, |
| "confidence_cutoff": 0.9711979045098904 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9166666666666666, |
| "confidence_cutoff": 0.8156208519156563 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 84, |
| "nll": 0.4638696687650698, |
| "acc": 0.9166666666666666, |
| "ece": 0.06166928073435357, |
| "brier": 0.14839515349641946, |
| "mean_conf": 0.9647966493434161, |
| "confident_error_rate": 0.047619047619047616, |
| "coverage_at_0_9": 0.9047619047619048, |
| "accuracy_at_0_9": 0.9473684210526315, |
| "coverage_at_5pct_error": 0.8214285714285714, |
| "coverage_at_1pct_error": 0.05952380952380952, |
| "aurc": 0.048011723747782474, |
| "error_rate_at_0_9": 0.05263157894736842, |
| "confidence_bias": 0.048129982676749505, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 4, |
| "error_rate": 0.05263157894736842 |
| }, |
| "0.95": { |
| "n": 72, |
| "errors": 4, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 3, |
| "error_rate": 0.04918032786885246 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9761904761904762, |
| "confidence_cutoff": 0.9974597361195856 |
| }, |
| "0.8": { |
| "coverage": 0.8095238095238095, |
| "accuracy": 0.9558823529411765, |
| "confidence_cutoff": 0.9721480404501962 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": {}, |
| "permutation": { |
| "n": 60, |
| "mean_max_delta": 0.018664570457196664, |
| "flip_rate": 0.016666666666666666 |
| }, |
| "temperature": 1.5157165665103978, |
| "calibrated_clean": { |
| "n": 1264, |
| "nll": 0.4416141187311443, |
| "acc": 0.8243670886075949, |
| "ece": 0.029334304090866264, |
| "brier": 0.24694275137320498, |
| "mean_conf": 0.8342071023922145, |
| "confident_error_rate": 0.022943037974683545, |
| "coverage_at_0_9": 0.567246835443038, |
| "accuracy_at_0_9": 0.9595536959553695, |
| "coverage_at_5pct_error": 0.622626582278481, |
| "coverage_at_1pct_error": 0.21044303797468356, |
| "aurc": 0.05692034604957053, |
| "error_rate_at_0_9": 0.040446304044630406, |
| "confidence_bias": 0.009840013784619628, |
| "top_bins": { |
| "0.9": { |
| "n": 717, |
| "errors": 29, |
| "error_rate": 0.040446304044630406 |
| }, |
| "0.95": { |
| "n": 615, |
| "errors": 19, |
| "error_rate": 0.030894308943089432 |
| }, |
| "0.99": { |
| "n": 326, |
| "errors": 6, |
| "error_rate": 0.018404907975460124 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9651898734177216, |
| "confidence_cutoff": 0.9445154395381901 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.8913043478260869, |
| "confidence_cutoff": 0.606953327797692 |
| } |
| }, |
| "score_mae": 0.4676383167623379, |
| "ranked_probability_score": 0.07005212248660367 |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 1204, |
| "requested_questions": 1468, |
| "evaluated_records": 1204, |
| "evaluated_questions": 1468, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 79.56905200080655, |
| "p95": 175.74176955040457 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 1.5157165665103978, |
| "logits_recorded": true |
| }, |
| "transfer": { |
| "objective": -0.8434221557083209, |
| "paired_flip": { |
| "pairs": 64, |
| "flip_rate": 0.40625, |
| "both_correct_rate": 0.34375, |
| "invariant_pairs": 24, |
| "invariance_rate": 0.9583333333333334, |
| "invariant_both_correct_rate": 0.5833333333333334 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 656, |
| "nll": 0.8332240113525762, |
| "acc": 0.6021341463414634, |
| "ece": 0.12798419967379476, |
| "brier": 0.5023300109226222, |
| "mean_conf": 0.7301183460152582, |
| "confident_error_rate": 0.04573170731707317, |
| "coverage_at_0_9": 0.3125, |
| "accuracy_at_0_9": 0.8536585365853658, |
| "coverage_at_5pct_error": 0.11890243902439024, |
| "coverage_at_1pct_error": 0.09451219512195122, |
| "aurc": 0.23059238555383313, |
| "error_rate_at_0_9": 0.14634146341463414, |
| "confidence_bias": 0.1279841996737947, |
| "top_bins": { |
| "0.9": { |
| "n": 205, |
| "errors": 30, |
| "error_rate": 0.14634146341463414 |
| }, |
| "0.95": { |
| "n": 151, |
| "errors": 20, |
| "error_rate": 0.13245033112582782 |
| }, |
| "0.99": { |
| "n": 79, |
| "errors": 4, |
| "error_rate": 0.05063291139240506 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.725609756097561, |
| "confidence_cutoff": 0.7209882521334812 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.6533333333333333, |
| "confidence_cutoff": 0.5341488281217432 |
| } |
| }, |
| "score_mae": 0.6641775577164577, |
| "ranked_probability_score": 0.29565575671388794 |
| }, |
| "tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.6427335354097619, |
| "acc": 0.65625, |
| "ece": 0.07911568686379761, |
| "brier": 0.45100153299560664, |
| "mean_conf": 0.5864401989835899, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.15625, |
| "coverage_at_1pct_error": 0.15625, |
| "aurc": 0.24615223624507382, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.0698098010164101, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.53125, |
| "accuracy": 0.6470588235294118, |
| "confidence_cutoff": 0.5770142502482459 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.6153846153846154, |
| "confidence_cutoff": 0.5480732917785645 |
| } |
| } |
| }, |
| "composition_held_conditional": { |
| "n": 32, |
| "nll": 0.7002559076723711, |
| "acc": 0.5, |
| "ece": 0.040716346048888484, |
| "brier": 0.5069657859898338, |
| "mean_conf": 0.5407163460488885, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.5428605550039391, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.04071634604888852, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5319679538002 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.46153846153846156, |
| "confidence_cutoff": 0.516680121421814 |
| } |
| } |
| }, |
| "composition_held_or_not": { |
| "n": 32, |
| "nll": 0.6868598188008299, |
| "acc": 0.5, |
| "ece": 0.11919972571689182, |
| "brier": 0.49389757171435983, |
| "mean_conf": 0.5462136768533761, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.09375, |
| "coverage_at_1pct_error": 0.09375, |
| "aurc": 0.39094494868742846, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.04621367685337607, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6875, |
| "confidence_cutoff": 0.5332706888479405 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.46153846153846156, |
| "confidence_cutoff": 0.5239105664518307 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 40, |
| "nll": 0.1012225013321368, |
| "acc": 0.975, |
| "ece": 0.0636128189986208, |
| "brier": 0.04847437535132595, |
| "mean_conf": 0.9158894408229405, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.725, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.975, |
| "aurc": 0.000625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.05911055917705943, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 23, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 20, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9909082970328174 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8867185513663616 |
| } |
| } |
| }, |
| "contrastive_deadline": { |
| "n": 40, |
| "nll": 2.3533720238091194, |
| "acc": 0.3, |
| "ece": 0.6268044472682741, |
| "brier": 1.1752809482099298, |
| "mean_conf": 0.902081250744253, |
| "confident_error_rate": 0.45, |
| "coverage_at_0_9": 0.65, |
| "accuracy_at_0_9": 0.3076923076923077, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.5450540740713118, |
| "error_rate_at_0_9": 0.6923076923076923, |
| "confidence_bias": 0.602081250744253, |
| "top_bins": { |
| "0.9": { |
| "n": 26, |
| "errors": 18, |
| "error_rate": 0.6923076923076923 |
| }, |
| "0.95": { |
| "n": 22, |
| "errors": 14, |
| "error_rate": 0.6363636363636364 |
| }, |
| "0.99": { |
| "n": 9, |
| "errors": 4, |
| "error_rate": 0.4444444444444444 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.4, |
| "confidence_cutoff": 0.9668620287297324 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.3125, |
| "confidence_cutoff": 0.8341740534671955 |
| } |
| }, |
| "score_mae": 0.6641775577164577, |
| "ranked_probability_score": 0.29565575671388794 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.1638223380813226, |
| "acc": 0.525, |
| "ece": 0.11298484196696625, |
| "brier": 0.5903870224663738, |
| "mean_conf": 0.6261382515881248, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.1125, |
| "accuracy_at_0_9": 0.8888888888888888, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.2782174494810715, |
| "error_rate_at_0_9": 0.1111111111111111, |
| "confidence_bias": 0.10113825158812473, |
| "top_bins": { |
| "0.9": { |
| "n": 9, |
| "errors": 1, |
| "error_rate": 0.1111111111111111 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.725, |
| "confidence_cutoff": 0.6017182592654616 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.609375, |
| "confidence_cutoff": 0.4732868788719531 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 1.2132248805897294, |
| "acc": 0.425, |
| "ece": 0.17901472294834253, |
| "brier": 0.6687983463380192, |
| "mean_conf": 0.5761396137039483, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.15, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.38430439973401825, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.1511396137039483, |
| "top_bins": { |
| "0.9": { |
| "n": 10, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6, |
| "confidence_cutoff": 0.5105295219311827 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.4375, |
| "confidence_cutoff": 0.385735625241166 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 1.00533805602315, |
| "acc": 0.3, |
| "ece": 0.3806088203975223, |
| "brier": 0.7785508948914812, |
| "mean_conf": 0.6776616149008904, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.0375, |
| "coverage_at_1pct_error": 0.0375, |
| "aurc": 0.7547032981712617, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.3776616149008904, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.15, |
| "confidence_cutoff": 0.6846514736219801 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.25, |
| "confidence_cutoff": 0.580651581287384 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.5594003098802613, |
| "acc": 0.7625, |
| "ece": 0.10667716921885279, |
| "brier": 0.3553818939712926, |
| "mean_conf": 0.818959518366006, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.4375, |
| "accuracy_at_0_9": 0.8571428571428571, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.14924964619005282, |
| "error_rate_at_0_9": 0.14285714285714285, |
| "confidence_bias": 0.056459518366006045, |
| "top_bins": { |
| "0.9": { |
| "n": 35, |
| "errors": 5, |
| "error_rate": 0.14285714285714285 |
| }, |
| "0.95": { |
| "n": 21, |
| "errors": 2, |
| "error_rate": 0.09523809523809523 |
| }, |
| "0.99": { |
| "n": 4, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.825, |
| "confidence_cutoff": 0.8597152316740018 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.675449788570404 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.18838356801169662, |
| "acc": 0.925, |
| "ece": 0.06864685652623173, |
| "brier": 0.10770728627230633, |
| "mean_conf": 0.895637017961667, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.7625, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.95, |
| "coverage_at_1pct_error": 0.7875, |
| "aurc": 0.007778982639311541, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.029362982038333096, |
| "top_bins": { |
| "0.9": { |
| "n": 61, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 54, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9910152874950432 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.8018387376851126 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.6630307731811524, |
| "acc": 0.7, |
| "ece": 0.11410098626654272, |
| "brier": 0.4256570275654813, |
| "mean_conf": 0.8141009862665427, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.425, |
| "accuracy_at_0_9": 0.8235294117647058, |
| "coverage_at_5pct_error": 0.0875, |
| "coverage_at_1pct_error": 0.0875, |
| "aurc": 0.20144605524801446, |
| "error_rate_at_0_9": 0.17647058823529413, |
| "confidence_bias": 0.11410098626654275, |
| "top_bins": { |
| "0.9": { |
| "n": 34, |
| "errors": 6, |
| "error_rate": 0.17647058823529413 |
| }, |
| "0.95": { |
| "n": 18, |
| "errors": 4, |
| "error_rate": 0.2222222222222222 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.825, |
| "confidence_cutoff": 0.8868192435698719 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.6436328483710542 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 656, |
| "nll": 0.8332240113525762, |
| "acc": 0.6021341463414634, |
| "ece": 0.12798419967379476, |
| "brier": 0.5023300109226222, |
| "mean_conf": 0.7301183460152582, |
| "confident_error_rate": 0.04573170731707317, |
| "coverage_at_0_9": 0.3125, |
| "accuracy_at_0_9": 0.8536585365853658, |
| "coverage_at_5pct_error": 0.11890243902439024, |
| "coverage_at_1pct_error": 0.09451219512195122, |
| "aurc": 0.23059238555383313, |
| "error_rate_at_0_9": 0.14634146341463414, |
| "confidence_bias": 0.1279841996737947, |
| "top_bins": { |
| "0.9": { |
| "n": 205, |
| "errors": 30, |
| "error_rate": 0.14634146341463414 |
| }, |
| "0.95": { |
| "n": 151, |
| "errors": 20, |
| "error_rate": 0.13245033112582782 |
| }, |
| "0.99": { |
| "n": 79, |
| "errors": 4, |
| "error_rate": 0.05063291139240506 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.725609756097561, |
| "confidence_cutoff": 0.7209882521334812 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.6533333333333333, |
| "confidence_cutoff": 0.5341488281217432 |
| } |
| }, |
| "score_mae": 0.6641775577164577, |
| "ranked_probability_score": 0.29565575671388794 |
| }, |
| "none_absent": { |
| "n": 36, |
| "nll": 0.6685620869807386, |
| "acc": 0.7777777777777778, |
| "ece": 0.18033089059255358, |
| "brier": 0.37205937297465375, |
| "mean_conf": 0.7657635384076253, |
| "confident_error_rate": 0.08333333333333333, |
| "coverage_at_0_9": 0.4166666666666667, |
| "accuracy_at_0_9": 0.8, |
| "coverage_at_5pct_error": 0.08333333333333333, |
| "coverage_at_1pct_error": 0.08333333333333333, |
| "aurc": 0.17909033623816945, |
| "error_rate_at_0_9": 0.2, |
| "confidence_bias": -0.012014239370152446, |
| "top_bins": { |
| "0.9": { |
| "n": 15, |
| "errors": 3, |
| "error_rate": 0.2 |
| }, |
| "0.95": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7777777777777778, |
| "confidence_cutoff": 0.8402871256038218 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.7931034482758621, |
| "confidence_cutoff": 0.5899206246083812 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 36, |
| "nll": 1.2061311893085476, |
| "acc": 0.5833333333333334, |
| "ece": 0.24336477102423715, |
| "brier": 0.5794150123852385, |
| "mean_conf": 0.7670494004663645, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.3333333333333333, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.4166666666666667, |
| "coverage_at_1pct_error": 0.4166666666666667, |
| "aurc": 0.1664477209679199, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.18371606713303112, |
| "top_bins": { |
| "0.9": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8888888888888888, |
| "confidence_cutoff": 0.8493899555479588 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.6551724137931034, |
| "confidence_cutoff": 0.5037377697189107 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 36, |
| "nll": 0.710652540300337, |
| "acc": 0.75, |
| "ece": 0.16435148952453013, |
| "brier": 0.37681908989783053, |
| "mean_conf": 0.7630775644796315, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.4166666666666667, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.4444444444444444, |
| "coverage_at_1pct_error": 0.4444444444444444, |
| "aurc": 0.09547117668697218, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.013077564479631487, |
| "top_bins": { |
| "0.9": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9444444444444444, |
| "confidence_cutoff": 0.8530542285676821 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.7586206896551724, |
| "confidence_cutoff": 0.4821188982489635 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.6427335354097619, |
| "acc": 0.65625, |
| "ece": 0.07911568686379761, |
| "brier": 0.45100153299560664, |
| "mean_conf": 0.5864401989835899, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.15625, |
| "coverage_at_1pct_error": 0.15625, |
| "aurc": 0.24615223624507382, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.0698098010164101, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.53125, |
| "accuracy": 0.6470588235294118, |
| "confidence_cutoff": 0.5770142502482459 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.6153846153846154, |
| "confidence_cutoff": 0.5480732917785645 |
| } |
| } |
| }, |
| "composition_held_conditional": { |
| "n": 32, |
| "nll": 0.7002559076723711, |
| "acc": 0.5, |
| "ece": 0.040716346048888484, |
| "brier": 0.5069657859898338, |
| "mean_conf": 0.5407163460488885, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.5428605550039391, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.04071634604888852, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5319679538002 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.46153846153846156, |
| "confidence_cutoff": 0.516680121421814 |
| } |
| } |
| }, |
| "composition_held_or_not": { |
| "n": 32, |
| "nll": 0.6868598188008299, |
| "acc": 0.5, |
| "ece": 0.11919972571689182, |
| "brier": 0.49389757171435983, |
| "mean_conf": 0.5462136768533761, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.09375, |
| "coverage_at_1pct_error": 0.09375, |
| "aurc": 0.39094494868742846, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.04621367685337607, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6875, |
| "confidence_cutoff": 0.5332706888479405 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.46153846153846156, |
| "confidence_cutoff": 0.5239105664518307 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 40, |
| "nll": 0.1012225013321368, |
| "acc": 0.975, |
| "ece": 0.0636128189986208, |
| "brier": 0.04847437535132595, |
| "mean_conf": 0.9158894408229405, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.725, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.975, |
| "aurc": 0.000625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.05911055917705943, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 23, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 20, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9909082970328174 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8867185513663616 |
| } |
| } |
| }, |
| "contrastive_deadline": { |
| "n": 40, |
| "nll": 2.3533720238091194, |
| "acc": 0.3, |
| "ece": 0.6268044472682741, |
| "brier": 1.1752809482099298, |
| "mean_conf": 0.902081250744253, |
| "confident_error_rate": 0.45, |
| "coverage_at_0_9": 0.65, |
| "accuracy_at_0_9": 0.3076923076923077, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.5450540740713118, |
| "error_rate_at_0_9": 0.6923076923076923, |
| "confidence_bias": 0.602081250744253, |
| "top_bins": { |
| "0.9": { |
| "n": 26, |
| "errors": 18, |
| "error_rate": 0.6923076923076923 |
| }, |
| "0.95": { |
| "n": 22, |
| "errors": 14, |
| "error_rate": 0.6363636363636364 |
| }, |
| "0.99": { |
| "n": 9, |
| "errors": 4, |
| "error_rate": 0.4444444444444444 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.4, |
| "confidence_cutoff": 0.9668620287297324 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.3125, |
| "confidence_cutoff": 0.8341740534671955 |
| } |
| }, |
| "score_mae": 0.6641775577164577, |
| "ranked_probability_score": 0.29565575671388794 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.1638223380813226, |
| "acc": 0.525, |
| "ece": 0.11298484196696625, |
| "brier": 0.5903870224663738, |
| "mean_conf": 0.6261382515881248, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.1125, |
| "accuracy_at_0_9": 0.8888888888888888, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.2782174494810715, |
| "error_rate_at_0_9": 0.1111111111111111, |
| "confidence_bias": 0.10113825158812473, |
| "top_bins": { |
| "0.9": { |
| "n": 9, |
| "errors": 1, |
| "error_rate": 0.1111111111111111 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.725, |
| "confidence_cutoff": 0.6017182592654616 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.609375, |
| "confidence_cutoff": 0.4732868788719531 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 1.2132248805897294, |
| "acc": 0.425, |
| "ece": 0.17901472294834253, |
| "brier": 0.6687983463380192, |
| "mean_conf": 0.5761396137039483, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.15, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.38430439973401825, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.1511396137039483, |
| "top_bins": { |
| "0.9": { |
| "n": 10, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6, |
| "confidence_cutoff": 0.5105295219311827 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.4375, |
| "confidence_cutoff": 0.385735625241166 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 1.00533805602315, |
| "acc": 0.3, |
| "ece": 0.3806088203975223, |
| "brier": 0.7785508948914812, |
| "mean_conf": 0.6776616149008904, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.0375, |
| "coverage_at_1pct_error": 0.0375, |
| "aurc": 0.7547032981712617, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.3776616149008904, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.15, |
| "confidence_cutoff": 0.6846514736219801 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.25, |
| "confidence_cutoff": 0.580651581287384 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.5594003098802613, |
| "acc": 0.7625, |
| "ece": 0.10667716921885279, |
| "brier": 0.3553818939712926, |
| "mean_conf": 0.818959518366006, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.4375, |
| "accuracy_at_0_9": 0.8571428571428571, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.14924964619005282, |
| "error_rate_at_0_9": 0.14285714285714285, |
| "confidence_bias": 0.056459518366006045, |
| "top_bins": { |
| "0.9": { |
| "n": 35, |
| "errors": 5, |
| "error_rate": 0.14285714285714285 |
| }, |
| "0.95": { |
| "n": 21, |
| "errors": 2, |
| "error_rate": 0.09523809523809523 |
| }, |
| "0.99": { |
| "n": 4, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.825, |
| "confidence_cutoff": 0.8597152316740018 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.675449788570404 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.18838356801169662, |
| "acc": 0.925, |
| "ece": 0.06864685652623173, |
| "brier": 0.10770728627230633, |
| "mean_conf": 0.895637017961667, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.7625, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.95, |
| "coverage_at_1pct_error": 0.7875, |
| "aurc": 0.007778982639311541, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.029362982038333096, |
| "top_bins": { |
| "0.9": { |
| "n": 61, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 54, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9910152874950432 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.8018387376851126 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.6630307731811524, |
| "acc": 0.7, |
| "ece": 0.11410098626654272, |
| "brier": 0.4256570275654813, |
| "mean_conf": 0.8141009862665427, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.425, |
| "accuracy_at_0_9": 0.8235294117647058, |
| "coverage_at_5pct_error": 0.0875, |
| "coverage_at_1pct_error": 0.0875, |
| "aurc": 0.20144605524801446, |
| "error_rate_at_0_9": 0.17647058823529413, |
| "confidence_bias": 0.11410098626654275, |
| "top_bins": { |
| "0.9": { |
| "n": 34, |
| "errors": 6, |
| "error_rate": 0.17647058823529413 |
| }, |
| "0.95": { |
| "n": 18, |
| "errors": 4, |
| "error_rate": 0.2222222222222222 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.825, |
| "confidence_cutoff": 0.8868192435698719 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.6436328483710542 |
| } |
| } |
| } |
| }, |
| "permutation": { |
| "n": 36, |
| "mean_max_delta": 0.07456167804296458, |
| "flip_rate": 0.1111111111111111 |
| }, |
| "temperature": 1.5157165665103978, |
| "calibrated_clean": { |
| "n": 656, |
| "nll": 0.7627521342590968, |
| "acc": 0.6021341463414634, |
| "ece": 0.07572062366206761, |
| "brier": 0.4747254916137418, |
| "mean_conf": 0.669677827611837, |
| "confident_error_rate": 0.021341463414634148, |
| "coverage_at_0_9": 0.18445121951219512, |
| "accuracy_at_0_9": 0.8842975206611571, |
| "coverage_at_5pct_error": 0.12652439024390244, |
| "coverage_at_1pct_error": 0.09298780487804878, |
| "aurc": 0.23002342930643124, |
| "error_rate_at_0_9": 0.11570247933884298, |
| "confidence_bias": 0.06754368127037358, |
| "top_bins": { |
| "0.9": { |
| "n": 121, |
| "errors": 14, |
| "error_rate": 0.11570247933884298 |
| }, |
| "0.95": { |
| "n": 75, |
| "errors": 4, |
| "error_rate": 0.05333333333333334 |
| }, |
| "0.99": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7164634146341463, |
| "confidence_cutoff": 0.641505976392506 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.6476190476190476, |
| "confidence_cutoff": 0.5125548604205254 |
| } |
| }, |
| "score_mae": 0.6487109061534533, |
| "ranked_probability_score": 0.2651482098948402 |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 764, |
| "requested_questions": 764, |
| "evaluated_records": 764, |
| "evaluated_questions": 764, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 79.30278050116613, |
| "p95": 83.82728769975074 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 1.5157165665103978, |
| "logits_recorded": true |
| }, |
| "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1" |
| }, |
| "provenance": { |
| "config": { |
| "epochs": 2, |
| "seed": 0, |
| "lr": 0.0001, |
| "lora": 16, |
| "accum": 2, |
| "batch": 4, |
| "perm_kl": 0.0, |
| "perm_frac": 0.3, |
| "ord_w": 0.0, |
| "p_none": 0.1, |
| "p_none_distract": 0.12, |
| "p_distract": 0.15, |
| "p_none_pair": 0.25, |
| "synthetic_repeat": 1, |
| "public_frac": 1.0, |
| "head_lr": 0.0, |
| "weight_decay": 0.01, |
| "anchor_w": 0.0, |
| "label_smoothing": 0.0, |
| "brier_w": 0.0, |
| "focal_gamma": 0.0, |
| "base": "google/gemma-4-E2B", |
| "base_revision": "d29ff6b45f081a49ee2733a859c9c9c2d95d1a6f", |
| "dtype": "bf16", |
| "weights_dtype": "bf16", |
| "checkpointing": 1 |
| }, |
| "config_sha256": "cb5576f6ebf3fffe10370041387b4b6810c03c2b112b925168c16ded6729c34c", |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", |
| "source_hashes": { |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", |
| "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", |
| "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", |
| "kev/benchmark.py": "9c442efd8b6b852c574a646192c61ee91afa1306e3b2978bb6391ae1b91a3fa0", |
| "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", |
| "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59", |
| "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", |
| "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", |
| "kev/data.py": "ef1be396e7218ba5c94f0f67de2ceff23a39656b57199cb3aa3192cde53e8320", |
| "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", |
| "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", |
| "kev/experiment.py": "d12ab7f0765de0f29364f3411b99f06bb80b1cbf120ca9ed2b03316ec766e828", |
| "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", |
| "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9", |
| "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", |
| "kev/model.py": "c2129026ab56395f78c7d3071410541d5423dd7d97f7371518570572f206c376", |
| "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", |
| "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3", |
| "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", |
| "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735", |
| "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", |
| "kev/suite.py": "d18d5f4f02d079344d61c2a31882c524c90012ebfdb660bf9160e145f37e3d42", |
| "kev/train.py": "e103e99d3a68ab99eb58f135320fbc46b18514fcc2b195d1c6879633836e99b9", |
| "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", |
| "modal_app.py": "7a13e9d29b3fc68f8236a0c1e4d67079b40a2a328a296f7ced1022817dff82aa", |
| "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d", |
| "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" |
| }, |
| "git_commit": "ccd6e8b646068a6689792d76e3cae3698e9a015a", |
| "platform": "Linux-6.12.103-127.188.amzn2023.x86_64-x86_64-with-glibc2.36", |
| "torch": "2.8.0+cu128", |
| "device": "cuda", |
| "gpu": "NVIDIA L4", |
| "legacy_checkpoint": false, |
| "measured_checkpoint": { |
| "requested": "/kev/runs/g4-e2b-e2-l4/00-trial-0/checkpoint", |
| "resolved": "/kev/runs/g4-e2b-e2-l4/00-trial-0/checkpoint", |
| "head_sha256": "dd2d8c5b1ac8bee3f413cd89802d93826674baf181dd4649004868b358605785", |
| "adapter_sha256": "e23dbe51453b6e926da0db70a38d680bc64644caff872cf0defcbbd8b1acbe46", |
| "inference_temperature": 1.0 |
| } |
| }, |
| "mechanism_checks": { |
| "n": 8, |
| "packed_max_delta": 0.0, |
| "sibling_max_delta": 0.0069029927253723145, |
| "tolerance": 0.001, |
| "passed": false |
| }, |
| "gates": { |
| "passed": false, |
| "checks": { |
| "complete_coverage": true, |
| "isolation_and_packing": false, |
| "transfer_complete": true, |
| "heldout_pairs_at_least_70pct": false, |
| "transfer_confident_errors_below_10pct": true |
| }, |
| "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." |
| }, |
| "calibration_fit": { |
| "temperature": 1.5157165665103978, |
| "aggregation": "micro", |
| "objective": "raw-logit NLL", |
| "split": "calibration", |
| "rows_sha256": "d51919a6df076619eb78186da47cebbf6b669444a4c356a5ff4a244184fa6433", |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", |
| "n": 1148 |
| }, |
| "wall_seconds": 10031.380025413, |
| "promotable": false, |
| "test_evaluated": false, |
| "training_resources": { |
| "wall_seconds": 9709.689131975174, |
| "records_seen": 30212, |
| "requested_records": 25012, |
| "truncated_records": 0, |
| "rejected_records": 0, |
| "optimizer_steps": 3128, |
| "forward_tokens": 6112066, |
| "peak_device_bytes": 14624471040, |
| "device": "cuda", |
| "dtype": "bf16", |
| "batch": 4, |
| "peak_rss_bytes": 11347705856 |
| } |
| } |
|
|
