{
  "objective": -0.5112698517216128,
  "paired_flip": {
    "pairs": 80,
    "flip_rate": 0.675,
    "both_correct_rate": 0.625,
    "invariant_pairs": 32,
    "invariance_rate": 0.96875,
    "invariant_both_correct_rate": 0.59375
  },
  "unknowable": null,
  "clean": {
    "n": 1264,
    "nll": 0.4989297577819544,
    "acc": 0.8243670886075949,
    "ece": 0.05704976909178335,
    "brier": 0.25535485232026645,
    "mean_conf": 0.8785235750178558,
    "confident_error_rate": 0.040348101265822785,
    "coverage_at_0_9": 0.6566455696202531,
    "accuracy_at_0_9": 0.9385542168674699,
    "coverage_at_5pct_error": 0.6337025316455697,
    "coverage_at_1pct_error": 0.18908227848101267,
    "aurc": 0.057098658886616284,
    "error_rate_at_0_9": 0.06144578313253012,
    "confidence_bias": 0.05415648641026094,
    "top_bins": {
      "0.9": {
        "n": 830,
        "errors": 51,
        "error_rate": 0.06144578313253012
      },
      "0.95": {
        "n": 765,
        "errors": 32,
        "error_rate": 0.04183006535947712
      },
      "0.99": {
        "n": 616,
        "errors": 21,
        "error_rate": 0.03409090909090909
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9651898734177216,
        "confidence_cutoff": 0.9884021799746481
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.8913043478260869,
        "confidence_cutoff": 0.7007823799483796
      }
    },
    "score_mae": 0.44288336559433733,
    "ranked_probability_score": 0.07198397780180195
  },
  "tasks": {
    "agnews": {
      "n": 80,
      "nll": 0.7033810609329125,
      "acc": 0.85,
      "ece": 0.12788153266590807,
      "brier": 0.2690347009363626,
      "mean_conf": 0.9626452504559462,
      "confident_error_rate": 0.1125,
      "coverage_at_0_9": 0.9,
      "accuracy_at_0_9": 0.875,
      "coverage_at_5pct_error": 0.5,
      "coverage_at_1pct_error": 0.175,
      "aurc": 0.06388995806168134,
      "error_rate_at_0_9": 0.125,
      "confidence_bias": 0.1126452504559462,
      "top_bins": {
        "0.9": {
          "n": 72,
          "errors": 9,
          "error_rate": 0.125
        },
        "0.95": {
          "n": 66,
          "errors": 6,
          "error_rate": 0.09090909090909091
        },
        "0.99": {
          "n": 61,
          "errors": 5,
          "error_rate": 0.08196721311475409
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.95,
          "confidence_cutoff": 0.9983029203561053
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.921875,
          "confidence_cutoff": 0.9797347727687692
        }
      }
    },
    "agnews_yn": {
      "n": 160,
      "nll": 0.218266217089547,
      "acc": 0.925,
      "ece": 0.053822844561075917,
      "brier": 0.12359771772019439,
      "mean_conf": 0.9648416375603937,
      "confident_error_rate": 0.03125,
      "coverage_at_0_9": 0.9,
      "accuracy_at_0_9": 0.9652777777777778,
      "coverage_at_5pct_error": 0.94375,
      "coverage_at_1pct_error": 0.5,
      "aurc": 0.013180196924715198,
      "error_rate_at_0_9": 0.034722222222222224,
      "confidence_bias": 0.039841637560393606,
      "top_bins": {
        "0.9": {
          "n": 144,
          "errors": 5,
          "error_rate": 0.034722222222222224
        },
        "0.95": {
          "n": 132,
          "errors": 3,
          "error_rate": 0.022727272727272728
        },
        "0.99": {
          "n": 98,
          "errors": 1,
          "error_rate": 0.01020408163265306
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9943423697163807
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9765625,
          "confidence_cutoff": 0.962791422719902
        }
      }
    },
    "amazon": {
      "n": 80,
      "nll": 0.9101254335951751,
      "acc": 0.65,
      "ece": 0.09852625629411788,
      "brier": 0.4737080597217712,
      "mean_conf": 0.7311588013609494,
      "confident_error_rate": 0.0125,
      "coverage_at_0_9": 0.2125,
      "accuracy_at_0_9": 0.9411764705882353,
      "coverage_at_5pct_error": 0.1875,
      "coverage_at_1pct_error": 0.1875,
      "aurc": 0.16015706656418405,
      "error_rate_at_0_9": 0.058823529411764705,
      "confidence_bias": 0.08115880136094933,
      "top_bins": {
        "0.9": {
          "n": 17,
          "errors": 1,
          "error_rate": 0.058823529411764705
        },
        "0.95": {
          "n": 12,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.85,
          "confidence_cutoff": 0.7263951073590981
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.75,
          "confidence_cutoff": 0.5548622064540625
        }
      },
      "score_mae": 0.45330552002269775,
      "ranked_probability_score": 0.07559174675562505
    },
    "banking77": {
      "n": 80,
      "nll": 0.3979839296889442,
      "acc": 0.875,
      "ece": 0.0922611325054158,
      "brier": 0.1822296579454367,
      "mean_conf": 0.8923134569904196,
      "confident_error_rate": 0.0375,
      "coverage_at_0_9": 0.725,
      "accuracy_at_0_9": 0.9482758620689655,
      "coverage_at_5pct_error": 0.825,
      "coverage_at_1pct_error": 0.675,
      "aurc": 0.02199412946893899,
      "error_rate_at_0_9": 0.05172413793103448,
      "confidence_bias": 0.01731345699041964,
      "top_bins": {
        "0.9": {
          "n": 58,
          "errors": 3,
          "error_rate": 0.05172413793103448
        },
        "0.95": {
          "n": 52,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 41,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9931638954155307
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.953125,
          "confidence_cutoff": 0.845837989712747
        }
      }
    },
    "boolq": {
      "n": 80,
      "nll": 0.5220096518672049,
      "acc": 0.8125,
      "ece": 0.14833387100163806,
      "brier": 0.2951043064516692,
      "mean_conf": 0.9369053870550708,
      "confident_error_rate": 0.0875,
      "coverage_at_0_9": 0.8,
      "accuracy_at_0_9": 0.890625,
      "coverage_at_5pct_error": 0.35,
      "coverage_at_1pct_error": 0.2625,
      "aurc": 0.07004308805415525,
      "error_rate_at_0_9": 0.109375,
      "confidence_bias": 0.12440538705507076,
      "top_bins": {
        "0.9": {
          "n": 64,
          "errors": 7,
          "error_rate": 0.109375
        },
        "0.95": {
          "n": 57,
          "errors": 4,
          "error_rate": 0.07017543859649122
        },
        "0.99": {
          "n": 33,
          "errors": 3,
          "error_rate": 0.09090909090909091
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.925,
          "confidence_cutoff": 0.9818334678368117
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.890625,
          "confidence_cutoff": 0.906916795952237
        }
      }
    },
    "composition_atom": {
      "n": 16,
      "nll": 0.52206201525939,
      "acc": 0.8125,
      "ece": 0.33102741792619084,
      "brier": 0.34727627266455574,
      "mean_conf": 0.678589841408767,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0625,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 0.5,
      "coverage_at_1pct_error": 0.5,
      "aurc": 0.1089238886113886,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.133910158591233,
      "top_bins": {
        "0.9": {
          "n": 1,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.7019797950393023
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.7692307692307693,
          "confidence_cutoff": 0.5925620021652579
        }
      }
    },
    "composition_conditional": {
      "n": 16,
      "nll": 0.7085893013145462,
      "acc": 0.5,
      "ece": 0.0392985537546805,
      "brier": 0.5153023405509731,
      "mean_conf": 0.5392985537546805,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.6738095238095239,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.0392985537546805,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.25,
          "confidence_cutoff": 0.5389107305136105
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.5714285714285714,
          "confidence_cutoff": 0.5247236050583006
        }
      }
    },
    "composition_conjunction": {
      "n": 16,
      "nll": 0.6954926808023254,
      "acc": 0.6875,
      "ece": 0.28356235633747884,
      "brier": 0.5005900249196042,
      "mean_conf": 0.5960339565951683,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.571116296897547,
      "error_rate_at_0_9": null,
      "confidence_bias": -0.09146604340483167,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.375,
          "confidence_cutoff": 0.5774681144198214
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.6428571428571429,
          "confidence_cutoff": 0.5598945450814203
        }
      }
    },
    "composition_disjunction": {
      "n": 16,
      "nll": 0.6223727318080233,
      "acc": 0.5625,
      "ece": 0.22532601951230213,
      "brier": 0.4331353693805437,
      "mean_conf": 0.5895230696045874,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.3125,
      "coverage_at_1pct_error": 0.3125,
      "aurc": 0.22883184523809524,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.027023069604587446,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5625,
          "accuracy": 0.6666666666666666,
          "confidence_cutoff": 0.5856707096099854
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.6428571428571429,
          "confidence_cutoff": 0.5193371772766113
        }
      }
    },
    "composition_exception": {
      "n": 16,
      "nll": 0.6747223985633183,
      "acc": 0.6875,
      "ece": 0.16649871606180544,
      "brier": 0.4815999757579885,
      "mean_conf": 0.5210012839381946,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.3167455808080808,
      "error_rate_at_0_9": null,
      "confidence_bias": -0.16649871606180544,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5625,
          "accuracy": 0.7777777777777778,
          "confidence_cutoff": 0.518760085105896
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.6428571428571429,
          "confidence_cutoff": 0.5035156757687234
        }
      }
    },
    "composition_negation": {
      "n": 16,
      "nll": 0.7403130749553563,
      "acc": 0.5,
      "ece": 0.08891041545292032,
      "brier": 0.5451303525712351,
      "mean_conf": 0.5889104154529203,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.6560387529137529,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.08891041545292033,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.375,
          "confidence_cutoff": 0.5939869638673642
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.46153846153846156,
          "confidence_cutoff": 0.5687558393928909
        }
      }
    },
    "composition_nested_and": {
      "n": 16,
      "nll": 0.6701464137220343,
      "acc": 0.5,
      "ece": 0.1743572557807083,
      "brier": 0.4801171644212976,
      "mean_conf": 0.5974301225035049,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.25,
      "coverage_at_1pct_error": 0.25,
      "aurc": 0.37770632145632144,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.09743012250350491,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.5,
          "confidence_cutoff": 0.5888296365737915
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.46153846153846156,
          "confidence_cutoff": 0.5511568948971299
        }
      }
    },
    "composition_nested_or": {
      "n": 16,
      "nll": 0.5812368508611689,
      "acc": 0.5625,
      "ece": 0.232265838198384,
      "brier": 0.39731880250019436,
      "mean_conf": 0.596988431257154,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.375,
      "coverage_at_1pct_error": 0.375,
      "aurc": 0.1931575369075369,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.03448843125715395,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.875,
          "confidence_cutoff": 0.6048937439918518
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.6923076923076923,
          "confidence_cutoff": 0.5087750402458
        }
      }
    },
    "contrastive_age_eligibility": {
      "n": 24,
      "nll": 0.00030557725656388685,
      "acc": 1.0,
      "ece": 0.0003054692741715215,
      "brier": 4.3159530440543707e-07,
      "mean_conf": 0.9996945307258285,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "aurc": 0.0,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.0003054692741715215,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9998034213326141
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9996426136047368
        }
      }
    },
    "contrastive_quantity_limit": {
      "n": 24,
      "nll": 0.0949641863436191,
      "acc": 0.9583333333333334,
      "ece": 0.06630730027669939,
      "brier": 0.061578113407729874,
      "mean_conf": 0.9503996922670156,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.8333333333333334,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.9166666666666666,
      "aurc": 0.0035477053140096615,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.007933641066317776,
      "top_bins": {
        "0.9": {
          "n": 20,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 20,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 14,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999154857901148
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9568661440200333
        }
      }
    },
    "contrastive_return_window": {
      "n": 24,
      "nll": 0.5242983013148222,
      "acc": 0.75,
      "ece": 0.13728604635195948,
      "brier": 0.3391539263874357,
      "mean_conf": 0.8872860463519596,
      "confident_error_rate": 0.08333333333333333,
      "coverage_at_0_9": 0.6666666666666666,
      "accuracy_at_0_9": 0.875,
      "coverage_at_5pct_error": 0.4583333333333333,
      "coverage_at_1pct_error": 0.4583333333333333,
      "aurc": 0.08703314878649715,
      "error_rate_at_0_9": 0.125,
      "confidence_bias": 0.13728604635195962,
      "top_bins": {
        "0.9": {
          "n": 16,
          "errors": 2,
          "error_rate": 0.125
        },
        "0.95": {
          "n": 11,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9166666666666666,
          "confidence_cutoff": 0.9489070242881575
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 0.8,
          "confidence_cutoff": 0.7711345531891699
        }
      }
    },
    "contrastive_spend_threshold": {
      "n": 24,
      "nll": 9.237405628065089e-06,
      "acc": 1.0,
      "ece": 9.237352093016682e-06,
      "brier": 2.021011459531347e-10,
      "mean_conf": 0.999990762647907,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "aurc": 0.0,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -9.237352093016682e-06,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999922291820006
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9999883904450512
        }
      }
    },
    "dbpedia14": {
      "n": 80,
      "nll": 0.3415167120067256,
      "acc": 0.9625,
      "ece": 0.043700092546254746,
      "brier": 0.07457610595560957,
      "mean_conf": 0.9898401932810058,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.9625,
      "accuracy_at_0_9": 0.974025974025974,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.1375,
      "aurc": 0.02543183035490516,
      "error_rate_at_0_9": 0.025974025974025976,
      "confidence_bias": 0.027340193281005787,
      "top_bins": {
        "0.9": {
          "n": 77,
          "errors": 2,
          "error_rate": 0.025974025974025976
        },
        "0.95": {
          "n": 77,
          "errors": 2,
          "error_rate": 0.025974025974025976
        },
        "0.99": {
          "n": 77,
          "errors": 2,
          "error_rate": 0.025974025974025976
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9999976727949713
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.999980934132584
        }
      }
    },
    "imdb": {
      "n": 80,
      "nll": 0.24985978507417891,
      "acc": 0.9375,
      "ece": 0.05457930309714365,
      "brier": 0.11189476167690375,
      "mean_conf": 0.977379170939226,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.95,
      "accuracy_at_0_9": 0.9473684210526315,
      "coverage_at_5pct_error": 0.9375,
      "coverage_at_1pct_error": 0.475,
      "aurc": 0.013302134857181047,
      "error_rate_at_0_9": 0.05263157894736842,
      "confidence_bias": 0.039879170939225994,
      "top_bins": {
        "0.9": {
          "n": 76,
          "errors": 4,
          "error_rate": 0.05263157894736842
        },
        "0.95": {
          "n": 74,
          "errors": 3,
          "error_rate": 0.04054054054054054
        },
        "0.99": {
          "n": 68,
          "errors": 1,
          "error_rate": 0.014705882352941176
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9992378023250749
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9947106490827637
        }
      }
    },
    "mnli": {
      "n": 80,
      "nll": 0.8964775304734388,
      "acc": 0.825,
      "ece": 0.16162789615757212,
      "brier": 0.33240517794745345,
      "mean_conf": 0.9328373683549026,
      "confident_error_rate": 0.1375,
      "coverage_at_0_9": 0.8,
      "accuracy_at_0_9": 0.828125,
      "coverage_at_5pct_error": 0.0625,
      "coverage_at_1pct_error": 0.0625,
      "aurc": 0.1279801828240891,
      "error_rate_at_0_9": 0.171875,
      "confidence_bias": 0.10783736835490265,
      "top_bins": {
        "0.9": {
          "n": 64,
          "errors": 11,
          "error_rate": 0.171875
        },
        "0.95": {
          "n": 62,
          "errors": 10,
          "error_rate": 0.16129032258064516
        },
        "0.99": {
          "n": 50,
          "errors": 7,
          "error_rate": 0.14
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.875,
          "confidence_cutoff": 0.9951284115930806
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.828125,
          "confidence_cutoff": 0.9465963914191013
        }
      }
    },
    "sst5": {
      "n": 80,
      "nll": 1.0021265818153486,
      "acc": 0.6125,
      "ece": 0.11978396394038553,
      "brier": 0.5446736304135753,
      "mean_conf": 0.700510761326975,
      "confident_error_rate": 0.0125,
      "coverage_at_0_9": 0.075,
      "accuracy_at_0_9": 0.8333333333333334,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.3072321995862583,
      "error_rate_at_0_9": 0.16666666666666666,
      "confidence_bias": 0.08801076132697494,
      "top_bins": {
        "0.9": {
          "n": 6,
          "errors": 1,
          "error_rate": 0.16666666666666666
        },
        "0.95": {
          "n": 1,
          "errors": 1,
          "error_rate": 1.0
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.675,
          "confidence_cutoff": 0.7306530300819783
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.65625,
          "confidence_cutoff": 0.5438117284883917
        }
      },
      "score_mae": 0.5114106925506208,
      "ranked_probability_score": 0.08295468856515971
    },
    "trec": {
      "n": 80,
      "nll": 0.2925501809265985,
      "acc": 0.9375,
      "ece": 0.05267290937483785,
      "brier": 0.09750309073685226,
      "mean_conf": 0.9901729093748377,
      "confident_error_rate": 0.0375,
      "coverage_at_0_9": 0.975,
      "accuracy_at_0_9": 0.9615384615384616,
      "coverage_at_5pct_error": 0.975,
      "coverage_at_1pct_error": 0.65,
      "aurc": 0.010066847867815351,
      "error_rate_at_0_9": 0.038461538461538464,
      "confidence_bias": 0.052672909374837706,
      "top_bins": {
        "0.9": {
          "n": 78,
          "errors": 3,
          "error_rate": 0.038461538461538464
        },
        "0.95": {
          "n": 77,
          "errors": 2,
          "error_rate": 0.025974025974025976
        },
        "0.99": {
          "n": 74,
          "errors": 2,
          "error_rate": 0.02702702702702703
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9998811671798993
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.96875,
          "confidence_cutoff": 0.9989895525699338
        }
      }
    },
    "yelp": {
      "n": 80,
      "nll": 0.724927007873615,
      "acc": 0.7125,
      "ece": 0.12184401403681792,
      "brier": 0.43845374961294875,
      "mean_conf": 0.7841603935118646,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.25,
      "accuracy_at_0_9": 0.9,
      "coverage_at_5pct_error": 0.1625,
      "coverage_at_1pct_error": 0.1625,
      "aurc": 0.18107778703006866,
      "error_rate_at_0_9": 0.1,
      "confidence_bias": 0.0716603935118646,
      "top_bins": {
        "0.9": {
          "n": 20,
          "errors": 2,
          "error_rate": 0.1
        },
        "0.95": {
          "n": 12,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 3,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.8,
          "confidence_cutoff": 0.8030074284504592
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.734375,
          "confidence_cutoff": 0.6472819362414997
        }
      },
      "score_mae": 0.36393388420969347,
      "ranked_probability_score": 0.0574054980846211
    },
    "yelp_yn": {
      "n": 80,
      "nll": 0.1767395803682206,
      "acc": 0.925,
      "ece": 0.04299234681271804,
      "brier": 0.10751418779018915,
      "mean_conf": 0.9602990730093268,
      "confident_error_rate": 0.0125,
      "coverage_at_0_9": 0.8625,
      "accuracy_at_0_9": 0.9855072463768116,
      "coverage_at_5pct_error": 0.9125,
      "coverage_at_1pct_error": 0.6875,
      "aurc": 0.009561420798710252,
      "error_rate_at_0_9": 0.014492753623188406,
      "confidence_bias": 0.035299073009326776,
      "top_bins": {
        "0.9": {
          "n": 69,
          "errors": 1,
          "error_rate": 0.014492753623188406
        },
        "0.95": {
          "n": 64,
          "errors": 1,
          "error_rate": 0.015625
        },
        "0.99": {
          "n": 49,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9945756192344355
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9567033266284315
        }
      }
    }
  },
  "variants": {
    "clean": {
      "n": 1264,
      "nll": 0.4989297577819544,
      "acc": 0.8243670886075949,
      "ece": 0.05704976909178335,
      "brier": 0.25535485232026645,
      "mean_conf": 0.8785235750178558,
      "confident_error_rate": 0.040348101265822785,
      "coverage_at_0_9": 0.6566455696202531,
      "accuracy_at_0_9": 0.9385542168674699,
      "coverage_at_5pct_error": 0.6337025316455697,
      "coverage_at_1pct_error": 0.18908227848101267,
      "aurc": 0.057098658886616284,
      "error_rate_at_0_9": 0.06144578313253012,
      "confidence_bias": 0.05415648641026094,
      "top_bins": {
        "0.9": {
          "n": 830,
          "errors": 51,
          "error_rate": 0.06144578313253012
        },
        "0.95": {
          "n": 765,
          "errors": 32,
          "error_rate": 0.04183006535947712
        },
        "0.99": {
          "n": 616,
          "errors": 21,
          "error_rate": 0.03409090909090909
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9651898734177216,
          "confidence_cutoff": 0.9884021799746481
        },
        "0.8": {
          "coverage": 0.8006329113924051,
          "accuracy": 0.8913043478260869,
          "confidence_cutoff": 0.7007823799483796
        }
      },
      "score_mae": 0.44288336559433733,
      "ranked_probability_score": 0.07198397780180195
    },
    "none_absent": {
      "n": 60,
      "nll": 0.3973810280057748,
      "acc": 0.8666666666666667,
      "ece": 0.06408010262843164,
      "brier": 0.19542789202339048,
      "mean_conf": 0.9298361792245137,
      "confident_error_rate": 0.016666666666666666,
      "coverage_at_0_9": 0.7666666666666667,
      "accuracy_at_0_9": 0.9782608695652174,
      "coverage_at_5pct_error": 0.8,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.09434286048325893,
      "error_rate_at_0_9": 0.021739130434782608,
      "confidence_bias": 0.06316951255784697,
      "top_bins": {
        "0.9": {
          "n": 46,
          "errors": 1,
          "error_rate": 0.021739130434782608
        },
        "0.95": {
          "n": 40,
          "errors": 1,
          "error_rate": 0.025
        },
        "0.99": {
          "n": 23,
          "errors": 1,
          "error_rate": 0.043478260869565216
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9666666666666667,
          "confidence_cutoff": 0.9822965689718635
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9583333333333334,
          "confidence_cutoff": 0.8703627257812488
        }
      }
    },
    "none_present": {
      "n": 60,
      "nll": 0.8331048448864015,
      "acc": 0.85,
      "ece": 0.09958344599948121,
      "brier": 0.25491634205576813,
      "mean_conf": 0.8976962829199532,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.7,
      "accuracy_at_0_9": 0.9285714285714286,
      "coverage_at_5pct_error": 0.6666666666666666,
      "coverage_at_1pct_error": 0.1,
      "aurc": 0.0707109577846649,
      "error_rate_at_0_9": 0.07142857142857142,
      "confidence_bias": 0.0476962829199532,
      "top_bins": {
        "0.9": {
          "n": 42,
          "errors": 3,
          "error_rate": 0.07142857142857142
        },
        "0.95": {
          "n": 36,
          "errors": 2,
          "error_rate": 0.05555555555555555
        },
        "0.99": {
          "n": 25,
          "errors": 1,
          "error_rate": 0.04
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9333333333333333,
          "confidence_cutoff": 0.9711979045098904
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9166666666666666,
          "confidence_cutoff": 0.8156208519156563
        }
      }
    },
    "permuted": {
      "n": 84,
      "nll": 0.4638696687650698,
      "acc": 0.9166666666666666,
      "ece": 0.06166928073435357,
      "brier": 0.14839515349641946,
      "mean_conf": 0.9647966493434161,
      "confident_error_rate": 0.047619047619047616,
      "coverage_at_0_9": 0.9047619047619048,
      "accuracy_at_0_9": 0.9473684210526315,
      "coverage_at_5pct_error": 0.8214285714285714,
      "coverage_at_1pct_error": 0.05952380952380952,
      "aurc": 0.048011723747782474,
      "error_rate_at_0_9": 0.05263157894736842,
      "confidence_bias": 0.048129982676749505,
      "top_bins": {
        "0.9": {
          "n": 76,
          "errors": 4,
          "error_rate": 0.05263157894736842
        },
        "0.95": {
          "n": 72,
          "errors": 4,
          "error_rate": 0.05555555555555555
        },
        "0.99": {
          "n": 61,
          "errors": 3,
          "error_rate": 0.04918032786885246
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9761904761904762,
          "confidence_cutoff": 0.9974597361195856
        },
        "0.8": {
          "coverage": 0.8095238095238095,
          "accuracy": 0.9558823529411765,
          "confidence_cutoff": 0.9721480404501962
        }
      }
    }
  },
  "heldout_tasks": {},
  "permutation": {
    "n": 60,
    "mean_max_delta": 0.018664570457196664,
    "flip_rate": 0.016666666666666666
  },
  "temperature": 1.5157165665103978,
  "calibrated_clean": {
    "n": 1264,
    "nll": 0.4416141187311443,
    "acc": 0.8243670886075949,
    "ece": 0.029334304090866264,
    "brier": 0.24694275137320498,
    "mean_conf": 0.8342071023922145,
    "confident_error_rate": 0.022943037974683545,
    "coverage_at_0_9": 0.567246835443038,
    "accuracy_at_0_9": 0.9595536959553695,
    "coverage_at_5pct_error": 0.622626582278481,
    "coverage_at_1pct_error": 0.21044303797468356,
    "aurc": 0.05692034604957053,
    "error_rate_at_0_9": 0.040446304044630406,
    "confidence_bias": 0.009840013784619628,
    "top_bins": {
      "0.9": {
        "n": 717,
        "errors": 29,
        "error_rate": 0.040446304044630406
      },
      "0.95": {
        "n": 615,
        "errors": 19,
        "error_rate": 0.030894308943089432
      },
      "0.99": {
        "n": 326,
        "errors": 6,
        "error_rate": 0.018404907975460124
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9651898734177216,
        "confidence_cutoff": 0.9445154395381901
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.8913043478260869,
        "confidence_cutoff": 0.606953327797692
      }
    },
    "score_mae": 0.4676383167623379,
    "ranked_probability_score": 0.07005212248660367
  },
  "metric_policy": {
    "version": 2,
    "selective_ties": "whole_confidence_groups",
    "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee",
    "aurc": "right-step integral over whole confidence groups",
    "confident_error_rate": "high-confidence errors divided by all questions",
    "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9",
    "nll": "exact from logits when recorded; otherwise from floored probabilities",
    "nll_floor": 1e-09,
    "renormalize_returned_probabilities": true,
    "raw_sums_outside_1e_5": 0,
    "returned_zeros": 0
  },
  "coverage": {
    "requested_records": 1204,
    "requested_questions": 1468,
    "evaluated_records": 1204,
    "evaluated_questions": 1468,
    "rejected_records": 0,
    "truncated_records": 0
  },
  "latency_ms": {
    "median": 79.56905200080655,
    "p95": 175.74176955040457
  },
  "calibration": {
    "inference_temperature": 1.0,
    "additional_temperature": 1.5157165665103978,
    "logits_recorded": true
  },
  "transfer": {
    "objective": -0.8434221557083209,
    "paired_flip": {
      "pairs": 64,
      "flip_rate": 0.40625,
      "both_correct_rate": 0.34375,
      "invariant_pairs": 24,
      "invariance_rate": 0.9583333333333334,
      "invariant_both_correct_rate": 0.5833333333333334
    },
    "unknowable": null,
    "clean": {
      "n": 656,
      "nll": 0.8332240113525762,
      "acc": 0.6021341463414634,
      "ece": 0.12798419967379476,
      "brier": 0.5023300109226222,
      "mean_conf": 0.7301183460152582,
      "confident_error_rate": 0.04573170731707317,
      "coverage_at_0_9": 0.3125,
      "accuracy_at_0_9": 0.8536585365853658,
      "coverage_at_5pct_error": 0.11890243902439024,
      "coverage_at_1pct_error": 0.09451219512195122,
      "aurc": 0.23059238555383313,
      "error_rate_at_0_9": 0.14634146341463414,
      "confidence_bias": 0.1279841996737947,
      "top_bins": {
        "0.9": {
          "n": 205,
          "errors": 30,
          "error_rate": 0.14634146341463414
        },
        "0.95": {
          "n": 151,
          "errors": 20,
          "error_rate": 0.13245033112582782
        },
        "0.99": {
          "n": 79,
          "errors": 4,
          "error_rate": 0.05063291139240506
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.725609756097561,
          "confidence_cutoff": 0.7209882521334812
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.6533333333333333,
          "confidence_cutoff": 0.5341488281217432
        }
      },
      "score_mae": 0.6641775577164577,
      "ranked_probability_score": 0.29565575671388794
    },
    "tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 0.6427335354097619,
        "acc": 0.65625,
        "ece": 0.07911568686379761,
        "brier": 0.45100153299560664,
        "mean_conf": 0.5864401989835899,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.15625,
        "coverage_at_1pct_error": 0.15625,
        "aurc": 0.24615223624507382,
        "error_rate_at_0_9": null,
        "confidence_bias": -0.0698098010164101,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.53125,
            "accuracy": 0.6470588235294118,
            "confidence_cutoff": 0.5770142502482459
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.6153846153846154,
            "confidence_cutoff": 0.5480732917785645
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.7002559076723711,
        "acc": 0.5,
        "ece": 0.040716346048888484,
        "brier": 0.5069657859898338,
        "mean_conf": 0.5407163460488885,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.5428605550039391,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.04071634604888852,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.5,
            "confidence_cutoff": 0.5319679538002
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.46153846153846156,
            "confidence_cutoff": 0.516680121421814
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.6868598188008299,
        "acc": 0.5,
        "ece": 0.11919972571689182,
        "brier": 0.49389757171435983,
        "mean_conf": 0.5462136768533761,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.09375,
        "coverage_at_1pct_error": 0.09375,
        "aurc": 0.39094494868742846,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.04621367685337607,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.6875,
            "confidence_cutoff": 0.5332706888479405
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.46153846153846156,
            "confidence_cutoff": 0.5239105664518307
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 0.1012225013321368,
        "acc": 0.975,
        "ece": 0.0636128189986208,
        "brier": 0.04847437535132595,
        "mean_conf": 0.9158894408229405,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.725,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 1.0,
        "coverage_at_1pct_error": 0.975,
        "aurc": 0.000625,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.05911055917705943,
        "top_bins": {
          "0.9": {
            "n": 29,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 23,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 20,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9909082970328174
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 1.0,
            "confidence_cutoff": 0.8867185513663616
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 2.3533720238091194,
        "acc": 0.3,
        "ece": 0.6268044472682741,
        "brier": 1.1752809482099298,
        "mean_conf": 0.902081250744253,
        "confident_error_rate": 0.45,
        "coverage_at_0_9": 0.65,
        "accuracy_at_0_9": 0.3076923076923077,
        "coverage_at_5pct_error": 0.075,
        "coverage_at_1pct_error": 0.075,
        "aurc": 0.5450540740713118,
        "error_rate_at_0_9": 0.6923076923076923,
        "confidence_bias": 0.602081250744253,
        "top_bins": {
          "0.9": {
            "n": 26,
            "errors": 18,
            "error_rate": 0.6923076923076923
          },
          "0.95": {
            "n": 22,
            "errors": 14,
            "error_rate": 0.6363636363636364
          },
          "0.99": {
            "n": 9,
            "errors": 4,
            "error_rate": 0.4444444444444444
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.4,
            "confidence_cutoff": 0.9668620287297324
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.3125,
            "confidence_cutoff": 0.8341740534671955
          }
        },
        "score_mae": 0.6641775577164577,
        "ranked_probability_score": 0.29565575671388794
      },
      "emotion": {
        "n": 80,
        "nll": 1.1638223380813226,
        "acc": 0.525,
        "ece": 0.11298484196696625,
        "brier": 0.5903870224663738,
        "mean_conf": 0.6261382515881248,
        "confident_error_rate": 0.0125,
        "coverage_at_0_9": 0.1125,
        "accuracy_at_0_9": 0.8888888888888888,
        "coverage_at_5pct_error": 0.0625,
        "coverage_at_1pct_error": 0.0625,
        "aurc": 0.2782174494810715,
        "error_rate_at_0_9": 0.1111111111111111,
        "confidence_bias": 0.10113825158812473,
        "top_bins": {
          "0.9": {
            "n": 9,
            "errors": 1,
            "error_rate": 0.1111111111111111
          },
          "0.95": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.725,
            "confidence_cutoff": 0.6017182592654616
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.609375,
            "confidence_cutoff": 0.4732868788719531
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 1.2132248805897294,
        "acc": 0.425,
        "ece": 0.17901472294834253,
        "brier": 0.6687983463380192,
        "mean_conf": 0.5761396137039483,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.125,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.15,
        "coverage_at_1pct_error": 0.15,
        "aurc": 0.38430439973401825,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.1511396137039483,
        "top_bins": {
          "0.9": {
            "n": 10,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.6,
            "confidence_cutoff": 0.5105295219311827
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.4375,
            "confidence_cutoff": 0.385735625241166
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 1.00533805602315,
        "acc": 0.3,
        "ece": 0.3806088203975223,
        "brier": 0.7785508948914812,
        "mean_conf": 0.6776616149008904,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0125,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.0375,
        "coverage_at_1pct_error": 0.0375,
        "aurc": 0.7547032981712617,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.3776616149008904,
        "top_bins": {
          "0.9": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.15,
            "confidence_cutoff": 0.6846514736219801
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.25,
            "confidence_cutoff": 0.580651581287384
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.5594003098802613,
        "acc": 0.7625,
        "ece": 0.10667716921885279,
        "brier": 0.3553818939712926,
        "mean_conf": 0.818959518366006,
        "confident_error_rate": 0.0625,
        "coverage_at_0_9": 0.4375,
        "accuracy_at_0_9": 0.8571428571428571,
        "coverage_at_5pct_error": 0.0625,
        "coverage_at_1pct_error": 0.0625,
        "aurc": 0.14924964619005282,
        "error_rate_at_0_9": 0.14285714285714285,
        "confidence_bias": 0.056459518366006045,
        "top_bins": {
          "0.9": {
            "n": 35,
            "errors": 5,
            "error_rate": 0.14285714285714285
          },
          "0.95": {
            "n": 21,
            "errors": 2,
            "error_rate": 0.09523809523809523
          },
          "0.99": {
            "n": 4,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.825,
            "confidence_cutoff": 0.8597152316740018
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.78125,
            "confidence_cutoff": 0.675449788570404
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.18838356801169662,
        "acc": 0.925,
        "ece": 0.06864685652623173,
        "brier": 0.10770728627230633,
        "mean_conf": 0.895637017961667,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.7625,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.95,
        "coverage_at_1pct_error": 0.7875,
        "aurc": 0.007778982639311541,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.029362982038333096,
        "top_bins": {
          "0.9": {
            "n": 61,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 54,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 40,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9910152874950432
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.984375,
            "confidence_cutoff": 0.8018387376851126
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.6630307731811524,
        "acc": 0.7,
        "ece": 0.11410098626654272,
        "brier": 0.4256570275654813,
        "mean_conf": 0.8141009862665427,
        "confident_error_rate": 0.075,
        "coverage_at_0_9": 0.425,
        "accuracy_at_0_9": 0.8235294117647058,
        "coverage_at_5pct_error": 0.0875,
        "coverage_at_1pct_error": 0.0875,
        "aurc": 0.20144605524801446,
        "error_rate_at_0_9": 0.17647058823529413,
        "confidence_bias": 0.11410098626654275,
        "top_bins": {
          "0.9": {
            "n": 34,
            "errors": 6,
            "error_rate": 0.17647058823529413
          },
          "0.95": {
            "n": 18,
            "errors": 4,
            "error_rate": 0.2222222222222222
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.825,
            "confidence_cutoff": 0.8868192435698719
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.75,
            "confidence_cutoff": 0.6436328483710542
          }
        }
      }
    },
    "variants": {
      "clean": {
        "n": 656,
        "nll": 0.8332240113525762,
        "acc": 0.6021341463414634,
        "ece": 0.12798419967379476,
        "brier": 0.5023300109226222,
        "mean_conf": 0.7301183460152582,
        "confident_error_rate": 0.04573170731707317,
        "coverage_at_0_9": 0.3125,
        "accuracy_at_0_9": 0.8536585365853658,
        "coverage_at_5pct_error": 0.11890243902439024,
        "coverage_at_1pct_error": 0.09451219512195122,
        "aurc": 0.23059238555383313,
        "error_rate_at_0_9": 0.14634146341463414,
        "confidence_bias": 0.1279841996737947,
        "top_bins": {
          "0.9": {
            "n": 205,
            "errors": 30,
            "error_rate": 0.14634146341463414
          },
          "0.95": {
            "n": 151,
            "errors": 20,
            "error_rate": 0.13245033112582782
          },
          "0.99": {
            "n": 79,
            "errors": 4,
            "error_rate": 0.05063291139240506
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.725609756097561,
            "confidence_cutoff": 0.7209882521334812
          },
          "0.8": {
            "coverage": 0.8003048780487805,
            "accuracy": 0.6533333333333333,
            "confidence_cutoff": 0.5341488281217432
          }
        },
        "score_mae": 0.6641775577164577,
        "ranked_probability_score": 0.29565575671388794
      },
      "none_absent": {
        "n": 36,
        "nll": 0.6685620869807386,
        "acc": 0.7777777777777778,
        "ece": 0.18033089059255358,
        "brier": 0.37205937297465375,
        "mean_conf": 0.7657635384076253,
        "confident_error_rate": 0.08333333333333333,
        "coverage_at_0_9": 0.4166666666666667,
        "accuracy_at_0_9": 0.8,
        "coverage_at_5pct_error": 0.08333333333333333,
        "coverage_at_1pct_error": 0.08333333333333333,
        "aurc": 0.17909033623816945,
        "error_rate_at_0_9": 0.2,
        "confidence_bias": -0.012014239370152446,
        "top_bins": {
          "0.9": {
            "n": 15,
            "errors": 3,
            "error_rate": 0.2
          },
          "0.95": {
            "n": 6,
            "errors": 1,
            "error_rate": 0.16666666666666666
          },
          "0.99": {
            "n": 2,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7777777777777778,
            "confidence_cutoff": 0.8402871256038218
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.7931034482758621,
            "confidence_cutoff": 0.5899206246083812
          }
        }
      },
      "none_present": {
        "n": 36,
        "nll": 1.2061311893085476,
        "acc": 0.5833333333333334,
        "ece": 0.24336477102423715,
        "brier": 0.5794150123852385,
        "mean_conf": 0.7670494004663645,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.3333333333333333,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.4166666666666667,
        "coverage_at_1pct_error": 0.4166666666666667,
        "aurc": 0.1664477209679199,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.18371606713303112,
        "top_bins": {
          "0.9": {
            "n": 12,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 9,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 2,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.8888888888888888,
            "confidence_cutoff": 0.8493899555479588
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.6551724137931034,
            "confidence_cutoff": 0.5037377697189107
          }
        }
      },
      "permuted": {
        "n": 36,
        "nll": 0.710652540300337,
        "acc": 0.75,
        "ece": 0.16435148952453013,
        "brier": 0.37681908989783053,
        "mean_conf": 0.7630775644796315,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.4166666666666667,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.4444444444444444,
        "coverage_at_1pct_error": 0.4444444444444444,
        "aurc": 0.09547117668697218,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.013077564479631487,
        "top_bins": {
          "0.9": {
            "n": 15,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 15,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.9444444444444444,
            "confidence_cutoff": 0.8530542285676821
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.7586206896551724,
            "confidence_cutoff": 0.4821188982489635
          }
        }
      }
    },
    "heldout_tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 0.6427335354097619,
        "acc": 0.65625,
        "ece": 0.07911568686379761,
        "brier": 0.45100153299560664,
        "mean_conf": 0.5864401989835899,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.15625,
        "coverage_at_1pct_error": 0.15625,
        "aurc": 0.24615223624507382,
        "error_rate_at_0_9": null,
        "confidence_bias": -0.0698098010164101,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.53125,
            "accuracy": 0.6470588235294118,
            "confidence_cutoff": 0.5770142502482459
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.6153846153846154,
            "confidence_cutoff": 0.5480732917785645
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.7002559076723711,
        "acc": 0.5,
        "ece": 0.040716346048888484,
        "brier": 0.5069657859898338,
        "mean_conf": 0.5407163460488885,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.5428605550039391,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.04071634604888852,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.5,
            "confidence_cutoff": 0.5319679538002
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.46153846153846156,
            "confidence_cutoff": 0.516680121421814
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.6868598188008299,
        "acc": 0.5,
        "ece": 0.11919972571689182,
        "brier": 0.49389757171435983,
        "mean_conf": 0.5462136768533761,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.09375,
        "coverage_at_1pct_error": 0.09375,
        "aurc": 0.39094494868742846,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.04621367685337607,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.6875,
            "confidence_cutoff": 0.5332706888479405
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.46153846153846156,
            "confidence_cutoff": 0.5239105664518307
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 0.1012225013321368,
        "acc": 0.975,
        "ece": 0.0636128189986208,
        "brier": 0.04847437535132595,
        "mean_conf": 0.9158894408229405,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.725,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 1.0,
        "coverage_at_1pct_error": 0.975,
        "aurc": 0.000625,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.05911055917705943,
        "top_bins": {
          "0.9": {
            "n": 29,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 23,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 20,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9909082970328174
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 1.0,
            "confidence_cutoff": 0.8867185513663616
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 2.3533720238091194,
        "acc": 0.3,
        "ece": 0.6268044472682741,
        "brier": 1.1752809482099298,
        "mean_conf": 0.902081250744253,
        "confident_error_rate": 0.45,
        "coverage_at_0_9": 0.65,
        "accuracy_at_0_9": 0.3076923076923077,
        "coverage_at_5pct_error": 0.075,
        "coverage_at_1pct_error": 0.075,
        "aurc": 0.5450540740713118,
        "error_rate_at_0_9": 0.6923076923076923,
        "confidence_bias": 0.602081250744253,
        "top_bins": {
          "0.9": {
            "n": 26,
            "errors": 18,
            "error_rate": 0.6923076923076923
          },
          "0.95": {
            "n": 22,
            "errors": 14,
            "error_rate": 0.6363636363636364
          },
          "0.99": {
            "n": 9,
            "errors": 4,
            "error_rate": 0.4444444444444444
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.4,
            "confidence_cutoff": 0.9668620287297324
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.3125,
            "confidence_cutoff": 0.8341740534671955
          }
        },
        "score_mae": 0.6641775577164577,
        "ranked_probability_score": 0.29565575671388794
      },
      "emotion": {
        "n": 80,
        "nll": 1.1638223380813226,
        "acc": 0.525,
        "ece": 0.11298484196696625,
        "brier": 0.5903870224663738,
        "mean_conf": 0.6261382515881248,
        "confident_error_rate": 0.0125,
        "coverage_at_0_9": 0.1125,
        "accuracy_at_0_9": 0.8888888888888888,
        "coverage_at_5pct_error": 0.0625,
        "coverage_at_1pct_error": 0.0625,
        "aurc": 0.2782174494810715,
        "error_rate_at_0_9": 0.1111111111111111,
        "confidence_bias": 0.10113825158812473,
        "top_bins": {
          "0.9": {
            "n": 9,
            "errors": 1,
            "error_rate": 0.1111111111111111
          },
          "0.95": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.725,
            "confidence_cutoff": 0.6017182592654616
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.609375,
            "confidence_cutoff": 0.4732868788719531
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 1.2132248805897294,
        "acc": 0.425,
        "ece": 0.17901472294834253,
        "brier": 0.6687983463380192,
        "mean_conf": 0.5761396137039483,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.125,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.15,
        "coverage_at_1pct_error": 0.15,
        "aurc": 0.38430439973401825,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.1511396137039483,
        "top_bins": {
          "0.9": {
            "n": 10,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.6,
            "confidence_cutoff": 0.5105295219311827
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.4375,
            "confidence_cutoff": 0.385735625241166
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 1.00533805602315,
        "acc": 0.3,
        "ece": 0.3806088203975223,
        "brier": 0.7785508948914812,
        "mean_conf": 0.6776616149008904,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0125,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.0375,
        "coverage_at_1pct_error": 0.0375,
        "aurc": 0.7547032981712617,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.3776616149008904,
        "top_bins": {
          "0.9": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.15,
            "confidence_cutoff": 0.6846514736219801
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.25,
            "confidence_cutoff": 0.580651581287384
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.5594003098802613,
        "acc": 0.7625,
        "ece": 0.10667716921885279,
        "brier": 0.3553818939712926,
        "mean_conf": 0.818959518366006,
        "confident_error_rate": 0.0625,
        "coverage_at_0_9": 0.4375,
        "accuracy_at_0_9": 0.8571428571428571,
        "coverage_at_5pct_error": 0.0625,
        "coverage_at_1pct_error": 0.0625,
        "aurc": 0.14924964619005282,
        "error_rate_at_0_9": 0.14285714285714285,
        "confidence_bias": 0.056459518366006045,
        "top_bins": {
          "0.9": {
            "n": 35,
            "errors": 5,
            "error_rate": 0.14285714285714285
          },
          "0.95": {
            "n": 21,
            "errors": 2,
            "error_rate": 0.09523809523809523
          },
          "0.99": {
            "n": 4,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.825,
            "confidence_cutoff": 0.8597152316740018
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.78125,
            "confidence_cutoff": 0.675449788570404
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.18838356801169662,
        "acc": 0.925,
        "ece": 0.06864685652623173,
        "brier": 0.10770728627230633,
        "mean_conf": 0.895637017961667,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.7625,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.95,
        "coverage_at_1pct_error": 0.7875,
        "aurc": 0.007778982639311541,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.029362982038333096,
        "top_bins": {
          "0.9": {
            "n": 61,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 54,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 40,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9910152874950432
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.984375,
            "confidence_cutoff": 0.8018387376851126
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.6630307731811524,
        "acc": 0.7,
        "ece": 0.11410098626654272,
        "brier": 0.4256570275654813,
        "mean_conf": 0.8141009862665427,
        "confident_error_rate": 0.075,
        "coverage_at_0_9": 0.425,
        "accuracy_at_0_9": 0.8235294117647058,
        "coverage_at_5pct_error": 0.0875,
        "coverage_at_1pct_error": 0.0875,
        "aurc": 0.20144605524801446,
        "error_rate_at_0_9": 0.17647058823529413,
        "confidence_bias": 0.11410098626654275,
        "top_bins": {
          "0.9": {
            "n": 34,
            "errors": 6,
            "error_rate": 0.17647058823529413
          },
          "0.95": {
            "n": 18,
            "errors": 4,
            "error_rate": 0.2222222222222222
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.825,
            "confidence_cutoff": 0.8868192435698719
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.75,
            "confidence_cutoff": 0.6436328483710542
          }
        }
      }
    },
    "permutation": {
      "n": 36,
      "mean_max_delta": 0.07456167804296458,
      "flip_rate": 0.1111111111111111
    },
    "temperature": 1.5157165665103978,
    "calibrated_clean": {
      "n": 656,
      "nll": 0.7627521342590968,
      "acc": 0.6021341463414634,
      "ece": 0.07572062366206761,
      "brier": 0.4747254916137418,
      "mean_conf": 0.669677827611837,
      "confident_error_rate": 0.021341463414634148,
      "coverage_at_0_9": 0.18445121951219512,
      "accuracy_at_0_9": 0.8842975206611571,
      "coverage_at_5pct_error": 0.12652439024390244,
      "coverage_at_1pct_error": 0.09298780487804878,
      "aurc": 0.23002342930643124,
      "error_rate_at_0_9": 0.11570247933884298,
      "confidence_bias": 0.06754368127037358,
      "top_bins": {
        "0.9": {
          "n": 121,
          "errors": 14,
          "error_rate": 0.11570247933884298
        },
        "0.95": {
          "n": 75,
          "errors": 4,
          "error_rate": 0.05333333333333334
        },
        "0.99": {
          "n": 15,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.7164634146341463,
          "confidence_cutoff": 0.641505976392506
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.6476190476190476,
          "confidence_cutoff": 0.5125548604205254
        }
      },
      "score_mae": 0.6487109061534533,
      "ranked_probability_score": 0.2651482098948402
    },
    "metric_policy": {
      "version": 2,
      "selective_ties": "whole_confidence_groups",
      "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee",
      "aurc": "right-step integral over whole confidence groups",
      "confident_error_rate": "high-confidence errors divided by all questions",
      "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9",
      "nll": "exact from logits when recorded; otherwise from floored probabilities",
      "nll_floor": 1e-09,
      "renormalize_returned_probabilities": true,
      "raw_sums_outside_1e_5": 0,
      "returned_zeros": 0
    },
    "coverage": {
      "requested_records": 764,
      "requested_questions": 764,
      "evaluated_records": 764,
      "evaluated_questions": 764,
      "rejected_records": 0,
      "truncated_records": 0
    },
    "latency_ms": {
      "median": 79.30278050116613,
      "p95": 83.82728769975074
    },
    "calibration": {
      "inference_temperature": 1.0,
      "additional_temperature": 1.5157165665103978,
      "logits_recorded": true
    },
    "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1"
  },
  "provenance": {
    "config": {
      "epochs": 2,
      "seed": 0,
      "lr": 0.0001,
      "lora": 16,
      "accum": 2,
      "batch": 4,
      "perm_kl": 0.0,
      "perm_frac": 0.3,
      "ord_w": 0.0,
      "p_none": 0.1,
      "p_none_distract": 0.12,
      "p_distract": 0.15,
      "p_none_pair": 0.25,
      "synthetic_repeat": 1,
      "public_frac": 1.0,
      "head_lr": 0.0,
      "weight_decay": 0.01,
      "anchor_w": 0.0,
      "label_smoothing": 0.0,
      "brier_w": 0.0,
      "focal_gamma": 0.0,
      "base": "google/gemma-4-E2B",
      "base_revision": "d29ff6b45f081a49ee2733a859c9c9c2d95d1a6f",
      "dtype": "bf16",
      "weights_dtype": "bf16",
      "checkpointing": 1
    },
    "config_sha256": "cb5576f6ebf3fffe10370041387b4b6810c03c2b112b925168c16ded6729c34c",
    "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2",
    "source_hashes": {
      "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
      "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb",
      "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582",
      "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a",
      "kev/benchmark.py": "9c442efd8b6b852c574a646192c61ee91afa1306e3b2978bb6391ae1b91a3fa0",
      "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519",
      "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59",
      "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435",
      "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
      "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf",
      "kev/data.py": "ef1be396e7218ba5c94f0f67de2ceff23a39656b57199cb3aa3192cde53e8320",
      "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4",
      "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c",
      "kev/experiment.py": "d12ab7f0765de0f29364f3411b99f06bb80b1cbf120ca9ed2b03316ec766e828",
      "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243",
      "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9",
      "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909",
      "kev/model.py": "c2129026ab56395f78c7d3071410541d5423dd7d97f7371518570572f206c376",
      "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02",
      "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3",
      "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa",
      "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735",
      "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3",
      "kev/suite.py": "d18d5f4f02d079344d61c2a31882c524c90012ebfdb660bf9160e145f37e3d42",
      "kev/train.py": "e103e99d3a68ab99eb58f135320fbc46b18514fcc2b195d1c6879633836e99b9",
      "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763",
      "modal_app.py": "7a13e9d29b3fc68f8236a0c1e4d67079b40a2a328a296f7ced1022817dff82aa",
      "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d",
      "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466"
    },
    "git_commit": "ccd6e8b646068a6689792d76e3cae3698e9a015a",
    "platform": "Linux-6.12.103-127.188.amzn2023.x86_64-x86_64-with-glibc2.36",
    "torch": "2.8.0+cu128",
    "device": "cuda",
    "gpu": "NVIDIA L4",
    "legacy_checkpoint": false,
    "measured_checkpoint": {
      "requested": "/kev/runs/g4-e2b-e2-l4/00-trial-0/checkpoint",
      "resolved": "/kev/runs/g4-e2b-e2-l4/00-trial-0/checkpoint",
      "head_sha256": "dd2d8c5b1ac8bee3f413cd89802d93826674baf181dd4649004868b358605785",
      "adapter_sha256": "e23dbe51453b6e926da0db70a38d680bc64644caff872cf0defcbbd8b1acbe46",
      "inference_temperature": 1.0
    }
  },
  "mechanism_checks": {
    "n": 8,
    "packed_max_delta": 0.0,
    "sibling_max_delta": 0.0069029927253723145,
    "tolerance": 0.001,
    "passed": false
  },
  "gates": {
    "passed": false,
    "checks": {
      "complete_coverage": true,
      "isolation_and_packing": false,
      "transfer_complete": true,
      "heldout_pairs_at_least_70pct": false,
      "transfer_confident_errors_below_10pct": true
    },
    "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release."
  },
  "calibration_fit": {
    "temperature": 1.5157165665103978,
    "aggregation": "micro",
    "objective": "raw-logit NLL",
    "split": "calibration",
    "rows_sha256": "d51919a6df076619eb78186da47cebbf6b669444a4c356a5ff4a244184fa6433",
    "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2",
    "n": 1148
  },
  "wall_seconds": 10031.380025413,
  "promotable": false,
  "test_evaluated": false,
  "training_resources": {
    "wall_seconds": 9709.689131975174,
    "records_seen": 30212,
    "requested_records": 25012,
    "truncated_records": 0,
    "rejected_records": 0,
    "optimizer_steps": 3128,
    "forward_tokens": 6112066,
    "peak_device_bytes": 14624471040,
    "device": "cuda",
    "dtype": "bf16",
    "batch": 4,
    "peak_rss_bytes": 11347705856
  }
}