{ "temperature": { "choice": 1.0779, "score": 1.0253, "null": 1.0253 }, "test": { "ag_news": { "type": "choice", "n": 488, "accuracy": 0.9303, "chance": 0.25, "majority": 0.2705, "nll": 0.201, "brier": 0.1047, "ece": 0.026 }, "ag_news_yn": { "type": "null", "n": 312, "accuracy": 0.9295, "chance": 0.5, "majority": 0.6314, "nll": 0.1621, "brier": 0.0975, "ece": 0.021 }, "amazon_stars": { "type": "score", "n": 800, "accuracy": 0.6162, "chance": 0.2, "majority": 0.2112, "nll": 0.8907, "brier": 0.4995, "ece": 0.0292, "mae": 0.5045 }, "anli": { "type": "choice", "n": 800, "accuracy": 0.84, "chance": 0.4169, "majority": 0.55, "nll": 0.4101, "brier": 0.2414, "ece": 0.0355 }, "app_reviews": { "type": "score", "n": 800, "accuracy": 0.7175, "chance": 0.2, "majority": 0.5763, "nll": 0.7942, "brier": 0.4037, "ece": 0.0362, "mae": 0.6299 }, "arc": { "type": "choice", "n": 218, "accuracy": 0.8486, "chance": 0.2504, "majority": 0.2706, "nll": 0.4031, "brier": 0.2142, "ece": 0.0582 }, "banking77": { "type": "choice", "n": 454, "accuracy": 0.8194, "chance": 0.013, "majority": 0.0264, "nll": 0.5496, "brier": 0.2576, "ece": 0.0486 }, "banking77_yn": { "type": "null", "n": 227, "accuracy": 0.978, "chance": 0.5, "majority": 0.6344, "nll": 0.0588, "brier": 0.0328, "ece": 0.0089 }, "bitext_category": { "type": "choice", "n": 262, "accuracy": 1.0, "chance": 0.0909, "majority": 0.2061, "nll": 0.0014, "brier": 0.0002, "ece": 0.0013 }, "bitext_category_yn": { "type": "null", "n": 146, "accuracy": 0.9863, "chance": 0.5, "majority": 0.5753, "nll": 0.0435, "brier": 0.0192, "ece": 0.0079 }, "bitext_intent": { "type": "choice", "n": 252, "accuracy": 0.9762, "chance": 0.037, "majority": 0.0794, "nll": 0.0729, "brier": 0.0356, "ece": 0.0214 }, "bitext_intent_yn": { "type": "null", "n": 140, "accuracy": 0.9929, "chance": 0.5, "majority": 0.7643, "nll": 0.026, "brier": 0.0132, "ece": 0.0053 }, "boolq": { "type": "null", "n": 651, "accuracy": 0.8356, "chance": 0.5, "majority": 0.6006, "nll": 0.3745, "brier": 0.2374, "ece": 0.0376 }, "clinc_oos": { "type": "choice", "n": 535, "accuracy": 0.9196, "chance": 0.0066, "majority": 0.0224, "nll": 0.2167, "brier": 0.1107, "ece": 0.0207 }, "clinc_oos_yn": { "type": "null", "n": 265, "accuracy": 0.9962, "chance": 0.5, "majority": 0.5132, "nll": 0.0189, "brier": 0.0088, "ece": 0.0114 }, "commonlit": { "type": "score", "n": 258, "accuracy": 0.4574, "chance": 0.2, "majority": 0.2519, "nll": 1.1151, "brier": 0.6129, "ece": 0.0607, "mae": 0.6485 }, "commonsense_qa": { "type": "choice", "n": 689, "accuracy": 0.7808, "chance": 0.2, "majority": 0.2163, "nll": 0.5728, "brier": 0.3027, "ece": 0.0297 }, "emotion": { "type": "choice", "n": 517, "accuracy": 0.9014, "chance": 0.1667, "majority": 0.3482, "nll": 0.1994, "brier": 0.1212, "ece": 0.0364 }, "emotion_yn": { "type": "null", "n": 283, "accuracy": 0.9576, "chance": 0.5, "majority": 0.5654, "nll": 0.1266, "brier": 0.0703, "ece": 0.0279 }, "fever": { "type": "choice", "n": 800, "accuracy": 0.9038, "chance": 0.3956, "majority": 0.5112, "nll": 0.2559, "brier": 0.1445, "ece": 0.0173 }, "hellaswag": { "type": "choice", "n": 800, "accuracy": 0.8812, "chance": 0.25, "majority": 0.2562, "nll": 0.3182, "brier": 0.1698, "ece": 0.013 }, "mnli": { "type": "choice", "n": 800, "accuracy": 0.8912, "chance": 0.4233, "majority": 0.5275, "nll": 0.2736, "brier": 0.157, "ece": 0.0177 }, "openbookqa": { "type": "choice", "n": 325, "accuracy": 0.8154, "chance": 0.25, "majority": 0.2769, "nll": 0.4333, "brier": 0.2293, "ece": 0.0634 }, "paws": { "type": "null", "n": 800, "accuracy": 0.9463, "chance": 0.5, "majority": 0.5487, "nll": 0.1553, "brier": 0.0874, "ece": 0.0176 }, "qqp": { "type": "null", "n": 800, "accuracy": 0.8862, "chance": 0.5, "majority": 0.665, "nll": 0.2803, "brier": 0.1696, "ece": 0.0329 }, "sick": { "type": "score", "n": 633, "accuracy": 0.684, "chance": 0.2, "majority": 0.4107, "nll": 0.6634, "brier": 0.4094, "ece": 0.0552, "mae": 0.3845 }, "snli": { "type": "choice", "n": 800, "accuracy": 0.9225, "chance": 0.4198, "majority": 0.4888, "nll": 0.2145, "brier": 0.1184, "ece": 0.0165 }, "sst2": { "type": "choice", "n": 800, "accuracy": 0.94, "chance": 0.5, "majority": 0.5175, "nll": 0.1648, "brier": 0.0935, "ece": 0.0137 }, "sts_more": { "type": "score", "n": 453, "accuracy": 0.6512, "chance": 0.1667, "majority": 0.3333, "nll": 0.8228, "brier": 0.4691, "ece": 0.0733, "mae": 0.4559 }, "stsb": { "type": "score", "n": 337, "accuracy": 0.6053, "chance": 0.1667, "majority": 0.273, "nll": 0.8552, "brier": 0.5083, "ece": 0.0609, "mae": 0.4576 }, "tripadvisor": { "type": "score", "n": 800, "accuracy": 0.6875, "chance": 0.2, "majority": 0.5125, "nll": 0.7356, "brier": 0.4309, "ece": 0.0281, "mae": 0.4326 }, "tweet_hate": { "type": "null", "n": 633, "accuracy": 0.8199, "chance": 0.5, "majority": 0.564, "nll": 0.4128, "brier": 0.2621, "ece": 0.0326 }, "tweet_sentiment": { "type": "choice", "n": 645, "accuracy": 0.7922, "chance": 0.4013, "majority": 0.3922, "nll": 0.4332, "brier": 0.2717, "ece": 0.0504 }, "tweet_sentiment_yn": { "type": "null", "n": 155, "accuracy": 0.7871, "chance": 0.5, "majority": 0.5484, "nll": 0.4168, "brier": 0.2763, "ece": 0.0659 }, "yelp": { "type": "score", "n": 800, "accuracy": 0.7163, "chance": 0.2, "majority": 0.2162, "nll": 0.6784, "brier": 0.3981, "ece": 0.0327, "mae": 0.3788 }, "_macro": { "accuracy": 0.8404, "chance": 0.3173, "majority": 0.4102, "nll": 0.3815, "ece": 0.0319 } }, "test_uncalibrated": { "ag_news": { "type": "choice", "n": 488, "accuracy": 0.9303, "chance": 0.25, "majority": 0.2705, "nll": 0.1983, "brier": 0.1041, "ece": 0.0203 }, "ag_news_yn": { "type": "null", "n": 312, "accuracy": 0.9295, "chance": 0.5, "majority": 0.6314, "nll": 0.1625, "brier": 0.0978, "ece": 0.02 }, "amazon_stars": { "type": "score", "n": 800, "accuracy": 0.6162, "chance": 0.2, "majority": 0.2112, "nll": 0.891, "brier": 0.4995, "ece": 0.03, "mae": 0.503 }, "anli": { "type": "choice", "n": 800, "accuracy": 0.84, "chance": 0.4169, "majority": 0.55, "nll": 0.4118, "brier": 0.242, "ece": 0.0413 }, "app_reviews": { "type": "score", "n": 800, "accuracy": 0.7175, "chance": 0.2, "majority": 0.5763, "nll": 0.7935, "brier": 0.4036, "ece": 0.025, "mae": 0.6229 }, "arc": { "type": "choice", "n": 218, "accuracy": 0.8486, "chance": 0.2504, "majority": 0.2706, "nll": 0.4019, "brier": 0.2139, "ece": 0.0309 }, "banking77": { "type": "choice", "n": 454, "accuracy": 0.8194, "chance": 0.013, "majority": 0.0264, "nll": 0.5565, "brier": 0.2594, "ece": 0.0705 }, "banking77_yn": { "type": "null", "n": 227, "accuracy": 0.978, "chance": 0.5, "majority": 0.6344, "nll": 0.0585, "brier": 0.0328, "ece": 0.0085 }, "bitext_category": { "type": "choice", "n": 262, "accuracy": 1.0, "chance": 0.0909, "majority": 0.2061, "nll": 0.0011, "brier": 0.0002, "ece": 0.001 }, "bitext_category_yn": { "type": "null", "n": 146, "accuracy": 0.9863, "chance": 0.5, "majority": 0.5753, "nll": 0.0435, "brier": 0.0192, "ece": 0.0076 }, "bitext_intent": { "type": "choice", "n": 252, "accuracy": 0.9762, "chance": 0.037, "majority": 0.0794, "nll": 0.0757, "brier": 0.0363, "ece": 0.0181 }, "bitext_intent_yn": { "type": "null", "n": 140, "accuracy": 0.9929, "chance": 0.5, "majority": 0.7643, "nll": 0.026, "brier": 0.0132, "ece": 0.0046 }, "boolq": { "type": "null", "n": 651, "accuracy": 0.8356, "chance": 0.5, "majority": 0.6006, "nll": 0.3761, "brier": 0.2379, "ece": 0.0372 }, "clinc_oos": { "type": "choice", "n": 535, "accuracy": 0.9196, "chance": 0.0066, "majority": 0.0224, "nll": 0.2184, "brier": 0.1112, "ece": 0.0227 }, "clinc_oos_yn": { "type": "null", "n": 265, "accuracy": 0.9962, "chance": 0.5, "majority": 0.5132, "nll": 0.0183, "brier": 0.0087, "ece": 0.0108 }, "commonlit": { "type": "score", "n": 258, "accuracy": 0.4574, "chance": 0.2, "majority": 0.2519, "nll": 1.1153, "brier": 0.6132, "ece": 0.0672, "mae": 0.6469 }, "commonsense_qa": { "type": "choice", "n": 689, "accuracy": 0.7808, "chance": 0.2, "majority": 0.2163, "nll": 0.5742, "brier": 0.3031, "ece": 0.0373 }, "emotion": { "type": "choice", "n": 517, "accuracy": 0.9014, "chance": 0.1667, "majority": 0.3482, "nll": 0.2027, "brier": 0.1226, "ece": 0.0347 }, "emotion_yn": { "type": "null", "n": 283, "accuracy": 0.9576, "chance": 0.5, "majority": 0.5654, "nll": 0.1273, "brier": 0.0704, "ece": 0.0289 }, "fever": { "type": "choice", "n": 800, "accuracy": 0.9038, "chance": 0.3956, "majority": 0.5112, "nll": 0.2582, "brier": 0.1455, "ece": 0.0231 }, "hellaswag": { "type": "choice", "n": 800, "accuracy": 0.8812, "chance": 0.25, "majority": 0.2562, "nll": 0.3205, "brier": 0.1703, "ece": 0.0304 }, "mnli": { "type": "choice", "n": 800, "accuracy": 0.8912, "chance": 0.4233, "majority": 0.5275, "nll": 0.2739, "brier": 0.1574, "ece": 0.0172 }, "openbookqa": { "type": "choice", "n": 325, "accuracy": 0.8154, "chance": 0.25, "majority": 0.2769, "nll": 0.4289, "brier": 0.2282, "ece": 0.0523 }, "paws": { "type": "null", "n": 800, "accuracy": 0.9463, "chance": 0.5, "majority": 0.5487, "nll": 0.1552, "brier": 0.0875, "ece": 0.0181 }, "qqp": { "type": "null", "n": 800, "accuracy": 0.8862, "chance": 0.5, "majority": 0.665, "nll": 0.2805, "brier": 0.1694, "ece": 0.025 }, "sick": { "type": "score", "n": 633, "accuracy": 0.684, "chance": 0.2, "majority": 0.4107, "nll": 0.6643, "brier": 0.41, "ece": 0.0511, "mae": 0.3832 }, "snli": { "type": "choice", "n": 800, "accuracy": 0.9225, "chance": 0.4198, "majority": 0.4888, "nll": 0.2146, "brier": 0.1185, "ece": 0.0156 }, "sst2": { "type": "choice", "n": 800, "accuracy": 0.94, "chance": 0.5, "majority": 0.5175, "nll": 0.1664, "brier": 0.0937, "ece": 0.015 }, "sts_more": { "type": "score", "n": 453, "accuracy": 0.6512, "chance": 0.1667, "majority": 0.3333, "nll": 0.8228, "brier": 0.4689, "ece": 0.0648, "mae": 0.455 }, "stsb": { "type": "score", "n": 337, "accuracy": 0.6053, "chance": 0.1667, "majority": 0.273, "nll": 0.8565, "brier": 0.5091, "ece": 0.0778, "mae": 0.4572 }, "tripadvisor": { "type": "score", "n": 800, "accuracy": 0.6875, "chance": 0.2, "majority": 0.5125, "nll": 0.7356, "brier": 0.4309, "ece": 0.0329, "mae": 0.4305 }, "tweet_hate": { "type": "null", "n": 633, "accuracy": 0.8199, "chance": 0.5, "majority": 0.564, "nll": 0.4139, "brier": 0.2625, "ece": 0.0343 }, "tweet_sentiment": { "type": "choice", "n": 645, "accuracy": 0.7922, "chance": 0.4013, "majority": 0.3922, "nll": 0.4359, "brier": 0.2735, "ece": 0.0621 }, "tweet_sentiment_yn": { "type": "null", "n": 155, "accuracy": 0.7871, "chance": 0.5, "majority": 0.5484, "nll": 0.4186, "brier": 0.2774, "ece": 0.0693 }, "yelp": { "type": "score", "n": 800, "accuracy": 0.7163, "chance": 0.2, "majority": 0.2162, "nll": 0.6778, "brier": 0.3979, "ece": 0.0294, "mae": 0.3774 }, "_macro": { "accuracy": 0.8404, "chance": 0.3173, "majority": 0.4102, "nll": 0.3822, "ece": 0.0324 } }, "heldout": { "counterfactual": { "type": "null", "n": 600, "accuracy": 0.9, "chance": 0.5, "majority": 0.8883, "nll": 0.2634, "brier": 0.1549, "ece": 0.0294 }, "dbpedia": { "type": "choice", "n": 600, "accuracy": 0.93, "chance": 0.0714, "majority": 0.1017, "nll": 0.2053, "brier": 0.1013, "ece": 0.0235 }, "enron_spam": { "type": "null", "n": 600, "accuracy": 0.615, "chance": 0.5, "majority": 0.5067, "nll": 0.7031, "brier": 0.487, "ece": 0.2422 }, "fin_tweets": { "type": "choice", "n": 600, "accuracy": 0.8317, "chance": 0.3333, "majority": 0.6867, "nll": 0.5174, "brier": 0.298, "ece": 0.1573 }, "massive": { "type": "choice", "n": 600, "accuracy": 0.735, "chance": 0.0169, "majority": 0.0717, "nll": 1.1317, "brier": 0.3809, "ece": 0.1136 }, "rotten_tomatoes": { "type": "null", "n": 600, "accuracy": 0.9217, "chance": 0.5, "majority": 0.505, "nll": 0.2313, "brier": 0.1316, "ece": 0.0288 }, "sst5": { "type": "score", "n": 600, "accuracy": 0.5067, "chance": 0.2, "majority": 0.2967, "nll": 1.0277, "brier": 0.6006, "ece": 0.0662, "mae": 0.5386 }, "toxic": { "type": "null", "n": 600, "accuracy": 0.8667, "chance": 0.5, "majority": 0.9017, "nll": 0.3263, "brier": 0.1945, "ece": 0.09 }, "trec": { "type": "choice", "n": 500, "accuracy": 0.818, "chance": 0.1667, "majority": 0.276, "nll": 0.5322, "brier": 0.2674, "ece": 0.0463 }, "_macro": { "accuracy": 0.7916, "chance": 0.3098, "majority": 0.4705, "nll": 0.5487, "ece": 0.0886 } }, "heldout_uncalibrated": { "counterfactual": { "type": "null", "n": 600, "accuracy": 0.9, "chance": 0.5, "majority": 0.8883, "nll": 0.2625, "brier": 0.1545, "ece": 0.0245 }, "dbpedia": { "type": "choice", "n": 600, "accuracy": 0.93, "chance": 0.0714, "majority": 0.1017, "nll": 0.2054, "brier": 0.1019, "ece": 0.0183 }, "enron_spam": { "type": "null", "n": 600, "accuracy": 0.615, "chance": 0.5, "majority": 0.5067, "nll": 0.7134, "brier": 0.4911, "ece": 0.2448 }, "fin_tweets": { "type": "choice", "n": 600, "accuracy": 0.8317, "chance": 0.3333, "majority": 0.6867, "nll": 0.5023, "brier": 0.2901, "ece": 0.1383 }, "massive": { "type": "choice", "n": 600, "accuracy": 0.735, "chance": 0.0169, "majority": 0.0717, "nll": 1.1827, "brier": 0.3862, "ece": 0.1292 }, "rotten_tomatoes": { "type": "null", "n": 600, "accuracy": 0.9217, "chance": 0.5, "majority": 0.505, "nll": 0.2325, "brier": 0.1317, "ece": 0.0299 }, "sst5": { "type": "score", "n": 600, "accuracy": 0.5067, "chance": 0.2, "majority": 0.2967, "nll": 1.0287, "brier": 0.6017, "ece": 0.0703, "mae": 0.5396 }, "toxic": { "type": "null", "n": 600, "accuracy": 0.8667, "chance": 0.5, "majority": 0.9017, "nll": 0.3232, "brier": 0.1931, "ece": 0.0868 }, "trec": { "type": "choice", "n": 500, "accuracy": 0.818, "chance": 0.1667, "majority": 0.276, "nll": 0.5334, "brier": 0.2668, "ece": 0.0399 }, "_macro": { "accuracy": 0.7916, "chance": 0.3098, "majority": 0.4705, "nll": 0.5538, "ece": 0.0869 } } }