{ "overall": { "count": 6113, "accuracy": 0.6679208244724358, "brier": 0.4544455845096957, "brier_soft": 0.4310818103173888, "nll": 0.8546754183783308, "nll_soft": 0.8720034822503786, "ece": 0.10337806078709712, "mean_confidence": 0.769263460145617, "confident_error_rate": 0.05300179944380828, "score_mae": 0.53153959018479, "evidence_auc": 0.9074727460810058 }, "by_type": { "bool": { "count": 1898, "accuracy": 0.7787144362486829, "brier": 0.32803778696812624, "brier_soft": 0.32512087658844907, "nll": 0.5308636866774319, "nll_soft": 0.5373907544543911, "ece": 0.0827987949639653, "mean_confidence": 0.8615132312126481, "confident_error_rate": 0.07586933614330875, "evidence_auc": 0.9039322111530974 }, "choice": { "count": 3248, "accuracy": 0.625615763546798, "brier": 0.5167065686871394, "brier_soft": 0.4935414744397849, "nll": 1.034598991800825, "nll_soft": 1.0506837306443417, "ece": 0.13399685761808722, "mean_confidence": 0.7569434392263682, "confident_error_rate": 0.05511083743842365, "evidence_auc": 0.9208606009429315 }, "score": { "count": 967, "accuracy": 0.5925542916235781, "brier": 0.4934304067698423, "brier_soft": 0.429266777378387, "nll": 0.8859090277806523, "nll_soft": 0.9286131105577126, "ece": 0.07366032000041674, "mean_confidence": 0.6295792434553326, "confident_error_rate": 0.001034126163391934, "score_mae": 0.53153959018479 } }, "by_source": { "ag_news": { "count": 100, "accuracy": 0.88, "brier": 0.1956816155108337, "brier_soft": 0.1956816155108337, "nll": 0.3569359667701418, "nll_soft": 0.3569359667701418, "ece": 0.09543950200080867, "mean_confidence": 0.957247873544693, "confident_error_rate": 0.06 }, "amazon_polarity": { "count": 100, "accuracy": 0.97, "brier": 0.06277382037310442, "brier_soft": 0.06277382037310442, "nll": 0.14048191098950258, "nll_soft": 0.14048191098950258, "ece": 0.028436743021011374, "mean_confidence": 0.9850584268569946, "confident_error_rate": 0.03 }, "amazon_stars": { "count": 100, "accuracy": 0.53, "brier": 0.576941525754432, "brier_soft": 0.576941525754432, "nll": 0.9323376081861305, "nll_soft": 0.9323376081861305, "ece": 0.16485882312059405, "mean_confidence": 0.6662529197335243, "confident_error_rate": 0.0, "score_mae": 0.5123922033898998 }, "anli": { "count": 100, "accuracy": 0.39, "brier": 1.0697171006877613, "brier_soft": 1.0697171006877613, "nll": 2.53446586810947, "nll_soft": 2.53446586810947, "ece": 0.49495991080999374, "mean_confidence": 0.8732111909985543, "confident_error_rate": 0.35 }, "arc_challenge": { "count": 100, "accuracy": 0.36, "brier": 0.85564326196725, "brier_soft": 0.85564326196725, "nll": 1.6615319409245108, "nll_soft": 1.6615319409245108, "ece": 0.30022275894880296, "mean_confidence": 0.6384962823987007, "confident_error_rate": 0.07 }, "arc_easy": { "count": 100, "accuracy": 0.45, "brier": 0.6727499751604388, "brier_soft": 0.6727499751604388, "nll": 1.3332344578928694, "nll_soft": 1.3332344578928694, "ece": 0.20594446569681168, "mean_confidence": 0.6355124202370643, "confident_error_rate": 0.03 }, "banking77": { "count": 100, "accuracy": 0.81, "brier": 0.27520138323214327, "brier_soft": 0.27520138323214327, "nll": 0.7042584741411863, "nll_soft": 0.7042584741411863, "ece": 0.07309853732585905, "mean_confidence": 0.7600076282024384, "confident_error_rate": 0.0 }, "boolq": { "count": 125, "accuracy": 0.568, "brier": 0.7086733078745265, "brier_soft": 0.7086733078745265, "nll": 1.0996114747593868, "nll_soft": 1.0996114747593868, "ece": 0.345228253364563, "mean_confidence": 0.8863685808181763, "confident_error_rate": 0.264, "evidence_auc": 0.9706666666666667 }, "cb": { "count": 28, "accuracy": 0.9285714285714286, "brier": 0.14187151621545094, "brier_soft": 0.14187151621545094, "nll": 0.3411688505063019, "nll_soft": 0.3411688505063019, "ece": 0.08071247168949669, "mean_confidence": 0.9737525028841836, "confident_error_rate": 0.07142857142857142 }, "civil_comments_severity": { "count": 100, "accuracy": 0.81, "brier": 0.2641616792613485, "brier_soft": 0.18398093610713928, "nll": 0.48525818979939045, "nll_soft": 0.6278173365191989, "ece": 0.0930167320370674, "mean_confidence": 0.7847322449088097, "confident_error_rate": 0.01, "score_mae": 0.31169346647828206 }, "civil_comments_toxic": { "count": 100, "accuracy": 0.94, "brier": 0.08438325557871443, "brier_soft": 0.02902029657244265, "nll": 0.151836350238126, "nll_soft": 0.27572009664481245, "ece": 0.04216815173625941, "mean_confidence": 0.9121545308828354, "confident_error_rate": 0.01 }, "clinc150": { "count": 100, "accuracy": 0.8, "brier": 0.2993320627901788, "brier_soft": 0.2993320627901788, "nll": 0.9652885590344843, "nll_soft": 0.9652885590344843, "ece": 0.11481654163450003, "mean_confidence": 0.7029852807894349, "confident_error_rate": 0.0 }, "cola": { "count": 100, "accuracy": 0.58, "brier": 0.49509596184792515, "brier_soft": 0.49509596184792515, "nll": 0.6953869891066888, "nll_soft": 0.6953869891066888, "ece": 0.15781798183918, "mean_confidence": 0.7369759422540665, "confident_error_rate": 0.02 }, "commonsense_qa": { "count": 100, "accuracy": 0.42, "brier": 0.7442126993153202, "brier_soft": 0.7442126993153202, "nll": 1.4986388927152847, "nll_soft": 1.4986388927152847, "ece": 0.2365888097882271, "mean_confidence": 0.6122629001736641, "confident_error_rate": 0.05 }, "cosmos_qa": { "count": 120, "accuracy": 0.4, "brier": 0.7216093164068191, "brier_soft": 0.7216093164068191, "nll": 1.3523367352044129, "nll_soft": 1.3523367352044129, "ece": 0.22780664563179015, "mean_confidence": 0.6138915047049522, "confident_error_rate": 0.016666666666666666, "evidence_auc": 0.836875 }, "dates": { "count": 300, "accuracy": 0.7633333333333333, "brier": 0.28494306618151344, "brier_soft": 0.28494306618151344, "nll": 0.461646918030587, "nll_soft": 0.461646918030587, "ece": 0.06982577522595727, "mean_confidence": 0.8165360381205876, "confident_error_rate": 0.02666666666666667, "score_mae": 0.42511637235032534 }, "dbpedia": { "count": 100, "accuracy": 0.99, "brier": 0.01313320294332829, "brier_soft": 0.01313320294332829, "nll": 0.022723471569511666, "nll_soft": 0.022723471569511666, "ece": 0.015564617514610263, "mean_confidence": 0.9883377414941787, "confident_error_rate": 0.0 }, "dynasent": { "count": 100, "accuracy": 0.53, "brier": 0.5607406074995769, "brier_soft": 0.42609746333683257, "nll": 0.9629181593054416, "nll_soft": 1.0097704657215225, "ece": 0.1584402248263359, "mean_confidence": 0.6659586235880852, "confident_error_rate": 0.0, "score_mae": 0.5633883766271174 }, "enron_spam": { "count": 100, "accuracy": 0.94, "brier": 0.08979354473427502, "brier_soft": 0.08979354473427502, "nll": 0.1666279776671364, "nll_soft": 0.1666279776671364, "ece": 0.0407087934017182, "mean_confidence": 0.9807087934017181, "confident_error_rate": 0.03 }, "go_emotions": { "count": 100, "accuracy": 0.54, "brier": 0.5986230077830135, "brier_soft": 0.23341383083194883, "nll": 1.5085460014590555, "nll_soft": 2.014061282637175, "ece": 0.1133653888106346, "mean_confidence": 0.47643122762441636, "confident_error_rate": 0.0 }, "hellaswag": { "count": 100, "accuracy": 0.49, "brier": 0.7303985853764171, "brier_soft": 0.7303985853764171, "nll": 1.4383446168093819, "nll_soft": 1.4383446168093819, "ece": 0.22660601407289505, "mean_confidence": 0.658343857228756, "confident_error_rate": 0.06 }, "imdb": { "count": 100, "accuracy": 0.95, "brier": 0.08479113838840116, "brier_soft": 0.08479113838840116, "nll": 0.20038125969079196, "nll_soft": 0.20038125969079196, "ece": 0.04454047501087191, "mean_confidence": 0.977543198466301, "confident_error_rate": 0.03 }, "ledgar": { "count": 100, "accuracy": 0.75, "brier": 0.37746571387860506, "brier_soft": 0.37746571387860506, "nll": 1.117831587098228, "nll_soft": 1.117831587098228, "ece": 0.09851389944553374, "mean_confidence": 0.7810949206352233, "confident_error_rate": 0.01 }, "liar": { "count": 100, "accuracy": 0.32, "brier": 0.7427717464748098, "brier_soft": 0.7427717464748098, "nll": 1.4772028411140192, "nll_soft": 1.4772028411140192, "ece": 0.10479136735200882, "mean_confidence": 0.3535214939713478, "confident_error_rate": 0.0, "score_mae": 0.9304733183607459 }, "measuring_hate_speech": { "count": 100, "accuracy": 0.83, "brier": 0.26393481691390774, "brier_soft": 0.1380680952422536, "nll": 0.5018035055388043, "nll_soft": 0.6680358527776807, "ece": 0.1381094029545784, "mean_confidence": 0.7157146391272545, "confident_error_rate": 0.0, "score_mae": 0.5079832007945515 }, "medqa": { "count": 100, "accuracy": 0.34, "brier": 0.8253335318993641, "brier_soft": 0.8253335318993641, "nll": 1.6004826115846618, "nll_soft": 1.6004826115846618, "ece": 0.24258146941661834, "mean_confidence": 0.5722464674711227, "confident_error_rate": 0.01 }, "mnli": { "count": 100, "accuracy": 0.78, "brier": 0.35049440453320635, "brier_soft": 0.35049440453320635, "nll": 0.7340184397638638, "nll_soft": 0.7340184397638638, "ece": 0.1504778018593788, "mean_confidence": 0.9077262398600578, "confident_error_rate": 0.09 }, "mrpc": { "count": 100, "accuracy": 0.81, "brier": 0.296402084274162, "brier_soft": 0.296402084274162, "nll": 0.49396724593180946, "nll_soft": 0.49396724593180946, "ece": 0.12271335303783416, "mean_confidence": 0.913284073472023, "confident_error_rate": 0.11 }, "multirc": { "count": 125, "accuracy": 0.664, "brier": 0.5778966045169998, "brier_soft": 0.5778966045169998, "nll": 1.0105245482832101, "nll_soft": 1.0105245482832101, "ece": 0.2395370769500732, "mean_confidence": 0.8928435230255127, "confident_error_rate": 0.232, "evidence_auc": 0.6994666666666667 }, "openbookqa": { "count": 100, "accuracy": 0.52, "brier": 0.6241696926704345, "brier_soft": 0.6241696926704345, "nll": 1.2342317482843357, "nll_soft": 1.2342317482843357, "ece": 0.1368167817592621, "mean_confidence": 0.6306584364175797, "confident_error_rate": 0.01 }, "piqa": { "count": 100, "accuracy": 0.61, "brier": 0.5360872271281376, "brier_soft": 0.5360872271281376, "nll": 0.7988801723216932, "nll_soft": 0.7988801723216932, "ece": 0.1281885802745819, "mean_confidence": 0.7082396709918976, "confident_error_rate": 0.07 }, "policy_application": { "count": 150, "accuracy": 0.7333333333333333, "brier": 0.3540402117052364, "brier_soft": 0.3540402117052364, "nll": 0.5500808292524187, "nll_soft": 0.5500808292524187, "ece": 0.07159115393956501, "mean_confidence": 0.7941183785597483, "confident_error_rate": 0.02666666666666667, "evidence_auc": 0.5696533682145193 }, "policy_hard": { "count": 300, "accuracy": 0.6966666666666667, "brier": 0.39680403372516454, "brier_soft": 0.39680403372516454, "nll": 0.6353135836137924, "nll_soft": 0.6353135836137924, "ece": 0.09296291559934615, "mean_confidence": 0.7867085169752439, "confident_error_rate": 0.03, "evidence_auc": 0.7445023148148148 }, "qqp": { "count": 100, "accuracy": 0.8, "brier": 0.33138385122291225, "brier_soft": 0.33138385122291225, "nll": 0.5997304360971085, "nll_soft": 0.5997304360971085, "ece": 0.160344073176384, "mean_confidence": 0.92669453561306, "confident_error_rate": 0.11 }, "race": { "count": 125, "accuracy": 0.464, "brier": 0.6912585134522173, "brier_soft": 0.6912585134522173, "nll": 1.3744160623714958, "nll_soft": 1.3744160623714958, "ece": 0.14605404186248777, "mean_confidence": 0.5852895030975341, "confident_error_rate": 0.016, "evidence_auc": 0.7856 }, "rotten_tomatoes": { "count": 100, "accuracy": 0.89, "brier": 0.2377622469757273, "brier_soft": 0.2377622469757273, "nll": 0.3884358803375364, "nll_soft": 0.3884358803375364, "ece": 0.13390481829643247, "mean_confidence": 0.7656551349163055, "confident_error_rate": 0.0 }, "rte": { "count": 100, "accuracy": 0.7, "brier": 0.48653410178986045, "brier_soft": 0.48653410178986045, "nll": 0.8552582691879445, "nll_soft": 0.8552582691879445, "ece": 0.20795070528984066, "mean_confidence": 0.9079507052898407, "confident_error_rate": 0.2 }, "scotus": { "count": 100, "accuracy": 0.63, "brier": 0.558399670895838, "brier_soft": 0.558399670895838, "nll": 1.3936418468284624, "nll_soft": 1.3936418468284624, "ece": 0.13719820365309715, "mean_confidence": 0.6636926798522472, "confident_error_rate": 0.01 }, "shp": { "count": 100, "accuracy": 0.5, "brier": 0.6054587301920193, "brier_soft": 0.2182656459890131, "nll": 0.8379870996959774, "nll_soft": 0.8549041361552857, "ece": 0.22597848415374755, "mean_confidence": 0.7121235334873199, "confident_error_rate": 0.04 }, "sms_spam": { "count": 100, "accuracy": 0.96, "brier": 0.1060751114641901, "brier_soft": 0.1060751114641901, "nll": 0.21970222462417918, "nll_soft": 0.21970222462417918, "ece": 0.1075866764783859, "mean_confidence": 0.852413323521614, "confident_error_rate": 0.0 }, "snli": { "count": 100, "accuracy": 0.79, "brier": 0.3950873837225307, "brier_soft": 0.3950873837225307, "nll": 0.8923272739656278, "nll_soft": 0.8923272739656278, "ece": 0.20375718295574188, "mean_confidence": 0.9457461875677109, "confident_error_rate": 0.16 }, "social_iqa": { "count": 100, "accuracy": 0.5, "brier": 0.7282720596366673, "brier_soft": 0.7282720596366673, "nll": 1.4898040089024713, "nll_soft": 1.4898040089024713, "ece": 0.2804051974415779, "mean_confidence": 0.766021781861782, "confident_error_rate": 0.11 }, "sst2": { "count": 100, "accuracy": 0.9, "brier": 0.22684034833869313, "brier_soft": 0.22684034833869313, "nll": 0.3794984833755713, "nll_soft": 0.3794984833755713, "ece": 0.14272936522960666, "mean_confidence": 0.7780753713846207, "confident_error_rate": 0.02 }, "sst5": { "count": 100, "accuracy": 0.56, "brier": 0.5610144765744216, "brier_soft": 0.5610144765744216, "nll": 1.012664727296375, "nll_soft": 1.012664727296375, "ece": 0.06806655168533324, "mean_confidence": 0.5649158990383149, "confident_error_rate": 0.0, "score_mae": 0.5462690752366325 }, "stance_abortion": { "count": 60, "accuracy": 0.6333333333333333, "brier": 0.524088736443351, "brier_soft": 0.524088736443351, "nll": 0.8961191172069207, "nll_soft": 0.8961191172069207, "ece": 0.21964709063371024, "mean_confidence": 0.8473132938146591, "confident_error_rate": 0.11666666666666667 }, "stance_atheism": { "count": 60, "accuracy": 0.7, "brier": 0.4295024180903305, "brier_soft": 0.4295024180903305, "nll": 0.7695298592508046, "nll_soft": 0.7695298592508046, "ece": 0.20804943144321447, "mean_confidence": 0.8740352998177211, "confident_error_rate": 0.11666666666666667 }, "stance_feminist": { "count": 60, "accuracy": 0.6833333333333333, "brier": 0.5069301963867938, "brier_soft": 0.5069301963867938, "nll": 0.8636041819841961, "nll_soft": 0.8636041819841961, "ece": 0.24766051123539606, "mean_confidence": 0.8220602666338285, "confident_error_rate": 0.08333333333333333 }, "stance_hillary": { "count": 60, "accuracy": 0.6666666666666666, "brier": 0.5074511953888173, "brier_soft": 0.5074511953888173, "nll": 0.9144219988342865, "nll_soft": 0.9144219988342865, "ece": 0.1895925045013428, "mean_confidence": 0.8227309505144755, "confident_error_rate": 0.08333333333333333 }, "stsb": { "count": 100, "accuracy": 0.45, "brier": 0.6911273352227522, "brier_soft": 0.4113556479959866, "nll": 1.3398778933038693, "nll_soft": 1.3971825733832768, "ece": 0.17724637597799303, "mean_confidence": 0.6206034556031227, "confident_error_rate": 0.0, "score_mae": 0.6759577402091236 }, "subjectivity": { "count": 100, "accuracy": 0.94, "brier": 0.0767058129522083, "brier_soft": 0.0767058129522083, "nll": 0.11295796720663813, "nll_soft": 0.11295796720663813, "ece": 0.04773887455463412, "mean_confidence": 0.9554775649309158, "confident_error_rate": 0.0 }, "swag": { "count": 100, "accuracy": 0.4, "brier": 0.8060758700239877, "brier_soft": 0.8060758700239877, "nll": 1.6377803414833798, "nll_soft": 1.6377803414833798, "ece": 0.313101019859314, "mean_confidence": 0.713101019859314, "confident_error_rate": 0.08 }, "trec": { "count": 100, "accuracy": 0.96, "brier": 0.0627841559130499, "brier_soft": 0.0627841559130499, "nll": 0.1128754384695211, "nll_soft": 0.1128754384695211, "ece": 0.03479043036699291, "mean_confidence": 0.9710938653349876, "confident_error_rate": 0.01 }, "tweet_hate": { "count": 100, "accuracy": 0.54, "brier": 0.4972376556033321, "brier_soft": 0.4972376556033321, "nll": 0.692663662352403, "nll_soft": 0.692663662352403, "ece": 0.11306986272335055, "mean_confidence": 0.6452960926294327, "confident_error_rate": 0.0 }, "tweet_sentiment": { "count": 100, "accuracy": 0.67, "brier": 0.4101129435493609, "brier_soft": 0.4101129435493609, "nll": 0.6681253383561087, "nll_soft": 0.6681253383561087, "ece": 0.06605960577726364, "mean_confidence": 0.6303303089737892, "confident_error_rate": 0.0, "score_mae": 0.3951122248638421 }, "vitaminc": { "count": 100, "accuracy": 0.55, "brier": 0.6667891043228811, "brier_soft": 0.6667891043228811, "nll": 1.3148888047347953, "nll_soft": 1.3148888047347953, "ece": 0.2980202031135559, "mean_confidence": 0.8368437862396241, "confident_error_rate": 0.16 }, "wic": { "count": 100, "accuracy": 0.61, "brier": 0.5528101823846308, "brier_soft": 0.5528101823846308, "nll": 0.8525569252877085, "nll_soft": 0.8525569252877085, "ece": 0.21445172727108003, "mean_confidence": 0.8244517272710801, "confident_error_rate": 0.12 }, "winogrande": { "count": 100, "accuracy": 0.57, "brier": 0.5353339564884243, "brier_soft": 0.5353339564884243, "nll": 0.7633381528774766, "nll_soft": 0.7633381528774766, "ece": 0.17423450052738187, "mean_confidence": 0.7043132489919662, "confident_error_rate": 0.04 }, "yelp": { "count": 100, "accuracy": 0.62, "brier": 0.41939405623046616, "brier_soft": 0.41939405623046616, "nll": 0.7083493947055206, "nll_soft": 0.7083493947055206, "ece": 0.12632228136062626, "mean_confidence": 0.6600254368782044, "confident_error_rate": 0.0, "score_mae": 0.4118902616520063 } }, "reliability": [ { "bin": "0.0-0.1", "count": 4, "confidence": 0.06522660795599222, "accuracy": 0.5 }, { "bin": "0.1-0.2", "count": 11, "confidence": 0.16455867480147968, "accuracy": 0.18181818181818182 }, { "bin": "0.2-0.3", "count": 88, "confidence": 0.2596369771794839, "accuracy": 0.29545454545454547 }, { "bin": "0.3-0.4", "count": 172, "confidence": 0.3557821914207104, "accuracy": 0.25 }, { "bin": "0.4-0.5", "count": 412, "confidence": 0.45668168980809093, "accuracy": 0.40048543689320387 }, { "bin": "0.5-0.6", "count": 828, "confidence": 0.5498056534839713, "accuracy": 0.4782608695652174 }, { "bin": "0.6-0.7", "count": 719, "confidence": 0.6487497972513605, "accuracy": 0.5465924895688457 }, { "bin": "0.7-0.8", "count": 714, "confidence": 0.7503752866212059, "accuracy": 0.6162464985994398 }, { "bin": "0.8-0.9", "count": 827, "confidence": 0.8522957843044793, "accuracy": 0.727932285368803 }, { "bin": "0.9-1.0", "count": 2338, "confidence": 0.9691763384613449, "accuracy": 0.8614200171086399 } ] }