| { |
| "overall": { |
| "count": 6113, |
| "accuracy": 0.6679208244724358, |
| "brier": 0.4544455845096957, |
| "brier_soft": 0.4310818103173888, |
| "nll": 0.8546754183783308, |
| "nll_soft": 0.8720034822503786, |
| "ece": 0.10337806078709712, |
| "mean_confidence": 0.769263460145617, |
| "confident_error_rate": 0.05300179944380828, |
| "score_mae": 0.53153959018479, |
| "evidence_auc": 0.9074727460810058 |
| }, |
| "by_type": { |
| "bool": { |
| "count": 1898, |
| "accuracy": 0.7787144362486829, |
| "brier": 0.32803778696812624, |
| "brier_soft": 0.32512087658844907, |
| "nll": 0.5308636866774319, |
| "nll_soft": 0.5373907544543911, |
| "ece": 0.0827987949639653, |
| "mean_confidence": 0.8615132312126481, |
| "confident_error_rate": 0.07586933614330875, |
| "evidence_auc": 0.9039322111530974 |
| }, |
| "choice": { |
| "count": 3248, |
| "accuracy": 0.625615763546798, |
| "brier": 0.5167065686871394, |
| "brier_soft": 0.4935414744397849, |
| "nll": 1.034598991800825, |
| "nll_soft": 1.0506837306443417, |
| "ece": 0.13399685761808722, |
| "mean_confidence": 0.7569434392263682, |
| "confident_error_rate": 0.05511083743842365, |
| "evidence_auc": 0.9208606009429315 |
| }, |
| "score": { |
| "count": 967, |
| "accuracy": 0.5925542916235781, |
| "brier": 0.4934304067698423, |
| "brier_soft": 0.429266777378387, |
| "nll": 0.8859090277806523, |
| "nll_soft": 0.9286131105577126, |
| "ece": 0.07366032000041674, |
| "mean_confidence": 0.6295792434553326, |
| "confident_error_rate": 0.001034126163391934, |
| "score_mae": 0.53153959018479 |
| } |
| }, |
| "by_source": { |
| "ag_news": { |
| "count": 100, |
| "accuracy": 0.88, |
| "brier": 0.1956816155108337, |
| "brier_soft": 0.1956816155108337, |
| "nll": 0.3569359667701418, |
| "nll_soft": 0.3569359667701418, |
| "ece": 0.09543950200080867, |
| "mean_confidence": 0.957247873544693, |
| "confident_error_rate": 0.06 |
| }, |
| "amazon_polarity": { |
| "count": 100, |
| "accuracy": 0.97, |
| "brier": 0.06277382037310442, |
| "brier_soft": 0.06277382037310442, |
| "nll": 0.14048191098950258, |
| "nll_soft": 0.14048191098950258, |
| "ece": 0.028436743021011374, |
| "mean_confidence": 0.9850584268569946, |
| "confident_error_rate": 0.03 |
| }, |
| "amazon_stars": { |
| "count": 100, |
| "accuracy": 0.53, |
| "brier": 0.576941525754432, |
| "brier_soft": 0.576941525754432, |
| "nll": 0.9323376081861305, |
| "nll_soft": 0.9323376081861305, |
| "ece": 0.16485882312059405, |
| "mean_confidence": 0.6662529197335243, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5123922033898998 |
| }, |
| "anli": { |
| "count": 100, |
| "accuracy": 0.39, |
| "brier": 1.0697171006877613, |
| "brier_soft": 1.0697171006877613, |
| "nll": 2.53446586810947, |
| "nll_soft": 2.53446586810947, |
| "ece": 0.49495991080999374, |
| "mean_confidence": 0.8732111909985543, |
| "confident_error_rate": 0.35 |
| }, |
| "arc_challenge": { |
| "count": 100, |
| "accuracy": 0.36, |
| "brier": 0.85564326196725, |
| "brier_soft": 0.85564326196725, |
| "nll": 1.6615319409245108, |
| "nll_soft": 1.6615319409245108, |
| "ece": 0.30022275894880296, |
| "mean_confidence": 0.6384962823987007, |
| "confident_error_rate": 0.07 |
| }, |
| "arc_easy": { |
| "count": 100, |
| "accuracy": 0.45, |
| "brier": 0.6727499751604388, |
| "brier_soft": 0.6727499751604388, |
| "nll": 1.3332344578928694, |
| "nll_soft": 1.3332344578928694, |
| "ece": 0.20594446569681168, |
| "mean_confidence": 0.6355124202370643, |
| "confident_error_rate": 0.03 |
| }, |
| "banking77": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.27520138323214327, |
| "brier_soft": 0.27520138323214327, |
| "nll": 0.7042584741411863, |
| "nll_soft": 0.7042584741411863, |
| "ece": 0.07309853732585905, |
| "mean_confidence": 0.7600076282024384, |
| "confident_error_rate": 0.0 |
| }, |
| "boolq": { |
| "count": 125, |
| "accuracy": 0.568, |
| "brier": 0.7086733078745265, |
| "brier_soft": 0.7086733078745265, |
| "nll": 1.0996114747593868, |
| "nll_soft": 1.0996114747593868, |
| "ece": 0.345228253364563, |
| "mean_confidence": 0.8863685808181763, |
| "confident_error_rate": 0.264, |
| "evidence_auc": 0.9706666666666667 |
| }, |
| "cb": { |
| "count": 28, |
| "accuracy": 0.9285714285714286, |
| "brier": 0.14187151621545094, |
| "brier_soft": 0.14187151621545094, |
| "nll": 0.3411688505063019, |
| "nll_soft": 0.3411688505063019, |
| "ece": 0.08071247168949669, |
| "mean_confidence": 0.9737525028841836, |
| "confident_error_rate": 0.07142857142857142 |
| }, |
| "civil_comments_severity": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.2641616792613485, |
| "brier_soft": 0.18398093610713928, |
| "nll": 0.48525818979939045, |
| "nll_soft": 0.6278173365191989, |
| "ece": 0.0930167320370674, |
| "mean_confidence": 0.7847322449088097, |
| "confident_error_rate": 0.01, |
| "score_mae": 0.31169346647828206 |
| }, |
| "civil_comments_toxic": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.08438325557871443, |
| "brier_soft": 0.02902029657244265, |
| "nll": 0.151836350238126, |
| "nll_soft": 0.27572009664481245, |
| "ece": 0.04216815173625941, |
| "mean_confidence": 0.9121545308828354, |
| "confident_error_rate": 0.01 |
| }, |
| "clinc150": { |
| "count": 100, |
| "accuracy": 0.8, |
| "brier": 0.2993320627901788, |
| "brier_soft": 0.2993320627901788, |
| "nll": 0.9652885590344843, |
| "nll_soft": 0.9652885590344843, |
| "ece": 0.11481654163450003, |
| "mean_confidence": 0.7029852807894349, |
| "confident_error_rate": 0.0 |
| }, |
| "cola": { |
| "count": 100, |
| "accuracy": 0.58, |
| "brier": 0.49509596184792515, |
| "brier_soft": 0.49509596184792515, |
| "nll": 0.6953869891066888, |
| "nll_soft": 0.6953869891066888, |
| "ece": 0.15781798183918, |
| "mean_confidence": 0.7369759422540665, |
| "confident_error_rate": 0.02 |
| }, |
| "commonsense_qa": { |
| "count": 100, |
| "accuracy": 0.42, |
| "brier": 0.7442126993153202, |
| "brier_soft": 0.7442126993153202, |
| "nll": 1.4986388927152847, |
| "nll_soft": 1.4986388927152847, |
| "ece": 0.2365888097882271, |
| "mean_confidence": 0.6122629001736641, |
| "confident_error_rate": 0.05 |
| }, |
| "cosmos_qa": { |
| "count": 120, |
| "accuracy": 0.4, |
| "brier": 0.7216093164068191, |
| "brier_soft": 0.7216093164068191, |
| "nll": 1.3523367352044129, |
| "nll_soft": 1.3523367352044129, |
| "ece": 0.22780664563179015, |
| "mean_confidence": 0.6138915047049522, |
| "confident_error_rate": 0.016666666666666666, |
| "evidence_auc": 0.836875 |
| }, |
| "dates": { |
| "count": 300, |
| "accuracy": 0.7633333333333333, |
| "brier": 0.28494306618151344, |
| "brier_soft": 0.28494306618151344, |
| "nll": 0.461646918030587, |
| "nll_soft": 0.461646918030587, |
| "ece": 0.06982577522595727, |
| "mean_confidence": 0.8165360381205876, |
| "confident_error_rate": 0.02666666666666667, |
| "score_mae": 0.42511637235032534 |
| }, |
| "dbpedia": { |
| "count": 100, |
| "accuracy": 0.99, |
| "brier": 0.01313320294332829, |
| "brier_soft": 0.01313320294332829, |
| "nll": 0.022723471569511666, |
| "nll_soft": 0.022723471569511666, |
| "ece": 0.015564617514610263, |
| "mean_confidence": 0.9883377414941787, |
| "confident_error_rate": 0.0 |
| }, |
| "dynasent": { |
| "count": 100, |
| "accuracy": 0.53, |
| "brier": 0.5607406074995769, |
| "brier_soft": 0.42609746333683257, |
| "nll": 0.9629181593054416, |
| "nll_soft": 1.0097704657215225, |
| "ece": 0.1584402248263359, |
| "mean_confidence": 0.6659586235880852, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5633883766271174 |
| }, |
| "enron_spam": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.08979354473427502, |
| "brier_soft": 0.08979354473427502, |
| "nll": 0.1666279776671364, |
| "nll_soft": 0.1666279776671364, |
| "ece": 0.0407087934017182, |
| "mean_confidence": 0.9807087934017181, |
| "confident_error_rate": 0.03 |
| }, |
| "go_emotions": { |
| "count": 100, |
| "accuracy": 0.54, |
| "brier": 0.5986230077830135, |
| "brier_soft": 0.23341383083194883, |
| "nll": 1.5085460014590555, |
| "nll_soft": 2.014061282637175, |
| "ece": 0.1133653888106346, |
| "mean_confidence": 0.47643122762441636, |
| "confident_error_rate": 0.0 |
| }, |
| "hellaswag": { |
| "count": 100, |
| "accuracy": 0.49, |
| "brier": 0.7303985853764171, |
| "brier_soft": 0.7303985853764171, |
| "nll": 1.4383446168093819, |
| "nll_soft": 1.4383446168093819, |
| "ece": 0.22660601407289505, |
| "mean_confidence": 0.658343857228756, |
| "confident_error_rate": 0.06 |
| }, |
| "imdb": { |
| "count": 100, |
| "accuracy": 0.95, |
| "brier": 0.08479113838840116, |
| "brier_soft": 0.08479113838840116, |
| "nll": 0.20038125969079196, |
| "nll_soft": 0.20038125969079196, |
| "ece": 0.04454047501087191, |
| "mean_confidence": 0.977543198466301, |
| "confident_error_rate": 0.03 |
| }, |
| "ledgar": { |
| "count": 100, |
| "accuracy": 0.75, |
| "brier": 0.37746571387860506, |
| "brier_soft": 0.37746571387860506, |
| "nll": 1.117831587098228, |
| "nll_soft": 1.117831587098228, |
| "ece": 0.09851389944553374, |
| "mean_confidence": 0.7810949206352233, |
| "confident_error_rate": 0.01 |
| }, |
| "liar": { |
| "count": 100, |
| "accuracy": 0.32, |
| "brier": 0.7427717464748098, |
| "brier_soft": 0.7427717464748098, |
| "nll": 1.4772028411140192, |
| "nll_soft": 1.4772028411140192, |
| "ece": 0.10479136735200882, |
| "mean_confidence": 0.3535214939713478, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.9304733183607459 |
| }, |
| "measuring_hate_speech": { |
| "count": 100, |
| "accuracy": 0.83, |
| "brier": 0.26393481691390774, |
| "brier_soft": 0.1380680952422536, |
| "nll": 0.5018035055388043, |
| "nll_soft": 0.6680358527776807, |
| "ece": 0.1381094029545784, |
| "mean_confidence": 0.7157146391272545, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5079832007945515 |
| }, |
| "medqa": { |
| "count": 100, |
| "accuracy": 0.34, |
| "brier": 0.8253335318993641, |
| "brier_soft": 0.8253335318993641, |
| "nll": 1.6004826115846618, |
| "nll_soft": 1.6004826115846618, |
| "ece": 0.24258146941661834, |
| "mean_confidence": 0.5722464674711227, |
| "confident_error_rate": 0.01 |
| }, |
| "mnli": { |
| "count": 100, |
| "accuracy": 0.78, |
| "brier": 0.35049440453320635, |
| "brier_soft": 0.35049440453320635, |
| "nll": 0.7340184397638638, |
| "nll_soft": 0.7340184397638638, |
| "ece": 0.1504778018593788, |
| "mean_confidence": 0.9077262398600578, |
| "confident_error_rate": 0.09 |
| }, |
| "mrpc": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.296402084274162, |
| "brier_soft": 0.296402084274162, |
| "nll": 0.49396724593180946, |
| "nll_soft": 0.49396724593180946, |
| "ece": 0.12271335303783416, |
| "mean_confidence": 0.913284073472023, |
| "confident_error_rate": 0.11 |
| }, |
| "multirc": { |
| "count": 125, |
| "accuracy": 0.664, |
| "brier": 0.5778966045169998, |
| "brier_soft": 0.5778966045169998, |
| "nll": 1.0105245482832101, |
| "nll_soft": 1.0105245482832101, |
| "ece": 0.2395370769500732, |
| "mean_confidence": 0.8928435230255127, |
| "confident_error_rate": 0.232, |
| "evidence_auc": 0.6994666666666667 |
| }, |
| "openbookqa": { |
| "count": 100, |
| "accuracy": 0.52, |
| "brier": 0.6241696926704345, |
| "brier_soft": 0.6241696926704345, |
| "nll": 1.2342317482843357, |
| "nll_soft": 1.2342317482843357, |
| "ece": 0.1368167817592621, |
| "mean_confidence": 0.6306584364175797, |
| "confident_error_rate": 0.01 |
| }, |
| "piqa": { |
| "count": 100, |
| "accuracy": 0.61, |
| "brier": 0.5360872271281376, |
| "brier_soft": 0.5360872271281376, |
| "nll": 0.7988801723216932, |
| "nll_soft": 0.7988801723216932, |
| "ece": 0.1281885802745819, |
| "mean_confidence": 0.7082396709918976, |
| "confident_error_rate": 0.07 |
| }, |
| "policy_application": { |
| "count": 150, |
| "accuracy": 0.7333333333333333, |
| "brier": 0.3540402117052364, |
| "brier_soft": 0.3540402117052364, |
| "nll": 0.5500808292524187, |
| "nll_soft": 0.5500808292524187, |
| "ece": 0.07159115393956501, |
| "mean_confidence": 0.7941183785597483, |
| "confident_error_rate": 0.02666666666666667, |
| "evidence_auc": 0.5696533682145193 |
| }, |
| "policy_hard": { |
| "count": 300, |
| "accuracy": 0.6966666666666667, |
| "brier": 0.39680403372516454, |
| "brier_soft": 0.39680403372516454, |
| "nll": 0.6353135836137924, |
| "nll_soft": 0.6353135836137924, |
| "ece": 0.09296291559934615, |
| "mean_confidence": 0.7867085169752439, |
| "confident_error_rate": 0.03, |
| "evidence_auc": 0.7445023148148148 |
| }, |
| "qqp": { |
| "count": 100, |
| "accuracy": 0.8, |
| "brier": 0.33138385122291225, |
| "brier_soft": 0.33138385122291225, |
| "nll": 0.5997304360971085, |
| "nll_soft": 0.5997304360971085, |
| "ece": 0.160344073176384, |
| "mean_confidence": 0.92669453561306, |
| "confident_error_rate": 0.11 |
| }, |
| "race": { |
| "count": 125, |
| "accuracy": 0.464, |
| "brier": 0.6912585134522173, |
| "brier_soft": 0.6912585134522173, |
| "nll": 1.3744160623714958, |
| "nll_soft": 1.3744160623714958, |
| "ece": 0.14605404186248777, |
| "mean_confidence": 0.5852895030975341, |
| "confident_error_rate": 0.016, |
| "evidence_auc": 0.7856 |
| }, |
| "rotten_tomatoes": { |
| "count": 100, |
| "accuracy": 0.89, |
| "brier": 0.2377622469757273, |
| "brier_soft": 0.2377622469757273, |
| "nll": 0.3884358803375364, |
| "nll_soft": 0.3884358803375364, |
| "ece": 0.13390481829643247, |
| "mean_confidence": 0.7656551349163055, |
| "confident_error_rate": 0.0 |
| }, |
| "rte": { |
| "count": 100, |
| "accuracy": 0.7, |
| "brier": 0.48653410178986045, |
| "brier_soft": 0.48653410178986045, |
| "nll": 0.8552582691879445, |
| "nll_soft": 0.8552582691879445, |
| "ece": 0.20795070528984066, |
| "mean_confidence": 0.9079507052898407, |
| "confident_error_rate": 0.2 |
| }, |
| "scotus": { |
| "count": 100, |
| "accuracy": 0.63, |
| "brier": 0.558399670895838, |
| "brier_soft": 0.558399670895838, |
| "nll": 1.3936418468284624, |
| "nll_soft": 1.3936418468284624, |
| "ece": 0.13719820365309715, |
| "mean_confidence": 0.6636926798522472, |
| "confident_error_rate": 0.01 |
| }, |
| "shp": { |
| "count": 100, |
| "accuracy": 0.5, |
| "brier": 0.6054587301920193, |
| "brier_soft": 0.2182656459890131, |
| "nll": 0.8379870996959774, |
| "nll_soft": 0.8549041361552857, |
| "ece": 0.22597848415374755, |
| "mean_confidence": 0.7121235334873199, |
| "confident_error_rate": 0.04 |
| }, |
| "sms_spam": { |
| "count": 100, |
| "accuracy": 0.96, |
| "brier": 0.1060751114641901, |
| "brier_soft": 0.1060751114641901, |
| "nll": 0.21970222462417918, |
| "nll_soft": 0.21970222462417918, |
| "ece": 0.1075866764783859, |
| "mean_confidence": 0.852413323521614, |
| "confident_error_rate": 0.0 |
| }, |
| "snli": { |
| "count": 100, |
| "accuracy": 0.79, |
| "brier": 0.3950873837225307, |
| "brier_soft": 0.3950873837225307, |
| "nll": 0.8923272739656278, |
| "nll_soft": 0.8923272739656278, |
| "ece": 0.20375718295574188, |
| "mean_confidence": 0.9457461875677109, |
| "confident_error_rate": 0.16 |
| }, |
| "social_iqa": { |
| "count": 100, |
| "accuracy": 0.5, |
| "brier": 0.7282720596366673, |
| "brier_soft": 0.7282720596366673, |
| "nll": 1.4898040089024713, |
| "nll_soft": 1.4898040089024713, |
| "ece": 0.2804051974415779, |
| "mean_confidence": 0.766021781861782, |
| "confident_error_rate": 0.11 |
| }, |
| "sst2": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.22684034833869313, |
| "brier_soft": 0.22684034833869313, |
| "nll": 0.3794984833755713, |
| "nll_soft": 0.3794984833755713, |
| "ece": 0.14272936522960666, |
| "mean_confidence": 0.7780753713846207, |
| "confident_error_rate": 0.02 |
| }, |
| "sst5": { |
| "count": 100, |
| "accuracy": 0.56, |
| "brier": 0.5610144765744216, |
| "brier_soft": 0.5610144765744216, |
| "nll": 1.012664727296375, |
| "nll_soft": 1.012664727296375, |
| "ece": 0.06806655168533324, |
| "mean_confidence": 0.5649158990383149, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5462690752366325 |
| }, |
| "stance_abortion": { |
| "count": 60, |
| "accuracy": 0.6333333333333333, |
| "brier": 0.524088736443351, |
| "brier_soft": 0.524088736443351, |
| "nll": 0.8961191172069207, |
| "nll_soft": 0.8961191172069207, |
| "ece": 0.21964709063371024, |
| "mean_confidence": 0.8473132938146591, |
| "confident_error_rate": 0.11666666666666667 |
| }, |
| "stance_atheism": { |
| "count": 60, |
| "accuracy": 0.7, |
| "brier": 0.4295024180903305, |
| "brier_soft": 0.4295024180903305, |
| "nll": 0.7695298592508046, |
| "nll_soft": 0.7695298592508046, |
| "ece": 0.20804943144321447, |
| "mean_confidence": 0.8740352998177211, |
| "confident_error_rate": 0.11666666666666667 |
| }, |
| "stance_feminist": { |
| "count": 60, |
| "accuracy": 0.6833333333333333, |
| "brier": 0.5069301963867938, |
| "brier_soft": 0.5069301963867938, |
| "nll": 0.8636041819841961, |
| "nll_soft": 0.8636041819841961, |
| "ece": 0.24766051123539606, |
| "mean_confidence": 0.8220602666338285, |
| "confident_error_rate": 0.08333333333333333 |
| }, |
| "stance_hillary": { |
| "count": 60, |
| "accuracy": 0.6666666666666666, |
| "brier": 0.5074511953888173, |
| "brier_soft": 0.5074511953888173, |
| "nll": 0.9144219988342865, |
| "nll_soft": 0.9144219988342865, |
| "ece": 0.1895925045013428, |
| "mean_confidence": 0.8227309505144755, |
| "confident_error_rate": 0.08333333333333333 |
| }, |
| "stsb": { |
| "count": 100, |
| "accuracy": 0.45, |
| "brier": 0.6911273352227522, |
| "brier_soft": 0.4113556479959866, |
| "nll": 1.3398778933038693, |
| "nll_soft": 1.3971825733832768, |
| "ece": 0.17724637597799303, |
| "mean_confidence": 0.6206034556031227, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6759577402091236 |
| }, |
| "subjectivity": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.0767058129522083, |
| "brier_soft": 0.0767058129522083, |
| "nll": 0.11295796720663813, |
| "nll_soft": 0.11295796720663813, |
| "ece": 0.04773887455463412, |
| "mean_confidence": 0.9554775649309158, |
| "confident_error_rate": 0.0 |
| }, |
| "swag": { |
| "count": 100, |
| "accuracy": 0.4, |
| "brier": 0.8060758700239877, |
| "brier_soft": 0.8060758700239877, |
| "nll": 1.6377803414833798, |
| "nll_soft": 1.6377803414833798, |
| "ece": 0.313101019859314, |
| "mean_confidence": 0.713101019859314, |
| "confident_error_rate": 0.08 |
| }, |
| "trec": { |
| "count": 100, |
| "accuracy": 0.96, |
| "brier": 0.0627841559130499, |
| "brier_soft": 0.0627841559130499, |
| "nll": 0.1128754384695211, |
| "nll_soft": 0.1128754384695211, |
| "ece": 0.03479043036699291, |
| "mean_confidence": 0.9710938653349876, |
| "confident_error_rate": 0.01 |
| }, |
| "tweet_hate": { |
| "count": 100, |
| "accuracy": 0.54, |
| "brier": 0.4972376556033321, |
| "brier_soft": 0.4972376556033321, |
| "nll": 0.692663662352403, |
| "nll_soft": 0.692663662352403, |
| "ece": 0.11306986272335055, |
| "mean_confidence": 0.6452960926294327, |
| "confident_error_rate": 0.0 |
| }, |
| "tweet_sentiment": { |
| "count": 100, |
| "accuracy": 0.67, |
| "brier": 0.4101129435493609, |
| "brier_soft": 0.4101129435493609, |
| "nll": 0.6681253383561087, |
| "nll_soft": 0.6681253383561087, |
| "ece": 0.06605960577726364, |
| "mean_confidence": 0.6303303089737892, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.3951122248638421 |
| }, |
| "vitaminc": { |
| "count": 100, |
| "accuracy": 0.55, |
| "brier": 0.6667891043228811, |
| "brier_soft": 0.6667891043228811, |
| "nll": 1.3148888047347953, |
| "nll_soft": 1.3148888047347953, |
| "ece": 0.2980202031135559, |
| "mean_confidence": 0.8368437862396241, |
| "confident_error_rate": 0.16 |
| }, |
| "wic": { |
| "count": 100, |
| "accuracy": 0.61, |
| "brier": 0.5528101823846308, |
| "brier_soft": 0.5528101823846308, |
| "nll": 0.8525569252877085, |
| "nll_soft": 0.8525569252877085, |
| "ece": 0.21445172727108003, |
| "mean_confidence": 0.8244517272710801, |
| "confident_error_rate": 0.12 |
| }, |
| "winogrande": { |
| "count": 100, |
| "accuracy": 0.57, |
| "brier": 0.5353339564884243, |
| "brier_soft": 0.5353339564884243, |
| "nll": 0.7633381528774766, |
| "nll_soft": 0.7633381528774766, |
| "ece": 0.17423450052738187, |
| "mean_confidence": 0.7043132489919662, |
| "confident_error_rate": 0.04 |
| }, |
| "yelp": { |
| "count": 100, |
| "accuracy": 0.62, |
| "brier": 0.41939405623046616, |
| "brier_soft": 0.41939405623046616, |
| "nll": 0.7083493947055206, |
| "nll_soft": 0.7083493947055206, |
| "ece": 0.12632228136062626, |
| "mean_confidence": 0.6600254368782044, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.4118902616520063 |
| } |
| }, |
| "reliability": [ |
| { |
| "bin": "0.0-0.1", |
| "count": 4, |
| "confidence": 0.06522660795599222, |
| "accuracy": 0.5 |
| }, |
| { |
| "bin": "0.1-0.2", |
| "count": 11, |
| "confidence": 0.16455867480147968, |
| "accuracy": 0.18181818181818182 |
| }, |
| { |
| "bin": "0.2-0.3", |
| "count": 88, |
| "confidence": 0.2596369771794839, |
| "accuracy": 0.29545454545454547 |
| }, |
| { |
| "bin": "0.3-0.4", |
| "count": 172, |
| "confidence": 0.3557821914207104, |
| "accuracy": 0.25 |
| }, |
| { |
| "bin": "0.4-0.5", |
| "count": 412, |
| "confidence": 0.45668168980809093, |
| "accuracy": 0.40048543689320387 |
| }, |
| { |
| "bin": "0.5-0.6", |
| "count": 828, |
| "confidence": 0.5498056534839713, |
| "accuracy": 0.4782608695652174 |
| }, |
| { |
| "bin": "0.6-0.7", |
| "count": 719, |
| "confidence": 0.6487497972513605, |
| "accuracy": 0.5465924895688457 |
| }, |
| { |
| "bin": "0.7-0.8", |
| "count": 714, |
| "confidence": 0.7503752866212059, |
| "accuracy": 0.6162464985994398 |
| }, |
| { |
| "bin": "0.8-0.9", |
| "count": 827, |
| "confidence": 0.8522957843044793, |
| "accuracy": 0.727932285368803 |
| }, |
| { |
| "bin": "0.9-1.0", |
| "count": 2338, |
| "confidence": 0.9691763384613449, |
| "accuracy": 0.8614200171086399 |
| } |
| ] |
| } |
