| { |
| "overall": { |
| "count": 5513, |
| "accuracy": 0.7395247596589879, |
| "brier": 0.35543221635694794, |
| "brier_soft": 0.32787097437269497, |
| "nll": 0.6798980190388866, |
| "nll_soft": 0.6939236417123891, |
| "ece": 0.03505060344509826, |
| "mean_confidence": 0.755017923742952, |
| "confident_error_rate": 0.02920370034463994, |
| "score_mae": 0.5833856387926468, |
| "evidence_auc": 0.9994043337529221 |
| }, |
| "by_type": { |
| "choice": { |
| "count": 2993, |
| "accuracy": 0.7213498162378884, |
| "brier": 0.3711201333461809, |
| "brier_soft": 0.3442827979081974, |
| "nll": 0.74394983123531, |
| "nll_soft": 0.7572803793083344, |
| "ece": 0.04781638815122127, |
| "mean_confidence": 0.7496169757884853, |
| "confident_error_rate": 0.024724356832609422, |
| "evidence_auc": 0.9993569870249167 |
| }, |
| "noul": { |
| "count": 1620, |
| "accuracy": 0.8345679012345679, |
| "brier": 0.253961842505383, |
| "brier_soft": 0.2492008712245667, |
| "nll": 0.44479597034730256, |
| "nll_soft": 0.44916685601652256, |
| "ece": 0.06477943753706905, |
| "mean_confidence": 0.8877689307040277, |
| "confident_error_rate": 0.053703703703703705, |
| "evidence_auc": 0.9991164126352993 |
| }, |
| "score": { |
| "count": 900, |
| "accuracy": 0.6288888888888889, |
| "brier": 0.485907849791127, |
| "brier_soft": 0.41489872910403874, |
| "nll": 0.8900738467905202, |
| "nll_soft": 0.9237895052708771, |
| "ece": 0.09814345782042143, |
| "mean_confidence": 0.5340272636882586, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5833856387926468 |
| } |
| }, |
| "by_source": { |
| "ag_news": { |
| "count": 100, |
| "accuracy": 0.88, |
| "brier": 0.1626150587313346, |
| "brier_soft": 0.1626150587313346, |
| "nll": 0.30714077176782956, |
| "nll_soft": 0.30714077176782956, |
| "ece": 0.06605552225768023, |
| "mean_confidence": 0.9010962930653971, |
| "confident_error_rate": 0.03 |
| }, |
| "amazon_polarity": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.0807016434249766, |
| "brier_soft": 0.0807016434249766, |
| "nll": 0.12384397712211918, |
| "nll_soft": 0.12384397712211918, |
| "ece": 0.035107052853219106, |
| "mean_confidence": 0.9569018952897135, |
| "confident_error_rate": 0.0 |
| }, |
| "amazon_stars": { |
| "count": 100, |
| "accuracy": 0.59, |
| "brier": 0.5273514854384125, |
| "brier_soft": 0.5273514854384125, |
| "nll": 0.9298323955017933, |
| "nll_soft": 0.9298323955017933, |
| "ece": 0.08849946708455993, |
| "mean_confidence": 0.525445663454732, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5409659532487517 |
| }, |
| "anli": { |
| "count": 100, |
| "accuracy": 0.48, |
| "brier": 0.7580110428459982, |
| "brier_soft": 0.7580110428459982, |
| "nll": 1.368622216757406, |
| "nll_soft": 1.368622216757406, |
| "ece": 0.3121258033428911, |
| "mean_confidence": 0.7723577728427032, |
| "confident_error_rate": 0.13 |
| }, |
| "arc_challenge": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.24055934761415848, |
| "brier_soft": 0.24055934761415848, |
| "nll": 0.4411706000362115, |
| "nll_soft": 0.4411706000362115, |
| "ece": 0.0673066479533461, |
| "mean_confidence": 0.8316750467569213, |
| "confident_error_rate": 0.0 |
| }, |
| "arc_easy": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.1375523622489258, |
| "brier_soft": 0.1375523622489258, |
| "nll": 0.2746098585419268, |
| "nll_soft": 0.2746098585419268, |
| "ece": 0.06593661835503903, |
| "mean_confidence": 0.8765803273420658, |
| "confident_error_rate": 0.0 |
| }, |
| "banking77": { |
| "count": 100, |
| "accuracy": 0.66, |
| "brier": 0.4431805276953609, |
| "brier_soft": 0.4431805276953609, |
| "nll": 1.1794235133838844, |
| "nll_soft": 1.1794235133838844, |
| "ece": 0.15041971992111725, |
| "mean_confidence": 0.7271155569329256, |
| "confident_error_rate": 0.02 |
| }, |
| "boolq": { |
| "count": 125, |
| "accuracy": 0.672, |
| "brier": 0.5943519230409933, |
| "brier_soft": 0.5943519230409933, |
| "nll": 1.198119047336582, |
| "nll_soft": 1.198119047336582, |
| "ece": 0.2934385165179548, |
| "mean_confidence": 0.9305267511946171, |
| "confident_error_rate": 0.264, |
| "evidence_auc": 1.0 |
| }, |
| "cb": { |
| "count": 28, |
| "accuracy": 0.9285714285714286, |
| "brier": 0.1206399809111701, |
| "brier_soft": 0.1206399809111701, |
| "nll": 0.21512117628452895, |
| "nll_soft": 0.21512117628452895, |
| "ece": 0.08924438988716607, |
| "mean_confidence": 0.9300469646070304, |
| "confident_error_rate": 0.0 |
| }, |
| "civil_comments_severity": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.2695113879269868, |
| "brier_soft": 0.1842131970836758, |
| "nll": 0.5509239546378076, |
| "nll_soft": 0.6722600300908189, |
| "ece": 0.14800686874253358, |
| "mean_confidence": 0.7138696562789161, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.4315058978345452 |
| }, |
| "civil_comments_toxic": { |
| "count": 100, |
| "accuracy": 0.92, |
| "brier": 0.12684197636177458, |
| "brier_soft": 0.04971424161255073, |
| "nll": 0.2250471634876677, |
| "nll_soft": 0.29585551132903215, |
| "ece": 0.053707607791622175, |
| "mean_confidence": 0.8804237395801962, |
| "confident_error_rate": 0.01 |
| }, |
| "clinc150": { |
| "count": 100, |
| "accuracy": 0.72, |
| "brier": 0.40672308629755116, |
| "brier_soft": 0.40672308629755116, |
| "nll": 1.2759781850846823, |
| "nll_soft": 1.2759781850846823, |
| "ece": 0.15851017418590968, |
| "mean_confidence": 0.7813792962015765, |
| "confident_error_rate": 0.04 |
| }, |
| "cola": { |
| "count": 100, |
| "accuracy": 0.72, |
| "brier": 0.4117765664582892, |
| "brier_soft": 0.4117765664582892, |
| "nll": 0.6207637174912584, |
| "nll_soft": 0.6207637174912584, |
| "ece": 0.13335671444872274, |
| "mean_confidence": 0.8533567144487229, |
| "confident_error_rate": 0.08 |
| }, |
| "commonsense_qa": { |
| "count": 100, |
| "accuracy": 0.79, |
| "brier": 0.28843537622683724, |
| "brier_soft": 0.28843537622683724, |
| "nll": 0.5630187107405263, |
| "nll_soft": 0.5630187107405263, |
| "ece": 0.08747740471844836, |
| "mean_confidence": 0.7833733258289846, |
| "confident_error_rate": 0.02 |
| }, |
| "cosmos_qa": { |
| "count": 120, |
| "accuracy": 0.5916666666666667, |
| "brier": 0.5201670608791354, |
| "brier_soft": 0.5201670608791354, |
| "nll": 0.9967133911913827, |
| "nll_soft": 0.9967133911913827, |
| "ece": 0.15986319837836777, |
| "mean_confidence": 0.738942816266573, |
| "confident_error_rate": 0.041666666666666664, |
| "evidence_auc": 0.9925 |
| }, |
| "dbpedia": { |
| "count": 100, |
| "accuracy": 0.99, |
| "brier": 0.011042004099136306, |
| "brier_soft": 0.011042004099136306, |
| "nll": 0.02104353360930288, |
| "nll_soft": 0.02104353360930288, |
| "ece": 0.01389831021923994, |
| "mean_confidence": 0.990020302708288, |
| "confident_error_rate": 0.0 |
| }, |
| "dynasent": { |
| "count": 100, |
| "accuracy": 0.66, |
| "brier": 0.4904603061732375, |
| "brier_soft": 0.3492007570551342, |
| "nll": 0.8315245856719652, |
| "nll_soft": 0.8558785430767095, |
| "ece": 0.13365715855069826, |
| "mean_confidence": 0.5811434655880939, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5566348787742988 |
| }, |
| "enron_spam": { |
| "count": 100, |
| "accuracy": 0.97, |
| "brier": 0.044212729061323425, |
| "brier_soft": 0.044212729061323425, |
| "nll": 0.10096603728495956, |
| "nll_soft": 0.10096603728495956, |
| "ece": 0.01711993302888352, |
| "mean_confidence": 0.9753496362181928, |
| "confident_error_rate": 0.01 |
| }, |
| "go_emotions": { |
| "count": 100, |
| "accuracy": 0.57, |
| "brier": 0.680845077018312, |
| "brier_soft": 0.2609842220598735, |
| "nll": 1.7377780873686146, |
| "nll_soft": 2.1303361224920976, |
| "ece": 0.26184079332116555, |
| "mean_confidence": 0.3081592066788344, |
| "confident_error_rate": 0.0 |
| }, |
| "hellaswag": { |
| "count": 100, |
| "accuracy": 0.82, |
| "brier": 0.24091543463389853, |
| "brier_soft": 0.24091543463389853, |
| "nll": 0.4584441520294868, |
| "nll_soft": 0.4584441520294868, |
| "ece": 0.07152204188632028, |
| "mean_confidence": 0.7853499832313032, |
| "confident_error_rate": 0.0 |
| }, |
| "imdb": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.10389960094912438, |
| "brier_soft": 0.10389960094912438, |
| "nll": 0.18813249438725196, |
| "nll_soft": 0.18813249438725196, |
| "ece": 0.05724087175277302, |
| "mean_confidence": 0.9601319692633947, |
| "confident_error_rate": 0.04 |
| }, |
| "ledgar": { |
| "count": 100, |
| "accuracy": 0.72, |
| "brier": 0.38034738636938825, |
| "brier_soft": 0.38034738636938825, |
| "nll": 1.063681018918801, |
| "nll_soft": 1.063681018918801, |
| "ece": 0.07838524729470606, |
| "mean_confidence": 0.7610558636919699, |
| "confident_error_rate": 0.02 |
| }, |
| "liar": { |
| "count": 100, |
| "accuracy": 0.27, |
| "brier": 0.7719490896567759, |
| "brier_soft": 0.7719490896567759, |
| "nll": 1.5958625278397007, |
| "nll_soft": 1.5958625278397007, |
| "ece": 0.033802663573387265, |
| "mean_confidence": 0.26573383000473283, |
| "confident_error_rate": 0.0, |
| "score_mae": 1.1542656605030175 |
| }, |
| "measuring_hate_speech": { |
| "count": 100, |
| "accuracy": 0.82, |
| "brier": 0.32241765969838126, |
| "brier_soft": 0.18209662000522567, |
| "nll": 0.5957862814286871, |
| "nll_soft": 0.7160770828698846, |
| "ece": 0.19588341038588403, |
| "mean_confidence": 0.6241165896141161, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6257405208951771 |
| }, |
| "medqa": { |
| "count": 100, |
| "accuracy": 0.52, |
| "brier": 0.6204688217755439, |
| "brier_soft": 0.6204688217755439, |
| "nll": 1.1441714418928755, |
| "nll_soft": 1.1441714418928755, |
| "ece": 0.07335443810322065, |
| "mean_confidence": 0.5279374556156311, |
| "confident_error_rate": 0.0 |
| }, |
| "mnli": { |
| "count": 100, |
| "accuracy": 0.89, |
| "brier": 0.17481065065993526, |
| "brier_soft": 0.17481065065993526, |
| "nll": 0.33065511645217294, |
| "nll_soft": 0.33065511645217294, |
| "ece": 0.07649681566926217, |
| "mean_confidence": 0.8625791144915169, |
| "confident_error_rate": 0.03 |
| }, |
| "mrpc": { |
| "count": 100, |
| "accuracy": 0.82, |
| "brier": 0.2540667926292604, |
| "brier_soft": 0.2540667926292604, |
| "nll": 0.4153874415696912, |
| "nll_soft": 0.4153874415696912, |
| "ece": 0.09072470674487898, |
| "mean_confidence": 0.7960669137761736, |
| "confident_error_rate": 0.01 |
| }, |
| "multirc": { |
| "count": 125, |
| "accuracy": 0.728, |
| "brier": 0.4982938010410545, |
| "brier_soft": 0.4982938010410545, |
| "nll": 1.1173533355293808, |
| "nll_soft": 1.1173533355293808, |
| "ece": 0.2368767428889841, |
| "mean_confidence": 0.9123281362860616, |
| "confident_error_rate": 0.192, |
| "evidence_auc": 0.9938666666666667 |
| }, |
| "openbookqa": { |
| "count": 100, |
| "accuracy": 0.73, |
| "brier": 0.35918319957697675, |
| "brier_soft": 0.35918319957697675, |
| "nll": 0.7108537776596335, |
| "nll_soft": 0.7108537776596335, |
| "ece": 0.11907511376015169, |
| "mean_confidence": 0.7974778297066559, |
| "confident_error_rate": 0.06 |
| }, |
| "piqa": { |
| "count": 100, |
| "accuracy": 0.79, |
| "brier": 0.32680077893343484, |
| "brier_soft": 0.32680077893343484, |
| "nll": 0.5077245732572071, |
| "nll_soft": 0.5077245732572071, |
| "ece": 0.11010356068721007, |
| "mean_confidence": 0.7685585521255288, |
| "confident_error_rate": 0.02 |
| }, |
| "policy_application": { |
| "count": 150, |
| "accuracy": 0.8466666666666667, |
| "brier": 0.21779322319452216, |
| "brier_soft": 0.21779322319452216, |
| "nll": 0.3452863418296437, |
| "nll_soft": 0.3452863418296437, |
| "ece": 0.0801742745253821, |
| "mean_confidence": 0.8475822661354353, |
| "confident_error_rate": 0.006666666666666667, |
| "evidence_auc": 0.9908436886854153 |
| }, |
| "qqp": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.18394250218095784, |
| "brier_soft": 0.18394250218095784, |
| "nll": 0.28979379627290724, |
| "nll_soft": 0.28979379627290724, |
| "ece": 0.09559768684175766, |
| "mean_confidence": 0.8727692218380053, |
| "confident_error_rate": 0.01 |
| }, |
| "race": { |
| "count": 125, |
| "accuracy": 0.688, |
| "brier": 0.40703814587255865, |
| "brier_soft": 0.40703814587255865, |
| "nll": 0.7537291180537239, |
| "nll_soft": 0.7537291180537239, |
| "ece": 0.09619744773901716, |
| "mean_confidence": 0.7766490781661468, |
| "confident_error_rate": 0.016, |
| "evidence_auc": 0.9928 |
| }, |
| "rotten_tomatoes": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.14010738440191914, |
| "brier_soft": 0.14010738440191914, |
| "nll": 0.2416655146105705, |
| "nll_soft": 0.2416655146105705, |
| "ece": 0.04597459019218179, |
| "mean_confidence": 0.9104264144844768, |
| "confident_error_rate": 0.02 |
| }, |
| "rte": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.22111515141522584, |
| "brier_soft": 0.22111515141522584, |
| "nll": 0.3433614260676825, |
| "nll_soft": 0.3433614260676825, |
| "ece": 0.05934966559327243, |
| "mean_confidence": 0.9061174995419405, |
| "confident_error_rate": 0.04 |
| }, |
| "scotus": { |
| "count": 100, |
| "accuracy": 0.26, |
| "brier": 0.8915802571770692, |
| "brier_soft": 0.8915802571770692, |
| "nll": 2.3878836951913804, |
| "nll_soft": 2.3878836951913804, |
| "ece": 0.10914746015346483, |
| "mean_confidence": 0.16521583892580075, |
| "confident_error_rate": 0.0 |
| }, |
| "shp": { |
| "count": 100, |
| "accuracy": 0.5, |
| "brier": 0.49990982000791506, |
| "brier_soft": 0.1165292253075058, |
| "nll": 0.6957887808957649, |
| "nll_soft": 0.7022140495979027, |
| "ece": 0.09007377936084067, |
| "mean_confidence": 0.5508245098606296, |
| "confident_error_rate": 0.0 |
| }, |
| "sms_spam": { |
| "count": 100, |
| "accuracy": 0.99, |
| "brier": 0.01945800378248445, |
| "brier_soft": 0.01945800378248445, |
| "nll": 0.0451107545075836, |
| "nll_soft": 0.0451107545075836, |
| "ece": 0.0038497034110411155, |
| "mean_confidence": 0.9921453540295321, |
| "confident_error_rate": 0.01 |
| }, |
| "snli": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.199062955024896, |
| "brier_soft": 0.199062955024896, |
| "nll": 0.3503996991590813, |
| "nll_soft": 0.3503996991590813, |
| "ece": 0.0664930712350235, |
| "mean_confidence": 0.8970722048593263, |
| "confident_error_rate": 0.05 |
| }, |
| "social_iqa": { |
| "count": 100, |
| "accuracy": 0.71, |
| "brier": 0.431037408095384, |
| "brier_soft": 0.431037408095384, |
| "nll": 0.7204327278269901, |
| "nll_soft": 0.7204327278269901, |
| "ece": 0.106641996998918, |
| "mean_confidence": 0.7628192791146897, |
| "confident_error_rate": 0.03 |
| }, |
| "sst2": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.09230732647344755, |
| "brier_soft": 0.09230732647344755, |
| "nll": 0.17918581044030002, |
| "nll_soft": 0.17918581044030002, |
| "ece": 0.03225442688620987, |
| "mean_confidence": 0.9634628465849301, |
| "confident_error_rate": 0.04 |
| }, |
| "sst5": { |
| "count": 100, |
| "accuracy": 0.61, |
| "brier": 0.5360312392825825, |
| "brier_soft": 0.5360312392825825, |
| "nll": 0.9667997383108035, |
| "nll_soft": 0.9667997383108035, |
| "ece": 0.13009597538068704, |
| "mean_confidence": 0.479904024619313, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5185995248390466 |
| }, |
| "stance_abortion": { |
| "count": 60, |
| "accuracy": 0.65, |
| "brier": 0.41941041091876696, |
| "brier_soft": 0.41941041091876696, |
| "nll": 0.6945807290012106, |
| "nll_soft": 0.6945807290012106, |
| "ece": 0.06253681805122369, |
| "mean_confidence": 0.6790044674495604, |
| "confident_error_rate": 0.0 |
| }, |
| "stance_atheism": { |
| "count": 60, |
| "accuracy": 0.7333333333333333, |
| "brier": 0.33573074707097367, |
| "brier_soft": 0.33573074707097367, |
| "nll": 0.5493844906099609, |
| "nll_soft": 0.5493844906099609, |
| "ece": 0.11774155629528504, |
| "mean_confidence": 0.7629887792123973, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "stance_feminist": { |
| "count": 60, |
| "accuracy": 0.65, |
| "brier": 0.42391371238817716, |
| "brier_soft": 0.42391371238817716, |
| "nll": 0.6858903532741335, |
| "nll_soft": 0.6858903532741335, |
| "ece": 0.09564019985317251, |
| "mean_confidence": 0.6729372426738106, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "stance_hillary": { |
| "count": 60, |
| "accuracy": 0.7166666666666667, |
| "brier": 0.3950376026145419, |
| "brier_soft": 0.3950376026145419, |
| "nll": 0.6753047409123483, |
| "nll_soft": 0.6753047409123483, |
| "ece": 0.14666177006731274, |
| "mean_confidence": 0.6947910284891209, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "stsb": { |
| "count": 100, |
| "accuracy": 0.53, |
| "brier": 0.562591826514361, |
| "brier_soft": 0.2903885199851368, |
| "nll": 1.0047635453769461, |
| "nll_soft": 1.042223637401206, |
| "ece": 0.05010116262109817, |
| "mean_confidence": 0.5006841341226713, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5846652815901485 |
| }, |
| "subjectivity": { |
| "count": 100, |
| "accuracy": 0.92, |
| "brier": 0.10356259237332947, |
| "brier_soft": 0.10356259237332947, |
| "nll": 0.1536756285071837, |
| "nll_soft": 0.1536756285071837, |
| "ece": 0.06302427253308573, |
| "mean_confidence": 0.9584267044094797, |
| "confident_error_rate": 0.02 |
| }, |
| "swag": { |
| "count": 100, |
| "accuracy": 0.71, |
| "brier": 0.39564301574865673, |
| "brier_soft": 0.39564301574865673, |
| "nll": 0.718116512775688, |
| "nll_soft": 0.718116512775688, |
| "ece": 0.1196927981046649, |
| "mean_confidence": 0.7007635426910775, |
| "confident_error_rate": 0.0 |
| }, |
| "trec": { |
| "count": 100, |
| "accuracy": 0.93, |
| "brier": 0.09869043173228237, |
| "brier_soft": 0.09869043173228237, |
| "nll": 0.2255870758700891, |
| "nll_soft": 0.2255870758700891, |
| "ece": 0.03541382371297605, |
| "mean_confidence": 0.92494501233904, |
| "confident_error_rate": 0.01 |
| }, |
| "tweet_hate": { |
| "count": 100, |
| "accuracy": 0.66, |
| "brier": 0.4774603068268896, |
| "brier_soft": 0.4774603068268896, |
| "nll": 0.6801771939569775, |
| "nll_soft": 0.6801771939569775, |
| "ece": 0.1890020966071993, |
| "mean_confidence": 0.7647629815710008, |
| "confident_error_rate": 0.02 |
| }, |
| "tweet_sentiment": { |
| "count": 100, |
| "accuracy": 0.71, |
| "brier": 0.4240819345210271, |
| "brier_soft": 0.4240819345210271, |
| "nll": 0.7214182225919487, |
| "nll_soft": 0.7214182225919487, |
| "ece": 0.14850151921708057, |
| "mean_confidence": 0.5748318141162528, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.3819244602989468 |
| }, |
| "vitaminc": { |
| "count": 100, |
| "accuracy": 0.67, |
| "brier": 0.4246865636582119, |
| "brier_soft": 0.4246865636582119, |
| "nll": 0.7577260270925018, |
| "nll_soft": 0.7577260270925018, |
| "ece": 0.13463918498472002, |
| "mean_confidence": 0.7641609989462581, |
| "confident_error_rate": 0.05 |
| }, |
| "wic": { |
| "count": 100, |
| "accuracy": 0.66, |
| "brier": 0.41189935927685106, |
| "brier_soft": 0.41189935927685106, |
| "nll": 0.5931627190857616, |
| "nll_soft": 0.5931627190857616, |
| "ece": 0.08766499905395608, |
| "mean_confidence": 0.6542995718931661, |
| "confident_error_rate": 0.0 |
| }, |
| "winogrande": { |
| "count": 100, |
| "accuracy": 0.58, |
| "brier": 0.5746250741107299, |
| "brier_soft": 0.5746250741107299, |
| "nll": 0.8577718414331656, |
| "nll_soft": 0.8577718414331656, |
| "ece": 0.19468332651889314, |
| "mean_confidence": 0.7540031893020952, |
| "confident_error_rate": 0.11 |
| }, |
| "yelp": { |
| "count": 100, |
| "accuracy": 0.62, |
| "brier": 0.4687757189083782, |
| "brier_soft": 0.4687757189083782, |
| "nll": 0.81375336975503, |
| "nll_soft": 0.81375336975503, |
| "ece": 0.12454148770521407, |
| "mean_confidence": 0.5405161953954996, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.45616857114989023 |
| } |
| }, |
| "reliability": [ |
| { |
| "bin": "0.0-0.1", |
| "count": 2, |
| "confidence": 0.09036682874041202, |
| "accuracy": 0.5 |
| }, |
| { |
| "bin": "0.1-0.2", |
| "count": 122, |
| "confidence": 0.16504008000438247, |
| "accuracy": 0.30327868852459017 |
| }, |
| { |
| "bin": "0.2-0.3", |
| "count": 124, |
| "confidence": 0.2443757508869671, |
| "accuracy": 0.2903225806451613 |
| }, |
| { |
| "bin": "0.3-0.4", |
| "count": 191, |
| "confidence": 0.35652105981608595, |
| "accuracy": 0.418848167539267 |
| }, |
| { |
| "bin": "0.4-0.5", |
| "count": 568, |
| "confidence": 0.46077405748501, |
| "accuracy": 0.47183098591549294 |
| }, |
| { |
| "bin": "0.5-0.6", |
| "count": 538, |
| "confidence": 0.5489642809707137, |
| "accuracy": 0.5557620817843866 |
| }, |
| { |
| "bin": "0.6-0.7", |
| "count": 556, |
| "confidence": 0.6459295074019494, |
| "accuracy": 0.6528776978417267 |
| }, |
| { |
| "bin": "0.7-0.8", |
| "count": 457, |
| "confidence": 0.7521871119337753, |
| "accuracy": 0.7177242888402626 |
| }, |
| { |
| "bin": "0.8-0.9", |
| "count": 678, |
| "confidence": 0.8530844565623106, |
| "accuracy": 0.8097345132743363 |
| }, |
| { |
| "bin": "0.9-1.0", |
| "count": 2277, |
| "confidence": 0.9685375709309736, |
| "accuracy": 0.9292929292929293 |
| } |
| ], |
| "seconds": 31.36005187034607, |
| "model": "runs/assay-1.7b-v2", |
| "data": "data/v2/dev.jsonl" |
| } |
