Invalid JSON:Unexpected token 'N', ..."fidence": NaN,
"... is not valid JSON
| { | |
| "overall": { | |
| "count": 5513, | |
| "accuracy": 0.8342100489751496, | |
| "brier": 0.24266185842494856, | |
| "brier_soft": 0.21710945136112134, | |
| "nll": 0.45114049174356935, | |
| "nll_soft": 0.4673940458514404, | |
| "ece": 0.040077075217092, | |
| "mean_confidence": 0.8356972425873123, | |
| "confident_error_rate": 0.020678396517322693, | |
| "score_mae": 0.49627245924055996, | |
| "evidence_auc": 0.9999878244320566 | |
| }, | |
| "by_type": { | |
| "bool": { | |
| "count": 1620, | |
| "accuracy": 0.8753086419753087, | |
| "brier": 0.1834097224750155, | |
| "brier_soft": 0.17999034550789694, | |
| "nll": 0.287108752618916, | |
| "nll_soft": 0.2929012295966247, | |
| "ece": 0.044949108830161205, | |
| "mean_confidence": 0.9199214072135956, | |
| "confident_error_rate": 0.027777777777777776, | |
| "evidence_auc": 0.9999940298151033 | |
| }, | |
| "choice": { | |
| "count": 2993, | |
| "accuracy": 0.8529903107250251, | |
| "brier": 0.21721720365580133, | |
| "brier_soft": 0.19167011910193568, | |
| "nll": 0.4385055987416552, | |
| "nll_soft": 0.45237298320260255, | |
| "ece": 0.03450840124038069, | |
| "mean_confidence": 0.871664572050265, | |
| "confident_error_rate": 0.023053792181757436, | |
| "evidence_auc": 0.9999655528763348 | |
| }, | |
| "score": { | |
| "count": 900, | |
| "accuracy": 0.6977777777777778, | |
| "brier": 0.4339333161615588, | |
| "brier_soft": 0.3685237546210836, | |
| "nll": 0.7884156607843117, | |
| "nll_soft": 0.8314344934522995, | |
| "ece": 0.13329538383179557, | |
| "mean_confidence": 0.5644823939459822, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.49627245924055996 | |
| } | |
| }, | |
| "by_source": { | |
| "ag_news": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.14421606169681028, | |
| "brier_soft": 0.14421606169681028, | |
| "nll": 0.27468413989897816, | |
| "nll_soft": 0.27468413989897816, | |
| "ece": 0.0614854983919261, | |
| "mean_confidence": 0.9180683161223662, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "amazon_polarity": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.05888264685815231, | |
| "brier_soft": 0.05888264685815231, | |
| "nll": 0.09794057175036192, | |
| "nll_soft": 0.09794057175036192, | |
| "ece": 0.01887097873912972, | |
| "mean_confidence": 0.9638854049157964, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "amazon_stars": { | |
| "count": 100, | |
| "accuracy": 0.69, | |
| "brier": 0.4871344044281428, | |
| "brier_soft": 0.4871344044281428, | |
| "nll": 0.8516013229732908, | |
| "nll_soft": 0.8516013229732908, | |
| "ece": 0.17589107107084062, | |
| "mean_confidence": 0.5336082819156109, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4720212013208948 | |
| }, | |
| "anli": { | |
| "count": 100, | |
| "accuracy": 0.74, | |
| "brier": 0.38152495298281364, | |
| "brier_soft": 0.38152495298281364, | |
| "nll": 0.661801793051201, | |
| "nll_soft": 0.661801793051201, | |
| "ece": 0.10857199956125599, | |
| "mean_confidence": 0.8285262554356989, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "arc_challenge": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.03293778487998213, | |
| "brier_soft": 0.03293778487998213, | |
| "nll": 0.06098433202804499, | |
| "nll_soft": 0.06098433202804499, | |
| "ece": 0.010299708559671938, | |
| "mean_confidence": 0.9801719901814279, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "arc_easy": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.0284256234087672, | |
| "brier_soft": 0.0284256234087672, | |
| "nll": 0.0519709495739374, | |
| "nll_soft": 0.0519709495739374, | |
| "ece": 0.023087990617371268, | |
| "mean_confidence": 0.9835592129939819, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "banking77": { | |
| "count": 100, | |
| "accuracy": 0.83, | |
| "brier": 0.27660070225206373, | |
| "brier_soft": 0.27660070225206373, | |
| "nll": 0.712525693075277, | |
| "nll_soft": 0.712525693075277, | |
| "ece": 0.10742287422893768, | |
| "mean_confidence": 0.8310200307491299, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "boolq": { | |
| "count": 125, | |
| "accuracy": 0.72, | |
| "brier": 0.39824933464474527, | |
| "brier_soft": 0.39824933464474527, | |
| "nll": 0.5661039342669112, | |
| "nll_soft": 0.5661039342669112, | |
| "ece": 0.19542354013210722, | |
| "mean_confidence": 0.9012351747266559, | |
| "confident_error_rate": 0.024, | |
| "evidence_auc": 1.0 | |
| }, | |
| "cb": { | |
| "count": 28, | |
| "accuracy": 0.9642857142857143, | |
| "brier": 0.0779562215366468, | |
| "brier_soft": 0.0779562215366468, | |
| "nll": 0.1411652211306288, | |
| "nll_soft": 0.1411652211306288, | |
| "ece": 0.0595834548195279, | |
| "mean_confidence": 0.9340672209362116, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "civil_comments_severity": { | |
| "count": 100, | |
| "accuracy": 0.85, | |
| "brier": 0.23714380547223418, | |
| "brier_soft": 0.15459555443470976, | |
| "nll": 0.46339746603398696, | |
| "nll_soft": 0.5929402263240431, | |
| "ece": 0.14721173331831938, | |
| "mean_confidence": 0.7201727500927853, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.34746314098192826 | |
| }, | |
| "civil_comments_toxic": { | |
| "count": 100, | |
| "accuracy": 0.96, | |
| "brier": 0.09337668620304003, | |
| "brier_soft": 0.037982779335719564, | |
| "nll": 0.1861470519288842, | |
| "nll_soft": 0.27998517896776515, | |
| "ece": 0.09812762637818251, | |
| "mean_confidence": 0.8618723736218176, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "clinc150": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.18462976225127634, | |
| "brier_soft": 0.18462976225127634, | |
| "nll": 0.5128245533780458, | |
| "nll_soft": 0.5128245533780458, | |
| "ece": 0.07346104900904875, | |
| "mean_confidence": 0.9269573388899676, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "cola": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.22831615379946643, | |
| "brier_soft": 0.22831615379946643, | |
| "nll": 0.3590869683317728, | |
| "nll_soft": 0.3590869683317728, | |
| "ece": 0.07450945927672789, | |
| "mean_confidence": 0.8555754148249904, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "commonsense_qa": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.15046780462930462, | |
| "brier_soft": 0.15046780462930462, | |
| "nll": 0.3438882238808393, | |
| "nll_soft": 0.3438882238808393, | |
| "ece": 0.03343498566763066, | |
| "mean_confidence": 0.9374398583651028, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "cosmos_qa": { | |
| "count": 120, | |
| "accuracy": 0.7166666666666667, | |
| "brier": 0.44494135470751833, | |
| "brier_soft": 0.44494135470751833, | |
| "nll": 0.9113581258768009, | |
| "nll_soft": 0.9113581258768009, | |
| "ece": 0.17210482016936163, | |
| "mean_confidence": 0.868085591538144, | |
| "confident_error_rate": 0.09166666666666666, | |
| "evidence_auc": 0.999375 | |
| }, | |
| "dbpedia": { | |
| "count": 100, | |
| "accuracy": 1.0, | |
| "brier": 0.003741281367652323, | |
| "brier_soft": 0.003741281367652323, | |
| "nll": 0.00958541989430656, | |
| "nll_soft": 0.00958541989430656, | |
| "ece": 0.008060135485155948, | |
| "mean_confidence": 0.9919398645148441, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "dynasent": { | |
| "count": 100, | |
| "accuracy": 0.73, | |
| "brier": 0.39455497884064294, | |
| "brier_soft": 0.27562515443314395, | |
| "nll": 0.6819105783878199, | |
| "nll_soft": 0.7407221597727708, | |
| "ece": 0.17197017219513994, | |
| "mean_confidence": 0.5922648429725653, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4612412387247546 | |
| }, | |
| "enron_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.020371589208394592, | |
| "brier_soft": 0.020371589208394592, | |
| "nll": 0.04072451183380424, | |
| "nll_soft": 0.04072451183380424, | |
| "ece": 0.02126407007058294, | |
| "mean_confidence": 0.9811057852953994, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "go_emotions": { | |
| "count": 100, | |
| "accuracy": 0.6, | |
| "brier": 0.6247055007996507, | |
| "brier_soft": 0.2298631426256989, | |
| "nll": 1.5618458274135032, | |
| "nll_soft": 1.9562813612135246, | |
| "ece": 0.2758994442781854, | |
| "mean_confidence": 0.33157592066393576, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "hellaswag": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.04435647337572136, | |
| "brier_soft": 0.04435647337572136, | |
| "nll": 0.08034530834752719, | |
| "nll_soft": 0.08034530834752719, | |
| "ece": 0.03405762729343661, | |
| "mean_confidence": 0.9562575630567488, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "imdb": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.04711217330218374, | |
| "brier_soft": 0.04711217330218374, | |
| "nll": 0.10119131895321937, | |
| "nll_soft": 0.10119131895321937, | |
| "ece": 0.032128265447002514, | |
| "mean_confidence": 0.9619559211052533, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "ledgar": { | |
| "count": 100, | |
| "accuracy": 0.78, | |
| "brier": 0.29751737785310295, | |
| "brier_soft": 0.29751737785310295, | |
| "nll": 0.7476464446710225, | |
| "nll_soft": 0.7476464446710225, | |
| "ece": 0.061867458329410076, | |
| "mean_confidence": 0.8241415790544235, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "liar": { | |
| "count": 100, | |
| "accuracy": 0.39, | |
| "brier": 0.7232980473841729, | |
| "brier_soft": 0.7232980473841729, | |
| "nll": 1.4529365835997112, | |
| "nll_soft": 1.4529365835997112, | |
| "ece": 0.09217707472940212, | |
| "mean_confidence": 0.3114025357085896, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.9575159525161713 | |
| }, | |
| "measuring_hate_speech": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.2793448439064333, | |
| "brier_soft": 0.1463219404313681, | |
| "nll": 0.5353710733933518, | |
| "nll_soft": 0.6702362126220086, | |
| "ece": 0.16927708181106907, | |
| "mean_confidence": 0.6707229181889309, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5462726519505992 | |
| }, | |
| "medqa": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.1503504657387938, | |
| "brier_soft": 0.1503504657387938, | |
| "nll": 0.2623507353639012, | |
| "nll_soft": 0.2623507353639012, | |
| "ece": 0.08144816899443122, | |
| "mean_confidence": 0.8978548497595163, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "mnli": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.06228021631952076, | |
| "brier_soft": 0.06228021631952076, | |
| "nll": 0.13270113391968807, | |
| "nll_soft": 0.13270113391968807, | |
| "ece": 0.08238346171353027, | |
| "mean_confidence": 0.9110402983650409, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "mrpc": { | |
| "count": 100, | |
| "accuracy": 0.9, | |
| "brier": 0.16585409895732134, | |
| "brier_soft": 0.16585409895732134, | |
| "nll": 0.2645067375959682, | |
| "nll_soft": 0.2645067375959682, | |
| "ece": 0.061724041454668824, | |
| "mean_confidence": 0.885295208394654, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "multirc": { | |
| "count": 125, | |
| "accuracy": 0.768, | |
| "brier": 0.39100223311737586, | |
| "brier_soft": 0.39100223311737586, | |
| "nll": 0.61268910167795, | |
| "nll_soft": 0.61268910167795, | |
| "ece": 0.18123682958658455, | |
| "mean_confidence": 0.9092147247418939, | |
| "confident_error_rate": 0.128, | |
| "evidence_auc": 0.9997333333333334 | |
| }, | |
| "openbookqa": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.036709257253909985, | |
| "brier_soft": 0.036709257253909985, | |
| "nll": 0.07752743679878692, | |
| "nll_soft": 0.07752743679878692, | |
| "ece": 0.016796433783453687, | |
| "mean_confidence": 0.9650003552691371, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "piqa": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.12286978168514663, | |
| "brier_soft": 0.12286978168514663, | |
| "nll": 0.2169208854703175, | |
| "nll_soft": 0.2169208854703175, | |
| "ece": 0.07068212570085167, | |
| "mean_confidence": 0.9379244741154052, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "policy_application": { | |
| "count": 150, | |
| "accuracy": 0.9466666666666667, | |
| "brier": 0.07417797153585332, | |
| "brier_soft": 0.07417797153585332, | |
| "nll": 0.11845208966989801, | |
| "nll_soft": 0.11845208966989801, | |
| "ece": 0.044110458372438946, | |
| "mean_confidence": 0.9800457791058929, | |
| "confident_error_rate": 0.03333333333333333, | |
| "evidence_auc": 1.0 | |
| }, | |
| "qqp": { | |
| "count": 100, | |
| "accuracy": 0.92, | |
| "brier": 0.13311880331318848, | |
| "brier_soft": 0.13311880331318848, | |
| "nll": 0.2320502253844424, | |
| "nll_soft": 0.2320502253844424, | |
| "ece": 0.07355407226380711, | |
| "mean_confidence": 0.8951777070215553, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "race": { | |
| "count": 125, | |
| "accuracy": 0.792, | |
| "brier": 0.2674134484204704, | |
| "brier_soft": 0.2674134484204704, | |
| "nll": 0.513583670296167, | |
| "nll_soft": 0.513583670296167, | |
| "ece": 0.07423129886353177, | |
| "mean_confidence": 0.8306386892918173, | |
| "confident_error_rate": 0.016, | |
| "evidence_auc": 1.0 | |
| }, | |
| "rotten_tomatoes": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.03743266007384592, | |
| "brier_soft": 0.03743266007384592, | |
| "nll": 0.08081407315197316, | |
| "nll_soft": 0.08081407315197316, | |
| "ece": 0.0357435013308515, | |
| "mean_confidence": 0.9567344445787098, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "rte": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.11037080326966946, | |
| "brier_soft": 0.11037080326966946, | |
| "nll": 0.18424526282282977, | |
| "nll_soft": 0.18424526282282977, | |
| "ece": 0.03139877402899702, | |
| "mean_confidence": 0.9613987740289969, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "scotus": { | |
| "count": 100, | |
| "accuracy": 0.61, | |
| "brier": 0.5896053540662425, | |
| "brier_soft": 0.5896053540662425, | |
| "nll": 1.4407866367191966, | |
| "nll_soft": 1.4407866367191966, | |
| "ece": 0.17632924711827827, | |
| "mean_confidence": 0.7863292471182782, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "shp": { | |
| "count": 100, | |
| "accuracy": 0.65, | |
| "brier": 0.47029834003111476, | |
| "brier_soft": 0.10051645750786714, | |
| "nll": 0.6632380214745927, | |
| "nll_soft": 0.6838533045907272, | |
| "ece": 0.11777214652077381, | |
| "mean_confidence": 0.5322278534792262, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "sms_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.01162780117821516, | |
| "brier_soft": 0.01162780117821516, | |
| "nll": 0.02331012246933196, | |
| "nll_soft": 0.02331012246933196, | |
| "ece": 0.016788442416964124, | |
| "mean_confidence": 0.988267077668862, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "snli": { | |
| "count": 100, | |
| "accuracy": 0.92, | |
| "brier": 0.12711939635362396, | |
| "brier_soft": 0.12711939635362396, | |
| "nll": 0.22879034755833566, | |
| "nll_soft": 0.22879034755833566, | |
| "ece": 0.0444196020597182, | |
| "mean_confidence": 0.9308260670907695, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "social_iqa": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.2560615224983722, | |
| "brier_soft": 0.2560615224983722, | |
| "nll": 0.4826577509065636, | |
| "nll_soft": 0.4826577509065636, | |
| "ece": 0.07618387345343328, | |
| "mean_confidence": 0.8898566355070517, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "sst2": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.04105489560088457, | |
| "brier_soft": 0.04105489560088457, | |
| "nll": 0.10883305537700429, | |
| "nll_soft": 0.10883305537700429, | |
| "ece": 0.011059711327280758, | |
| "mean_confidence": 0.9782684225353241, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "sst5": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.497079796594402, | |
| "brier_soft": 0.497079796594402, | |
| "nll": 0.8742453682539675, | |
| "nll_soft": 0.8742453682539675, | |
| "ece": 0.16584622118782563, | |
| "mean_confidence": 0.5079338519681397, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.449198760613678 | |
| }, | |
| "stance_abortion": { | |
| "count": 60, | |
| "accuracy": 0.65, | |
| "brier": 0.5170044623404539, | |
| "brier_soft": 0.5170044623404539, | |
| "nll": 0.8217929893106105, | |
| "nll_soft": 0.8217929893106105, | |
| "ece": 0.19912084090351673, | |
| "mean_confidence": 0.8303765398473246, | |
| "confident_error_rate": 0.06666666666666667 | |
| }, | |
| "stance_atheism": { | |
| "count": 60, | |
| "accuracy": 0.7833333333333333, | |
| "brier": 0.3001419959341259, | |
| "brier_soft": 0.3001419959341259, | |
| "nll": 0.47098576663159153, | |
| "nll_soft": 0.47098576663159153, | |
| "ece": 0.14294118305587333, | |
| "mean_confidence": 0.8563605992498861, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_feminist": { | |
| "count": 60, | |
| "accuracy": 0.7833333333333333, | |
| "brier": 0.2920330154544224, | |
| "brier_soft": 0.2920330154544224, | |
| "nll": 0.49062532038555723, | |
| "nll_soft": 0.49062532038555723, | |
| "ece": 0.06563462797049141, | |
| "mean_confidence": 0.8395306016627094, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_hillary": { | |
| "count": 60, | |
| "accuracy": 0.7166666666666667, | |
| "brier": 0.36920640328234444, | |
| "brier_soft": 0.36920640328234444, | |
| "nll": 0.5975554352281234, | |
| "nll_soft": 0.5975554352281234, | |
| "ece": 0.08165786929705794, | |
| "mean_confidence": 0.7783727969746086, | |
| "confident_error_rate": 0.03333333333333333 | |
| }, | |
| "stsb": { | |
| "count": 100, | |
| "accuracy": 0.65, | |
| "brier": 0.4598476608257509, | |
| "brier_soft": 0.20566258588156333, | |
| "nll": 0.8174855825848395, | |
| "nll_soft": 0.8814355956930671, | |
| "ece": 0.12916616422774185, | |
| "mean_confidence": 0.5881235918154974, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4591854752360724 | |
| }, | |
| "subjectivity": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.025245447097925453, | |
| "brier_soft": 0.025245447097925453, | |
| "nll": 0.06443650373540663, | |
| "nll_soft": 0.06443650373540663, | |
| "ece": 0.05522051201824588, | |
| "mean_confidence": 0.9472574338913156, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "swag": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.24280604965542157, | |
| "brier_soft": 0.24280604965542157, | |
| "nll": 0.43879576779754104, | |
| "nll_soft": 0.43879576779754104, | |
| "ece": 0.07809132699403978, | |
| "mean_confidence": 0.8768457190873439, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "trec": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.044109563870200826, | |
| "brier_soft": 0.044109563870200826, | |
| "nll": 0.1119764191390441, | |
| "nll_soft": 0.1119764191390441, | |
| "ece": 0.03157924188503147, | |
| "mean_confidence": 0.9651043539300282, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "tweet_hate": { | |
| "count": 100, | |
| "accuracy": 0.5, | |
| "brier": 0.6361912887410113, | |
| "brier_soft": 0.6361912887410113, | |
| "nll": 0.87589744678152, | |
| "nll_soft": 0.87589744678152, | |
| "ece": 0.30966203984896934, | |
| "mean_confidence": 0.7996620398489693, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "tweet_sentiment": { | |
| "count": 100, | |
| "accuracy": 0.74, | |
| "brier": 0.4046324049729258, | |
| "brier_soft": 0.4046324049729258, | |
| "nll": 0.6904489665258657, | |
| "nll_soft": 0.6904489665258657, | |
| "ece": 0.17709361051488434, | |
| "mean_confidence": 0.5807963141231423, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3678134887808973 | |
| }, | |
| "vitaminc": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.22414852755241488, | |
| "brier_soft": 0.22414852755241488, | |
| "nll": 0.4106686914678594, | |
| "nll_soft": 0.4106686914678594, | |
| "ece": 0.05939190838622676, | |
| "mean_confidence": 0.8887477138644124, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "wic": { | |
| "count": 100, | |
| "accuracy": 0.78, | |
| "brier": 0.32325615773124416, | |
| "brier_soft": 0.32325615773124416, | |
| "nll": 0.49691056539654466, | |
| "nll_soft": 0.49691056539654466, | |
| "ece": 0.11144833190894102, | |
| "mean_confidence": 0.8764042284964352, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "winogrande": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.1699682822388243, | |
| "brier_soft": 0.1699682822388243, | |
| "nll": 0.32014269821420394, | |
| "nll_soft": 0.32014269821420394, | |
| "ece": 0.09579456110528843, | |
| "mean_confidence": 0.9299164455248426, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "yelp": { | |
| "count": 100, | |
| "accuracy": 0.73, | |
| "brier": 0.4223639030293232, | |
| "brier_soft": 0.4223639030293232, | |
| "nll": 0.7283440053059708, | |
| "nll_soft": 0.7283440053059708, | |
| "ece": 0.1725769529348749, | |
| "mean_confidence": 0.5753164587285784, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.40574022304004387 | |
| } | |
| }, | |
| "reliability": [ | |
| { | |
| "bin": "0.0-0.1", | |
| "count": 0, | |
| "confidence": NaN, | |
| "accuracy": NaN | |
| }, | |
| { | |
| "bin": "0.1-0.2", | |
| "count": 14, | |
| "confidence": 0.1695548747932898, | |
| "accuracy": 0.14285714285714285 | |
| }, | |
| { | |
| "bin": "0.2-0.3", | |
| "count": 94, | |
| "confidence": 0.250037071579069, | |
| "accuracy": 0.35106382978723405 | |
| }, | |
| { | |
| "bin": "0.3-0.4", | |
| "count": 120, | |
| "confidence": 0.3580054189178388, | |
| "accuracy": 0.475 | |
| }, | |
| { | |
| "bin": "0.4-0.5", | |
| "count": 317, | |
| "confidence": 0.45936274075655936, | |
| "accuracy": 0.5394321766561514 | |
| }, | |
| { | |
| "bin": "0.5-0.6", | |
| "count": 462, | |
| "confidence": 0.5473715655718976, | |
| "accuracy": 0.6363636363636364 | |
| }, | |
| { | |
| "bin": "0.6-0.7", | |
| "count": 311, | |
| "confidence": 0.6482043563078829, | |
| "accuracy": 0.7009646302250804 | |
| }, | |
| { | |
| "bin": "0.7-0.8", | |
| "count": 373, | |
| "confidence": 0.7494447313036959, | |
| "accuracy": 0.7372654155495979 | |
| }, | |
| { | |
| "bin": "0.8-0.9", | |
| "count": 603, | |
| "confidence": 0.8560699667307652, | |
| "accuracy": 0.736318407960199 | |
| }, | |
| { | |
| "bin": "0.9-1.0", | |
| "count": 3219, | |
| "confidence": 0.976238805062891, | |
| "accuracy": 0.9645852749301025 | |
| } | |
| ], | |
| "seconds": 387.77550768852234, | |
| "model": "runs/assay-27b", | |
| "data": "data/v2/dev.jsonl" | |
| } |