Invalid JSON:Unexpected token 'N', ..."fidence": NaN,
"... is not valid JSON
| { | |
| "overall": { | |
| "count": 6113, | |
| "accuracy": 0.8081138557173237, | |
| "brier": 0.2661494358472481, | |
| "brier_soft": 0.24260286062875672, | |
| "nll": 0.49575501723827897, | |
| "nll_soft": 0.5104445448249281, | |
| "ece": 0.024109620444869757, | |
| "mean_confidence": 0.8061256466219281, | |
| "confident_error_rate": 0.019466710289546867, | |
| "score_mae": 0.5130282987896484, | |
| "evidence_auc": 0.9938195273334924 | |
| }, | |
| "by_type": { | |
| "bool": { | |
| "count": 1898, | |
| "accuracy": 0.8730242360379347, | |
| "brier": 0.17989218003516222, | |
| "brier_soft": 0.17618496083298144, | |
| "nll": 0.28048398585407075, | |
| "nll_soft": 0.28450098328570933, | |
| "ece": 0.021932010938761043, | |
| "mean_confidence": 0.8927613096867237, | |
| "confident_error_rate": 0.02107481559536354, | |
| "evidence_auc": 0.993037902210008 | |
| }, | |
| "choice": { | |
| "count": 3248, | |
| "accuracy": 0.8075738916256158, | |
| "brier": 0.265855969310664, | |
| "brier_soft": 0.24213587499460246, | |
| "nll": 0.531371360305088, | |
| "nll_soft": 0.5459336605013914, | |
| "ece": 0.031153686026911636, | |
| "mean_confidence": 0.8267506704226892, | |
| "confident_error_rate": 0.02432266009852217, | |
| "evidence_auc": 0.9943635212159595 | |
| }, | |
| "score": { | |
| "count": 967, | |
| "accuracy": 0.6825232678386763, | |
| "brier": 0.4364384232745116, | |
| "brier_soft": 0.3745345495140871, | |
| "nll": 0.7986531923016, | |
| "nll_soft": 0.834717794136494, | |
| "ece": 0.12188649461723472, | |
| "mean_confidence": 0.5668034482746123, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5130282987896484 | |
| } | |
| }, | |
| "by_source": { | |
| "ag_news": { | |
| "count": 100, | |
| "accuracy": 0.89, | |
| "brier": 0.16845385243161523, | |
| "brier_soft": 0.16845385243161523, | |
| "nll": 0.31026867024327776, | |
| "nll_soft": 0.31026867024327776, | |
| "ece": 0.07299394846970284, | |
| "mean_confidence": 0.9300757047424044, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "amazon_polarity": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.06600077240643171, | |
| "brier_soft": 0.06600077240643171, | |
| "nll": 0.11152031928475888, | |
| "nll_soft": 0.11152031928475888, | |
| "ece": 0.045533713599151, | |
| "mean_confidence": 0.957777396191883, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "amazon_stars": { | |
| "count": 100, | |
| "accuracy": 0.58, | |
| "brier": 0.5194019417386688, | |
| "brier_soft": 0.5194019417386688, | |
| "nll": 0.9101515599872236, | |
| "nll_soft": 0.9101515599872236, | |
| "ece": 0.06545616524854728, | |
| "mean_confidence": 0.5352706927405049, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5297926624004965 | |
| }, | |
| "anli": { | |
| "count": 100, | |
| "accuracy": 0.63, | |
| "brier": 0.5421020492879414, | |
| "brier_soft": 0.5421020492879414, | |
| "nll": 0.9525621883553892, | |
| "nll_soft": 0.9525621883553892, | |
| "ece": 0.16033795682789762, | |
| "mean_confidence": 0.7903379568278975, | |
| "confident_error_rate": 0.08 | |
| }, | |
| "arc_challenge": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.10067166282744358, | |
| "brier_soft": 0.10067166282744358, | |
| "nll": 0.1765881280453366, | |
| "nll_soft": 0.1765881280453366, | |
| "ece": 0.049984724800788485, | |
| "mean_confidence": 0.9541562154058013, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "arc_easy": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.04413162229222551, | |
| "brier_soft": 0.04413162229222551, | |
| "nll": 0.09823945269698767, | |
| "nll_soft": 0.09823945269698767, | |
| "ece": 0.032215481404806866, | |
| "mean_confidence": 0.9643697174383512, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "banking77": { | |
| "count": 100, | |
| "accuracy": 0.75, | |
| "brier": 0.32482412936478505, | |
| "brier_soft": 0.32482412936478505, | |
| "nll": 0.7967586871695171, | |
| "nll_soft": 0.7967586871695171, | |
| "ece": 0.10943588766244959, | |
| "mean_confidence": 0.7685198614733331, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "boolq": { | |
| "count": 125, | |
| "accuracy": 0.824, | |
| "brier": 0.2463948134922965, | |
| "brier_soft": 0.2463948134922965, | |
| "nll": 0.37807162862812616, | |
| "nll_soft": 0.37807162862812616, | |
| "ece": 0.062208085455247614, | |
| "mean_confidence": 0.8090593917787792, | |
| "confident_error_rate": 0.024, | |
| "evidence_auc": 1.0 | |
| }, | |
| "cb": { | |
| "count": 28, | |
| "accuracy": 1.0, | |
| "brier": 0.022647583217287822, | |
| "brier_soft": 0.022647583217287822, | |
| "nll": 0.08633507267229382, | |
| "nll_soft": 0.08633507267229382, | |
| "ece": 0.0780946990230204, | |
| "mean_confidence": 0.9219053009769796, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "civil_comments_severity": { | |
| "count": 100, | |
| "accuracy": 0.86, | |
| "brier": 0.2456675530425123, | |
| "brier_soft": 0.1728906396682275, | |
| "nll": 0.503941587874954, | |
| "nll_soft": 0.638649591753842, | |
| "ece": 0.16160389502613104, | |
| "mean_confidence": 0.7384045083345092, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3738099519276761 | |
| }, | |
| "civil_comments_toxic": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.11602869397484142, | |
| "brier_soft": 0.04566567351745008, | |
| "nll": 0.21750172983061675, | |
| "nll_soft": 0.29374434108311787, | |
| "ece": 0.10875554606138138, | |
| "mean_confidence": 0.8681357944166203, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "clinc150": { | |
| "count": 100, | |
| "accuracy": 0.86, | |
| "brier": 0.22305660509274966, | |
| "brier_soft": 0.22305660509274966, | |
| "nll": 0.6025872624555403, | |
| "nll_soft": 0.6025872624555403, | |
| "ece": 0.09807589988409433, | |
| "mean_confidence": 0.8736587746039237, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "cola": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.2523992057795118, | |
| "brier_soft": 0.2523992057795118, | |
| "nll": 0.39240989420568745, | |
| "nll_soft": 0.39240989420568745, | |
| "ece": 0.05921553584016465, | |
| "mean_confidence": 0.8461464770037992, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "commonsense_qa": { | |
| "count": 100, | |
| "accuracy": 0.85, | |
| "brier": 0.20557339122351911, | |
| "brier_soft": 0.20557339122351911, | |
| "nll": 0.37316502903223514, | |
| "nll_soft": 0.37316502903223514, | |
| "ece": 0.06105498864803392, | |
| "mean_confidence": 0.8862737420051069, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "cosmos_qa": { | |
| "count": 120, | |
| "accuracy": 0.75, | |
| "brier": 0.36662150662163717, | |
| "brier_soft": 0.36662150662163717, | |
| "nll": 0.7134435721818425, | |
| "nll_soft": 0.7134435721818425, | |
| "ece": 0.1059972659018534, | |
| "mean_confidence": 0.8341453226399168, | |
| "confident_error_rate": 0.041666666666666664, | |
| "evidence_auc": 0.999375 | |
| }, | |
| "dates": { | |
| "count": 300, | |
| "accuracy": 0.9133333333333333, | |
| "brier": 0.13085868085462893, | |
| "brier_soft": 0.13085868085462893, | |
| "nll": 0.21953151134718468, | |
| "nll_soft": 0.21953151134718468, | |
| "ece": 0.041686146446700025, | |
| "mean_confidence": 0.8794316282366511, | |
| "confident_error_rate": 0.006666666666666667, | |
| "score_mae": 0.25286787804313066 | |
| }, | |
| "dbpedia": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.0054752057851738724, | |
| "brier_soft": 0.0054752057851738724, | |
| "nll": 0.01716145009688713, | |
| "nll_soft": 0.01716145009688713, | |
| "ece": 0.014245544114448125, | |
| "mean_confidence": 0.9852099690835066, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "dynasent": { | |
| "count": 100, | |
| "accuracy": 0.72, | |
| "brier": 0.41284497184536767, | |
| "brier_soft": 0.2900889097829367, | |
| "nll": 0.7092853395788147, | |
| "nll_soft": 0.7606595306372346, | |
| "ece": 0.19711298483299192, | |
| "mean_confidence": 0.5908801557778465, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4826536482310852 | |
| }, | |
| "enron_spam": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.04113819214598982, | |
| "brier_soft": 0.04113819214598982, | |
| "nll": 0.08761406998679618, | |
| "nll_soft": 0.08761406998679618, | |
| "ece": 0.015903681566536102, | |
| "mean_confidence": 0.9741391053291784, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "go_emotions": { | |
| "count": 100, | |
| "accuracy": 0.59, | |
| "brier": 0.6395508984210346, | |
| "brier_soft": 0.24193510902830995, | |
| "nll": 1.5803867250206407, | |
| "nll_soft": 2.0354749292162237, | |
| "ece": 0.22517856265972647, | |
| "mean_confidence": 0.36482143734027345, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "hellaswag": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.1078292724737536, | |
| "brier_soft": 0.1078292724737536, | |
| "nll": 0.20589254603888488, | |
| "nll_soft": 0.20589254603888488, | |
| "ece": 0.06967430987777687, | |
| "mean_confidence": 0.9220518174558482, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "imdb": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.07385005732170005, | |
| "brier_soft": 0.07385005732170005, | |
| "nll": 0.1359626492953412, | |
| "nll_soft": 0.1359626492953412, | |
| "ece": 0.04029954444016761, | |
| "mean_confidence": 0.9642720814018544, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "ledgar": { | |
| "count": 100, | |
| "accuracy": 0.78, | |
| "brier": 0.3281153648327422, | |
| "brier_soft": 0.3281153648327422, | |
| "nll": 0.7741302141152906, | |
| "nll_soft": 0.7741302141152906, | |
| "ece": 0.0873475859579324, | |
| "mean_confidence": 0.7839166768781682, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "liar": { | |
| "count": 100, | |
| "accuracy": 0.32, | |
| "brier": 0.7446807924003528, | |
| "brier_soft": 0.7446807924003528, | |
| "nll": 1.5117587745951337, | |
| "nll_soft": 1.5117587745951337, | |
| "ece": 0.08097392903128141, | |
| "mean_confidence": 0.2986578187328796, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 1.036142943391838 | |
| }, | |
| "measuring_hate_speech": { | |
| "count": 100, | |
| "accuracy": 0.86, | |
| "brier": 0.3124419981758317, | |
| "brier_soft": 0.16390942252786078, | |
| "nll": 0.5875051718312956, | |
| "nll_soft": 0.6936316854447891, | |
| "ece": 0.2250050213791679, | |
| "mean_confidence": 0.6349949786208321, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.6163252940584951 | |
| }, | |
| "medqa": { | |
| "count": 100, | |
| "accuracy": 0.65, | |
| "brier": 0.4597154624116973, | |
| "brier_soft": 0.4597154624116973, | |
| "nll": 0.8711963917666296, | |
| "nll_soft": 0.8711963917666296, | |
| "ece": 0.1116766692538281, | |
| "mean_confidence": 0.72033221601947, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "mnli": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.10529594164239192, | |
| "brier_soft": 0.10529594164239192, | |
| "nll": 0.2117929876710798, | |
| "nll_soft": 0.2117929876710798, | |
| "ece": 0.05925569835632346, | |
| "mean_confidence": 0.8817122608104968, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "mrpc": { | |
| "count": 100, | |
| "accuracy": 0.81, | |
| "brier": 0.23545523687620534, | |
| "brier_soft": 0.23545523687620534, | |
| "nll": 0.36065976461182353, | |
| "nll_soft": 0.36065976461182353, | |
| "ece": 0.08009349827050247, | |
| "mean_confidence": 0.840161282202333, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "multirc": { | |
| "count": 125, | |
| "accuracy": 0.712, | |
| "brier": 0.33425679170758704, | |
| "brier_soft": 0.33425679170758704, | |
| "nll": 0.47015119453748944, | |
| "nll_soft": 0.47015119453748944, | |
| "ece": 0.12508080340790598, | |
| "mean_confidence": 0.798369093709533, | |
| "confident_error_rate": 0.0, | |
| "evidence_auc": 0.9816 | |
| }, | |
| "openbookqa": { | |
| "count": 100, | |
| "accuracy": 0.89, | |
| "brier": 0.16850017640548665, | |
| "brier_soft": 0.16850017640548665, | |
| "nll": 0.3071681993046363, | |
| "nll_soft": 0.3071681993046363, | |
| "ece": 0.0681169423745702, | |
| "mean_confidence": 0.8801206851845036, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "piqa": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.22419783176420943, | |
| "brier_soft": 0.22419783176420943, | |
| "nll": 0.36462868341861826, | |
| "nll_soft": 0.36462868341861826, | |
| "ece": 0.08037792149433066, | |
| "mean_confidence": 0.87560922670375, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "policy_application": { | |
| "count": 150, | |
| "accuracy": 0.9066666666666666, | |
| "brier": 0.1126946527275131, | |
| "brier_soft": 0.1126946527275131, | |
| "nll": 0.16724892277415848, | |
| "nll_soft": 0.16724892277415848, | |
| "ece": 0.0675511235449082, | |
| "mean_confidence": 0.9432567076192507, | |
| "confident_error_rate": 0.0, | |
| "evidence_auc": 1.0 | |
| }, | |
| "policy_hard": { | |
| "count": 300, | |
| "accuracy": 0.87, | |
| "brier": 0.1993095979231174, | |
| "brier_soft": 0.1993095979231174, | |
| "nll": 0.32845201396386725, | |
| "nll_soft": 0.32845201396386725, | |
| "ece": 0.05610869195965445, | |
| "mean_confidence": 0.9002004387015582, | |
| "confident_error_rate": 0.043333333333333335, | |
| "evidence_auc": 0.9759837962962963 | |
| }, | |
| "qqp": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.14536355743868318, | |
| "brier_soft": 0.14536355743868318, | |
| "nll": 0.236511504425006, | |
| "nll_soft": 0.236511504425006, | |
| "ece": 0.06902310544783366, | |
| "mean_confidence": 0.863890066837007, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "race": { | |
| "count": 125, | |
| "accuracy": 0.712, | |
| "brier": 0.33794214086609586, | |
| "brier_soft": 0.33794214086609586, | |
| "nll": 0.6353569038250544, | |
| "nll_soft": 0.6353569038250544, | |
| "ece": 0.11330167076929255, | |
| "mean_confidence": 0.8167852186501845, | |
| "confident_error_rate": 0.032, | |
| "evidence_auc": 0.9816 | |
| }, | |
| "rotten_tomatoes": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.07359394082826959, | |
| "brier_soft": 0.07359394082826959, | |
| "nll": 0.12453968361648274, | |
| "nll_soft": 0.12453968361648274, | |
| "ece": 0.043066136326572566, | |
| "mean_confidence": 0.9365716100101855, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "rte": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.1378394419526845, | |
| "brier_soft": 0.1378394419526845, | |
| "nll": 0.22826701855192236, | |
| "nll_soft": 0.22826701855192236, | |
| "ece": 0.06151258102601688, | |
| "mean_confidence": 0.9404203748836127, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "scotus": { | |
| "count": 100, | |
| "accuracy": 0.2, | |
| "brier": 0.8824845341223795, | |
| "brier_soft": 0.8824845341223795, | |
| "nll": 2.3725589448613023, | |
| "nll_soft": 2.3725589448613023, | |
| "ece": 0.03287297194524475, | |
| "mean_confidence": 0.2235796914649564, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "shp": { | |
| "count": 100, | |
| "accuracy": 0.59, | |
| "brier": 0.4797397521419983, | |
| "brier_soft": 0.10692687814904282, | |
| "nll": 0.6725667723983078, | |
| "nll_soft": 0.6904620785786614, | |
| "ece": 0.0708668070573588, | |
| "mean_confidence": 0.539133192942641, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "sms_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.01941783931224156, | |
| "brier_soft": 0.01941783931224156, | |
| "nll": 0.04414426991615675, | |
| "nll_soft": 0.04414426991615675, | |
| "ece": 0.023315330523776823, | |
| "mean_confidence": 0.9825703907007697, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "snli": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.08293153330688835, | |
| "brier_soft": 0.08293153330688835, | |
| "nll": 0.171101802859479, | |
| "nll_soft": 0.171101802859479, | |
| "ece": 0.04866259143581518, | |
| "mean_confidence": 0.9194179507114802, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "social_iqa": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.2572472931446341, | |
| "brier_soft": 0.2572472931446341, | |
| "nll": 0.4914033563092567, | |
| "nll_soft": 0.4914033563092567, | |
| "ece": 0.07708066338472419, | |
| "mean_confidence": 0.862248560054147, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "sst2": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.05812558128312039, | |
| "brier_soft": 0.05812558128312039, | |
| "nll": 0.12278980073652958, | |
| "nll_soft": 0.12278980073652958, | |
| "ece": 0.03465141864763422, | |
| "mean_confidence": 0.9609904656090619, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "sst5": { | |
| "count": 100, | |
| "accuracy": 0.67, | |
| "brier": 0.4940931300179605, | |
| "brier_soft": 0.4940931300179605, | |
| "nll": 0.8709395956738571, | |
| "nll_soft": 0.8709395956738571, | |
| "ece": 0.1772210597159328, | |
| "mean_confidence": 0.49880477007066004, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.43722177939425927 | |
| }, | |
| "stance_abortion": { | |
| "count": 60, | |
| "accuracy": 0.6333333333333333, | |
| "brier": 0.5098698490864988, | |
| "brier_soft": 0.5098698490864988, | |
| "nll": 0.8114548457536739, | |
| "nll_soft": 0.8114548457536739, | |
| "ece": 0.20824389335951496, | |
| "mean_confidence": 0.8298381203954579, | |
| "confident_error_rate": 0.06666666666666667 | |
| }, | |
| "stance_atheism": { | |
| "count": 60, | |
| "accuracy": 0.7666666666666667, | |
| "brier": 0.3626489360126296, | |
| "brier_soft": 0.3626489360126296, | |
| "nll": 0.6279343615381726, | |
| "nll_soft": 0.6279343615381726, | |
| "ece": 0.13401842415402584, | |
| "mean_confidence": 0.8509966632145957, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "stance_feminist": { | |
| "count": 60, | |
| "accuracy": 0.7166666666666667, | |
| "brier": 0.36714295306448386, | |
| "brier_soft": 0.36714295306448386, | |
| "nll": 0.6301224389630461, | |
| "nll_soft": 0.6301224389630461, | |
| "ece": 0.09630169928292202, | |
| "mean_confidence": 0.7805527340063932, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_hillary": { | |
| "count": 60, | |
| "accuracy": 0.6833333333333333, | |
| "brier": 0.43392563993709476, | |
| "brier_soft": 0.43392563993709476, | |
| "nll": 0.7006138108687522, | |
| "nll_soft": 0.7006138108687522, | |
| "ece": 0.14445342515385268, | |
| "mean_confidence": 0.7761897463072162, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stsb": { | |
| "count": 100, | |
| "accuracy": 0.67, | |
| "brier": 0.47941600616535696, | |
| "brier_soft": 0.22487109798673896, | |
| "nll": 0.8814722777779251, | |
| "nll_soft": 0.938008268970549, | |
| "ece": 0.13853521132174568, | |
| "mean_confidence": 0.5474027968940396, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.49143445522479867 | |
| }, | |
| "subjectivity": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.060069737216477195, | |
| "brier_soft": 0.060069737216477195, | |
| "nll": 0.10706919888260735, | |
| "nll_soft": 0.10706919888260735, | |
| "ece": 0.047342527386868574, | |
| "mean_confidence": 0.9464740773328073, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "swag": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.26526580978725744, | |
| "brier_soft": 0.26526580978725744, | |
| "nll": 0.524827253241994, | |
| "nll_soft": 0.524827253241994, | |
| "ece": 0.07704899673765067, | |
| "mean_confidence": 0.8086559627591525, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "trec": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.06165980860279739, | |
| "brier_soft": 0.06165980860279739, | |
| "nll": 0.1440644009426513, | |
| "nll_soft": 0.1440644009426513, | |
| "ece": 0.034644018323219124, | |
| "mean_confidence": 0.953222112770253, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "tweet_hate": { | |
| "count": 100, | |
| "accuracy": 0.59, | |
| "brier": 0.6099733449281726, | |
| "brier_soft": 0.6099733449281726, | |
| "nll": 0.9084071924004988, | |
| "nll_soft": 0.9084071924004988, | |
| "ece": 0.24263082120266097, | |
| "mean_confidence": 0.8118554592243875, | |
| "confident_error_rate": 0.14 | |
| }, | |
| "tweet_sentiment": { | |
| "count": 100, | |
| "accuracy": 0.64, | |
| "brier": 0.4367689299986772, | |
| "brier_soft": 0.4367689299986772, | |
| "nll": 0.7338407156604912, | |
| "nll_soft": 0.7338407156604912, | |
| "ece": 0.12957176644981752, | |
| "mean_confidence": 0.5765707988516637, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3922079327893301 | |
| }, | |
| "vitaminc": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.2858661765372957, | |
| "brier_soft": 0.2858661765372957, | |
| "nll": 0.5383008018865654, | |
| "nll_soft": 0.5383008018865654, | |
| "ece": 0.097544971218811, | |
| "mean_confidence": 0.860535493801815, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "wic": { | |
| "count": 100, | |
| "accuracy": 0.76, | |
| "brier": 0.3297731392352389, | |
| "brier_soft": 0.3297731392352389, | |
| "nll": 0.5004091591132902, | |
| "nll_soft": 0.5004091591132902, | |
| "ece": 0.05507502517150548, | |
| "mean_confidence": 0.7671851329825314, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "winogrande": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.24850296736454125, | |
| "brier_soft": 0.24850296736454125, | |
| "nll": 0.41686536630288906, | |
| "nll_soft": 0.41686536630288906, | |
| "ece": 0.07463382751134487, | |
| "mean_confidence": 0.8865171061775413, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "yelp": { | |
| "count": 100, | |
| "accuracy": 0.69, | |
| "brier": 0.4359510518792043, | |
| "brier_soft": 0.4359510518792043, | |
| "nll": 0.7529076116320128, | |
| "nll_soft": 0.7529076116320128, | |
| "ece": 0.11329441101759959, | |
| "mean_confidence": 0.5767055889824003, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.431973503589024 | |
| } | |
| }, | |
| "reliability": [ | |
| { | |
| "bin": "0.0-0.1", | |
| "count": 0, | |
| "confidence": NaN, | |
| "accuracy": NaN | |
| }, | |
| { | |
| "bin": "0.1-0.2", | |
| "count": 32, | |
| "confidence": 0.18775015679946586, | |
| "accuracy": 0.1875 | |
| }, | |
| { | |
| "bin": "0.2-0.3", | |
| "count": 167, | |
| "confidence": 0.23931850114374761, | |
| "accuracy": 0.2874251497005988 | |
| }, | |
| { | |
| "bin": "0.3-0.4", | |
| "count": 139, | |
| "confidence": 0.3597909667718615, | |
| "accuracy": 0.38848920863309355 | |
| }, | |
| { | |
| "bin": "0.4-0.5", | |
| "count": 441, | |
| "confidence": 0.4603512037898012, | |
| "accuracy": 0.5328798185941043 | |
| }, | |
| { | |
| "bin": "0.5-0.6", | |
| "count": 584, | |
| "confidence": 0.5552843800701477, | |
| "accuracy": 0.5941780821917808 | |
| }, | |
| { | |
| "bin": "0.6-0.7", | |
| "count": 443, | |
| "confidence": 0.6568538439207049, | |
| "accuracy": 0.6975169300225733 | |
| }, | |
| { | |
| "bin": "0.7-0.8", | |
| "count": 428, | |
| "confidence": 0.7512629337446123, | |
| "accuracy": 0.7289719626168224 | |
| }, | |
| { | |
| "bin": "0.8-0.9", | |
| "count": 658, | |
| "confidence": 0.8510231288751523, | |
| "accuracy": 0.8009118541033434 | |
| }, | |
| { | |
| "bin": "0.9-1.0", | |
| "count": 3221, | |
| "confidence": 0.972387448797977, | |
| "accuracy": 0.9630549518782987 | |
| } | |
| ], | |
| "seconds": 110.24949526786804, | |
| "model": "runs/assay-8b", | |
| "data": "data/v4/dev.jsonl" | |
| } |