Invalid JSON:Unexpected token 'N', ..."fidence": NaN,
"... is not valid JSON
| { | |
| "overall": { | |
| "count": 6113, | |
| "accuracy": 0.8081138557173237, | |
| "brier": 0.26832227363299765, | |
| "brier_soft": 0.24630130870612255, | |
| "nll": 0.505385205242332, | |
| "nll_soft": 0.523684859731946, | |
| "ece": 0.03634856478866526, | |
| "mean_confidence": 0.8375686736383623, | |
| "confident_error_rate": 0.03140847374447898, | |
| "score_mae": 0.485172839677863, | |
| "evidence_auc": 0.9938195273334924 | |
| }, | |
| "by_type": { | |
| "bool": { | |
| "count": 1898, | |
| "accuracy": 0.8730242360379347, | |
| "brier": 0.18502372315079466, | |
| "brier_soft": 0.18166443929411372, | |
| "nll": 0.293633727648158, | |
| "nll_soft": 0.2986379499634176, | |
| "ece": 0.04287967945534256, | |
| "mean_confidence": 0.9132292502672847, | |
| "confident_error_rate": 0.033192834562697573, | |
| "evidence_auc": 0.993037902210008 | |
| }, | |
| "choice": { | |
| "count": 3248, | |
| "accuracy": 0.8075738916256158, | |
| "brier": 0.27162731616245755, | |
| "brier_soft": 0.2490854051600207, | |
| "nll": 0.5523529039003323, | |
| "nll_soft": 0.5704940630171196, | |
| "ece": 0.05672168252811685, | |
| "mean_confidence": 0.8598830180546172, | |
| "confident_error_rate": 0.03940886699507389, | |
| "evidence_auc": 0.9943635212159595 | |
| }, | |
| "score": { | |
| "count": 967, | |
| "accuracy": 0.6825232678386763, | |
| "brier": 0.42071717609373765, | |
| "brier_soft": 0.3638173716448313, | |
| "nll": 0.763247893176724, | |
| "nll_soft": 0.8081758033414839, | |
| "ece": 0.07701368480022007, | |
| "mean_confidence": 0.6141139010368198, | |
| "confident_error_rate": 0.001034126163391934, | |
| "score_mae": 0.485172839677863 | |
| } | |
| }, | |
| "by_source": { | |
| "ag_news": { | |
| "count": 100, | |
| "accuracy": 0.89, | |
| "brier": 0.17744726851892056, | |
| "brier_soft": 0.17744726851892056, | |
| "nll": 0.33952497205533755, | |
| "nll_soft": 0.33952497205533755, | |
| "ece": 0.08231704690785716, | |
| "mean_confidence": 0.9545400192391296, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "amazon_polarity": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.06681896606224931, | |
| "brier_soft": 0.06681896606224931, | |
| "nll": 0.11474157606024799, | |
| "nll_soft": 0.11474157606024799, | |
| "ece": 0.04592720580103715, | |
| "mean_confidence": 0.9684370974150603, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "amazon_stars": { | |
| "count": 100, | |
| "accuracy": 0.58, | |
| "brier": 0.513726379766378, | |
| "brier_soft": 0.513726379766378, | |
| "nll": 0.8877784154246966, | |
| "nll_soft": 0.8877784154246966, | |
| "ece": 0.07482866258617463, | |
| "mean_confidence": 0.5826355888936225, | |
| "confident_error_rate": 0.01, | |
| "score_mae": 0.5120021688214367 | |
| }, | |
| "anli": { | |
| "count": 100, | |
| "accuracy": 0.63, | |
| "brier": 0.5701719988732347, | |
| "brier_soft": 0.5701719988732347, | |
| "nll": 1.0697121018694993, | |
| "nll_soft": 1.0697121018694993, | |
| "ece": 0.21497286575145572, | |
| "mean_confidence": 0.8324542993352486, | |
| "confident_error_rate": 0.13 | |
| }, | |
| "arc_challenge": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.10599146992155316, | |
| "brier_soft": 0.10599146992155316, | |
| "nll": 0.19019848146359147, | |
| "nll_soft": 0.19019848146359147, | |
| "ece": 0.060081904951066994, | |
| "mean_confidence": 0.9680451934575867, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "arc_easy": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.04437057638950536, | |
| "brier_soft": 0.04437057638950536, | |
| "nll": 0.10036992791451768, | |
| "nll_soft": 0.10036992791451768, | |
| "ece": 0.0319046333779032, | |
| "mean_confidence": 0.9737156977945386, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "banking77": { | |
| "count": 100, | |
| "accuracy": 0.75, | |
| "brier": 0.3398126842344269, | |
| "brier_soft": 0.3398126842344269, | |
| "nll": 0.8138418054666235, | |
| "nll_soft": 0.8138418054666235, | |
| "ece": 0.10284948389955295, | |
| "mean_confidence": 0.8351027846874414, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "boolq": { | |
| "count": 125, | |
| "accuracy": 0.824, | |
| "brier": 0.2450309380139414, | |
| "brier_soft": 0.2450309380139414, | |
| "nll": 0.3832584543059262, | |
| "nll_soft": 0.3832584543059262, | |
| "ece": 0.08305327395905993, | |
| "mean_confidence": 0.8318427620613265, | |
| "confident_error_rate": 0.024, | |
| "evidence_auc": 1.0 | |
| }, | |
| "cb": { | |
| "count": 28, | |
| "accuracy": 1.0, | |
| "brier": 0.012525034629445196, | |
| "brier_soft": 0.012525034629445196, | |
| "nll": 0.052362560175484685, | |
| "nll_soft": 0.052362560175484685, | |
| "ece": 0.04811603778085669, | |
| "mean_confidence": 0.9518839622191433, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "civil_comments_severity": { | |
| "count": 100, | |
| "accuracy": 0.86, | |
| "brier": 0.23187375352715037, | |
| "brier_soft": 0.17771031506273136, | |
| "nll": 0.47878826513277256, | |
| "nll_soft": 0.6466024107213819, | |
| "ece": 0.1323208998113436, | |
| "mean_confidence": 0.7884055630186311, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.32065261707897563 | |
| }, | |
| "civil_comments_toxic": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.10944319163588359, | |
| "brier_soft": 0.04568398403607942, | |
| "nll": 0.19590720265259998, | |
| "nll_soft": 0.29088734219622764, | |
| "ece": 0.06555426270198156, | |
| "mean_confidence": 0.9010713536348429, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "clinc150": { | |
| "count": 100, | |
| "accuracy": 0.86, | |
| "brier": 0.22210552087365046, | |
| "brier_soft": 0.22210552087365046, | |
| "nll": 0.6395453706356091, | |
| "nll_soft": 0.6395453706356091, | |
| "ece": 0.11099497317528545, | |
| "mean_confidence": 0.9251752116271217, | |
| "confident_error_rate": 0.07 | |
| }, | |
| "cola": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.259117219091911, | |
| "brier_soft": 0.259117219091911, | |
| "nll": 0.40935600764726915, | |
| "nll_soft": 0.40935600764726915, | |
| "ece": 0.08932037436110461, | |
| "mean_confidence": 0.8750779518376584, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "commonsense_qa": { | |
| "count": 100, | |
| "accuracy": 0.85, | |
| "brier": 0.21088569401090138, | |
| "brier_soft": 0.21088569401090138, | |
| "nll": 0.3965346936975742, | |
| "nll_soft": 0.3965346936975742, | |
| "ece": 0.07310374823833152, | |
| "mean_confidence": 0.9136859527085365, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "cosmos_qa": { | |
| "count": 120, | |
| "accuracy": 0.75, | |
| "brier": 0.3841359793377875, | |
| "brier_soft": 0.3841359793377875, | |
| "nll": 0.7882912921850803, | |
| "nll_soft": 0.7882912921850803, | |
| "ece": 0.14439795786405063, | |
| "mean_confidence": 0.8702594151182701, | |
| "confident_error_rate": 0.08333333333333333, | |
| "evidence_auc": 0.999375 | |
| }, | |
| "dates": { | |
| "count": 300, | |
| "accuracy": 0.9133333333333333, | |
| "brier": 0.12682090218682965, | |
| "brier_soft": 0.12682090218682965, | |
| "nll": 0.20972857100463424, | |
| "nll_soft": 0.20972857100463424, | |
| "ece": 0.022666054970448136, | |
| "mean_confidence": 0.9010661321285901, | |
| "confident_error_rate": 0.016666666666666666, | |
| "score_mae": 0.22207703752612915 | |
| }, | |
| "dbpedia": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.005118529799186151, | |
| "brier_soft": 0.005118529799186151, | |
| "nll": 0.009824460705460551, | |
| "nll_soft": 0.009824460705460551, | |
| "ece": 0.007555830713154134, | |
| "mean_confidence": 0.9922426538540887, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "dynasent": { | |
| "count": 100, | |
| "accuracy": 0.72, | |
| "brier": 0.3964167951058033, | |
| "brier_soft": 0.27899306375845145, | |
| "nll": 0.6748527018027392, | |
| "nll_soft": 0.738852701627455, | |
| "ece": 0.13431695655660056, | |
| "mean_confidence": 0.6383355955935446, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.45075000945645066 | |
| }, | |
| "enron_spam": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.04102532628851713, | |
| "brier_soft": 0.04102532628851713, | |
| "nll": 0.0898814377234004, | |
| "nll_soft": 0.0898814377234004, | |
| "ece": 0.0173370851081312, | |
| "mean_confidence": 0.9860527259793832, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "go_emotions": { | |
| "count": 100, | |
| "accuracy": 0.59, | |
| "brier": 0.6037759399411283, | |
| "brier_soft": 0.24043395414551444, | |
| "nll": 1.4698625936544292, | |
| "nll_soft": 2.0367941435838968, | |
| "ece": 0.19190927347525633, | |
| "mean_confidence": 0.4642116066087375, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "hellaswag": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.11090716468636642, | |
| "brier_soft": 0.11090716468636642, | |
| "nll": 0.2050347391855065, | |
| "nll_soft": 0.2050347391855065, | |
| "ece": 0.05729168206322114, | |
| "mean_confidence": 0.9446148727839713, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "imdb": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.07627635078317603, | |
| "brier_soft": 0.07627635078317603, | |
| "nll": 0.14854959274335647, | |
| "nll_soft": 0.14854959274335647, | |
| "ece": 0.04784298973029209, | |
| "mean_confidence": 0.9738336965806963, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "ledgar": { | |
| "count": 100, | |
| "accuracy": 0.78, | |
| "brier": 0.3408363523310183, | |
| "brier_soft": 0.3408363523310183, | |
| "nll": 0.7893043506756048, | |
| "nll_soft": 0.7893043506756048, | |
| "ece": 0.11682861577561969, | |
| "mean_confidence": 0.8561558831040632, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "liar": { | |
| "count": 100, | |
| "accuracy": 0.32, | |
| "brier": 0.7419574626836368, | |
| "brier_soft": 0.7419574626836368, | |
| "nll": 1.491062320585033, | |
| "nll_soft": 1.491062320585033, | |
| "ece": 0.0781747414321995, | |
| "mean_confidence": 0.32746267464577467, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 1.0138855311286923 | |
| }, | |
| "measuring_hate_speech": { | |
| "count": 100, | |
| "accuracy": 0.86, | |
| "brier": 0.2815976144439547, | |
| "brier_soft": 0.1480667816419444, | |
| "nll": 0.5342900214990065, | |
| "nll_soft": 0.6664983532839511, | |
| "ece": 0.17104543438427827, | |
| "mean_confidence": 0.6889545656157217, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5597911792807024 | |
| }, | |
| "medqa": { | |
| "count": 100, | |
| "accuracy": 0.65, | |
| "brier": 0.4757799375152343, | |
| "brier_soft": 0.4757799375152343, | |
| "nll": 0.9227869245729243, | |
| "nll_soft": 0.9227869245729243, | |
| "ece": 0.12644466791999856, | |
| "mean_confidence": 0.76931472502784, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "mnli": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.0996316035037227, | |
| "brier_soft": 0.0996316035037227, | |
| "nll": 0.19154369357221818, | |
| "nll_soft": 0.19154369357221818, | |
| "ece": 0.04375298064064905, | |
| "mean_confidence": 0.9138691136500888, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "mrpc": { | |
| "count": 100, | |
| "accuracy": 0.81, | |
| "brier": 0.24199403452777526, | |
| "brier_soft": 0.24199403452777526, | |
| "nll": 0.3648608292269973, | |
| "nll_soft": 0.3648608292269973, | |
| "ece": 0.07234249682670134, | |
| "mean_confidence": 0.8719535628687496, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "multirc": { | |
| "count": 125, | |
| "accuracy": 0.712, | |
| "brier": 0.3503515136663073, | |
| "brier_soft": 0.3503515136663073, | |
| "nll": 0.49143203629023763, | |
| "nll_soft": 0.49143203629023763, | |
| "ece": 0.14635532511730132, | |
| "mean_confidence": 0.8244143309088223, | |
| "confident_error_rate": 0.016, | |
| "evidence_auc": 0.9816 | |
| }, | |
| "openbookqa": { | |
| "count": 100, | |
| "accuracy": 0.89, | |
| "brier": 0.17270474379906137, | |
| "brier_soft": 0.17270474379906137, | |
| "nll": 0.31487342081140474, | |
| "nll_soft": 0.31487342081140474, | |
| "ece": 0.0706087790568628, | |
| "mean_confidence": 0.9002089284314065, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "piqa": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.22901713881739658, | |
| "brier_soft": 0.22901713881739658, | |
| "nll": 0.3914267480993911, | |
| "nll_soft": 0.3914267480993911, | |
| "ece": 0.09250240078656738, | |
| "mean_confidence": 0.8972098554800523, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "policy_application": { | |
| "count": 150, | |
| "accuracy": 0.9066666666666666, | |
| "brier": 0.1202855020589476, | |
| "brier_soft": 0.1202855020589476, | |
| "nll": 0.17614015484592835, | |
| "nll_soft": 0.17614015484592835, | |
| "ece": 0.06743290575220404, | |
| "mean_confidence": 0.9566756649191206, | |
| "confident_error_rate": 0.013333333333333334, | |
| "evidence_auc": 1.0 | |
| }, | |
| "policy_hard": { | |
| "count": 300, | |
| "accuracy": 0.87, | |
| "brier": 0.20656161280894106, | |
| "brier_soft": 0.20656161280894106, | |
| "nll": 0.35075682790318247, | |
| "nll_soft": 0.35075682790318247, | |
| "ece": 0.07365686249726952, | |
| "mean_confidence": 0.9254342343056455, | |
| "confident_error_rate": 0.05333333333333334, | |
| "evidence_auc": 0.9759837962962963 | |
| }, | |
| "qqp": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.14255690249181868, | |
| "brier_soft": 0.14255690249181868, | |
| "nll": 0.22598062142518732, | |
| "nll_soft": 0.22598062142518732, | |
| "ece": 0.05481505613728288, | |
| "mean_confidence": 0.8876121401144598, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "race": { | |
| "count": 125, | |
| "accuracy": 0.712, | |
| "brier": 0.3546762617536465, | |
| "brier_soft": 0.3546762617536465, | |
| "nll": 0.6863510995706414, | |
| "nll_soft": 0.6863510995706414, | |
| "ece": 0.1374830827190989, | |
| "mean_confidence": 0.8452280203211845, | |
| "confident_error_rate": 0.032, | |
| "evidence_auc": 0.9816 | |
| }, | |
| "rotten_tomatoes": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.07396981420351864, | |
| "brier_soft": 0.07396981420351864, | |
| "nll": 0.11963696118574361, | |
| "nll_soft": 0.11963696118574361, | |
| "ece": 0.04639630588091059, | |
| "mean_confidence": 0.9517830907375209, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "rte": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.1434888267368668, | |
| "brier_soft": 0.1434888267368668, | |
| "nll": 0.2490209554286854, | |
| "nll_soft": 0.2490209554286854, | |
| "ece": 0.058570730504756724, | |
| "mean_confidence": 0.9568197067803338, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "scotus": { | |
| "count": 100, | |
| "accuracy": 0.2, | |
| "brier": 0.8886954446327406, | |
| "brier_soft": 0.8886954446327406, | |
| "nll": 2.419535738172022, | |
| "nll_soft": 2.419535738172022, | |
| "ece": 0.07677489821988327, | |
| "mean_confidence": 0.26455679810075017, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "shp": { | |
| "count": 100, | |
| "accuracy": 0.59, | |
| "brier": 0.4767881507616933, | |
| "brier_soft": 0.10796886719815929, | |
| "nll": 0.6694358409882295, | |
| "nll_soft": 0.6917291391720062, | |
| "ece": 0.06946449609206626, | |
| "mean_confidence": 0.5484305322637995, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "sms_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.01965210244272633, | |
| "brier_soft": 0.01965210244272633, | |
| "nll": 0.04081128588729513, | |
| "nll_soft": 0.04081128588729513, | |
| "ece": 0.006474000923922583, | |
| "mean_confidence": 0.9920199980802447, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "snli": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.08146867312596515, | |
| "brier_soft": 0.08146867312596515, | |
| "nll": 0.158465964646708, | |
| "nll_soft": 0.158465964646708, | |
| "ece": 0.028524302433964217, | |
| "mean_confidence": 0.9484107171835535, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "social_iqa": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.2614318560423284, | |
| "brier_soft": 0.2614318560423284, | |
| "nll": 0.5328960290627591, | |
| "nll_soft": 0.5328960290627591, | |
| "ece": 0.08443231200903581, | |
| "mean_confidence": 0.8928271961877579, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "sst2": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.05705390850807199, | |
| "brier_soft": 0.05705390850807199, | |
| "nll": 0.12873286798535566, | |
| "nll_soft": 0.12873286798535566, | |
| "ece": 0.03444267451074121, | |
| "mean_confidence": 0.973252100448412, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "sst5": { | |
| "count": 100, | |
| "accuracy": 0.67, | |
| "brier": 0.4731484379764573, | |
| "brier_soft": 0.4731484379764573, | |
| "nll": 0.820745620437561, | |
| "nll_soft": 0.820745620437561, | |
| "ece": 0.133823772664781, | |
| "mean_confidence": 0.5465827882093774, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.42073229181882404 | |
| }, | |
| "stance_abortion": { | |
| "count": 60, | |
| "accuracy": 0.6333333333333333, | |
| "brier": 0.5461372518947913, | |
| "brier_soft": 0.5461372518947913, | |
| "nll": 0.9076919478704142, | |
| "nll_soft": 0.9076919478704142, | |
| "ece": 0.27791953126303576, | |
| "mean_confidence": 0.8731201720022546, | |
| "confident_error_rate": 0.13333333333333333 | |
| }, | |
| "stance_atheism": { | |
| "count": 60, | |
| "accuracy": 0.7666666666666667, | |
| "brier": 0.3807376010727825, | |
| "brier_soft": 0.3807376010727825, | |
| "nll": 0.6894866431293326, | |
| "nll_soft": 0.6894866431293326, | |
| "ece": 0.1623851166899183, | |
| "mean_confidence": 0.8907779015958398, | |
| "confident_error_rate": 0.11666666666666667 | |
| }, | |
| "stance_feminist": { | |
| "count": 60, | |
| "accuracy": 0.7166666666666667, | |
| "brier": 0.3819212632872141, | |
| "brier_soft": 0.3819212632872141, | |
| "nll": 0.6580171893898042, | |
| "nll_soft": 0.6580171893898042, | |
| "ece": 0.13298127336044635, | |
| "mean_confidence": 0.825324515184345, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "stance_hillary": { | |
| "count": 60, | |
| "accuracy": 0.6833333333333333, | |
| "brier": 0.45702756655176746, | |
| "brier_soft": 0.45702756655176746, | |
| "nll": 0.7477193618494712, | |
| "nll_soft": 0.7477193618494712, | |
| "ece": 0.1903802570948809, | |
| "mean_confidence": 0.8185082158591793, | |
| "confident_error_rate": 0.03333333333333333 | |
| }, | |
| "stsb": { | |
| "count": 100, | |
| "accuracy": 0.67, | |
| "brier": 0.4637369660396809, | |
| "brier_soft": 0.21863385963253795, | |
| "nll": 0.8565117839821994, | |
| "nll_soft": 0.9269421980771559, | |
| "ece": 0.10975654106260692, | |
| "mean_confidence": 0.5983604068227878, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4717016493911864 | |
| }, | |
| "subjectivity": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.06213683667359613, | |
| "brier_soft": 0.06213683667359613, | |
| "nll": 0.09836556526336807, | |
| "nll_soft": 0.09836556526336807, | |
| "ece": 0.037214204902439964, | |
| "mean_confidence": 0.965239654729141, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "swag": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.261763729502724, | |
| "brier_soft": 0.261763729502724, | |
| "nll": 0.5414241498614379, | |
| "nll_soft": 0.5414241498614379, | |
| "ece": 0.07772512379591902, | |
| "mean_confidence": 0.8475851911820763, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "trec": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.06072142112442041, | |
| "brier_soft": 0.06072142112442041, | |
| "nll": 0.1440798856068137, | |
| "nll_soft": 0.1440798856068137, | |
| "ece": 0.02690207486651098, | |
| "mean_confidence": 0.9736191389408069, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "tweet_hate": { | |
| "count": 100, | |
| "accuracy": 0.59, | |
| "brier": 0.6465587241335488, | |
| "brier_soft": 0.6465587241335488, | |
| "nll": 1.033942009148757, | |
| "nll_soft": 1.033942009148757, | |
| "ece": 0.26985585995709155, | |
| "mean_confidence": 0.8472034199650343, | |
| "confident_error_rate": 0.2 | |
| }, | |
| "tweet_sentiment": { | |
| "count": 100, | |
| "accuracy": 0.64, | |
| "brier": 0.4223186571536932, | |
| "brier_soft": 0.4223186571536932, | |
| "nll": 0.7004327844815618, | |
| "nll_soft": 0.7004327844815618, | |
| "ece": 0.07372919978481107, | |
| "mean_confidence": 0.6227698543011895, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3842789231307415 | |
| }, | |
| "vitaminc": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.29574730745541433, | |
| "brier_soft": 0.29574730745541433, | |
| "nll": 0.5826839900533985, | |
| "nll_soft": 0.5826839900533985, | |
| "ece": 0.11659560936407634, | |
| "mean_confidence": 0.9001589414913586, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "wic": { | |
| "count": 100, | |
| "accuracy": 0.76, | |
| "brier": 0.33426080682454734, | |
| "brier_soft": 0.33426080682454734, | |
| "nll": 0.5083526804189769, | |
| "nll_soft": 0.5083526804189769, | |
| "ece": 0.06690578964005471, | |
| "mean_confidence": 0.8040500963252548, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "winogrande": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.25538046579627915, | |
| "brier_soft": 0.25538046579627915, | |
| "nll": 0.45847041026936064, | |
| "nll_soft": 0.45847041026936064, | |
| "ece": 0.08929165536123829, | |
| "mean_confidence": 0.9104076795837307, | |
| "confident_error_rate": 0.07 | |
| }, | |
| "yelp": { | |
| "count": 100, | |
| "accuracy": 0.69, | |
| "brier": 0.4207442539736879, | |
| "brier_soft": 0.4207442539736879, | |
| "nll": 0.7108912277650302, | |
| "nll_soft": 0.7108912277650302, | |
| "ece": 0.09729612455914759, | |
| "mean_confidence": 0.629268115548149, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4090353744354196 | |
| } | |
| }, | |
| "reliability": [ | |
| { | |
| "bin": "0.0-0.1", | |
| "count": 0, | |
| "confidence": NaN, | |
| "accuracy": NaN | |
| }, | |
| { | |
| "bin": "0.1-0.2", | |
| "count": 5, | |
| "confidence": 0.19399414131767895, | |
| "accuracy": 0.2 | |
| }, | |
| { | |
| "bin": "0.2-0.3", | |
| "count": 150, | |
| "confidence": 0.2521147949419595, | |
| "accuracy": 0.22666666666666666 | |
| }, | |
| { | |
| "bin": "0.3-0.4", | |
| "count": 99, | |
| "confidence": 0.3545488043431464, | |
| "accuracy": 0.3939393939393939 | |
| }, | |
| { | |
| "bin": "0.4-0.5", | |
| "count": 348, | |
| "confidence": 0.464996527886961, | |
| "accuracy": 0.4942528735632184 | |
| }, | |
| { | |
| "bin": "0.5-0.6", | |
| "count": 484, | |
| "confidence": 0.5522991218018702, | |
| "accuracy": 0.5516528925619835 | |
| }, | |
| { | |
| "bin": "0.6-0.7", | |
| "count": 425, | |
| "confidence": 0.6444857209509922, | |
| "accuracy": 0.6494117647058824 | |
| }, | |
| { | |
| "bin": "0.7-0.8", | |
| "count": 432, | |
| "confidence": 0.748695643337274, | |
| "accuracy": 0.7013888888888888 | |
| }, | |
| { | |
| "bin": "0.8-0.9", | |
| "count": 547, | |
| "confidence": 0.8524781590758739, | |
| "accuracy": 0.7623400365630713 | |
| }, | |
| { | |
| "bin": "0.9-1.0", | |
| "count": 3623, | |
| "confidence": 0.980786561241046, | |
| "accuracy": 0.9470052442727022 | |
| } | |
| ], | |
| "seconds": 110.56856632232666, | |
| "model": "runs/assay-8b", | |
| "data": "data/v4/dev.jsonl" | |
| } |