Invalid JSON:Unexpected token 'N', ..."fidence": NaN,
"... is not valid JSON
| { | |
| "overall": { | |
| "count": 5513, | |
| "accuracy": 0.7395247596589879, | |
| "brier": 0.35891411049374133, | |
| "brier_soft": 0.3325900319955616, | |
| "nll": 0.7000357437946032, | |
| "nll_soft": 0.7170249935885112, | |
| "ece": 0.055324478614689324, | |
| "mean_confidence": 0.7869682151212168, | |
| "confident_error_rate": 0.04117540359151097, | |
| "score_mae": 0.5581918609054473, | |
| "evidence_auc": 0.9994043337529221 | |
| }, | |
| "by_type": { | |
| "choice": { | |
| "count": 2993, | |
| "accuracy": 0.7213498162378884, | |
| "brier": 0.377699802879719, | |
| "brier_soft": 0.35180044438730496, | |
| "nll": 0.765764452542442, | |
| "nll_soft": 0.7819117596312863, | |
| "ece": 0.07580803751223453, | |
| "mean_confidence": 0.7862548693547208, | |
| "confident_error_rate": 0.03942532576010692, | |
| "evidence_auc": 0.9993569870249167 | |
| }, | |
| "noul": { | |
| "count": 1620, | |
| "accuracy": 0.8345679012345679, | |
| "brier": 0.25971340795463804, | |
| "brier_soft": 0.2552280015387969, | |
| "nll": 0.48789057074198444, | |
| "nll_soft": 0.49318502851467866, | |
| "ece": 0.08318764369782508, | |
| "mean_confidence": 0.9068540703713514, | |
| "confident_error_rate": 0.06728395061728396, | |
| "evidence_auc": 0.9991164126352993 | |
| }, | |
| "score": { | |
| "count": 900, | |
| "accuracy": 0.6288888888888889, | |
| "brier": 0.4750025113849818, | |
| "brier_soft": 0.4079563931638626, | |
| "nll": 0.8633125827534488, | |
| "nll_soft": 0.9041523854258257, | |
| "ece": 0.06167508369139869, | |
| "mean_confidence": 0.573545946647778, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5581918609054473 | |
| } | |
| }, | |
| "by_source": { | |
| "ag_news": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.16595892610945714, | |
| "brier_soft": 0.16595892610945714, | |
| "nll": 0.317619005762231, | |
| "nll_soft": 0.317619005762231, | |
| "ece": 0.09714434528932915, | |
| "mean_confidence": 0.9271270369579904, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "amazon_polarity": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.08482015835915871, | |
| "brier_soft": 0.08482015835915871, | |
| "nll": 0.1269236993763182, | |
| "nll_soft": 0.1269236993763182, | |
| "ece": 0.041405803486552054, | |
| "mean_confidence": 0.9682691791456501, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "amazon_stars": { | |
| "count": 100, | |
| "accuracy": 0.59, | |
| "brier": 0.5217159065322495, | |
| "brier_soft": 0.5217159065322495, | |
| "nll": 0.90655071807393, | |
| "nll_soft": 0.90655071807393, | |
| "ece": 0.09689129521679905, | |
| "mean_confidence": 0.5675670253578411, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5244947503958304 | |
| }, | |
| "anli": { | |
| "count": 100, | |
| "accuracy": 0.48, | |
| "brier": 0.7963336642886663, | |
| "brier_soft": 0.7963336642886663, | |
| "nll": 1.5469344891195587, | |
| "nll_soft": 1.5469344891195587, | |
| "ece": 0.3310566451772958, | |
| "mean_confidence": 0.8110566451772957, | |
| "confident_error_rate": 0.21 | |
| }, | |
| "arc_challenge": { | |
| "count": 100, | |
| "accuracy": 0.81, | |
| "brier": 0.24633883275451762, | |
| "brier_soft": 0.24633883275451762, | |
| "nll": 0.4466973186010664, | |
| "nll_soft": 0.4466973186010664, | |
| "ece": 0.06410845564719533, | |
| "mean_confidence": 0.8628771978503359, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "arc_easy": { | |
| "count": 100, | |
| "accuracy": 0.9, | |
| "brier": 0.13677828266154088, | |
| "brier_soft": 0.13677828266154088, | |
| "nll": 0.265735472190341, | |
| "nll_soft": 0.265735472190341, | |
| "ece": 0.06328248053925811, | |
| "mean_confidence": 0.9029465101494548, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "banking77": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.4637771706673255, | |
| "brier_soft": 0.4637771706673255, | |
| "nll": 1.2276487123499176, | |
| "nll_soft": 1.2276487123499176, | |
| "ece": 0.15956483965367682, | |
| "mean_confidence": 0.8062828546524757, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "boolq": { | |
| "count": 125, | |
| "accuracy": 0.672, | |
| "brier": 0.6070634046798411, | |
| "brier_soft": 0.6070634046798411, | |
| "nll": 1.4168346082089363, | |
| "nll_soft": 1.4168346082089363, | |
| "ece": 0.30354649748023976, | |
| "mean_confidence": 0.9458572833588131, | |
| "confident_error_rate": 0.288, | |
| "evidence_auc": 1.0 | |
| }, | |
| "cb": { | |
| "count": 28, | |
| "accuracy": 0.9285714285714286, | |
| "brier": 0.12532849323262302, | |
| "brier_soft": 0.12532849323262302, | |
| "nll": 0.2153673382500027, | |
| "nll_soft": 0.2153673382500027, | |
| "ece": 0.08797639003481685, | |
| "mean_confidence": 0.9568322560301521, | |
| "confident_error_rate": 0.03571428571428571 | |
| }, | |
| "civil_comments_severity": { | |
| "count": 100, | |
| "accuracy": 0.85, | |
| "brier": 0.25539906222762987, | |
| "brier_soft": 0.18607564031826235, | |
| "nll": 0.5235735283062534, | |
| "nll_soft": 0.6705480040818754, | |
| "ece": 0.12916842803595735, | |
| "mean_confidence": 0.7606838855531521, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3739653312049361 | |
| }, | |
| "civil_comments_toxic": { | |
| "count": 100, | |
| "accuracy": 0.92, | |
| "brier": 0.1244634426466273, | |
| "brier_soft": 0.051799858710000625, | |
| "nll": 0.2133707491202998, | |
| "nll_soft": 0.2991409650379465, | |
| "ece": 0.04793496244874154, | |
| "mean_confidence": 0.9069060768240247, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "clinc150": { | |
| "count": 100, | |
| "accuracy": 0.72, | |
| "brier": 0.4240880520787745, | |
| "brier_soft": 0.4240880520787745, | |
| "nll": 1.3707408511728065, | |
| "nll_soft": 1.3707408511728065, | |
| "ece": 0.18535569602870228, | |
| "mean_confidence": 0.8346231087184794, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "cola": { | |
| "count": 100, | |
| "accuracy": 0.72, | |
| "brier": 0.4325226256789194, | |
| "brier_soft": 0.4325226256789194, | |
| "nll": 0.6804222452824308, | |
| "nll_soft": 0.6804222452824308, | |
| "ece": 0.16408725417487102, | |
| "mean_confidence": 0.884087254174871, | |
| "confident_error_rate": 0.13 | |
| }, | |
| "commonsense_qa": { | |
| "count": 100, | |
| "accuracy": 0.79, | |
| "brier": 0.2879336103250024, | |
| "brier_soft": 0.2879336103250024, | |
| "nll": 0.5679404731283431, | |
| "nll_soft": 0.5679404731283431, | |
| "ece": 0.06999279818542659, | |
| "mean_confidence": 0.8233452649244275, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "cosmos_qa": { | |
| "count": 120, | |
| "accuracy": 0.5916666666666667, | |
| "brier": 0.5430136874921789, | |
| "brier_soft": 0.5430136874921789, | |
| "nll": 1.074075795624674, | |
| "nll_soft": 1.074075795624674, | |
| "ece": 0.19056260243873563, | |
| "mean_confidence": 0.7822292691054024, | |
| "confident_error_rate": 0.06666666666666667, | |
| "evidence_auc": 0.9925 | |
| }, | |
| "dbpedia": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.012088062630857132, | |
| "brier_soft": 0.012088062630857132, | |
| "nll": 0.01793650723546934, | |
| "nll_soft": 0.01793650723546934, | |
| "ece": 0.010164549918044101, | |
| "mean_confidence": 0.9949775540034673, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "dynasent": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.48594524162593145, | |
| "brier_soft": 0.346234914599779, | |
| "nll": 0.8224392534906353, | |
| "nll_soft": 0.8519392531744227, | |
| "ece": 0.09168704014757871, | |
| "mean_confidence": 0.6220737263754066, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5377038785936414 | |
| }, | |
| "enron_spam": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.04407558581320349, | |
| "brier_soft": 0.04407558581320349, | |
| "nll": 0.10800463572408005, | |
| "nll_soft": 0.10800463572408005, | |
| "ece": 0.020701951320388896, | |
| "mean_confidence": 0.9832870047313784, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "go_emotions": { | |
| "count": 100, | |
| "accuracy": 0.57, | |
| "brier": 0.6423581009956649, | |
| "brier_soft": 0.24900484568580272, | |
| "nll": 1.6278569291251241, | |
| "nll_soft": 2.103362881896529, | |
| "ece": 0.19085840262746412, | |
| "mean_confidence": 0.38353292815495565, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "hellaswag": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.2382844971411162, | |
| "brier_soft": 0.2382844971411162, | |
| "nll": 0.4449554075221306, | |
| "nll_soft": 0.4449554075221306, | |
| "ece": 0.06424349905061746, | |
| "mean_confidence": 0.8227664908027684, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "imdb": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.10672909810875578, | |
| "brier_soft": 0.10672909810875578, | |
| "nll": 0.20608689578509665, | |
| "nll_soft": 0.20608689578509665, | |
| "ece": 0.06029207899584085, | |
| "mean_confidence": 0.9714224898960039, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "ledgar": { | |
| "count": 100, | |
| "accuracy": 0.72, | |
| "brier": 0.39749454268307993, | |
| "brier_soft": 0.39749454268307993, | |
| "nll": 1.105951261520073, | |
| "nll_soft": 1.105951261520073, | |
| "ece": 0.12604416845897642, | |
| "mean_confidence": 0.8357689870014333, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "liar": { | |
| "count": 100, | |
| "accuracy": 0.27, | |
| "brier": 0.7678109993862653, | |
| "brier_soft": 0.7678109993862653, | |
| "nll": 1.5776155917960453, | |
| "nll_soft": 1.5776155917960453, | |
| "ece": 0.03512032448929137, | |
| "mean_confidence": 0.28709838151462497, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 1.1204754718941812 | |
| }, | |
| "measuring_hate_speech": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.2975627280485236, | |
| "brier_soft": 0.17057811194913736, | |
| "nll": 0.549532489247843, | |
| "nll_soft": 0.6952408193109864, | |
| "ece": 0.15596998903458828, | |
| "mean_confidence": 0.6674488231979009, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.580115784356518 | |
| }, | |
| "medqa": { | |
| "count": 100, | |
| "accuracy": 0.52, | |
| "brier": 0.6283885076925132, | |
| "brier_soft": 0.6283885076925132, | |
| "nll": 1.1593420652053754, | |
| "nll_soft": 1.1593420652053754, | |
| "ece": 0.10174980260960248, | |
| "mean_confidence": 0.574442080508045, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "mnli": { | |
| "count": 100, | |
| "accuracy": 0.89, | |
| "brier": 0.1718816447782862, | |
| "brier_soft": 0.1718816447782862, | |
| "nll": 0.32824266416424464, | |
| "nll_soft": 0.32824266416424464, | |
| "ece": 0.0852239640391438, | |
| "mean_confidence": 0.8939806868250493, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "mrpc": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.25076123522083993, | |
| "brier_soft": 0.25076123522083993, | |
| "nll": 0.4133734348931764, | |
| "nll_soft": 0.4133734348931764, | |
| "ece": 0.08357678747719188, | |
| "mean_confidence": 0.827315098378607, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "multirc": { | |
| "count": 125, | |
| "accuracy": 0.728, | |
| "brier": 0.505724757364016, | |
| "brier_soft": 0.505724757364016, | |
| "nll": 1.312838385375779, | |
| "nll_soft": 1.312838385375779, | |
| "ece": 0.2501538619832258, | |
| "mean_confidence": 0.9280461062866522, | |
| "confident_error_rate": 0.2, | |
| "evidence_auc": 0.9938666666666667 | |
| }, | |
| "openbookqa": { | |
| "count": 100, | |
| "accuracy": 0.73, | |
| "brier": 0.3668734371778269, | |
| "brier_soft": 0.3668734371778269, | |
| "nll": 0.7540166865354884, | |
| "nll_soft": 0.7540166865354884, | |
| "ece": 0.15139395643328224, | |
| "mean_confidence": 0.8355156248733749, | |
| "confident_error_rate": 0.07 | |
| }, | |
| "piqa": { | |
| "count": 100, | |
| "accuracy": 0.79, | |
| "brier": 0.32694164004786186, | |
| "brier_soft": 0.32694164004786186, | |
| "nll": 0.5180996363143726, | |
| "nll_soft": 0.5180996363143726, | |
| "ece": 0.11247283410685574, | |
| "mean_confidence": 0.7981778076696955, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "policy_application": { | |
| "count": 150, | |
| "accuracy": 0.8466666666666667, | |
| "brier": 0.22223428758579872, | |
| "brier_soft": 0.22223428758579872, | |
| "nll": 0.3469153470049193, | |
| "nll_soft": 0.3469153470049193, | |
| "ece": 0.08065517066529573, | |
| "mean_confidence": 0.873683047126326, | |
| "confident_error_rate": 0.02, | |
| "evidence_auc": 0.9908436886854153 | |
| }, | |
| "qqp": { | |
| "count": 100, | |
| "accuracy": 0.9, | |
| "brier": 0.18692819863784732, | |
| "brier_soft": 0.18692819863784732, | |
| "nll": 0.29128078782999794, | |
| "nll_soft": 0.29128078782999794, | |
| "ece": 0.09064689206058686, | |
| "mean_confidence": 0.8968450141922143, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "race": { | |
| "count": 125, | |
| "accuracy": 0.688, | |
| "brier": 0.4242329241788974, | |
| "brier_soft": 0.4242329241788974, | |
| "nll": 0.7993514526469424, | |
| "nll_soft": 0.7993514526469424, | |
| "ece": 0.1449716612385065, | |
| "mean_confidence": 0.8115467336901924, | |
| "confident_error_rate": 0.04, | |
| "evidence_auc": 0.9928 | |
| }, | |
| "rotten_tomatoes": { | |
| "count": 100, | |
| "accuracy": 0.9, | |
| "brier": 0.14135503273237815, | |
| "brier_soft": 0.14135503273237815, | |
| "nll": 0.2503500682963772, | |
| "nll_soft": 0.2503500682963772, | |
| "ece": 0.04641989479613032, | |
| "mean_confidence": 0.9276372803949359, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "rte": { | |
| "count": 100, | |
| "accuracy": 0.85, | |
| "brier": 0.2306702691840198, | |
| "brier_soft": 0.2306702691840198, | |
| "nll": 0.36936737768302536, | |
| "nll_soft": 0.36936737768302536, | |
| "ece": 0.08101771753892531, | |
| "mean_confidence": 0.9265637146952472, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "scotus": { | |
| "count": 100, | |
| "accuracy": 0.26, | |
| "brier": 0.8889236149471939, | |
| "brier_soft": 0.8889236149471939, | |
| "nll": 2.376624371079669, | |
| "nll_soft": 2.376624371079669, | |
| "ece": 0.10195170965346324, | |
| "mean_confidence": 0.18935687669674867, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "shp": { | |
| "count": 100, | |
| "accuracy": 0.5, | |
| "brier": 0.5014794776803457, | |
| "brier_soft": 0.11966493331225726, | |
| "nll": 0.6992988018417869, | |
| "nll_soft": 0.7070817502394954, | |
| "ece": 0.08970654421887113, | |
| "mean_confidence": 0.5607733615565005, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "sms_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.019680706975513457, | |
| "brier_soft": 0.019680706975513457, | |
| "nll": 0.04841511394573284, | |
| "nll_soft": 0.04841511394573284, | |
| "ece": 0.0066156662697345325, | |
| "mean_confidence": 0.9966156662697345, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "snli": { | |
| "count": 100, | |
| "accuracy": 0.85, | |
| "brier": 0.20592408056668393, | |
| "brier_soft": 0.20592408056668393, | |
| "nll": 0.36888655335733217, | |
| "nll_soft": 0.36888655335733217, | |
| "ece": 0.10215280064869335, | |
| "mean_confidence": 0.922910036477836, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "social_iqa": { | |
| "count": 100, | |
| "accuracy": 0.71, | |
| "brier": 0.4464141973857656, | |
| "brier_soft": 0.4464141973857656, | |
| "nll": 0.7565779421223005, | |
| "nll_soft": 0.7565779421223005, | |
| "ece": 0.1362994068405467, | |
| "mean_confidence": 0.803044630431075, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "sst2": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.09458072010854306, | |
| "brier_soft": 0.09458072010854306, | |
| "nll": 0.19510168063804795, | |
| "nll_soft": 0.19510168063804795, | |
| "ece": 0.04233138537219727, | |
| "mean_confidence": 0.976039135118701, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "sst5": { | |
| "count": 100, | |
| "accuracy": 0.61, | |
| "brier": 0.5215090991589044, | |
| "brier_soft": 0.5215090991589044, | |
| "nll": 0.9352095146683718, | |
| "nll_soft": 0.9352095146683718, | |
| "ece": 0.1058405134838416, | |
| "mean_confidence": 0.5193849682693964, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5064625990954423 | |
| }, | |
| "stance_abortion": { | |
| "count": 60, | |
| "accuracy": 0.65, | |
| "brier": 0.42215429052282005, | |
| "brier_soft": 0.42215429052282005, | |
| "nll": 0.6954065175846141, | |
| "nll_soft": 0.6954065175846141, | |
| "ece": 0.1120315155546204, | |
| "mean_confidence": 0.7198534026566822, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_atheism": { | |
| "count": 60, | |
| "accuracy": 0.7333333333333333, | |
| "brier": 0.3399983824095354, | |
| "brier_soft": 0.3399983824095354, | |
| "nll": 0.5524210799908459, | |
| "nll_soft": 0.5524210799908459, | |
| "ece": 0.1144791457233202, | |
| "mean_confidence": 0.8005444999199404, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_feminist": { | |
| "count": 60, | |
| "accuracy": 0.65, | |
| "brier": 0.4284771001653723, | |
| "brier_soft": 0.4284771001653723, | |
| "nll": 0.6909320319691683, | |
| "nll_soft": 0.6909320319691683, | |
| "ece": 0.11512288881592876, | |
| "mean_confidence": 0.7075273754165499, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_hillary": { | |
| "count": 60, | |
| "accuracy": 0.7166666666666667, | |
| "brier": 0.39654058995650643, | |
| "brier_soft": 0.39654058995650643, | |
| "nll": 0.6784103477966172, | |
| "nll_soft": 0.6784103477966172, | |
| "ece": 0.1533237783580024, | |
| "mean_confidence": 0.7318050382015089, | |
| "confident_error_rate": 0.03333333333333333 | |
| }, | |
| "stsb": { | |
| "count": 100, | |
| "accuracy": 0.53, | |
| "brier": 0.5593050243129714, | |
| "brier_soft": 0.2919083253578051, | |
| "nll": 0.9862805521087811, | |
| "nll_soft": 1.0316559706376203, | |
| "ece": 0.044542105927844744, | |
| "mean_confidence": 0.5431262130898866, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5677600431430505 | |
| }, | |
| "subjectivity": { | |
| "count": 100, | |
| "accuracy": 0.92, | |
| "brier": 0.10998461343405579, | |
| "brier_soft": 0.10998461343405579, | |
| "nll": 0.1651519507144023, | |
| "nll_soft": 0.1651519507144023, | |
| "ece": 0.04798910875726077, | |
| "mean_confidence": 0.9679891087572606, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "swag": { | |
| "count": 100, | |
| "accuracy": 0.71, | |
| "brier": 0.4003799807391956, | |
| "brier_soft": 0.4003799807391956, | |
| "nll": 0.715850991098381, | |
| "nll_soft": 0.715850991098381, | |
| "ece": 0.10425936796862421, | |
| "mean_confidence": 0.7495599288646645, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "trec": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.0973401897205966, | |
| "brier_soft": 0.0973401897205966, | |
| "nll": 0.23106466126006978, | |
| "nll_soft": 0.23106466126006978, | |
| "ece": 0.03301735640409483, | |
| "mean_confidence": 0.9457145921252907, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "tweet_hate": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.499521442971358, | |
| "brier_soft": 0.499521442971358, | |
| "nll": 0.7246301877586945, | |
| "nll_soft": 0.7246301877586945, | |
| "ece": 0.2203497182309466, | |
| "mean_confidence": 0.7959954828308132, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "tweet_sentiment": { | |
| "count": 100, | |
| "accuracy": 0.71, | |
| "brier": 0.4093804097708304, | |
| "brier_soft": 0.4093804097708304, | |
| "nll": 0.6906869092595, | |
| "nll_soft": 0.6906869092595, | |
| "ece": 0.11551745886123974, | |
| "mean_confidence": 0.6126727871823638, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.37256540101600827 | |
| }, | |
| "vitaminc": { | |
| "count": 100, | |
| "accuracy": 0.67, | |
| "brier": 0.43661602738993105, | |
| "brier_soft": 0.43661602738993105, | |
| "nll": 0.7999046209455473, | |
| "nll_soft": 0.7999046209455473, | |
| "ece": 0.15065187251617596, | |
| "mean_confidence": 0.806123844596811, | |
| "confident_error_rate": 0.07 | |
| }, | |
| "wic": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.41257324470815915, | |
| "brier_soft": 0.41257324470815915, | |
| "nll": 0.5911202500978636, | |
| "nll_soft": 0.5911202500978636, | |
| "ece": 0.09333030231242981, | |
| "mean_confidence": 0.6790619026658443, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "winogrande": { | |
| "count": 100, | |
| "accuracy": 0.58, | |
| "brier": 0.5970109839281106, | |
| "brier_soft": 0.5970109839281106, | |
| "nll": 0.9404834376569532, | |
| "nll_soft": 0.9404834376569532, | |
| "ece": 0.2111694877129903, | |
| "mean_confidence": 0.7809421535184294, | |
| "confident_error_rate": 0.15 | |
| }, | |
| "yelp": { | |
| "count": 100, | |
| "accuracy": 0.62, | |
| "brier": 0.4563941314015302, | |
| "brier_soft": 0.4563941314015302, | |
| "nll": 0.7779246878296787, | |
| "nll_soft": 0.7779246878296787, | |
| "ece": 0.13593820054296052, | |
| "mean_confidence": 0.5818577092894286, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4401834884494164 | |
| } | |
| }, | |
| "reliability": [ | |
| { | |
| "bin": "0.0-0.1", | |
| "count": 0, | |
| "confidence": NaN, | |
| "accuracy": NaN | |
| }, | |
| { | |
| "bin": "0.1-0.2", | |
| "count": 76, | |
| "confidence": 0.17221104488237657, | |
| "accuracy": 0.32894736842105265 | |
| }, | |
| { | |
| "bin": "0.2-0.3", | |
| "count": 132, | |
| "confidence": 0.2395367660809224, | |
| "accuracy": 0.2727272727272727 | |
| }, | |
| { | |
| "bin": "0.3-0.4", | |
| "count": 140, | |
| "confidence": 0.3559224153152535, | |
| "accuracy": 0.3142857142857143 | |
| }, | |
| { | |
| "bin": "0.4-0.5", | |
| "count": 455, | |
| "confidence": 0.46425223118313025, | |
| "accuracy": 0.46593406593406594 | |
| }, | |
| { | |
| "bin": "0.5-0.6", | |
| "count": 537, | |
| "confidence": 0.5499579597258507, | |
| "accuracy": 0.5288640595903166 | |
| }, | |
| { | |
| "bin": "0.6-0.7", | |
| "count": 506, | |
| "confidence": 0.6456601586886904, | |
| "accuracy": 0.6146245059288538 | |
| }, | |
| { | |
| "bin": "0.7-0.8", | |
| "count": 428, | |
| "confidence": 0.751258870528824, | |
| "accuracy": 0.6822429906542056 | |
| }, | |
| { | |
| "bin": "0.8-0.9", | |
| "count": 566, | |
| "confidence": 0.8520923292712668, | |
| "accuracy": 0.7544169611307421 | |
| }, | |
| { | |
| "bin": "0.9-1.0", | |
| "count": 2673, | |
| "confidence": 0.9752826241952612, | |
| "accuracy": 0.9150766928544707 | |
| } | |
| ], | |
| "seconds": 31.28231954574585, | |
| "model": "runs/assay-1.7b-v2", | |
| "data": "data/v2/dev.jsonl" | |
| } |