{ "overall": { "count": 5513, "accuracy": 0.7395247596589879, "brier": 0.35891411049374133, "brier_soft": 0.3325900319955616, "nll": 0.7000357437946032, "nll_soft": 0.7170249935885112, "ece": 0.055324478614689324, "mean_confidence": 0.7869682151212168, "confident_error_rate": 0.04117540359151097, "score_mae": 0.5581918609054473, "evidence_auc": 0.9994043337529221 }, "by_type": { "choice": { "count": 2993, "accuracy": 0.7213498162378884, "brier": 0.377699802879719, "brier_soft": 0.35180044438730496, "nll": 0.765764452542442, "nll_soft": 0.7819117596312863, "ece": 0.07580803751223453, "mean_confidence": 0.7862548693547208, "confident_error_rate": 0.03942532576010692, "evidence_auc": 0.9993569870249167 }, "noul": { "count": 1620, "accuracy": 0.8345679012345679, "brier": 0.25971340795463804, "brier_soft": 0.2552280015387969, "nll": 0.48789057074198444, "nll_soft": 0.49318502851467866, "ece": 0.08318764369782508, "mean_confidence": 0.9068540703713514, "confident_error_rate": 0.06728395061728396, "evidence_auc": 0.9991164126352993 }, "score": { "count": 900, "accuracy": 0.6288888888888889, "brier": 0.4750025113849818, "brier_soft": 0.4079563931638626, "nll": 0.8633125827534488, "nll_soft": 0.9041523854258257, "ece": 0.06167508369139869, "mean_confidence": 0.573545946647778, "confident_error_rate": 0.0, "score_mae": 0.5581918609054473 } }, "by_source": { "ag_news": { "count": 100, "accuracy": 0.88, "brier": 0.16595892610945714, "brier_soft": 0.16595892610945714, "nll": 0.317619005762231, "nll_soft": 0.317619005762231, "ece": 0.09714434528932915, "mean_confidence": 0.9271270369579904, "confident_error_rate": 0.04 }, "amazon_polarity": { "count": 100, "accuracy": 0.94, "brier": 0.08482015835915871, "brier_soft": 0.08482015835915871, "nll": 0.1269236993763182, "nll_soft": 0.1269236993763182, "ece": 0.041405803486552054, "mean_confidence": 0.9682691791456501, "confident_error_rate": 0.01 }, "amazon_stars": { "count": 100, "accuracy": 0.59, "brier": 0.5217159065322495, "brier_soft": 0.5217159065322495, "nll": 0.90655071807393, "nll_soft": 0.90655071807393, "ece": 0.09689129521679905, "mean_confidence": 0.5675670253578411, "confident_error_rate": 0.0, "score_mae": 0.5244947503958304 }, "anli": { "count": 100, "accuracy": 0.48, "brier": 0.7963336642886663, "brier_soft": 0.7963336642886663, "nll": 1.5469344891195587, "nll_soft": 1.5469344891195587, "ece": 0.3310566451772958, "mean_confidence": 0.8110566451772957, "confident_error_rate": 0.21 }, "arc_challenge": { "count": 100, "accuracy": 0.81, "brier": 0.24633883275451762, "brier_soft": 0.24633883275451762, "nll": 0.4466973186010664, "nll_soft": 0.4466973186010664, "ece": 0.06410845564719533, "mean_confidence": 0.8628771978503359, "confident_error_rate": 0.01 }, "arc_easy": { "count": 100, "accuracy": 0.9, "brier": 0.13677828266154088, "brier_soft": 0.13677828266154088, "nll": 0.265735472190341, "nll_soft": 0.265735472190341, "ece": 0.06328248053925811, "mean_confidence": 0.9029465101494548, "confident_error_rate": 0.01 }, "banking77": { "count": 100, "accuracy": 0.66, "brier": 0.4637771706673255, "brier_soft": 0.4637771706673255, "nll": 1.2276487123499176, "nll_soft": 1.2276487123499176, "ece": 0.15956483965367682, "mean_confidence": 0.8062828546524757, "confident_error_rate": 0.03 }, "boolq": { "count": 125, "accuracy": 0.672, "brier": 0.6070634046798411, "brier_soft": 0.6070634046798411, "nll": 1.4168346082089363, "nll_soft": 1.4168346082089363, "ece": 0.30354649748023976, "mean_confidence": 0.9458572833588131, "confident_error_rate": 0.288, "evidence_auc": 1.0 }, "cb": { "count": 28, "accuracy": 0.9285714285714286, "brier": 0.12532849323262302, "brier_soft": 0.12532849323262302, "nll": 0.2153673382500027, "nll_soft": 0.2153673382500027, "ece": 0.08797639003481685, "mean_confidence": 0.9568322560301521, "confident_error_rate": 0.03571428571428571 }, "civil_comments_severity": { "count": 100, "accuracy": 0.85, "brier": 0.25539906222762987, "brier_soft": 0.18607564031826235, "nll": 0.5235735283062534, "nll_soft": 0.6705480040818754, "ece": 0.12916842803595735, "mean_confidence": 0.7606838855531521, "confident_error_rate": 0.0, "score_mae": 0.3739653312049361 }, "civil_comments_toxic": { "count": 100, "accuracy": 0.92, "brier": 0.1244634426466273, "brier_soft": 0.051799858710000625, "nll": 0.2133707491202998, "nll_soft": 0.2991409650379465, "ece": 0.04793496244874154, "mean_confidence": 0.9069060768240247, "confident_error_rate": 0.02 }, "clinc150": { "count": 100, "accuracy": 0.72, "brier": 0.4240880520787745, "brier_soft": 0.4240880520787745, "nll": 1.3707408511728065, "nll_soft": 1.3707408511728065, "ece": 0.18535569602870228, "mean_confidence": 0.8346231087184794, "confident_error_rate": 0.06 }, "cola": { "count": 100, "accuracy": 0.72, "brier": 0.4325226256789194, "brier_soft": 0.4325226256789194, "nll": 0.6804222452824308, "nll_soft": 0.6804222452824308, "ece": 0.16408725417487102, "mean_confidence": 0.884087254174871, "confident_error_rate": 0.13 }, "commonsense_qa": { "count": 100, "accuracy": 0.79, "brier": 0.2879336103250024, "brier_soft": 0.2879336103250024, "nll": 0.5679404731283431, "nll_soft": 0.5679404731283431, "ece": 0.06999279818542659, "mean_confidence": 0.8233452649244275, "confident_error_rate": 0.03 }, "cosmos_qa": { "count": 120, "accuracy": 0.5916666666666667, "brier": 0.5430136874921789, "brier_soft": 0.5430136874921789, "nll": 1.074075795624674, "nll_soft": 1.074075795624674, "ece": 0.19056260243873563, "mean_confidence": 0.7822292691054024, "confident_error_rate": 0.06666666666666667, "evidence_auc": 0.9925 }, "dbpedia": { "count": 100, "accuracy": 0.99, "brier": 0.012088062630857132, "brier_soft": 0.012088062630857132, "nll": 0.01793650723546934, "nll_soft": 0.01793650723546934, "ece": 0.010164549918044101, "mean_confidence": 0.9949775540034673, "confident_error_rate": 0.0 }, "dynasent": { "count": 100, "accuracy": 0.66, "brier": 0.48594524162593145, "brier_soft": 0.346234914599779, "nll": 0.8224392534906353, "nll_soft": 0.8519392531744227, "ece": 0.09168704014757871, "mean_confidence": 0.6220737263754066, "confident_error_rate": 0.0, "score_mae": 0.5377038785936414 }, "enron_spam": { "count": 100, "accuracy": 0.97, "brier": 0.04407558581320349, "brier_soft": 0.04407558581320349, "nll": 0.10800463572408005, "nll_soft": 0.10800463572408005, "ece": 0.020701951320388896, "mean_confidence": 0.9832870047313784, "confident_error_rate": 0.02 }, "go_emotions": { "count": 100, "accuracy": 0.57, "brier": 0.6423581009956649, "brier_soft": 0.24900484568580272, "nll": 1.6278569291251241, "nll_soft": 2.103362881896529, "ece": 0.19085840262746412, "mean_confidence": 0.38353292815495565, "confident_error_rate": 0.0 }, "hellaswag": { "count": 100, "accuracy": 0.82, "brier": 0.2382844971411162, "brier_soft": 0.2382844971411162, "nll": 0.4449554075221306, "nll_soft": 0.4449554075221306, "ece": 0.06424349905061746, "mean_confidence": 0.8227664908027684, "confident_error_rate": 0.01 }, "imdb": { "count": 100, "accuracy": 0.94, "brier": 0.10672909810875578, "brier_soft": 0.10672909810875578, "nll": 0.20608689578509665, "nll_soft": 0.20608689578509665, "ece": 0.06029207899584085, "mean_confidence": 0.9714224898960039, "confident_error_rate": 0.04 }, "ledgar": { "count": 100, "accuracy": 0.72, "brier": 0.39749454268307993, "brier_soft": 0.39749454268307993, "nll": 1.105951261520073, "nll_soft": 1.105951261520073, "ece": 0.12604416845897642, "mean_confidence": 0.8357689870014333, "confident_error_rate": 0.04 }, "liar": { "count": 100, "accuracy": 0.27, "brier": 0.7678109993862653, "brier_soft": 0.7678109993862653, "nll": 1.5776155917960453, "nll_soft": 1.5776155917960453, "ece": 0.03512032448929137, "mean_confidence": 0.28709838151462497, "confident_error_rate": 0.0, "score_mae": 1.1204754718941812 }, "measuring_hate_speech": { "count": 100, "accuracy": 0.82, "brier": 0.2975627280485236, "brier_soft": 0.17057811194913736, "nll": 0.549532489247843, "nll_soft": 0.6952408193109864, "ece": 0.15596998903458828, "mean_confidence": 0.6674488231979009, "confident_error_rate": 0.0, "score_mae": 0.580115784356518 }, "medqa": { "count": 100, "accuracy": 0.52, "brier": 0.6283885076925132, "brier_soft": 0.6283885076925132, "nll": 1.1593420652053754, "nll_soft": 1.1593420652053754, "ece": 0.10174980260960248, "mean_confidence": 0.574442080508045, "confident_error_rate": 0.0 }, "mnli": { "count": 100, "accuracy": 0.89, "brier": 0.1718816447782862, "brier_soft": 0.1718816447782862, "nll": 0.32824266416424464, "nll_soft": 0.32824266416424464, "ece": 0.0852239640391438, "mean_confidence": 0.8939806868250493, "confident_error_rate": 0.04 }, "mrpc": { "count": 100, "accuracy": 0.82, "brier": 0.25076123522083993, "brier_soft": 0.25076123522083993, "nll": 0.4133734348931764, "nll_soft": 0.4133734348931764, "ece": 0.08357678747719188, "mean_confidence": 0.827315098378607, "confident_error_rate": 0.03 }, "multirc": { "count": 125, "accuracy": 0.728, "brier": 0.505724757364016, "brier_soft": 0.505724757364016, "nll": 1.312838385375779, "nll_soft": 1.312838385375779, "ece": 0.2501538619832258, "mean_confidence": 0.9280461062866522, "confident_error_rate": 0.2, "evidence_auc": 0.9938666666666667 }, "openbookqa": { "count": 100, "accuracy": 0.73, "brier": 0.3668734371778269, "brier_soft": 0.3668734371778269, "nll": 0.7540166865354884, "nll_soft": 0.7540166865354884, "ece": 0.15139395643328224, "mean_confidence": 0.8355156248733749, "confident_error_rate": 0.07 }, "piqa": { "count": 100, "accuracy": 0.79, "brier": 0.32694164004786186, "brier_soft": 0.32694164004786186, "nll": 0.5180996363143726, "nll_soft": 0.5180996363143726, "ece": 0.11247283410685574, "mean_confidence": 0.7981778076696955, "confident_error_rate": 0.04 }, "policy_application": { "count": 150, "accuracy": 0.8466666666666667, "brier": 0.22223428758579872, "brier_soft": 0.22223428758579872, "nll": 0.3469153470049193, "nll_soft": 0.3469153470049193, "ece": 0.08065517066529573, "mean_confidence": 0.873683047126326, "confident_error_rate": 0.02, "evidence_auc": 0.9908436886854153 }, "qqp": { "count": 100, "accuracy": 0.9, "brier": 0.18692819863784732, "brier_soft": 0.18692819863784732, "nll": 0.29128078782999794, "nll_soft": 0.29128078782999794, "ece": 0.09064689206058686, "mean_confidence": 0.8968450141922143, "confident_error_rate": 0.01 }, "race": { "count": 125, "accuracy": 0.688, "brier": 0.4242329241788974, "brier_soft": 0.4242329241788974, "nll": 0.7993514526469424, "nll_soft": 0.7993514526469424, "ece": 0.1449716612385065, "mean_confidence": 0.8115467336901924, "confident_error_rate": 0.04, "evidence_auc": 0.9928 }, "rotten_tomatoes": { "count": 100, "accuracy": 0.9, "brier": 0.14135503273237815, "brier_soft": 0.14135503273237815, "nll": 0.2503500682963772, "nll_soft": 0.2503500682963772, "ece": 0.04641989479613032, "mean_confidence": 0.9276372803949359, "confident_error_rate": 0.03 }, "rte": { "count": 100, "accuracy": 0.85, "brier": 0.2306702691840198, "brier_soft": 0.2306702691840198, "nll": 0.36936737768302536, "nll_soft": 0.36936737768302536, "ece": 0.08101771753892531, "mean_confidence": 0.9265637146952472, "confident_error_rate": 0.06 }, "scotus": { "count": 100, "accuracy": 0.26, "brier": 0.8889236149471939, "brier_soft": 0.8889236149471939, "nll": 2.376624371079669, "nll_soft": 2.376624371079669, "ece": 0.10195170965346324, "mean_confidence": 0.18935687669674867, "confident_error_rate": 0.0 }, "shp": { "count": 100, "accuracy": 0.5, "brier": 0.5014794776803457, "brier_soft": 0.11966493331225726, "nll": 0.6992988018417869, "nll_soft": 0.7070817502394954, "ece": 0.08970654421887113, "mean_confidence": 0.5607733615565005, "confident_error_rate": 0.01 }, "sms_spam": { "count": 100, "accuracy": 0.99, "brier": 0.019680706975513457, "brier_soft": 0.019680706975513457, "nll": 0.04841511394573284, "nll_soft": 0.04841511394573284, "ece": 0.0066156662697345325, "mean_confidence": 0.9966156662697345, "confident_error_rate": 0.01 }, "snli": { "count": 100, "accuracy": 0.85, "brier": 0.20592408056668393, "brier_soft": 0.20592408056668393, "nll": 0.36888655335733217, "nll_soft": 0.36888655335733217, "ece": 0.10215280064869335, "mean_confidence": 0.922910036477836, "confident_error_rate": 0.05 }, "social_iqa": { "count": 100, "accuracy": 0.71, "brier": 0.4464141973857656, "brier_soft": 0.4464141973857656, "nll": 0.7565779421223005, "nll_soft": 0.7565779421223005, "ece": 0.1362994068405467, "mean_confidence": 0.803044630431075, "confident_error_rate": 0.05 }, "sst2": { "count": 100, "accuracy": 0.94, "brier": 0.09458072010854306, "brier_soft": 0.09458072010854306, "nll": 0.19510168063804795, "nll_soft": 0.19510168063804795, "ece": 0.04233138537219727, "mean_confidence": 0.976039135118701, "confident_error_rate": 0.04 }, "sst5": { "count": 100, "accuracy": 0.61, "brier": 0.5215090991589044, "brier_soft": 0.5215090991589044, "nll": 0.9352095146683718, "nll_soft": 0.9352095146683718, "ece": 0.1058405134838416, "mean_confidence": 0.5193849682693964, "confident_error_rate": 0.0, "score_mae": 0.5064625990954423 }, "stance_abortion": { "count": 60, "accuracy": 0.65, "brier": 0.42215429052282005, "brier_soft": 0.42215429052282005, "nll": 0.6954065175846141, "nll_soft": 0.6954065175846141, "ece": 0.1120315155546204, "mean_confidence": 0.7198534026566822, "confident_error_rate": 0.016666666666666666 }, "stance_atheism": { "count": 60, "accuracy": 0.7333333333333333, "brier": 0.3399983824095354, "brier_soft": 0.3399983824095354, "nll": 0.5524210799908459, "nll_soft": 0.5524210799908459, "ece": 0.1144791457233202, "mean_confidence": 0.8005444999199404, "confident_error_rate": 0.016666666666666666 }, "stance_feminist": { "count": 60, "accuracy": 0.65, "brier": 0.4284771001653723, "brier_soft": 0.4284771001653723, "nll": 0.6909320319691683, "nll_soft": 0.6909320319691683, "ece": 0.11512288881592876, "mean_confidence": 0.7075273754165499, "confident_error_rate": 0.016666666666666666 }, "stance_hillary": { "count": 60, "accuracy": 0.7166666666666667, "brier": 0.39654058995650643, "brier_soft": 0.39654058995650643, "nll": 0.6784103477966172, "nll_soft": 0.6784103477966172, "ece": 0.1533237783580024, "mean_confidence": 0.7318050382015089, "confident_error_rate": 0.03333333333333333 }, "stsb": { "count": 100, "accuracy": 0.53, "brier": 0.5593050243129714, "brier_soft": 0.2919083253578051, "nll": 0.9862805521087811, "nll_soft": 1.0316559706376203, "ece": 0.044542105927844744, "mean_confidence": 0.5431262130898866, "confident_error_rate": 0.0, "score_mae": 0.5677600431430505 }, "subjectivity": { "count": 100, "accuracy": 0.92, "brier": 0.10998461343405579, "brier_soft": 0.10998461343405579, "nll": 0.1651519507144023, "nll_soft": 0.1651519507144023, "ece": 0.04798910875726077, "mean_confidence": 0.9679891087572606, "confident_error_rate": 0.03 }, "swag": { "count": 100, "accuracy": 0.71, "brier": 0.4003799807391956, "brier_soft": 0.4003799807391956, "nll": 0.715850991098381, "nll_soft": 0.715850991098381, "ece": 0.10425936796862421, "mean_confidence": 0.7495599288646645, "confident_error_rate": 0.01 }, "trec": { "count": 100, "accuracy": 0.93, "brier": 0.0973401897205966, "brier_soft": 0.0973401897205966, "nll": 0.23106466126006978, "nll_soft": 0.23106466126006978, "ece": 0.03301735640409483, "mean_confidence": 0.9457145921252907, "confident_error_rate": 0.02 }, "tweet_hate": { "count": 100, "accuracy": 0.66, "brier": 0.499521442971358, "brier_soft": 0.499521442971358, "nll": 0.7246301877586945, "nll_soft": 0.7246301877586945, "ece": 0.2203497182309466, "mean_confidence": 0.7959954828308132, "confident_error_rate": 0.06 }, "tweet_sentiment": { "count": 100, "accuracy": 0.71, "brier": 0.4093804097708304, "brier_soft": 0.4093804097708304, "nll": 0.6906869092595, "nll_soft": 0.6906869092595, "ece": 0.11551745886123974, "mean_confidence": 0.6126727871823638, "confident_error_rate": 0.0, "score_mae": 0.37256540101600827 }, "vitaminc": { "count": 100, "accuracy": 0.67, "brier": 0.43661602738993105, "brier_soft": 0.43661602738993105, "nll": 0.7999046209455473, "nll_soft": 0.7999046209455473, "ece": 0.15065187251617596, "mean_confidence": 0.806123844596811, "confident_error_rate": 0.07 }, "wic": { "count": 100, "accuracy": 0.66, "brier": 0.41257324470815915, "brier_soft": 0.41257324470815915, "nll": 0.5911202500978636, "nll_soft": 0.5911202500978636, "ece": 0.09333030231242981, "mean_confidence": 0.6790619026658443, "confident_error_rate": 0.0 }, "winogrande": { "count": 100, "accuracy": 0.58, "brier": 0.5970109839281106, "brier_soft": 0.5970109839281106, "nll": 0.9404834376569532, "nll_soft": 0.9404834376569532, "ece": 0.2111694877129903, "mean_confidence": 0.7809421535184294, "confident_error_rate": 0.15 }, "yelp": { "count": 100, "accuracy": 0.62, "brier": 0.4563941314015302, "brier_soft": 0.4563941314015302, "nll": 0.7779246878296787, "nll_soft": 0.7779246878296787, "ece": 0.13593820054296052, "mean_confidence": 0.5818577092894286, "confident_error_rate": 0.0, "score_mae": 0.4401834884494164 } }, "reliability": [ { "bin": "0.0-0.1", "count": 0, "confidence": NaN, "accuracy": NaN }, { "bin": "0.1-0.2", "count": 76, "confidence": 0.17221104488237657, "accuracy": 0.32894736842105265 }, { "bin": "0.2-0.3", "count": 132, "confidence": 0.2395367660809224, "accuracy": 0.2727272727272727 }, { "bin": "0.3-0.4", "count": 140, "confidence": 0.3559224153152535, "accuracy": 0.3142857142857143 }, { "bin": "0.4-0.5", "count": 455, "confidence": 0.46425223118313025, "accuracy": 0.46593406593406594 }, { "bin": "0.5-0.6", "count": 537, "confidence": 0.5499579597258507, "accuracy": 0.5288640595903166 }, { "bin": "0.6-0.7", "count": 506, "confidence": 0.6456601586886904, "accuracy": 0.6146245059288538 }, { "bin": "0.7-0.8", "count": 428, "confidence": 0.751258870528824, "accuracy": 0.6822429906542056 }, { "bin": "0.8-0.9", "count": 566, "confidence": 0.8520923292712668, "accuracy": 0.7544169611307421 }, { "bin": "0.9-1.0", "count": 2673, "confidence": 0.9752826241952612, "accuracy": 0.9150766928544707 } ], "seconds": 31.28231954574585, "model": "runs/assay-1.7b-v2", "data": "data/v2/dev.jsonl" }