Invalid JSON:Unexpected token 'N', ..."fidence": NaN,
"... is not valid JSON
| { | |
| "overall": { | |
| "count": 6113, | |
| "accuracy": 0.7906101750368068, | |
| "brier": 0.2865185962820081, | |
| "brier_soft": 0.26187629556812925, | |
| "nll": 0.531438177468442, | |
| "nll_soft": 0.5442780755003573, | |
| "ece": 0.027146574970523422, | |
| "mean_confidence": 0.7952999926910861, | |
| "confident_error_rate": 0.017176509079011942, | |
| "score_mae": 0.5416698742073451, | |
| "evidence_auc": 0.9997787857086019 | |
| }, | |
| "by_type": { | |
| "bool": { | |
| "count": 1898, | |
| "accuracy": 0.8535300316122234, | |
| "brier": 0.20239683530024727, | |
| "brier_soft": 0.1986991046710315, | |
| "nll": 0.3164081539673983, | |
| "nll_soft": 0.320539412862896, | |
| "ece": 0.03776428708532474, | |
| "mean_confidence": 0.891294318697548, | |
| "confident_error_rate": 0.02476290832455216, | |
| "evidence_auc": 0.9991618000687049 | |
| }, | |
| "choice": { | |
| "count": 3248, | |
| "accuracy": 0.7924876847290641, | |
| "brier": 0.284974599802582, | |
| "brier_soft": 0.26098893765800385, | |
| "nll": 0.5679457318609575, | |
| "nll_soft": 0.5810034227604031, | |
| "ece": 0.030920124238874132, | |
| "mean_confidence": 0.8102787767075924, | |
| "confident_error_rate": 0.017857142857142856, | |
| "evidence_auc": 0.999926113573992 | |
| }, | |
| "score": { | |
| "count": 967, | |
| "accuracy": 0.6608066184074457, | |
| "brier": 0.45681642762488106, | |
| "brier_soft": 0.38885917748620424, | |
| "nll": 0.8308698713030757, | |
| "nll_soft": 0.860071305888437, | |
| "ece": 0.1165943047968045, | |
| "mean_confidence": 0.5565737039156183, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5416698742073451 | |
| } | |
| }, | |
| "by_source": { | |
| "ag_news": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.14107379849283894, | |
| "brier_soft": 0.14107379849283894, | |
| "nll": 0.2626451906685023, | |
| "nll_soft": 0.2626451906685023, | |
| "ece": 0.04884664500994877, | |
| "mean_confidence": 0.9267561964206152, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "amazon_polarity": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.08932649085635297, | |
| "brier_soft": 0.08932649085635297, | |
| "nll": 0.1286234055208405, | |
| "nll_soft": 0.1286234055208405, | |
| "ece": 0.0598796401142421, | |
| "mean_confidence": 0.9619265198646406, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "amazon_stars": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.5146504076696957, | |
| "brier_soft": 0.5146504076696957, | |
| "nll": 0.9000029896261006, | |
| "nll_soft": 0.9000029896261006, | |
| "ece": 0.15304034642163666, | |
| "mean_confidence": 0.5267374344918574, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5121194916103842 | |
| }, | |
| "anli": { | |
| "count": 100, | |
| "accuracy": 0.58, | |
| "brier": 0.5707019496568548, | |
| "brier_soft": 0.5707019496568548, | |
| "nll": 0.9733660613496383, | |
| "nll_soft": 0.9733660613496383, | |
| "ece": 0.20588943546049263, | |
| "mean_confidence": 0.781207916786596, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "arc_challenge": { | |
| "count": 100, | |
| "accuracy": 0.92, | |
| "brier": 0.138929125961344, | |
| "brier_soft": 0.138929125961344, | |
| "nll": 0.24651074556516497, | |
| "nll_soft": 0.24651074556516497, | |
| "ece": 0.07689403844076734, | |
| "mean_confidence": 0.9312481481859443, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "arc_easy": { | |
| "count": 100, | |
| "accuracy": 0.96, | |
| "brier": 0.06105675437719455, | |
| "brier_soft": 0.06105675437719455, | |
| "nll": 0.1295253280350266, | |
| "nll_soft": 0.1295253280350266, | |
| "ece": 0.05505374327055725, | |
| "mean_confidence": 0.9480651360916019, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "banking77": { | |
| "count": 100, | |
| "accuracy": 0.73, | |
| "brier": 0.3608873627059889, | |
| "brier_soft": 0.3608873627059889, | |
| "nll": 0.9405229381993333, | |
| "nll_soft": 0.9405229381993333, | |
| "ece": 0.08553226405095403, | |
| "mean_confidence": 0.7356290730340809, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "boolq": { | |
| "count": 125, | |
| "accuracy": 0.768, | |
| "brier": 0.29296620551098923, | |
| "brier_soft": 0.29296620551098923, | |
| "nll": 0.4262508125125178, | |
| "nll_soft": 0.4262508125125178, | |
| "ece": 0.06190941058686133, | |
| "mean_confidence": 0.7972549868134068, | |
| "confident_error_rate": 0.008, | |
| "evidence_auc": 1.0 | |
| }, | |
| "cb": { | |
| "count": 28, | |
| "accuracy": 0.9285714285714286, | |
| "brier": 0.08452813725424528, | |
| "brier_soft": 0.08452813725424528, | |
| "nll": 0.15482732377842925, | |
| "nll_soft": 0.15482732377842925, | |
| "ece": 0.0891914767253553, | |
| "mean_confidence": 0.9239020966140715, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "civil_comments_severity": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.2854281637857832, | |
| "brier_soft": 0.18639637414257712, | |
| "nll": 0.5742660112377674, | |
| "nll_soft": 0.6703868491946997, | |
| "ece": 0.17662348482102086, | |
| "mean_confidence": 0.683961388574109, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4394353714638848 | |
| }, | |
| "civil_comments_toxic": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.11580263606349772, | |
| "brier_soft": 0.045619708720982234, | |
| "nll": 0.2174061558177036, | |
| "nll_soft": 0.2958174496542495, | |
| "ece": 0.07866423697912733, | |
| "mean_confidence": 0.8840980723553749, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "clinc150": { | |
| "count": 100, | |
| "accuracy": 0.87, | |
| "brier": 0.21760990606015762, | |
| "brier_soft": 0.21760990606015762, | |
| "nll": 0.6578076778338765, | |
| "nll_soft": 0.6578076778338765, | |
| "ece": 0.0802484366750311, | |
| "mean_confidence": 0.8693127305899719, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "cola": { | |
| "count": 100, | |
| "accuracy": 0.75, | |
| "brier": 0.3132800525413747, | |
| "brier_soft": 0.3132800525413747, | |
| "nll": 0.4747007129405839, | |
| "nll_soft": 0.4747007129405839, | |
| "ece": 0.11109424747890481, | |
| "mean_confidence": 0.8610942474789048, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "commonsense_qa": { | |
| "count": 100, | |
| "accuracy": 0.87, | |
| "brier": 0.19463212383948517, | |
| "brier_soft": 0.19463212383948517, | |
| "nll": 0.36843752512773237, | |
| "nll_soft": 0.36843752512773237, | |
| "ece": 0.04958840880497705, | |
| "mean_confidence": 0.8938470752943246, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "cosmos_qa": { | |
| "count": 120, | |
| "accuracy": 0.7333333333333333, | |
| "brier": 0.3977299966460981, | |
| "brier_soft": 0.3977299966460981, | |
| "nll": 0.7969430905955159, | |
| "nll_soft": 0.7969430905955159, | |
| "ece": 0.11449862878263802, | |
| "mean_confidence": 0.7867019719064755, | |
| "confident_error_rate": 0.016666666666666666, | |
| "evidence_auc": 0.9990625 | |
| }, | |
| "dates": { | |
| "count": 300, | |
| "accuracy": 0.9066666666666666, | |
| "brier": 0.1366547604463823, | |
| "brier_soft": 0.1366547604463823, | |
| "nll": 0.22760700905623565, | |
| "nll_soft": 0.22760700905623565, | |
| "ece": 0.042681842028014755, | |
| "mean_confidence": 0.8748727759673026, | |
| "confident_error_rate": 0.006666666666666667, | |
| "score_mae": 0.2819932523078008 | |
| }, | |
| "dbpedia": { | |
| "count": 100, | |
| "accuracy": 1.0, | |
| "brier": 0.00018795749398129978, | |
| "brier_soft": 0.00018795749398129978, | |
| "nll": 0.008357058821629917, | |
| "nll_soft": 0.008357058821629917, | |
| "ece": 0.00828557394777929, | |
| "mean_confidence": 0.9917144260522207, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "dynasent": { | |
| "count": 100, | |
| "accuracy": 0.69, | |
| "brier": 0.4323214023458542, | |
| "brier_soft": 0.30704619662116733, | |
| "nll": 0.7412403519607587, | |
| "nll_soft": 0.790334108879113, | |
| "ece": 0.11594177635826394, | |
| "mean_confidence": 0.5767413146517898, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5042056048814512 | |
| }, | |
| "enron_spam": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.061530910842468194, | |
| "brier_soft": 0.061530910842468194, | |
| "nll": 0.09922311989131462, | |
| "nll_soft": 0.09922311989131462, | |
| "ece": 0.04473396168787776, | |
| "mean_confidence": 0.9767910137716824, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "go_emotions": { | |
| "count": 100, | |
| "accuracy": 0.54, | |
| "brier": 0.6541717455466549, | |
| "brier_soft": 0.25009065256646124, | |
| "nll": 1.652207905091813, | |
| "nll_soft": 2.060946796357424, | |
| "ece": 0.18927007018484554, | |
| "mean_confidence": 0.35338819562567303, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "hellaswag": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.16797113212596798, | |
| "brier_soft": 0.16797113212596798, | |
| "nll": 0.3132057670542855, | |
| "nll_soft": 0.3132057670542855, | |
| "ece": 0.07032309961111621, | |
| "mean_confidence": 0.9228263155853855, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "imdb": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.09338526335431517, | |
| "brier_soft": 0.09338526335431517, | |
| "nll": 0.15406285631818842, | |
| "nll_soft": 0.15406285631818842, | |
| "ece": 0.043104711718764975, | |
| "mean_confidence": 0.959424845682162, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "ledgar": { | |
| "count": 100, | |
| "accuracy": 0.75, | |
| "brier": 0.3506827260603255, | |
| "brier_soft": 0.3506827260603255, | |
| "nll": 0.9000206841298396, | |
| "nll_soft": 0.9000206841298396, | |
| "ece": 0.08742405102661228, | |
| "mean_confidence": 0.7466266913748092, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "liar": { | |
| "count": 100, | |
| "accuracy": 0.24, | |
| "brier": 0.7684441156135192, | |
| "brier_soft": 0.7684441156135192, | |
| "nll": 1.5789539284601737, | |
| "nll_soft": 1.5789539284601737, | |
| "ece": 0.09676656231782328, | |
| "mean_confidence": 0.30260542335327906, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 1.127111411685787 | |
| }, | |
| "measuring_hate_speech": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.30059922331895095, | |
| "brier_soft": 0.15922982151650258, | |
| "nll": 0.5642970152096843, | |
| "nll_soft": 0.6878727674961508, | |
| "ece": 0.196925863260578, | |
| "mean_confidence": 0.6452313291559981, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.6025989257232528 | |
| }, | |
| "medqa": { | |
| "count": 100, | |
| "accuracy": 0.61, | |
| "brier": 0.501600476205162, | |
| "brier_soft": 0.501600476205162, | |
| "nll": 0.9209206864946542, | |
| "nll_soft": 0.9209206864946542, | |
| "ece": 0.10855441863453118, | |
| "mean_confidence": 0.6787774204404297, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "mnli": { | |
| "count": 100, | |
| "accuracy": 0.9, | |
| "brier": 0.13846939435764263, | |
| "brier_soft": 0.13846939435764263, | |
| "nll": 0.2625242881689338, | |
| "nll_soft": 0.2625242881689338, | |
| "ece": 0.05978230532069495, | |
| "mean_confidence": 0.867065953881717, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "mrpc": { | |
| "count": 100, | |
| "accuracy": 0.81, | |
| "brier": 0.26125558901505913, | |
| "brier_soft": 0.26125558901505913, | |
| "nll": 0.3984524563684818, | |
| "nll_soft": 0.3984524563684818, | |
| "ece": 0.09004270964182014, | |
| "mean_confidence": 0.8107605511150068, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "multirc": { | |
| "count": 125, | |
| "accuracy": 0.752, | |
| "brier": 0.3193987297602871, | |
| "brier_soft": 0.3193987297602871, | |
| "nll": 0.47067874987325165, | |
| "nll_soft": 0.47067874987325165, | |
| "ece": 0.0981009103037617, | |
| "mean_confidence": 0.79820435302511, | |
| "confident_error_rate": 0.016, | |
| "evidence_auc": 0.9989333333333333 | |
| }, | |
| "openbookqa": { | |
| "count": 100, | |
| "accuracy": 0.87, | |
| "brier": 0.1793119275121386, | |
| "brier_soft": 0.1793119275121386, | |
| "nll": 0.3212052656163398, | |
| "nll_soft": 0.3212052656163398, | |
| "ece": 0.048824714093196483, | |
| "mean_confidence": 0.8795181984743128, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "piqa": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.24895208825020151, | |
| "brier_soft": 0.24895208825020151, | |
| "nll": 0.389222560310709, | |
| "nll_soft": 0.389222560310709, | |
| "ece": 0.08642289363272926, | |
| "mean_confidence": 0.838094799621635, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "policy_application": { | |
| "count": 150, | |
| "accuracy": 0.9066666666666666, | |
| "brier": 0.139494695459916, | |
| "brier_soft": 0.139494695459916, | |
| "nll": 0.21704375203377746, | |
| "nll_soft": 0.21704375203377746, | |
| "ece": 0.059896815803595904, | |
| "mean_confidence": 0.9398540889590621, | |
| "confident_error_rate": 0.013333333333333334, | |
| "evidence_auc": 1.0 | |
| }, | |
| "policy_hard": { | |
| "count": 300, | |
| "accuracy": 0.8433333333333334, | |
| "brier": 0.20894731883691625, | |
| "brier_soft": 0.20894731883691625, | |
| "nll": 0.34797745624001836, | |
| "nll_soft": 0.34797745624001836, | |
| "ece": 0.040093495814151, | |
| "mean_confidence": 0.880565356561097, | |
| "confident_error_rate": 0.02666666666666667, | |
| "evidence_auc": 0.9817708333333334 | |
| }, | |
| "qqp": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.13989046875222017, | |
| "brier_soft": 0.13989046875222017, | |
| "nll": 0.234108067003616, | |
| "nll_soft": 0.234108067003616, | |
| "ece": 0.05421047729337747, | |
| "mean_confidence": 0.8683763009029548, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "race": { | |
| "count": 125, | |
| "accuracy": 0.72, | |
| "brier": 0.3468329954783986, | |
| "brier_soft": 0.3468329954783986, | |
| "nll": 0.6595321747170682, | |
| "nll_soft": 0.6595321747170682, | |
| "ece": 0.08253622206480249, | |
| "mean_confidence": 0.7759816152635733, | |
| "confident_error_rate": 0.024, | |
| "evidence_auc": 0.9992 | |
| }, | |
| "rotten_tomatoes": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.07896598050795499, | |
| "brier_soft": 0.07896598050795499, | |
| "nll": 0.1295703707449942, | |
| "nll_soft": 0.1295703707449942, | |
| "ece": 0.06612016556692232, | |
| "mean_confidence": 0.9349814985533582, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "rte": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.1785855924400104, | |
| "brier_soft": 0.1785855924400104, | |
| "nll": 0.3144416684299057, | |
| "nll_soft": 0.3144416684299057, | |
| "ece": 0.08369600771650051, | |
| "mean_confidence": 0.9264092676999237, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "scotus": { | |
| "count": 100, | |
| "accuracy": 0.21, | |
| "brier": 0.8933246384582364, | |
| "brier_soft": 0.8933246384582364, | |
| "nll": 2.4025798961358653, | |
| "nll_soft": 2.4025798961358653, | |
| "ece": 0.026259098193438857, | |
| "mean_confidence": 0.20252531874914392, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "shp": { | |
| "count": 100, | |
| "accuracy": 0.68, | |
| "brier": 0.47749577158017964, | |
| "brier_soft": 0.1025225581044758, | |
| "nll": 0.6706076521115167, | |
| "nll_soft": 0.6859825612598958, | |
| "ece": 0.1478788162760493, | |
| "mean_confidence": 0.5369806117053606, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "sms_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.018716052570869043, | |
| "brier_soft": 0.018716052570869043, | |
| "nll": 0.04010322700735336, | |
| "nll_soft": 0.04010322700735336, | |
| "ece": 0.002744809770715926, | |
| "mean_confidence": 0.9907091112815634, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "snli": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.16836393178606746, | |
| "brier_soft": 0.16836393178606746, | |
| "nll": 0.2924678063682347, | |
| "nll_soft": 0.2924678063682347, | |
| "ece": 0.034565203155391265, | |
| "mean_confidence": 0.8986885605645243, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "social_iqa": { | |
| "count": 100, | |
| "accuracy": 0.78, | |
| "brier": 0.30628460928922563, | |
| "brier_soft": 0.30628460928922563, | |
| "nll": 0.5561253043672776, | |
| "nll_soft": 0.5561253043672776, | |
| "ece": 0.10795976232448823, | |
| "mean_confidence": 0.8395705446442819, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "sst2": { | |
| "count": 100, | |
| "accuracy": 0.96, | |
| "brier": 0.07928032002666491, | |
| "brier_soft": 0.07928032002666491, | |
| "nll": 0.15856545014783774, | |
| "nll_soft": 0.15856545014783774, | |
| "ece": 0.03715473937547079, | |
| "mean_confidence": 0.959105934637087, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "sst5": { | |
| "count": 100, | |
| "accuracy": 0.67, | |
| "brier": 0.5047456075070107, | |
| "brier_soft": 0.5047456075070107, | |
| "nll": 0.9046301136653718, | |
| "nll_soft": 0.9046301136653718, | |
| "ece": 0.1989545581620594, | |
| "mean_confidence": 0.4993328109434478, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.46473458149541735 | |
| }, | |
| "stance_abortion": { | |
| "count": 60, | |
| "accuracy": 0.6666666666666666, | |
| "brier": 0.4318499648737531, | |
| "brier_soft": 0.4318499648737531, | |
| "nll": 0.6693344974312939, | |
| "nll_soft": 0.6693344974312939, | |
| "ece": 0.16791879623099454, | |
| "mean_confidence": 0.7610590780360661, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_atheism": { | |
| "count": 60, | |
| "accuracy": 0.7833333333333333, | |
| "brier": 0.3150497260676467, | |
| "brier_soft": 0.3150497260676467, | |
| "nll": 0.5028852417656949, | |
| "nll_soft": 0.5028852417656949, | |
| "ece": 0.1430124566231118, | |
| "mean_confidence": 0.8368338633078285, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_feminist": { | |
| "count": 60, | |
| "accuracy": 0.75, | |
| "brier": 0.3648047830860601, | |
| "brier_soft": 0.3648047830860601, | |
| "nll": 0.5867590544526551, | |
| "nll_soft": 0.5867590544526551, | |
| "ece": 0.13802642902758205, | |
| "mean_confidence": 0.779361470377674, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "stance_hillary": { | |
| "count": 60, | |
| "accuracy": 0.6666666666666666, | |
| "brier": 0.39158167992187276, | |
| "brier_soft": 0.39158167992187276, | |
| "nll": 0.6568302459427436, | |
| "nll_soft": 0.6568302459427436, | |
| "ece": 0.13117570539181508, | |
| "mean_confidence": 0.7419570046865254, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "stsb": { | |
| "count": 100, | |
| "accuracy": 0.54, | |
| "brier": 0.5570610302874376, | |
| "brier_soft": 0.2655908186167737, | |
| "nll": 0.9685020049260296, | |
| "nll_soft": 0.9820895302047219, | |
| "ece": 0.07001749408544906, | |
| "mean_confidence": 0.539751064816704, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.565603055088638 | |
| }, | |
| "subjectivity": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.09908461321243932, | |
| "brier_soft": 0.09908461321243932, | |
| "nll": 0.16222277170135949, | |
| "nll_soft": 0.16222277170135949, | |
| "ece": 0.04927684333663743, | |
| "mean_confidence": 0.9490918012704767, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "swag": { | |
| "count": 100, | |
| "accuracy": 0.77, | |
| "brier": 0.3000022704804431, | |
| "brier_soft": 0.3000022704804431, | |
| "nll": 0.5425527902770044, | |
| "nll_soft": 0.5425527902770044, | |
| "ece": 0.07660043607599595, | |
| "mean_confidence": 0.7782203540696726, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "trec": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.061975700748181185, | |
| "brier_soft": 0.061975700748181185, | |
| "nll": 0.15411059891893825, | |
| "nll_soft": 0.15411059891893825, | |
| "ece": 0.017816031807190158, | |
| "mean_confidence": 0.9619683086236548, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "tweet_hate": { | |
| "count": 100, | |
| "accuracy": 0.48, | |
| "brier": 0.7473889756686806, | |
| "brier_soft": 0.7473889756686806, | |
| "nll": 1.1776300863149485, | |
| "nll_soft": 1.1776300863149485, | |
| "ece": 0.36222919958424477, | |
| "mean_confidence": 0.8422291995842447, | |
| "confident_error_rate": 0.21 | |
| }, | |
| "tweet_sentiment": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.4403312511848521, | |
| "brier_soft": 0.4403312511848521, | |
| "nll": 0.7376234445010855, | |
| "nll_soft": 0.7376234445010855, | |
| "ece": 0.10092449734474333, | |
| "mean_confidence": 0.5835192824734373, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3924732269672393 | |
| }, | |
| "vitaminc": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.26744036733920845, | |
| "brier_soft": 0.26744036733920845, | |
| "nll": 0.5030591509155484, | |
| "nll_soft": 0.5030591509155484, | |
| "ece": 0.08253130737011233, | |
| "mean_confidence": 0.8672411180861649, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "wic": { | |
| "count": 100, | |
| "accuracy": 0.71, | |
| "brier": 0.35517493029561253, | |
| "brier_soft": 0.35517493029561253, | |
| "nll": 0.5259271797133195, | |
| "nll_soft": 0.5259271797133195, | |
| "ece": 0.06847744577518106, | |
| "mean_confidence": 0.748921172428449, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "winogrande": { | |
| "count": 100, | |
| "accuracy": 0.75, | |
| "brier": 0.37336827230181, | |
| "brier_soft": 0.37336827230181, | |
| "nll": 0.6051925483123668, | |
| "nll_soft": 0.6051925483123668, | |
| "ece": 0.1422213440002727, | |
| "mean_confidence": 0.8710624725593938, | |
| "confident_error_rate": 0.07 | |
| }, | |
| "yelp": { | |
| "count": 100, | |
| "accuracy": 0.68, | |
| "brier": 0.4571774209762717, | |
| "brier_soft": 0.4571774209762717, | |
| "nll": 0.7820219928182245, | |
| "nll_soft": 0.7820219928182245, | |
| "ece": 0.1450263335660819, | |
| "mean_confidence": 0.5498948735721532, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.44073053562274644 | |
| } | |
| }, | |
| "reliability": [ | |
| { | |
| "bin": "0.0-0.1", | |
| "count": 0, | |
| "confidence": NaN, | |
| "accuracy": NaN | |
| }, | |
| { | |
| "bin": "0.1-0.2", | |
| "count": 74, | |
| "confidence": 0.17221642633416331, | |
| "accuracy": 0.1891891891891892 | |
| }, | |
| { | |
| "bin": "0.2-0.3", | |
| "count": 143, | |
| "confidence": 0.2449173694172205, | |
| "accuracy": 0.25874125874125875 | |
| }, | |
| { | |
| "bin": "0.3-0.4", | |
| "count": 158, | |
| "confidence": 0.35688169328104796, | |
| "accuracy": 0.34177215189873417 | |
| }, | |
| { | |
| "bin": "0.4-0.5", | |
| "count": 437, | |
| "confidence": 0.46054809633778104, | |
| "accuracy": 0.5560640732265446 | |
| }, | |
| { | |
| "bin": "0.5-0.6", | |
| "count": 621, | |
| "confidence": 0.5505989146559541, | |
| "accuracy": 0.5861513687600645 | |
| }, | |
| { | |
| "bin": "0.6-0.7", | |
| "count": 475, | |
| "confidence": 0.6529554400217262, | |
| "accuracy": 0.6589473684210526 | |
| }, | |
| { | |
| "bin": "0.7-0.8", | |
| "count": 485, | |
| "confidence": 0.752970305163874, | |
| "accuracy": 0.709278350515464 | |
| }, | |
| { | |
| "bin": "0.8-0.9", | |
| "count": 677, | |
| "confidence": 0.8544566232414949, | |
| "accuracy": 0.7769571639586411 | |
| }, | |
| { | |
| "bin": "0.9-1.0", | |
| "count": 3043, | |
| "confidence": 0.9728956090557227, | |
| "accuracy": 0.9654945777193559 | |
| } | |
| ], | |
| "seconds": 67.44323062896729, | |
| "model": "runs/assay-4b-v4", | |
| "data": "data/v4/dev.jsonl" | |
| } |