Invalid JSON:Unexpected token 'N', ..."fidence": NaN,
"... is not valid JSON
| { | |
| "overall": { | |
| "count": 6113, | |
| "accuracy": 0.7906101750368068, | |
| "brier": 0.2905109878230025, | |
| "brier_soft": 0.26728262886573473, | |
| "nll": 0.5449381413787723, | |
| "nll_soft": 0.5611535112867304, | |
| "ece": 0.045661348853598224, | |
| "mean_confidence": 0.8293620655723313, | |
| "confident_error_rate": 0.032717160150498935, | |
| "score_mae": 0.5152794260773468, | |
| "evidence_auc": 0.9997787857086019 | |
| }, | |
| "by_type": { | |
| "bool": { | |
| "count": 1898, | |
| "accuracy": 0.8535300316122234, | |
| "brier": 0.20983104954609286, | |
| "brier_soft": 0.20647029718217128, | |
| "nll": 0.34005517640404503, | |
| "nll_soft": 0.3452724980816234, | |
| "ece": 0.058970879163590936, | |
| "mean_confidence": 0.9125009107758142, | |
| "confident_error_rate": 0.04056902002107481, | |
| "evidence_auc": 0.9991618000687049 | |
| }, | |
| "choice": { | |
| "count": 3248, | |
| "accuracy": 0.7924876847290641, | |
| "brier": 0.29241979456448064, | |
| "brier_soft": 0.26966333263184994, | |
| "nll": 0.5891846494040984, | |
| "nll_soft": 0.6056750666591111, | |
| "ece": 0.056842395140909024, | |
| "mean_confidence": 0.8473377443208175, | |
| "confident_error_rate": 0.037561576354679806, | |
| "evidence_auc": 0.999926113573992 | |
| }, | |
| "score": { | |
| "count": 967, | |
| "accuracy": 0.6608066184074457, | |
| "brier": 0.4424558880848984, | |
| "brier_soft": 0.3786469305235023, | |
| "nll": 0.7984595575688168, | |
| "nll_soft": 0.8353377421179621, | |
| "ece": 0.07923403877747387, | |
| "mean_confidence": 0.6058020523652026, | |
| "confident_error_rate": 0.001034126163391934, | |
| "score_mae": 0.5152794260773468 | |
| } | |
| }, | |
| "by_source": { | |
| "ag_news": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.1457681115044795, | |
| "brier_soft": 0.1457681115044795, | |
| "nll": 0.2806835892729357, | |
| "nll_soft": 0.2806835892729357, | |
| "ece": 0.06551793024708885, | |
| "mean_confidence": 0.9522883211811144, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "amazon_polarity": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.09681691910594509, | |
| "brier_soft": 0.09681691910594509, | |
| "nll": 0.14046548898129674, | |
| "nll_soft": 0.14046548898129674, | |
| "ece": 0.052496080969926015, | |
| "mean_confidence": 0.9707786674013664, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "amazon_stars": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.5044599603848816, | |
| "brier_soft": 0.5044599603848816, | |
| "nll": 0.869633277428378, | |
| "nll_soft": 0.869633277428378, | |
| "ece": 0.11863087715317772, | |
| "mean_confidence": 0.576783671003821, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.49169127512149585 | |
| }, | |
| "anli": { | |
| "count": 100, | |
| "accuracy": 0.58, | |
| "brier": 0.6073661288342754, | |
| "brier_soft": 0.6073661288342754, | |
| "nll": 1.0969887313858857, | |
| "nll_soft": 1.0969887313858857, | |
| "ece": 0.2611784174019323, | |
| "mean_confidence": 0.8288514520699011, | |
| "confident_error_rate": 0.11 | |
| }, | |
| "arc_challenge": { | |
| "count": 100, | |
| "accuracy": 0.92, | |
| "brier": 0.14429682116011844, | |
| "brier_soft": 0.14429682116011844, | |
| "nll": 0.2640587153943526, | |
| "nll_soft": 0.2640587153943526, | |
| "ece": 0.0850214039559398, | |
| "mean_confidence": 0.9523708441098319, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "arc_easy": { | |
| "count": 100, | |
| "accuracy": 0.96, | |
| "brier": 0.06183016851108757, | |
| "brier_soft": 0.06183016851108757, | |
| "nll": 0.12801979988318146, | |
| "nll_soft": 0.12801979988318146, | |
| "ece": 0.05157003044933564, | |
| "mean_confidence": 0.9644089935643387, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "banking77": { | |
| "count": 100, | |
| "accuracy": 0.73, | |
| "brier": 0.37647411046583196, | |
| "brier_soft": 0.37647411046583196, | |
| "nll": 0.967146808339886, | |
| "nll_soft": 0.967146808339886, | |
| "ece": 0.10248049282910536, | |
| "mean_confidence": 0.8103718617199586, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "boolq": { | |
| "count": 125, | |
| "accuracy": 0.768, | |
| "brier": 0.3015155723338054, | |
| "brier_soft": 0.3015155723338054, | |
| "nll": 0.4403017247566234, | |
| "nll_soft": 0.4403017247566234, | |
| "ece": 0.0691713155356551, | |
| "mean_confidence": 0.8214496526402538, | |
| "confident_error_rate": 0.008, | |
| "evidence_auc": 1.0 | |
| }, | |
| "cb": { | |
| "count": 28, | |
| "accuracy": 0.9285714285714286, | |
| "brier": 0.0885260800587447, | |
| "brier_soft": 0.0885260800587447, | |
| "nll": 0.1389439819866921, | |
| "nll_soft": 0.1389439819866921, | |
| "ece": 0.07268257534162591, | |
| "mean_confidence": 0.9543104112884626, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "civil_comments_severity": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.26092584575590455, | |
| "brier_soft": 0.18027036778246955, | |
| "nll": 0.5385317230811851, | |
| "nll_soft": 0.6599217016387524, | |
| "ece": 0.10881889299660845, | |
| "mean_confidence": 0.7390225605257076, | |
| "confident_error_rate": 0.01, | |
| "score_mae": 0.3756541661037601 | |
| }, | |
| "civil_comments_toxic": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.1116733017198554, | |
| "brier_soft": 0.04788622185262424, | |
| "nll": 0.20032056546099006, | |
| "nll_soft": 0.299345330901428, | |
| "ece": 0.05349609414767832, | |
| "mean_confidence": 0.9194762200522314, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "clinc150": { | |
| "count": 100, | |
| "accuracy": 0.87, | |
| "brier": 0.22371522830971066, | |
| "brier_soft": 0.22371522830971066, | |
| "nll": 0.7134262722233493, | |
| "nll_soft": 0.7134262722233493, | |
| "ece": 0.09005755998184956, | |
| "mean_confidence": 0.9235783766174436, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "cola": { | |
| "count": 100, | |
| "accuracy": 0.75, | |
| "brier": 0.3315955099695127, | |
| "brier_soft": 0.3315955099695127, | |
| "nll": 0.5199970361758424, | |
| "nll_soft": 0.5199970361758424, | |
| "ece": 0.14225371716410648, | |
| "mean_confidence": 0.8922537171641064, | |
| "confident_error_rate": 0.08 | |
| }, | |
| "commonsense_qa": { | |
| "count": 100, | |
| "accuracy": 0.87, | |
| "brier": 0.2043245094910104, | |
| "brier_soft": 0.2043245094910104, | |
| "nll": 0.3885151902545537, | |
| "nll_soft": 0.3885151902545537, | |
| "ece": 0.07296244357028275, | |
| "mean_confidence": 0.9304146401653779, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "cosmos_qa": { | |
| "count": 120, | |
| "accuracy": 0.7333333333333333, | |
| "brier": 0.4125665910418836, | |
| "brier_soft": 0.4125665910418836, | |
| "nll": 0.8681569441961412, | |
| "nll_soft": 0.8681569441961412, | |
| "ece": 0.13699584744487603, | |
| "mean_confidence": 0.8332889952145706, | |
| "confident_error_rate": 0.06666666666666667, | |
| "evidence_auc": 0.9990625 | |
| }, | |
| "dates": { | |
| "count": 300, | |
| "accuracy": 0.9066666666666666, | |
| "brier": 0.1324410444899149, | |
| "brier_soft": 0.1324410444899149, | |
| "nll": 0.21817804469288452, | |
| "nll_soft": 0.21817804469288452, | |
| "ece": 0.03892548374314649, | |
| "mean_confidence": 0.8972209231665249, | |
| "confident_error_rate": 0.013333333333333334, | |
| "score_mae": 0.2506003548185511 | |
| }, | |
| "dbpedia": { | |
| "count": 100, | |
| "accuracy": 1.0, | |
| "brier": 1.6184818300542076e-05, | |
| "brier_soft": 1.6184818300542076e-05, | |
| "nll": 0.001827317797337679, | |
| "nll_soft": 0.001827317797337679, | |
| "ece": 0.0018219140015888202, | |
| "mean_confidence": 0.9981780859984112, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "dynasent": { | |
| "count": 100, | |
| "accuracy": 0.69, | |
| "brier": 0.4149544413021475, | |
| "brier_soft": 0.2949069170758868, | |
| "nll": 0.70767730702296, | |
| "nll_soft": 0.7696773071306522, | |
| "ece": 0.1306706361394352, | |
| "mean_confidence": 0.6253540882792548, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.47241350842045404 | |
| }, | |
| "enron_spam": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.06551335557649704, | |
| "brier_soft": 0.06551335557649704, | |
| "nll": 0.10981263791638528, | |
| "nll_soft": 0.10981263791638528, | |
| "ece": 0.03485533983225947, | |
| "mean_confidence": 0.9848553398322596, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "go_emotions": { | |
| "count": 100, | |
| "accuracy": 0.54, | |
| "brier": 0.6194861171793581, | |
| "brier_soft": 0.2513976651604882, | |
| "nll": 1.5335175515779613, | |
| "nll_soft": 2.049709515080812, | |
| "ece": 0.13643982953461417, | |
| "mean_confidence": 0.4588293132172613, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "hellaswag": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.1787256334271709, | |
| "brier_soft": 0.1787256334271709, | |
| "nll": 0.343730914235669, | |
| "nll_soft": 0.343730914235669, | |
| "ece": 0.07519650793085166, | |
| "mean_confidence": 0.9438893610557416, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "imdb": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.09857211855143005, | |
| "brier_soft": 0.09857211855143005, | |
| "nll": 0.16853336129576013, | |
| "nll_soft": 0.16853336129576013, | |
| "ece": 0.05390522915327187, | |
| "mean_confidence": 0.9718127557710655, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "ledgar": { | |
| "count": 100, | |
| "accuracy": 0.75, | |
| "brier": 0.36107508218738843, | |
| "brier_soft": 0.36107508218738843, | |
| "nll": 0.9228652156026649, | |
| "nll_soft": 0.9228652156026649, | |
| "ece": 0.12147796710352471, | |
| "mean_confidence": 0.8216070461871731, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "liar": { | |
| "count": 100, | |
| "accuracy": 0.24, | |
| "brier": 0.7720101279378543, | |
| "brier_soft": 0.7720101279378543, | |
| "nll": 1.5780722835852494, | |
| "nll_soft": 1.5780722835852494, | |
| "ece": 0.11389948643175861, | |
| "mean_confidence": 0.3331357181858688, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 1.1239995544225962 | |
| }, | |
| "measuring_hate_speech": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.269793616526203, | |
| "brier_soft": 0.14445387048134756, | |
| "nll": 0.50668914734155, | |
| "nll_soft": 0.6627516478286205, | |
| "ece": 0.15277715704079395, | |
| "mean_confidence": 0.7027064112837865, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.542979770335009 | |
| }, | |
| "medqa": { | |
| "count": 100, | |
| "accuracy": 0.61, | |
| "brier": 0.5232384116814891, | |
| "brier_soft": 0.5232384116814891, | |
| "nll": 0.9666950597140125, | |
| "nll_soft": 0.9666950597140125, | |
| "ece": 0.15310429795340835, | |
| "mean_confidence": 0.7327647249737831, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "mnli": { | |
| "count": 100, | |
| "accuracy": 0.9, | |
| "brier": 0.1349148983200619, | |
| "brier_soft": 0.1349148983200619, | |
| "nll": 0.2432668425098134, | |
| "nll_soft": 0.2432668425098134, | |
| "ece": 0.049021119706413305, | |
| "mean_confidence": 0.906729724583111, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "mrpc": { | |
| "count": 100, | |
| "accuracy": 0.81, | |
| "brier": 0.2675980359243206, | |
| "brier_soft": 0.2675980359243206, | |
| "nll": 0.399899604734159, | |
| "nll_soft": 0.399899604734159, | |
| "ece": 0.09618112086460426, | |
| "mean_confidence": 0.8472228896042645, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "multirc": { | |
| "count": 125, | |
| "accuracy": 0.752, | |
| "brier": 0.3283689232391426, | |
| "brier_soft": 0.3283689232391426, | |
| "nll": 0.4942773812599929, | |
| "nll_soft": 0.4942773812599929, | |
| "ece": 0.11061435237291556, | |
| "mean_confidence": 0.8251074372391113, | |
| "confident_error_rate": 0.032, | |
| "evidence_auc": 0.9989333333333333 | |
| }, | |
| "openbookqa": { | |
| "count": 100, | |
| "accuracy": 0.87, | |
| "brier": 0.18715713115610605, | |
| "brier_soft": 0.18715713115610605, | |
| "nll": 0.32547382828638893, | |
| "nll_soft": 0.32547382828638893, | |
| "ece": 0.07002363675922497, | |
| "mean_confidence": 0.910568385075042, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "piqa": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.2539820134589331, | |
| "brier_soft": 0.2539820134589331, | |
| "nll": 0.4068688007847875, | |
| "nll_soft": 0.4068688007847875, | |
| "ece": 0.08311327451943891, | |
| "mean_confidence": 0.8653291206132601, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "policy_application": { | |
| "count": 150, | |
| "accuracy": 0.9066666666666666, | |
| "brier": 0.14813577608521744, | |
| "brier_soft": 0.14813577608521744, | |
| "nll": 0.23572050653690194, | |
| "nll_soft": 0.23572050653690194, | |
| "ece": 0.06691158593503817, | |
| "mean_confidence": 0.9575142657331253, | |
| "confident_error_rate": 0.02666666666666667, | |
| "evidence_auc": 1.0 | |
| }, | |
| "policy_hard": { | |
| "count": 300, | |
| "accuracy": 0.8433333333333334, | |
| "brier": 0.21603088941877208, | |
| "brier_soft": 0.21603088941877208, | |
| "nll": 0.3671480630817236, | |
| "nll_soft": 0.3671480630817236, | |
| "ece": 0.07672543440705276, | |
| "mean_confidence": 0.909976742390077, | |
| "confident_error_rate": 0.04666666666666667, | |
| "evidence_auc": 0.9817708333333334 | |
| }, | |
| "qqp": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.1356641944974391, | |
| "brier_soft": 0.1356641944974391, | |
| "nll": 0.2231311517649759, | |
| "nll_soft": 0.2231311517649759, | |
| "ece": 0.06575801900138287, | |
| "mean_confidence": 0.8946227147846244, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "race": { | |
| "count": 125, | |
| "accuracy": 0.72, | |
| "brier": 0.35765456161571446, | |
| "brier_soft": 0.35765456161571446, | |
| "nll": 0.6983141916139135, | |
| "nll_soft": 0.6983141916139135, | |
| "ece": 0.0932914444873349, | |
| "mean_confidence": 0.8087819135327284, | |
| "confident_error_rate": 0.032, | |
| "evidence_auc": 0.9992 | |
| }, | |
| "rotten_tomatoes": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.08124824522675228, | |
| "brier_soft": 0.08124824522675228, | |
| "nll": 0.12389669533170551, | |
| "nll_soft": 0.12389669533170551, | |
| "ece": 0.05773620838858384, | |
| "mean_confidence": 0.9526650174313278, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "rte": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.18311153542500938, | |
| "brier_soft": 0.18311153542500938, | |
| "nll": 0.3545909140092871, | |
| "nll_soft": 0.3545909140092871, | |
| "ece": 0.10124014569716058, | |
| "mean_confidence": 0.943752464163045, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "scotus": { | |
| "count": 100, | |
| "accuracy": 0.21, | |
| "brier": 0.89769193671844, | |
| "brier_soft": 0.89769193671844, | |
| "nll": 2.415346637641461, | |
| "nll_soft": 2.415346637641461, | |
| "ece": 0.07353758881534735, | |
| "mean_confidence": 0.24746621955170325, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "shp": { | |
| "count": 100, | |
| "accuracy": 0.68, | |
| "brier": 0.4733767173132663, | |
| "brier_soft": 0.10233528576029081, | |
| "nll": 0.6665194253969489, | |
| "nll_soft": 0.6859362143369068, | |
| "ece": 0.14897214933195024, | |
| "mean_confidence": 0.546426399891975, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "sms_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.01930635801208172, | |
| "brier_soft": 0.01930635801208172, | |
| "nll": 0.04207345377972822, | |
| "nll_soft": 0.04207345377972822, | |
| "ece": 0.0067073655477912775, | |
| "mean_confidence": 0.9967073655477913, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "snli": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.17582400400290324, | |
| "brier_soft": 0.17582400400290324, | |
| "nll": 0.30302966915953083, | |
| "nll_soft": 0.30302966915953083, | |
| "ece": 0.0709306654852658, | |
| "mean_confidence": 0.9286928509331307, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "social_iqa": { | |
| "count": 100, | |
| "accuracy": 0.78, | |
| "brier": 0.3177565365861535, | |
| "brier_soft": 0.3177565365861535, | |
| "nll": 0.607449232287739, | |
| "nll_soft": 0.607449232287739, | |
| "ece": 0.12725246333797624, | |
| "mean_confidence": 0.8733738051484262, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "sst2": { | |
| "count": 100, | |
| "accuracy": 0.96, | |
| "brier": 0.07971217148658985, | |
| "brier_soft": 0.07971217148658985, | |
| "nll": 0.17264823580876176, | |
| "nll_soft": 0.17264823580876176, | |
| "ece": 0.043530750660772013, | |
| "mean_confidence": 0.9733277126761404, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "sst5": { | |
| "count": 100, | |
| "accuracy": 0.67, | |
| "brier": 0.48402215895940337, | |
| "brier_soft": 0.48402215895940337, | |
| "nll": 0.8604305286260937, | |
| "nll_soft": 0.8604305286260937, | |
| "ece": 0.16751759519570414, | |
| "mean_confidence": 0.5497045612334717, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4519901699493576 | |
| }, | |
| "stance_abortion": { | |
| "count": 60, | |
| "accuracy": 0.6666666666666666, | |
| "brier": 0.45936729035760654, | |
| "brier_soft": 0.45936729035760654, | |
| "nll": 0.705900344737038, | |
| "nll_soft": 0.705900344737038, | |
| "ece": 0.15772369018233034, | |
| "mean_confidence": 0.8077103918863352, | |
| "confident_error_rate": 0.03333333333333333 | |
| }, | |
| "stance_atheism": { | |
| "count": 60, | |
| "accuracy": 0.7833333333333333, | |
| "brier": 0.3328091281497823, | |
| "brier_soft": 0.3328091281497823, | |
| "nll": 0.5367039070600078, | |
| "nll_soft": 0.5367039070600078, | |
| "ece": 0.14772804545430168, | |
| "mean_confidence": 0.8721167817648096, | |
| "confident_error_rate": 0.06666666666666667 | |
| }, | |
| "stance_feminist": { | |
| "count": 60, | |
| "accuracy": 0.75, | |
| "brier": 0.38284189015448966, | |
| "brier_soft": 0.38284189015448966, | |
| "nll": 0.6097841885696503, | |
| "nll_soft": 0.6097841885696503, | |
| "ece": 0.15592787843210057, | |
| "mean_confidence": 0.8234037784819485, | |
| "confident_error_rate": 0.016666666666666666 | |
| }, | |
| "stance_hillary": { | |
| "count": 60, | |
| "accuracy": 0.6666666666666666, | |
| "brier": 0.4047196431521728, | |
| "brier_soft": 0.4047196431521728, | |
| "nll": 0.6834731284888856, | |
| "nll_soft": 0.6834731284888856, | |
| "ece": 0.1269529390022274, | |
| "mean_confidence": 0.7863250004418558, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "stsb": { | |
| "count": 100, | |
| "accuracy": 0.54, | |
| "brier": 0.5614097077774185, | |
| "brier_soft": 0.2704198364032701, | |
| "nll": 0.9670080291366133, | |
| "nll_soft": 0.9841675945745174, | |
| "ece": 0.08449101891072369, | |
| "mean_confidence": 0.5913493599636942, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.5504846148923596 | |
| }, | |
| "subjectivity": { | |
| "count": 100, | |
| "accuracy": 0.94, | |
| "brier": 0.10407316114282592, | |
| "brier_soft": 0.10407316114282592, | |
| "nll": 0.17088805058804918, | |
| "nll_soft": 0.17088805058804918, | |
| "ece": 0.040859098694625375, | |
| "mean_confidence": 0.9664050958509474, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "swag": { | |
| "count": 100, | |
| "accuracy": 0.77, | |
| "brier": 0.3051043063037333, | |
| "brier_soft": 0.3051043063037333, | |
| "nll": 0.5433260597527682, | |
| "nll_soft": 0.5433260597527682, | |
| "ece": 0.08123775745211292, | |
| "mean_confidence": 0.8281818811177126, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "trec": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.06155337791663981, | |
| "brier_soft": 0.06155337791663981, | |
| "nll": 0.1634662721991823, | |
| "nll_soft": 0.1634662721991823, | |
| "ece": 0.02569771263554021, | |
| "mean_confidence": 0.9807213871260368, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "tweet_hate": { | |
| "count": 100, | |
| "accuracy": 0.48, | |
| "brier": 0.7950030937517019, | |
| "brier_soft": 0.7950030937517019, | |
| "nll": 1.3997674961537496, | |
| "nll_soft": 1.3997674961537496, | |
| "ece": 0.40343398490604676, | |
| "mean_confidence": 0.8739591757981232, | |
| "confident_error_rate": 0.28 | |
| }, | |
| "tweet_sentiment": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.42840573743009913, | |
| "brier_soft": 0.42840573743009913, | |
| "nll": 0.7082548515314017, | |
| "nll_soft": 0.7082548515314017, | |
| "ece": 0.1119492420242185, | |
| "mean_confidence": 0.6316418997585348, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3841625395096181 | |
| }, | |
| "vitaminc": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.27409571807544036, | |
| "brier_soft": 0.27409571807544036, | |
| "nll": 0.546105934849192, | |
| "nll_soft": 0.546105934849192, | |
| "ece": 0.10942523312978639, | |
| "mean_confidence": 0.9069065792619518, | |
| "confident_error_rate": 0.09 | |
| }, | |
| "wic": { | |
| "count": 100, | |
| "accuracy": 0.71, | |
| "brier": 0.36201195057161817, | |
| "brier_soft": 0.36201195057161817, | |
| "nll": 0.5375649578022699, | |
| "nll_soft": 0.5375649578022699, | |
| "ece": 0.08803995502979516, | |
| "mean_confidence": 0.7847677951555196, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "winogrande": { | |
| "count": 100, | |
| "accuracy": 0.75, | |
| "brier": 0.392578222900395, | |
| "brier_soft": 0.392578222900395, | |
| "nll": 0.694012713399598, | |
| "nll_soft": 0.694012713399598, | |
| "ece": 0.1698561596606002, | |
| "mean_confidence": 0.8960738105444639, | |
| "confident_error_rate": 0.11 | |
| }, | |
| "yelp": { | |
| "count": 100, | |
| "accuracy": 0.68, | |
| "brier": 0.44095049816750126, | |
| "brier_soft": 0.44095049816750126, | |
| "nll": 0.7369559085064258, | |
| "nll_soft": 0.7369559085064258, | |
| "ece": 0.10563042684119633, | |
| "mean_confidence": 0.6006492517646783, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4214742136848633 | |
| } | |
| }, | |
| "reliability": [ | |
| { | |
| "bin": "0.0-0.1", | |
| "count": 0, | |
| "confidence": NaN, | |
| "accuracy": NaN | |
| }, | |
| { | |
| "bin": "0.1-0.2", | |
| "count": 26, | |
| "confidence": 0.1786501163201734, | |
| "accuracy": 0.23076923076923078 | |
| }, | |
| { | |
| "bin": "0.2-0.3", | |
| "count": 138, | |
| "confidence": 0.2481680800607115, | |
| "accuracy": 0.21014492753623187 | |
| }, | |
| { | |
| "bin": "0.3-0.4", | |
| "count": 126, | |
| "confidence": 0.3559411667797675, | |
| "accuracy": 0.2857142857142857 | |
| }, | |
| { | |
| "bin": "0.4-0.5", | |
| "count": 344, | |
| "confidence": 0.4660676106444815, | |
| "accuracy": 0.502906976744186 | |
| }, | |
| { | |
| "bin": "0.5-0.6", | |
| "count": 542, | |
| "confidence": 0.5533377851748736, | |
| "accuracy": 0.566420664206642 | |
| }, | |
| { | |
| "bin": "0.6-0.7", | |
| "count": 418, | |
| "confidence": 0.6479330715953778, | |
| "accuracy": 0.6004784688995215 | |
| }, | |
| { | |
| "bin": "0.7-0.8", | |
| "count": 466, | |
| "confidence": 0.7491832278985806, | |
| "accuracy": 0.6888412017167382 | |
| }, | |
| { | |
| "bin": "0.8-0.9", | |
| "count": 564, | |
| "confidence": 0.8562632478879275, | |
| "accuracy": 0.7464539007092199 | |
| }, | |
| { | |
| "bin": "0.9-1.0", | |
| "count": 3489, | |
| "confidence": 0.9810906862120456, | |
| "accuracy": 0.942676984809401 | |
| } | |
| ], | |
| "seconds": 67.76366090774536, | |
| "model": "runs/assay-4b-v4", | |
| "data": "data/v4/dev.jsonl" | |
| } |