| { |
| "selected_step": 4153, |
| "mixture_test": { |
| "test_completed": true, |
| "calibration_completed": true, |
| "raw": { |
| "overall": { |
| "count": 3693, |
| "soft_ce": 0.7085598693044783, |
| "kl": 0.3931482496012985, |
| "brier": 0.09769070277607729, |
| "argmax_agreement": 0.7733549959382616, |
| "hard_count": 2134, |
| "hard_ece": 0.042129272019540774, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.1280767528435369, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.18404859928386386, |
| "reference_agreement": 0.2222222222222222 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.23661101358223402, |
| "reference_agreement": 0.2608695652173913 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.30428753988952256, |
| "reference_agreement": 0.1891891891891892 |
| }, |
| { |
| "count": 36, |
| "confidence": 0.36932558263805926, |
| "reference_agreement": 0.3888888888888889 |
| }, |
| { |
| "count": 57, |
| "confidence": 0.43265722650360866, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 71, |
| "confidence": 0.5035452847115158, |
| "reference_agreement": 0.5070422535211268 |
| }, |
| { |
| "count": 93, |
| "confidence": 0.5702882440958308, |
| "reference_agreement": 0.5053763440860215 |
| }, |
| { |
| "count": 113, |
| "confidence": 0.6289013423061501, |
| "reference_agreement": 0.5398230088495575 |
| }, |
| { |
| "count": 88, |
| "confidence": 0.7035225652548067, |
| "reference_agreement": 0.5909090909090909 |
| }, |
| { |
| "count": 97, |
| "confidence": 0.7663997989026462, |
| "reference_agreement": 0.6804123711340206 |
| }, |
| { |
| "count": 133, |
| "confidence": 0.8331041320580328, |
| "reference_agreement": 0.7218045112781954 |
| }, |
| { |
| "count": 213, |
| "confidence": 0.9010564345576307, |
| "reference_agreement": 0.812206572769953 |
| }, |
| { |
| "count": 1154, |
| "confidence": 0.9892346259941478, |
| "reference_agreement": 0.9809358752166378 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.1280767528435369, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.18189946160649148, |
| "reference_agreement": 0.253859649122807 |
| }, |
| { |
| "count": 52, |
| "confidence": 0.24099144225648178, |
| "reference_agreement": 0.2604555167055167 |
| }, |
| { |
| "count": 105, |
| "confidence": 0.30435695062773516, |
| "reference_agreement": 0.2666363084654525 |
| }, |
| { |
| "count": 151, |
| "confidence": 0.36784610221152053, |
| "reference_agreement": 0.38072070275124964 |
| }, |
| { |
| "count": 219, |
| "confidence": 0.4348046806054027, |
| "reference_agreement": 0.4029960976805746 |
| }, |
| { |
| "count": 268, |
| "confidence": 0.5012982178117764, |
| "reference_agreement": 0.4966141654778031 |
| }, |
| { |
| "count": 287, |
| "confidence": 0.5665963085542071, |
| "reference_agreement": 0.5549424985680554 |
| }, |
| { |
| "count": 284, |
| "confidence": 0.6310971171485777, |
| "reference_agreement": 0.600541924578161 |
| }, |
| { |
| "count": 280, |
| "confidence": 0.7030994468429518, |
| "reference_agreement": 0.6497546304223978 |
| }, |
| { |
| "count": 218, |
| "confidence": 0.7682645359325981, |
| "reference_agreement": 0.7226513752826121 |
| }, |
| { |
| "count": 230, |
| "confidence": 0.8322149342184884, |
| "reference_agreement": 0.7612036827968656 |
| }, |
| { |
| "count": 324, |
| "confidence": 0.8996811349428281, |
| "reference_agreement": 0.8247895057612304 |
| }, |
| { |
| "count": 1252, |
| "confidence": 0.9876585496182678, |
| "reference_agreement": 0.9778121922719432 |
| } |
| ], |
| "expected_score_mae": 0.4570759707537796 |
| }, |
| "type": { |
| "choice": { |
| "count": 2526, |
| "soft_ce": 0.6822833143719118, |
| "kl": 0.4189869815862902, |
| "brier": 0.10139614023942027, |
| "argmax_agreement": 0.7862232779097387, |
| "hard_count": 1737, |
| "hard_ece": 0.04277428653676444, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.1280767528435369, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.17932821619602474, |
| "reference_agreement": 0.4 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.2380902472918557, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.3081997411534075, |
| "reference_agreement": 0.2857142857142857 |
| }, |
| { |
| "count": 24, |
| "confidence": 0.3697200232466156, |
| "reference_agreement": 0.4166666666666667 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.43214316524587604, |
| "reference_agreement": 0.3 |
| }, |
| { |
| "count": 58, |
| "confidence": 0.5047682692169494, |
| "reference_agreement": 0.5344827586206896 |
| }, |
| { |
| "count": 76, |
| "confidence": 0.5702475474241634, |
| "reference_agreement": 0.5131578947368421 |
| }, |
| { |
| "count": 85, |
| "confidence": 0.6294861640993207, |
| "reference_agreement": 0.5411764705882353 |
| }, |
| { |
| "count": 61, |
| "confidence": 0.7034878822257229, |
| "reference_agreement": 0.6229508196721312 |
| }, |
| { |
| "count": 73, |
| "confidence": 0.7662886786794736, |
| "reference_agreement": 0.6712328767123288 |
| }, |
| { |
| "count": 109, |
| "confidence": 0.8327800034747108, |
| "reference_agreement": 0.7155963302752294 |
| }, |
| { |
| "count": 163, |
| "confidence": 0.9005930750387896, |
| "reference_agreement": 0.7852760736196319 |
| }, |
| { |
| "count": 1024, |
| "confidence": 0.9905297995078783, |
| "reference_agreement": 0.9814453125 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.1280767528435369, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.1761727165792652, |
| "reference_agreement": 0.39832358674463936 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.23727536848341457, |
| "reference_agreement": 0.23968434343434342 |
| }, |
| { |
| "count": 56, |
| "confidence": 0.30649961126616326, |
| "reference_agreement": 0.2928895108131916 |
| }, |
| { |
| "count": 112, |
| "confidence": 0.36821444135421544, |
| "reference_agreement": 0.393456101968026 |
| }, |
| { |
| "count": 148, |
| "confidence": 0.43446487817770657, |
| "reference_agreement": 0.4016674390496405 |
| }, |
| { |
| "count": 173, |
| "confidence": 0.4989247303895505, |
| "reference_agreement": 0.5024153201745094 |
| }, |
| { |
| "count": 161, |
| "confidence": 0.5682084517346793, |
| "reference_agreement": 0.5282341373438668 |
| }, |
| { |
| "count": 170, |
| "confidence": 0.6297812369278754, |
| "reference_agreement": 0.5908322297566935 |
| }, |
| { |
| "count": 148, |
| "confidence": 0.700524650342172, |
| "reference_agreement": 0.6783159155834284 |
| }, |
| { |
| "count": 123, |
| "confidence": 0.7663341138612425, |
| "reference_agreement": 0.708765336806816 |
| }, |
| { |
| "count": 150, |
| "confidence": 0.8331362507431996, |
| "reference_agreement": 0.7423155758796159 |
| }, |
| { |
| "count": 207, |
| "confidence": 0.8989791587743893, |
| "reference_agreement": 0.7958650265103496 |
| }, |
| { |
| "count": 1048, |
| "confidence": 0.9898586616987579, |
| "reference_agreement": 0.9796660513168818 |
| } |
| ] |
| }, |
| "noul": { |
| "count": 582, |
| "soft_ce": 0.485984791408039, |
| "kl": 0.1677006591643438, |
| "brier": 0.056889637384882515, |
| "argmax_agreement": 0.8316151202749141, |
| "hard_count": 154, |
| "hard_ece": 0.05817309278238439, |
| "hard_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.5156046866868782, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.5744020664194579, |
| "reference_agreement": 0.8 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.6251505169037502, |
| "reference_agreement": 0.4666666666666667 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.7097643525488307, |
| "reference_agreement": 0.5333333333333333 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.7837242799754484, |
| "reference_agreement": 0.7272727272727273 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.833779143408664, |
| "reference_agreement": 0.9 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.9027589227901665, |
| "reference_agreement": 0.9090909090909091 |
| }, |
| { |
| "count": 74, |
| "confidence": 0.9777466873130052, |
| "reference_agreement": 0.9594594594594594 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 34, |
| "confidence": 0.5192844401029223, |
| "reference_agreement": 0.502892126954275 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.5665664402627255, |
| "reference_agreement": 0.6365357818181818 |
| }, |
| { |
| "count": 72, |
| "confidence": 0.6353817301417451, |
| "reference_agreement": 0.617865484172874 |
| }, |
| { |
| "count": 84, |
| "confidence": 0.7075900367197564, |
| "reference_agreement": 0.6229389882371539 |
| }, |
| { |
| "count": 58, |
| "confidence": 0.7769928207174944, |
| "reference_agreement": 0.7633095172413793 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.829995783859167, |
| "reference_agreement": 0.8141728722666914 |
| }, |
| { |
| "count": 83, |
| "confidence": 0.9004798674802651, |
| "reference_agreement": 0.8669205460521514 |
| }, |
| { |
| "count": 141, |
| "confidence": 0.975388672803986, |
| "reference_agreement": 0.9554196875487998 |
| } |
| ] |
| }, |
| "score": { |
| "count": 585, |
| "soft_ce": 1.043454353228223, |
| "kl": 0.5058693789008174, |
| "brier": 0.12228264299021577, |
| "argmax_agreement": 0.6598290598290598, |
| "hard_count": 243, |
| "hard_ece": 0.07115621447701632, |
| "hard_reliability": [ |
| { |
| "count": 13, |
| "confidence": 0.18586413124072504, |
| "reference_agreement": 0.15384615384615385 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.2362995959591558, |
| "reference_agreement": 0.2631578947368421 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.3019061999897666, |
| "reference_agreement": 0.13043478260869565 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.3685367014209464, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.4338667824041559, |
| "reference_agreement": 0.4117647058823529 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.49490420241461824, |
| "reference_agreement": 0.36363636363636365 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.5688318970482136, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.6294053829689576, |
| "reference_agreement": 0.6153846153846154 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.6958966365351196, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.7523646054019367, |
| "reference_agreement": 0.6923076923076923 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.8351455536348746, |
| "reference_agreement": 0.6428571428571429 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.9024161795738924, |
| "reference_agreement": 0.8928571428571429 |
| }, |
| { |
| "count": 56, |
| "confidence": 0.9807319435717284, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 13, |
| "confidence": 0.18586413124072504, |
| "reference_agreement": 0.15384615384615385 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.24331398836464874, |
| "reference_agreement": 0.2734375 |
| }, |
| { |
| "count": 49, |
| "confidence": 0.3019081956123887, |
| "reference_agreement": 0.23663264863946495 |
| }, |
| { |
| "count": 39, |
| "confidence": 0.3667883077504478, |
| "reference_agreement": 0.34414724859025103 |
| }, |
| { |
| "count": 71, |
| "confidence": 0.43551300115891045, |
| "reference_agreement": 0.4057656959535077 |
| }, |
| { |
| "count": 61, |
| "confidence": 0.49800447627318833, |
| "reference_agreement": 0.47666251871173404 |
| }, |
| { |
| "count": 71, |
| "confidence": 0.5629637410045656, |
| "reference_agreement": 0.5523004644601318 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.6290782481488474, |
| "reference_agreement": 0.610145539550307 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.7031798704359478, |
| "reference_agreement": 0.6086180416667305 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.7609997088313216, |
| "reference_agreement": 0.7050786860640831 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.8315691658607272, |
| "reference_agreement": 0.7580001074667474 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.9020050839504612, |
| "reference_agreement": 0.8980392369608144 |
| }, |
| { |
| "count": 63, |
| "confidence": 0.978521012641448, |
| "reference_agreement": 0.9970899523809524 |
| } |
| ], |
| "expected_score_mae": 0.4570759707537796 |
| } |
| }, |
| "category": { |
| "engineering": { |
| "count": 500, |
| "soft_ce": 0.0170732927832887, |
| "kl": 0.0170732927832887, |
| "brier": 0.004504996059548485, |
| "argmax_agreement": 0.996, |
| "hard_count": 500, |
| "hard_ece": 0.005887712376320884, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.48051708584209724, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.5901215718541692, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.659498582567833, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7592606819949799, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.809986801374795, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.9033714425633134, |
| "reference_agreement": 0.8333333333333334 |
| }, |
| { |
| "count": 485, |
| "confidence": 0.9982748543344034, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.48051708584209724, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.5901215718541692, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.659498582567833, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7592606819949799, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.809986801374795, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.9033714425633134, |
| "reference_agreement": 0.8333333333333334 |
| }, |
| { |
| "count": 485, |
| "confidence": 0.9982748543344034, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "expected_score_mae": 0.36947776287918127 |
| }, |
| "data_analytics": { |
| "count": 500, |
| "soft_ce": 0.63641646085155, |
| "kl": 0.6331406095589392, |
| "brier": 0.15179326704736298, |
| "argmax_agreement": 0.76, |
| "hard_count": 497, |
| "hard_ece": 0.05635174490474885, |
| "hard_reliability": [ |
| { |
| "count": 8, |
| "confidence": 0.18512452516415256, |
| "reference_agreement": 0.125 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.23685294357226974, |
| "reference_agreement": 0.23076923076923078 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.310619358628449, |
| "reference_agreement": 0.125 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.36840624755576634, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.4338411624493362, |
| "reference_agreement": 0.3076923076923077 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.49562565013910687, |
| "reference_agreement": 0.42857142857142855 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.5716282587437927, |
| "reference_agreement": 0.47368421052631576 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.6307835105473234, |
| "reference_agreement": 0.4444444444444444 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.7034512348597252, |
| "reference_agreement": 0.5333333333333333 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.7610956537573322, |
| "reference_agreement": 0.64 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.8287917395783113, |
| "reference_agreement": 0.7037037037037037 |
| }, |
| { |
| "count": 60, |
| "confidence": 0.9017458338366022, |
| "reference_agreement": 0.8 |
| }, |
| { |
| "count": 254, |
| "confidence": 0.9885657799728477, |
| "reference_agreement": 0.984251968503937 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 8, |
| "confidence": 0.18512452516415256, |
| "reference_agreement": 0.125 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.23685294357226974, |
| "reference_agreement": 0.23076923076923078 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.310619358628449, |
| "reference_agreement": 0.125 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.36840624755576634, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.4338411624493362, |
| "reference_agreement": 0.3076923076923077 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.49562565013910687, |
| "reference_agreement": 0.42857142857142855 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.5710235531343882, |
| "reference_agreement": 0.4762625909090909 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.6307835105473234, |
| "reference_agreement": 0.4444444444444444 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.7034512348597252, |
| "reference_agreement": 0.5333333333333333 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.7610956537573322, |
| "reference_agreement": 0.64 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.8287917395783113, |
| "reference_agreement": 0.7037037037037037 |
| }, |
| { |
| "count": 60, |
| "confidence": 0.9017458338366022, |
| "reference_agreement": 0.8 |
| }, |
| { |
| "count": 254, |
| "confidence": 0.9885657799728477, |
| "reference_agreement": 0.984251968503937 |
| } |
| ], |
| "expected_score_mae": 0.8379162320369495 |
| }, |
| "finance": { |
| "count": 500, |
| "soft_ce": 0.8325307227564824, |
| "kl": 0.21741036887290283, |
| "brier": 0.05805591006171185, |
| "argmax_agreement": 0.766, |
| "hard_count": 52, |
| "hard_ece": 0.12560551004414863, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.42545552219647026, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.5063873523524167, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.5575393887409873, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.6185993089535619, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7072795840269844, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.7583805784749291, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.8432613843014515, |
| "reference_agreement": 0.6 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.8980962253086776, |
| "reference_agreement": 0.9 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.9891833973182879, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 11, |
| "confidence": 0.32509882113799443, |
| "reference_agreement": 0.44772725303034394 |
| }, |
| { |
| "count": 31, |
| "confidence": 0.36165303383874403, |
| "reference_agreement": 0.32344085419356344 |
| }, |
| { |
| "count": 56, |
| "confidence": 0.433194585459043, |
| "reference_agreement": 0.39091664392857783 |
| }, |
| { |
| "count": 52, |
| "confidence": 0.49251118240718983, |
| "reference_agreement": 0.5019289031468974 |
| }, |
| { |
| "count": 50, |
| "confidence": 0.5634722409257146, |
| "reference_agreement": 0.6018555810889367 |
| }, |
| { |
| "count": 65, |
| "confidence": 0.6301825914444824, |
| "reference_agreement": 0.6561282074359387 |
| }, |
| { |
| "count": 68, |
| "confidence": 0.7056494781490086, |
| "reference_agreement": 0.627431364951034 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.7683118932263894, |
| "reference_agreement": 0.6799459059550426 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.8290161206378067, |
| "reference_agreement": 0.7909373143137849 |
| }, |
| { |
| "count": 35, |
| "confidence": 0.8969496853528575, |
| "reference_agreement": 0.9005142873333625 |
| }, |
| { |
| "count": 61, |
| "confidence": 0.9760249311137575, |
| "reference_agreement": 0.9830546393442623 |
| } |
| ], |
| "expected_score_mae": 0.28921449751489353 |
| }, |
| "trust_safety": { |
| "count": 500, |
| "soft_ce": 0.6965650575760396, |
| "kl": 0.3262561593110357, |
| "brier": 0.09241847396917179, |
| "argmax_agreement": 0.734, |
| "hard_count": 222, |
| "hard_ece": 0.06416689376484794, |
| "hard_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.37375993751238135, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.43472235444626844, |
| "reference_agreement": 0.4 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.502873413716155, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.5716600619610657, |
| "reference_agreement": 0.8181818181818182 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.6273743231263287, |
| "reference_agreement": 0.5714285714285714 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.7048754950230958, |
| "reference_agreement": 0.6428571428571429 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.772673527518938, |
| "reference_agreement": 0.625 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.8352234500546492, |
| "reference_agreement": 0.6956521739130435 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.9039014670840353, |
| "reference_agreement": 0.8181818181818182 |
| }, |
| { |
| "count": 113, |
| "confidence": 0.9792441416955855, |
| "reference_agreement": 0.9557522123893806 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 6, |
| "confidence": 0.23458907659811615, |
| "reference_agreement": 0.2075 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.29972115715656333, |
| "reference_agreement": 0.255416625 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.36836999088901834, |
| "reference_agreement": 0.33265160656977494 |
| }, |
| { |
| "count": 24, |
| "confidence": 0.4353651177707092, |
| "reference_agreement": 0.40927912136242695 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.5034541101303811, |
| "reference_agreement": 0.4637698182486976 |
| }, |
| { |
| "count": 44, |
| "confidence": 0.5639370342777291, |
| "reference_agreement": 0.5822456558441558 |
| }, |
| { |
| "count": 43, |
| "confidence": 0.6301712982399027, |
| "reference_agreement": 0.5623823154308876 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.7009662006930144, |
| "reference_agreement": 0.625547008811918 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.7682502104322542, |
| "reference_agreement": 0.6588631578947368 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.8340236494740164, |
| "reference_agreement": 0.7355623967695999 |
| }, |
| { |
| "count": 53, |
| "confidence": 0.9011086418913816, |
| "reference_agreement": 0.8135774190795747 |
| }, |
| { |
| "count": 151, |
| "confidence": 0.9780184864854932, |
| "reference_agreement": 0.9430664742453473 |
| } |
| ], |
| "expected_score_mae": 0.3245526679121535 |
| }, |
| "documents_meetings": { |
| "count": 500, |
| "soft_ce": 0.9479271825455599, |
| "kl": 0.27931837942071347, |
| "brier": 0.06632900701852151, |
| "argmax_agreement": 0.754, |
| "hard_count": 111, |
| "hard_ece": 0.07985648898540725, |
| "hard_reliability": [ |
| { |
| "count": 4, |
| "confidence": 0.5112290740361165, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.5755079266886977, |
| "reference_agreement": 0.625 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.6215870747073032, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.7132942799377675, |
| "reference_agreement": 0.5714285714285714 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7689218976142511, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.8325792906268336, |
| "reference_agreement": 0.7692307692307693 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.894684750793525, |
| "reference_agreement": 0.8461538461538461 |
| }, |
| { |
| "count": 54, |
| "confidence": 0.980788268448271, |
| "reference_agreement": 0.9444444444444444 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 16, |
| "confidence": 0.2529896903034424, |
| "reference_agreement": 0.2572032828282828 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.30085803531102356, |
| "reference_agreement": 0.2784615384615385 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.3672361246048308, |
| "reference_agreement": 0.373009828009828 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.43667169969021885, |
| "reference_agreement": 0.49312530062530063 |
| }, |
| { |
| "count": 64, |
| "confidence": 0.5041356739419488, |
| "reference_agreement": 0.576640625 |
| }, |
| { |
| "count": 57, |
| "confidence": 0.5696819547588704, |
| "reference_agreement": 0.6160136452241716 |
| }, |
| { |
| "count": 52, |
| "confidence": 0.6318176118679493, |
| "reference_agreement": 0.6767307692307691 |
| }, |
| { |
| "count": 51, |
| "confidence": 0.7024724227358164, |
| "reference_agreement": 0.7055555555555556 |
| }, |
| { |
| "count": 38, |
| "confidence": 0.7733050801429199, |
| "reference_agreement": 0.7889473684210526 |
| }, |
| { |
| "count": 33, |
| "confidence": 0.8353726880577852, |
| "reference_agreement": 0.8224242424242424 |
| }, |
| { |
| "count": 30, |
| "confidence": 0.8930073021992985, |
| "reference_agreement": 0.8578333333333333 |
| }, |
| { |
| "count": 54, |
| "confidence": 0.980788268448271, |
| "reference_agreement": 0.9444444444444444 |
| } |
| ], |
| "expected_score_mae": 0.3935315866553138 |
| }, |
| "customer_support": { |
| "count": 500, |
| "soft_ce": 1.1372885709258729, |
| "kl": 0.5544979078550764, |
| "brier": 0.11033544189723725, |
| "argmax_agreement": 0.706, |
| "hard_count": 142, |
| "hard_ece": 0.14171122927377042, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.1280767528435369, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.17932821619602474, |
| "reference_agreement": 0.4 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.22071972996391087, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.3112827985653901, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.37505887333787874, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.4293526333096114, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.49253722456655336, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.5647034564825082, |
| "reference_agreement": 0.18181818181818182 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.6399711185141979, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.6905290585463988, |
| "reference_agreement": 0.5714285714285714 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.7733585610196265, |
| "reference_agreement": 0.6 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.8359032378172329, |
| "reference_agreement": 0.7647058823529411 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.9086037470245399, |
| "reference_agreement": 0.6 |
| }, |
| { |
| "count": 51, |
| "confidence": 0.9778779954883188, |
| "reference_agreement": 0.9411764705882353 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.1280767528435369, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.1761727165792652, |
| "reference_agreement": 0.39832358674463936 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.23185648880822557, |
| "reference_agreement": 0.2729292929292929 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.3010096491709431, |
| "reference_agreement": 0.27153918422154916 |
| }, |
| { |
| "count": 33, |
| "confidence": 0.3716936308219563, |
| "reference_agreement": 0.43249653598146726 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.43566624910724644, |
| "reference_agreement": 0.3784143440302738 |
| }, |
| { |
| "count": 49, |
| "confidence": 0.5004958652365038, |
| "reference_agreement": 0.464408579758434 |
| }, |
| { |
| "count": 64, |
| "confidence": 0.5647801490500414, |
| "reference_agreement": 0.4939270455676706 |
| }, |
| { |
| "count": 35, |
| "confidence": 0.6338884106064454, |
| "reference_agreement": 0.6524762 |
| }, |
| { |
| "count": 49, |
| "confidence": 0.7014723835777643, |
| "reference_agreement": 0.6855101955782364 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.7717612201235851, |
| "reference_agreement": 0.7027941176470588 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.8313773086286068, |
| "reference_agreement": 0.763963945945946 |
| }, |
| { |
| "count": 47, |
| "confidence": 0.9031494358129605, |
| "reference_agreement": 0.7396099361702129 |
| }, |
| { |
| "count": 63, |
| "confidence": 0.9737046850235944, |
| "reference_agreement": 0.9396507939418092 |
| } |
| ], |
| "expected_score_mae": 0.33143847468388904 |
| }, |
| "ai_agents_evals": { |
| "count": 500, |
| "soft_ce": 0.765946732434316, |
| "kl": 0.6764210789434673, |
| "brier": 0.18685146874516365, |
| "argmax_agreement": 0.68, |
| "hard_count": 417, |
| "hard_ece": 0.06936429979471131, |
| "hard_reliability": [ |
| { |
| "count": 5, |
| "confidence": 0.187047500963241, |
| "reference_agreement": 0.2 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.23585635105072375, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.2982191389855985, |
| "reference_agreement": 0.2631578947368421 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.36932960679831744, |
| "reference_agreement": 0.5714285714285714 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.43134028318033907, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.5068791830358712, |
| "reference_agreement": 0.4375 |
| }, |
| { |
| "count": 33, |
| "confidence": 0.5698085895150488, |
| "reference_agreement": 0.5151515151515151 |
| }, |
| { |
| "count": 53, |
| "confidence": 0.6282809191581149, |
| "reference_agreement": 0.49056603773584906 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.7055160692185535, |
| "reference_agreement": 0.6176470588235294 |
| }, |
| { |
| "count": 33, |
| "confidence": 0.765969600593059, |
| "reference_agreement": 0.8181818181818182 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.8332225392168197, |
| "reference_agreement": 0.7857142857142857 |
| }, |
| { |
| "count": 54, |
| "confidence": 0.8969179427292381, |
| "reference_agreement": 0.8703703703703703 |
| }, |
| { |
| "count": 77, |
| "confidence": 0.9715351814332892, |
| "reference_agreement": 0.922077922077922 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 5, |
| "confidence": 0.187047500963241, |
| "reference_agreement": 0.2 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.23585635105072375, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.2982191389855985, |
| "reference_agreement": 0.2631578947368421 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.3728244454729145, |
| "reference_agreement": 0.5458333125 |
| }, |
| { |
| "count": 35, |
| "confidence": 0.43292344830072604, |
| "reference_agreement": 0.35496548479120027 |
| }, |
| { |
| "count": 41, |
| "confidence": 0.5085266662477907, |
| "reference_agreement": 0.45982781975131765 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.5678684243563111, |
| "reference_agreement": 0.5322624638888933 |
| }, |
| { |
| "count": 58, |
| "confidence": 0.6299378167372339, |
| "reference_agreement": 0.5120080436781644 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.7054146769488767, |
| "reference_agreement": 0.6565114530755077 |
| }, |
| { |
| "count": 44, |
| "confidence": 0.7651372249377204, |
| "reference_agreement": 0.8189414169069195 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.833874270098686, |
| "reference_agreement": 0.8076943700618164 |
| }, |
| { |
| "count": 65, |
| "confidence": 0.8966887344775478, |
| "reference_agreement": 0.8630389169038999 |
| }, |
| { |
| "count": 85, |
| "confidence": 0.9707936989016759, |
| "reference_agreement": 0.9253352246481347 |
| } |
| ], |
| "expected_score_mae": 0.8450261299297381 |
| }, |
| "legal": { |
| "count": 122, |
| "soft_ce": 0.5668266819917146, |
| "kl": 0.5668266819917146, |
| "brier": 0.13410281322654918, |
| "argmax_agreement": 0.819672131147541, |
| "hard_count": 122, |
| "hard_ece": 0.07370299961080379, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.25539450758217525, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.3631581357093971, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.4544914341376851, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.503085478379179, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.5760999183430778, |
| "reference_agreement": 0.4 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.6285356936618459, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.6943533513117489, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.7758294858466038, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.8221451030950648, |
| "reference_agreement": 0.42857142857142855 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.9086579271812519, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 72, |
| "confidence": 0.9849048761433361, |
| "reference_agreement": 0.9861111111111112 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.25539450758217525, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.3631581357093971, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.4544914341376851, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.503085478379179, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.5760999183430778, |
| "reference_agreement": 0.4 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.6285356936618459, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.6943533513117489, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.7758294858466038, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.8221451030950648, |
| "reference_agreement": 0.42857142857142855 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.9086579271812519, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 72, |
| "confidence": 0.9849048761433361, |
| "reference_agreement": 0.9861111111111112 |
| } |
| ], |
| "expected_score_mae": 0.05627741100384848 |
| }, |
| "sales_commerce": { |
| "count": 71, |
| "soft_ce": 0.43217932678724524, |
| "kl": 0.43217932678724524, |
| "brier": 0.13052027801486862, |
| "argmax_agreement": 0.8169014084507042, |
| "hard_count": 71, |
| "hard_ece": 0.07210090880407366, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.43246792381509247, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5125808974731219, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5624424423010923, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.6350879180799406, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.6908680440851511, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.7547308258492205, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.8418109162556351, |
| "reference_agreement": 0.7272727272727273 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.8995504132439553, |
| "reference_agreement": 0.875 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.9753556391806691, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.43246792381509247, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5125808974731219, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5624424423010923, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.6350879180799406, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.6908680440851511, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.7547308258492205, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.8418109162556351, |
| "reference_agreement": 0.7272727272727273 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.8995504132439553, |
| "reference_agreement": 0.875 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.9753556391806691, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "expected_score_mae": 0.3227189172274408 |
| } |
| } |
| }, |
| "calibrated": { |
| "overall": { |
| "count": 3693, |
| "soft_ce": 0.7041168734088186, |
| "kl": 0.3887052537056387, |
| "brier": 0.09690042536340486, |
| "argmax_agreement": 0.7733549959382616, |
| "hard_count": 2134, |
| "hard_ece": 0.022770969448581948, |
| "hard_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.11589315469434829, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.17569618187345318, |
| "reference_agreement": 0.17647058823529413 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.23574174336686576, |
| "reference_agreement": 0.23076923076923078 |
| }, |
| { |
| "count": 43, |
| "confidence": 0.30201610783522903, |
| "reference_agreement": 0.23255813953488372 |
| }, |
| { |
| "count": 51, |
| "confidence": 0.3728587574318181, |
| "reference_agreement": 0.35294117647058826 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.43468960041354515, |
| "reference_agreement": 0.36363636363636365 |
| }, |
| { |
| "count": 78, |
| "confidence": 0.5068967125125762, |
| "reference_agreement": 0.47435897435897434 |
| }, |
| { |
| "count": 116, |
| "confidence": 0.5692509240339776, |
| "reference_agreement": 0.5689655172413793 |
| }, |
| { |
| "count": 92, |
| "confidence": 0.6274691893699693, |
| "reference_agreement": 0.5760869565217391 |
| }, |
| { |
| "count": 113, |
| "confidence": 0.6976255782232683, |
| "reference_agreement": 0.584070796460177 |
| }, |
| { |
| "count": 125, |
| "confidence": 0.7655885050647548, |
| "reference_agreement": 0.72 |
| }, |
| { |
| "count": 153, |
| "confidence": 0.8348620973385892, |
| "reference_agreement": 0.7777777777777778 |
| }, |
| { |
| "count": 193, |
| "confidence": 0.9005488041523733, |
| "reference_agreement": 0.8860103626943006 |
| }, |
| { |
| "count": 1069, |
| "confidence": 0.9873898699630395, |
| "reference_agreement": 0.9850327408793265 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.11589315469434829, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.1712254393277906, |
| "reference_agreement": 0.22601507061461293 |
| }, |
| { |
| "count": 65, |
| "confidence": 0.241401549839449, |
| "reference_agreement": 0.27101087801087803 |
| }, |
| { |
| "count": 136, |
| "confidence": 0.3047942264115045, |
| "reference_agreement": 0.30327285409683113 |
| }, |
| { |
| "count": 180, |
| "confidence": 0.3704360020830201, |
| "reference_agreement": 0.3727716890584122 |
| }, |
| { |
| "count": 239, |
| "confidence": 0.4341484282824121, |
| "reference_agreement": 0.4200372081322403 |
| }, |
| { |
| "count": 268, |
| "confidence": 0.5028656676116453, |
| "reference_agreement": 0.5155282699000787 |
| }, |
| { |
| "count": 335, |
| "confidence": 0.5660902452493901, |
| "reference_agreement": 0.600639823533436 |
| }, |
| { |
| "count": 269, |
| "confidence": 0.631949793983103, |
| "reference_agreement": 0.6359665011640041 |
| }, |
| { |
| "count": 261, |
| "confidence": 0.6984239785753936, |
| "reference_agreement": 0.6471423835974844 |
| }, |
| { |
| "count": 249, |
| "confidence": 0.7657109724234291, |
| "reference_agreement": 0.7478915785725628 |
| }, |
| { |
| "count": 253, |
| "confidence": 0.8344639775274048, |
| "reference_agreement": 0.8022606606356278 |
| }, |
| { |
| "count": 261, |
| "confidence": 0.8995167793570855, |
| "reference_agreement": 0.8815173885724905 |
| }, |
| { |
| "count": 1151, |
| "confidence": 0.9856462873311037, |
| "reference_agreement": 0.9822842533704442 |
| } |
| ], |
| "expected_score_mae": 0.4584616360574015 |
| }, |
| "type": { |
| "choice": { |
| "count": 2526, |
| "soft_ce": 0.6766963060796009, |
| "kl": 0.4133999732939792, |
| "brier": 0.10042181389472693, |
| "argmax_agreement": 0.7862232779097387, |
| "hard_count": 1737, |
| "hard_ece": 0.024066863233766642, |
| "hard_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.11589315469434829, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.1466965963178435, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.23979741242633804, |
| "reference_agreement": 0.14285714285714285 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.30543179732871933, |
| "reference_agreement": 0.35 |
| }, |
| { |
| "count": 39, |
| "confidence": 0.3752333136919975, |
| "reference_agreement": 0.358974358974359 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.43632325318545506, |
| "reference_agreement": 0.35135135135135137 |
| }, |
| { |
| "count": 64, |
| "confidence": 0.5078595631087047, |
| "reference_agreement": 0.484375 |
| }, |
| { |
| "count": 91, |
| "confidence": 0.5663438578130047, |
| "reference_agreement": 0.6043956043956044 |
| }, |
| { |
| "count": 72, |
| "confidence": 0.6259376203313807, |
| "reference_agreement": 0.5555555555555556 |
| }, |
| { |
| "count": 87, |
| "confidence": 0.6985076075501826, |
| "reference_agreement": 0.6091954022988506 |
| }, |
| { |
| "count": 98, |
| "confidence": 0.7672216909611733, |
| "reference_agreement": 0.7142857142857143 |
| }, |
| { |
| "count": 126, |
| "confidence": 0.835107717990711, |
| "reference_agreement": 0.7777777777777778 |
| }, |
| { |
| "count": 140, |
| "confidence": 0.8998423906655577, |
| "reference_agreement": 0.8857142857142857 |
| }, |
| { |
| "count": 949, |
| "confidence": 0.988617468252103, |
| "reference_agreement": 0.9852476290832455 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.11589315469434829, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.15381363979618545, |
| "reference_agreement": 0.3198346624136098 |
| }, |
| { |
| "count": 31, |
| "confidence": 0.2398905686918626, |
| "reference_agreement": 0.2732486151840991 |
| }, |
| { |
| "count": 88, |
| "confidence": 0.3066439829194413, |
| "reference_agreement": 0.33710350424812785 |
| }, |
| { |
| "count": 140, |
| "confidence": 0.37159211827431626, |
| "reference_agreement": 0.3838011523963886 |
| }, |
| { |
| "count": 167, |
| "confidence": 0.4341350320260209, |
| "reference_agreement": 0.42457801395752337 |
| }, |
| { |
| "count": 166, |
| "confidence": 0.5007513721327872, |
| "reference_agreement": 0.5253523924620044 |
| }, |
| { |
| "count": 197, |
| "confidence": 0.565477477433576, |
| "reference_agreement": 0.619517122506086 |
| }, |
| { |
| "count": 158, |
| "confidence": 0.6298800441075835, |
| "reference_agreement": 0.6306406317730678 |
| }, |
| { |
| "count": 141, |
| "confidence": 0.6982422404036815, |
| "reference_agreement": 0.6697869686413617 |
| }, |
| { |
| "count": 136, |
| "confidence": 0.7667611071607225, |
| "reference_agreement": 0.7399642556669485 |
| }, |
| { |
| "count": 165, |
| "confidence": 0.8340530433682095, |
| "reference_agreement": 0.7888326080745547 |
| }, |
| { |
| "count": 164, |
| "confidence": 0.899220153771901, |
| "reference_agreement": 0.8851424152139722 |
| }, |
| { |
| "count": 960, |
| "confidence": 0.988268968121057, |
| "reference_agreement": 0.9841545132135424 |
| } |
| ] |
| }, |
| "noul": { |
| "count": 582, |
| "soft_ce": 0.4827658039249084, |
| "kl": 0.16448167168121322, |
| "brier": 0.05629811292264301, |
| "argmax_agreement": 0.8316151202749141, |
| "hard_count": 154, |
| "hard_ece": 0.03838079417551199, |
| "hard_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.51409214953238, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.5848970391247416, |
| "reference_agreement": 0.5833333333333334 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.6345947956878246, |
| "reference_agreement": 0.6 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.6959874383006549, |
| "reference_agreement": 0.46153846153846156 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.7675695442215852, |
| "reference_agreement": 0.7857142857142857 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.8339093290833637, |
| "reference_agreement": 0.9230769230769231 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.9051654333633907, |
| "reference_agreement": 0.88 |
| }, |
| { |
| "count": 65, |
| "confidence": 0.9758087375812551, |
| "reference_agreement": 0.9692307692307692 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 34, |
| "confidence": 0.5174138851791591, |
| "reference_agreement": 0.502892126954275 |
| }, |
| { |
| "count": 71, |
| "confidence": 0.5685861678053349, |
| "reference_agreement": 0.6089361690140845 |
| }, |
| { |
| "count": 72, |
| "confidence": 0.6374148978709203, |
| "reference_agreement": 0.6503378173406389 |
| }, |
| { |
| "count": 72, |
| "confidence": 0.6973135426818078, |
| "reference_agreement": 0.6177620419978036 |
| }, |
| { |
| "count": 76, |
| "confidence": 0.7668883954141004, |
| "reference_agreement": 0.7746528812652387 |
| }, |
| { |
| "count": 63, |
| "confidence": 0.8361374834836659, |
| "reference_agreement": 0.8580618729922205 |
| }, |
| { |
| "count": 65, |
| "confidence": 0.899478685155311, |
| "reference_agreement": 0.8626053588050548 |
| }, |
| { |
| "count": 129, |
| "confidence": 0.9701079851807889, |
| "reference_agreement": 0.9612726817393857 |
| } |
| ] |
| }, |
| "score": { |
| "count": 585, |
| "soft_ce": 1.0427334127476895, |
| "kl": 0.505148438420284, |
| "brier": 0.12208934572306966, |
| "argmax_agreement": 0.6598290598290598, |
| "hard_count": 243, |
| "hard_ece": 0.0736454437457963, |
| "hard_reliability": [ |
| { |
| "count": 13, |
| "confidence": 0.18461913127517923, |
| "reference_agreement": 0.15384615384615385 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.23424754950284965, |
| "reference_agreement": 0.2631578947368421 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.299045943058281, |
| "reference_agreement": 0.13043478260869565 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.365141449586235, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.43133153638239685, |
| "reference_agreement": 0.3888888888888889 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.5005622698299229, |
| "reference_agreement": 0.4166666666666667 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.5751578198046974, |
| "reference_agreement": 0.3076923076923077 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.6313708801299527, |
| "reference_agreement": 0.7 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.6933609064965334, |
| "reference_agreement": 0.5384615384615384 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.7511433692151679, |
| "reference_agreement": 0.6923076923076923 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.8335362248493442, |
| "reference_agreement": 0.6428571428571429 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.8999588812194718, |
| "reference_agreement": 0.8928571428571429 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.9798950122993066, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 13, |
| "confidence": 0.18461913127517923, |
| "reference_agreement": 0.15384615384615385 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.24277920912107187, |
| "reference_agreement": 0.26897058823529413 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.30140300614695376, |
| "reference_agreement": 0.24124999548612047 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.3663895954134836, |
| "reference_agreement": 0.3341685673754948 |
| }, |
| { |
| "count": 72, |
| "confidence": 0.4341795001548752, |
| "reference_agreement": 0.40950506128748676 |
| }, |
| { |
| "count": 68, |
| "confidence": 0.5007529272027481, |
| "reference_agreement": 0.49786392453063255 |
| }, |
| { |
| "count": 67, |
| "confidence": 0.5652470326858571, |
| "reference_agreement": 0.5363432798507779 |
| }, |
| { |
| "count": 39, |
| "confidence": 0.6302455119423146, |
| "reference_agreement": 0.6310114396011891 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.7006234882951765, |
| "reference_agreement": 0.6246944274306163 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.759432527246053, |
| "reference_agreement": 0.7220606842622975 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.8329589079683152, |
| "reference_agreement": 0.7502667524000977 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.9011143643285106, |
| "reference_agreement": 0.9013541875 |
| }, |
| { |
| "count": 62, |
| "confidence": 0.9773667295736088, |
| "reference_agreement": 0.9970430161290323 |
| } |
| ], |
| "expected_score_mae": 0.4584616360574015 |
| } |
| }, |
| "category": { |
| "engineering": { |
| "count": 500, |
| "soft_ce": 0.018681013106648606, |
| "kl": 0.018681013106648606, |
| "brier": 0.004596240452886235, |
| "argmax_agreement": 0.996, |
| "hard_count": 500, |
| "hard_ece": 0.010272557015768891, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.4716738957978698, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.5539038973137703, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.655163400891637, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.7052527420032781, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7571008671839193, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.8521442972732137, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.9071717975027496, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 479, |
| "confidence": 0.9978224823124803, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.4716738957978698, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.5539038973137703, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.655163400891637, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.7052527420032781, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7571008671839193, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.8521442972732137, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.9071717975027496, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 479, |
| "confidence": 0.9978224823124803, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "expected_score_mae": 0.3754904932299786 |
| }, |
| "data_analytics": { |
| "count": 500, |
| "soft_ce": 0.6222487807440032, |
| "kl": 0.6189729294513925, |
| "brier": 0.1492023277961967, |
| "argmax_agreement": 0.76, |
| "hard_count": 497, |
| "hard_ece": 0.04324263356987873, |
| "hard_reliability": [ |
| { |
| "count": 8, |
| "confidence": 0.1836431426040265, |
| "reference_agreement": 0.125 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.2326095606931366, |
| "reference_agreement": 0.23076923076923078 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.3080856226638128, |
| "reference_agreement": 0.19047619047619047 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.37224859620757544, |
| "reference_agreement": 0.36363636363636365 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.43216064330256254, |
| "reference_agreement": 0.2777777777777778 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.49955516847278475, |
| "reference_agreement": 0.375 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.5648218776742052, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.6284579234052629, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.7007100933753049, |
| "reference_agreement": 0.5517241379310345 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.7611781134778828, |
| "reference_agreement": 0.68 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.8391754153900324, |
| "reference_agreement": 0.7857142857142857 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.9013655316115832, |
| "reference_agreement": 0.8888888888888888 |
| }, |
| { |
| "count": 235, |
| "confidence": 0.9854872248925048, |
| "reference_agreement": 0.9914893617021276 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 8, |
| "confidence": 0.1836431426040265, |
| "reference_agreement": 0.125 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.2326095606931366, |
| "reference_agreement": 0.23076923076923078 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.3080856226638128, |
| "reference_agreement": 0.19047619047619047 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.37224859620757544, |
| "reference_agreement": 0.36363636363636365 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.43216064330256254, |
| "reference_agreement": 0.2777777777777778 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.49955516847278475, |
| "reference_agreement": 0.375 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.56429092950471, |
| "reference_agreement": 0.49903378260869563 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.6284579234052629, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.7007100933753049, |
| "reference_agreement": 0.5517241379310345 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.7611781134778828, |
| "reference_agreement": 0.68 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.8391754153900324, |
| "reference_agreement": 0.7857142857142857 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.9013655316115832, |
| "reference_agreement": 0.8888888888888888 |
| }, |
| { |
| "count": 235, |
| "confidence": 0.9854872248925048, |
| "reference_agreement": 0.9914893617021276 |
| } |
| ], |
| "expected_score_mae": 0.8394948703521851 |
| }, |
| "finance": { |
| "count": 500, |
| "soft_ce": 0.8372534729882183, |
| "kl": 0.22213311910463876, |
| "brier": 0.05824555646969622, |
| "argmax_agreement": 0.766, |
| "hard_count": 52, |
| "hard_ece": 0.11310128402928836, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.3723071041294397, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.50016902311118, |
| "reference_agreement": 0.6 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5753408310146032, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.6182583123935925, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.6819092006332838, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.7665259528880326, |
| "reference_agreement": 0.4 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.838982843566272, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.881833000901765, |
| "reference_agreement": 0.875 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.9797645091294379, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 16, |
| "confidence": 0.3226339567135692, |
| "reference_agreement": 0.3984374629166975 |
| }, |
| { |
| "count": 41, |
| "confidence": 0.36674248561146494, |
| "reference_agreement": 0.33682928918700217 |
| }, |
| { |
| "count": 62, |
| "confidence": 0.4340519346364484, |
| "reference_agreement": 0.4111594596595161 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.49924896849347133, |
| "reference_agreement": 0.5386290065109846 |
| }, |
| { |
| "count": 70, |
| "confidence": 0.5686484601862719, |
| "reference_agreement": 0.655761905714329 |
| }, |
| { |
| "count": 50, |
| "confidence": 0.6308002075172696, |
| "reference_agreement": 0.6414333730000463 |
| }, |
| { |
| "count": 54, |
| "confidence": 0.6999272500653496, |
| "reference_agreement": 0.6493579925926575 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.7648365922391855, |
| "reference_agreement": 0.7116637471812303 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.8315695075811502, |
| "reference_agreement": 0.8268750565625763 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.8916310587032179, |
| "reference_agreement": 0.9127121363636364 |
| }, |
| { |
| "count": 59, |
| "confidence": 0.9675218038967256, |
| "reference_agreement": 0.9836666610169491 |
| } |
| ], |
| "expected_score_mae": 0.29112017853517996 |
| }, |
| "trust_safety": { |
| "count": 500, |
| "soft_ce": 0.6856993182154376, |
| "kl": 0.31539041995043376, |
| "brier": 0.09066598215826954, |
| "argmax_agreement": 0.734, |
| "hard_count": 222, |
| "hard_ece": 0.06329846877237999, |
| "hard_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.3683034743483865, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.43137594222756087, |
| "reference_agreement": 0.2857142857142857 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.505779039854673, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.569920504948867, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.6316407404385947, |
| "reference_agreement": 0.6428571428571429 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.6972827104351492, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.7672982199470547, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.826724451152989, |
| "reference_agreement": 0.6842105263157895 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.9010283785718083, |
| "reference_agreement": 0.84 |
| }, |
| { |
| "count": 104, |
| "confidence": 0.9729483427736801, |
| "reference_agreement": 0.9711538461538461 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 9, |
| "confidence": 0.2361890508436565, |
| "reference_agreement": 0.27388888888888885 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.30342299595800304, |
| "reference_agreement": 0.2961904285714286 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.3674761849264621, |
| "reference_agreement": 0.3014857858344078 |
| }, |
| { |
| "count": 33, |
| "confidence": 0.4359383368744807, |
| "reference_agreement": 0.3778090549302529 |
| }, |
| { |
| "count": 35, |
| "confidence": 0.5047182741304667, |
| "reference_agreement": 0.517333303803201 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.5629326069104805, |
| "reference_agreement": 0.5935795651642476 |
| }, |
| { |
| "count": 49, |
| "confidence": 0.6337639434256919, |
| "reference_agreement": 0.6102933127493159 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.693334781004934, |
| "reference_agreement": 0.5768357275211178 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.7684317409650803, |
| "reference_agreement": 0.7117894730881641 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.8344109799487145, |
| "reference_agreement": 0.7692592592237804 |
| }, |
| { |
| "count": 44, |
| "confidence": 0.8999478793240552, |
| "reference_agreement": 0.8112662345983765 |
| }, |
| { |
| "count": 139, |
| "confidence": 0.9715298963628789, |
| "reference_agreement": 0.9553096231010606 |
| } |
| ], |
| "expected_score_mae": 0.3249617918461017 |
| }, |
| "documents_meetings": { |
| "count": 500, |
| "soft_ce": 0.9505308054596932, |
| "kl": 0.2819220023348468, |
| "brier": 0.06778096624550085, |
| "argmax_agreement": 0.754, |
| "hard_count": 111, |
| "hard_ece": 0.0745149993383762, |
| "hard_reliability": [ |
| { |
| "count": 7, |
| "confidence": 0.5130857536757218, |
| "reference_agreement": 0.8571428571428571 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.5759649564141678, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.631365854228681, |
| "reference_agreement": 0.8 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.7057283821613531, |
| "reference_agreement": 0.375 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.7734135275347481, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.8276341762810223, |
| "reference_agreement": 0.7857142857142857 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.9061937668671157, |
| "reference_agreement": 0.9090909090909091 |
| }, |
| { |
| "count": 49, |
| "confidence": 0.974518301452977, |
| "reference_agreement": 0.9591836734693877 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 20, |
| "confidence": 0.2507030605824935, |
| "reference_agreement": 0.2815126262626263 |
| }, |
| { |
| "count": 38, |
| "confidence": 0.3027780535641847, |
| "reference_agreement": 0.30187799043062197 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.36910116911685614, |
| "reference_agreement": 0.4304395304395305 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.43197305120456336, |
| "reference_agreement": 0.5305434782608696 |
| }, |
| { |
| "count": 70, |
| "confidence": 0.4998801989425534, |
| "reference_agreement": 0.5903253968253969 |
| }, |
| { |
| "count": 63, |
| "confidence": 0.564395576933704, |
| "reference_agreement": 0.65 |
| }, |
| { |
| "count": 52, |
| "confidence": 0.6360817173907356, |
| "reference_agreement": 0.7249679487179487 |
| }, |
| { |
| "count": 41, |
| "confidence": 0.6999872254915769, |
| "reference_agreement": 0.672560975609756 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.769121243337486, |
| "reference_agreement": 0.8225 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.8294780729236917, |
| "reference_agreement": 0.8451470588235294 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.9032976162782315, |
| "reference_agreement": 0.8896875 |
| }, |
| { |
| "count": 49, |
| "confidence": 0.974518301452977, |
| "reference_agreement": 0.9591836734693877 |
| } |
| ], |
| "expected_score_mae": 0.39489489482681317 |
| }, |
| "customer_support": { |
| "count": 500, |
| "soft_ce": 1.1285135288404535, |
| "kl": 0.545722865769657, |
| "brier": 0.108362788973766, |
| "argmax_agreement": 0.706, |
| "hard_count": 142, |
| "hard_ece": 0.11241881164831904, |
| "hard_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.11589315469434829, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.1466965963178435, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.23363389271036505, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.3018620938804334, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.37797570812443015, |
| "reference_agreement": 0.2222222222222222 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.4272157242978048, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.5005016740966861, |
| "reference_agreement": 0.14285714285714285 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.5675968327842555, |
| "reference_agreement": 0.8571428571428571 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.6248700358768576, |
| "reference_agreement": 0.5714285714285714 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.7018524259315515, |
| "reference_agreement": 0.5555555555555556 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.763652193538776, |
| "reference_agreement": 0.7222222222222222 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.8335790279408464, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.9023386004425975, |
| "reference_agreement": 0.8125 |
| }, |
| { |
| "count": 39, |
| "confidence": 0.9678176556415846, |
| "reference_agreement": 0.9487179487179487 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.11589315469434829, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.15381363979618545, |
| "reference_agreement": 0.3198346624136098 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.24077671068363551, |
| "reference_agreement": 0.29337121212121214 |
| }, |
| { |
| "count": 35, |
| "confidence": 0.3022639463465406, |
| "reference_agreement": 0.3521546318325211 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.37365140372838623, |
| "reference_agreement": 0.35323562863250324 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.4350034398300646, |
| "reference_agreement": 0.36927072142944645 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.5024000185598814, |
| "reference_agreement": 0.4825757636363636 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.5652814585293939, |
| "reference_agreement": 0.631175707569653 |
| }, |
| { |
| "count": 47, |
| "confidence": 0.6319760416224046, |
| "reference_agreement": 0.6904964592198635 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.7024876007617367, |
| "reference_agreement": 0.669458325 |
| }, |
| { |
| "count": 43, |
| "confidence": 0.766737205353825, |
| "reference_agreement": 0.7386433953488373 |
| }, |
| { |
| "count": 33, |
| "confidence": 0.83741488958767, |
| "reference_agreement": 0.7505555454545454 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.8973418194210278, |
| "reference_agreement": 0.8589099459459459 |
| }, |
| { |
| "count": 43, |
| "confidence": 0.965788577174672, |
| "reference_agreement": 0.9501550236821856 |
| } |
| ], |
| "expected_score_mae": 0.33319003384682117 |
| }, |
| "ai_agents_evals": { |
| "count": 500, |
| "soft_ce": 0.7625932921761905, |
| "kl": 0.6730676386853419, |
| "brier": 0.18645192181998924, |
| "argmax_agreement": 0.68, |
| "hard_count": 417, |
| "hard_ece": 0.052463098512104685, |
| "hard_reliability": [ |
| { |
| "count": 5, |
| "confidence": 0.18618071314902357, |
| "reference_agreement": 0.2 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.23838633417547514, |
| "reference_agreement": 0.2 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.295781111577869, |
| "reference_agreement": 0.2777777777777778 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.37139215520169777, |
| "reference_agreement": 0.4230769230769231 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.43633042832674923, |
| "reference_agreement": 0.45454545454545453 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.5119118632329336, |
| "reference_agreement": 0.4482758620689655 |
| }, |
| { |
| "count": 51, |
| "confidence": 0.5699148465657553, |
| "reference_agreement": 0.47058823529411764 |
| }, |
| { |
| "count": 38, |
| "confidence": 0.6258474222785992, |
| "reference_agreement": 0.5263157894736842 |
| }, |
| { |
| "count": 42, |
| "confidence": 0.692690594223072, |
| "reference_agreement": 0.7380952380952381 |
| }, |
| { |
| "count": 35, |
| "confidence": 0.7681140161954323, |
| "reference_agreement": 0.7714285714285715 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.8344200795661016, |
| "reference_agreement": 0.85 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.9006389466100536, |
| "reference_agreement": 0.9347826086956522 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.9717353590779876, |
| "reference_agreement": 0.8909090909090909 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 5, |
| "confidence": 0.18618071314902357, |
| "reference_agreement": 0.2 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.23838633417547514, |
| "reference_agreement": 0.2 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.295781111577869, |
| "reference_agreement": 0.2777777777777778 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.3733951639465118, |
| "reference_agreement": 0.41724137931034483 |
| }, |
| { |
| "count": 30, |
| "confidence": 0.4336998105448617, |
| "reference_agreement": 0.44690416558973367 |
| }, |
| { |
| "count": 39, |
| "confidence": 0.5120708414698647, |
| "reference_agreement": 0.4701608617898467 |
| }, |
| { |
| "count": 58, |
| "confidence": 0.5690122989604566, |
| "reference_agreement": 0.4917532841954055 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.6283615036372431, |
| "reference_agreement": 0.5640713243854648 |
| }, |
| { |
| "count": 54, |
| "confidence": 0.6932351995199476, |
| "reference_agreement": 0.7547641003856951 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.7641095557632885, |
| "reference_agreement": 0.7842272393489996 |
| }, |
| { |
| "count": 51, |
| "confidence": 0.8318454353186426, |
| "reference_agreement": 0.8372494792826007 |
| }, |
| { |
| "count": 55, |
| "confidence": 0.9002777292404353, |
| "reference_agreement": 0.9324725290016627 |
| }, |
| { |
| "count": 60, |
| "confidence": 0.9709295090707568, |
| "reference_agreement": 0.8988023166666667 |
| } |
| ], |
| "expected_score_mae": 0.8456565717140646 |
| }, |
| "legal": { |
| "count": 122, |
| "soft_ce": 0.5436294554804777, |
| "kl": 0.5436294554804777, |
| "brier": 0.1304449623060641, |
| "argmax_agreement": 0.819672131147541, |
| "hard_count": 122, |
| "hard_ece": 0.08307026110171992, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.2542299113522521, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.28724827093183936, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.377085304774584, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.4441052741464433, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5174342366971163, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.5644377124854338, |
| "reference_agreement": 0.5714285714285714 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.6251477859811808, |
| "reference_agreement": 0.8 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.7044969749942575, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.7587080210898536, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.8354103639312468, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.8963652039349991, |
| "reference_agreement": 0.7692307692307693 |
| }, |
| { |
| "count": 66, |
| "confidence": 0.9815239804604637, |
| "reference_agreement": 0.9848484848484849 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.2542299113522521, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.28724827093183936, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.377085304774584, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.4441052741464433, |
| "reference_agreement": 0.3333333333333333 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5174342366971163, |
| "reference_agreement": 0.0 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.5644377124854338, |
| "reference_agreement": 0.5714285714285714 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.6251477859811808, |
| "reference_agreement": 0.8 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.7044969749942575, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.7587080210898536, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.8354103639312468, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.8963652039349991, |
| "reference_agreement": 0.7692307692307693 |
| }, |
| { |
| "count": 66, |
| "confidence": 0.9815239804604637, |
| "reference_agreement": 0.9848484848484849 |
| } |
| ], |
| "expected_score_mae": 0.05800823336689319 |
| }, |
| "sales_commerce": { |
| "count": 71, |
| "soft_ce": 0.43972836851867453, |
| "kl": 0.43972836851867453, |
| "brier": 0.13079004940228145, |
| "argmax_agreement": 0.8169014084507042, |
| "hard_count": 71, |
| "hard_ece": 0.08143885493164209, |
| "hard_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.4480862145923521, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5225530044071978, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.5799241321545268, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.6273970582753347, |
| "reference_agreement": 0.2 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.6962808034900051, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7775016660973879, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.8388575355152714, |
| "reference_agreement": 0.7 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.9005931196007729, |
| "reference_agreement": 0.9 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.9756813796187571, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.4480862145923521, |
| "reference_agreement": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5225530044071978, |
| "reference_agreement": 0.5 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.5799241321545268, |
| "reference_agreement": 0.6666666666666666 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.6273970582753347, |
| "reference_agreement": 0.2 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.6962808034900051, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7775016660973879, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.8388575355152714, |
| "reference_agreement": 0.7 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.9005931196007729, |
| "reference_agreement": 0.9 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.9756813796187571, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "expected_score_mae": 0.3234410160549905 |
| } |
| } |
| }, |
| "permutation": { |
| "count": 256, |
| "completed": true, |
| "mean_total_variation": 0.061152504018688186, |
| "answer_flip_rate": 0.0625 |
| } |
| }, |
| "typed_decisions": { |
| "dataset": "LocalLLaMA/typed-decisions", |
| "revision": "7189f46b02248bd2569df7325c9df8e3b24e90a7", |
| "config": "all", |
| "split": "test", |
| "cases": 400, |
| "decisions": 2000, |
| "errors": 0, |
| "raw": { |
| "overall": { |
| "count": 2000, |
| "soft_ce": 0.8982348703026427, |
| "kl": 0.13087631467831382, |
| "brier": 0.03733508375955404, |
| "argmax_agreement": 0.7055, |
| "hard_count": 2, |
| "hard_ece": 0.02813636220158644, |
| "hard_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.9718636377984136, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 24, |
| "confidence": 0.31520772849577633, |
| "reference_agreement": 0.3355554526389121 |
| }, |
| { |
| "count": 171, |
| "confidence": 0.3746159687360105, |
| "reference_agreement": 0.38106237192013953 |
| }, |
| { |
| "count": 368, |
| "confidence": 0.43486926682677546, |
| "reference_agreement": 0.41427729467341934 |
| }, |
| { |
| "count": 330, |
| "confidence": 0.5005758970577506, |
| "reference_agreement": 0.4898888885960213 |
| }, |
| { |
| "count": 259, |
| "confidence": 0.5661637493278858, |
| "reference_agreement": 0.5401029315058599 |
| }, |
| { |
| "count": 208, |
| "confidence": 0.6334207148716826, |
| "reference_agreement": 0.5813301291346679 |
| }, |
| { |
| "count": 176, |
| "confidence": 0.6976026942523854, |
| "reference_agreement": 0.6650378634451258 |
| }, |
| { |
| "count": 117, |
| "confidence": 0.765488790351476, |
| "reference_agreement": 0.6715926028234322 |
| }, |
| { |
| "count": 95, |
| "confidence": 0.8268994900267613, |
| "reference_agreement": 0.7595845223159148 |
| }, |
| { |
| "count": 90, |
| "confidence": 0.8962419341009388, |
| "reference_agreement": 0.9119985310000607 |
| }, |
| { |
| "count": 162, |
| "confidence": 0.9675863551773324, |
| "reference_agreement": 0.9661111104013173 |
| } |
| ], |
| "expected_score_mae": 0.2689357269785638, |
| "gold_label_correct": 1409, |
| "gold_label_accuracy": 0.7045, |
| "gold_label_ece_15bin_all_decisions": 0.10329466050980009, |
| "full_vector_brier": 0.07467016751910809 |
| }, |
| "type": { |
| "choice": { |
| "count": 600, |
| "soft_ce": 1.0297958099292934, |
| "kl": 0.1660012130824713, |
| "brier": 0.041920794479682215, |
| "argmax_agreement": 0.6883333333333334, |
| "hard_count": 1, |
| "hard_ece": 0.023186746014468573, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9768132539855314, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 14, |
| "confidence": 0.3173165797051884, |
| "reference_agreement": 0.39047612547625926 |
| }, |
| { |
| "count": 114, |
| "confidence": 0.3727881473417713, |
| "reference_agreement": 0.3832163580409846 |
| }, |
| { |
| "count": 160, |
| "confidence": 0.4341875923805457, |
| "reference_agreement": 0.4184011179583752 |
| }, |
| { |
| "count": 92, |
| "confidence": 0.4936694315907819, |
| "reference_agreement": 0.511159424456586 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.5673671375615373, |
| "reference_agreement": 0.5801388408334563 |
| }, |
| { |
| "count": 31, |
| "confidence": 0.6339963551108706, |
| "reference_agreement": 0.6433333275269537 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.6907448914335091, |
| "reference_agreement": 0.7217737262976213 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.764159840829608, |
| "reference_agreement": 0.6959089121213682 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.8290876216067908, |
| "reference_agreement": 0.7640788558334829 |
| }, |
| { |
| "count": 33, |
| "confidence": 0.8944038265429061, |
| "reference_agreement": 0.9230363914142908 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.9583538062949253, |
| "reference_agreement": 0.955179537115826 |
| } |
| ], |
| "gold_label_correct": 413, |
| "gold_label_accuracy": 0.6883333333333334, |
| "gold_label_ece_15bin_all_decisions": 0.1452815073319075, |
| "full_vector_brier": 0.08384158895936443 |
| }, |
| "noul": { |
| "count": 600, |
| "soft_ce": 0.5376753319725679, |
| "kl": 0.08395308862000342, |
| "brier": 0.034802117110535095, |
| "argmax_agreement": 0.7666666666666667, |
| "hard_count": 1, |
| "hard_ece": 0.033085978388704196, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9669140216112958, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 59, |
| "confidence": 0.5179912574843223, |
| "reference_agreement": 0.48372877966101696 |
| }, |
| { |
| "count": 86, |
| "confidence": 0.5678122996718532, |
| "reference_agreement": 0.525968976744186 |
| }, |
| { |
| "count": 85, |
| "confidence": 0.6369483441744728, |
| "reference_agreement": 0.5698431176470588 |
| }, |
| { |
| "count": 85, |
| "confidence": 0.6995096580475858, |
| "reference_agreement": 0.6567059176470588 |
| }, |
| { |
| "count": 63, |
| "confidence": 0.7673512554565463, |
| "reference_agreement": 0.6198413174603175 |
| }, |
| { |
| "count": 47, |
| "confidence": 0.8295192389350612, |
| "reference_agreement": 0.740141914893617 |
| }, |
| { |
| "count": 54, |
| "confidence": 0.8971869226053684, |
| "reference_agreement": 0.9035740740740741 |
| }, |
| { |
| "count": 121, |
| "confidence": 0.9707946773144384, |
| "reference_agreement": 0.9677850826446281 |
| } |
| ], |
| "gold_label_correct": 459, |
| "gold_label_accuracy": 0.765, |
| "gold_label_ece_15bin_all_decisions": 0.037528508953695645, |
| "full_vector_brier": 0.06960423422107019 |
| }, |
| "score": { |
| "count": 800, |
| "soft_ce": 1.0699838193302107, |
| "kl": 0.1397250604189285, |
| "brier": 0.03579552570622214, |
| "argmax_agreement": 0.6725, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 10, |
| "confidence": 0.3122553368025993, |
| "reference_agreement": 0.25866651066662605 |
| }, |
| { |
| "count": 57, |
| "confidence": 0.3782716115244891, |
| "reference_agreement": 0.3767543996784493 |
| }, |
| { |
| "count": 208, |
| "confidence": 0.4353936317854138, |
| "reference_agreement": 0.41110512291576096 |
| }, |
| { |
| "count": 179, |
| "confidence": 0.4983853303415126, |
| "reference_agreement": 0.4809869731099504 |
| }, |
| { |
| "count": 125, |
| "confidence": 0.564567445609514, |
| "reference_agreement": 0.5344533032000944 |
| }, |
| { |
| "count": 92, |
| "confidence": 0.6299675264135088, |
| "reference_agreement": 0.5710507468116885 |
| }, |
| { |
| "count": 63, |
| "confidence": 0.6980777189561393, |
| "reference_agreement": 0.6510634385715672 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.7627357149721534, |
| "reference_agreement": 0.7567604832397332 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.8148277144485708, |
| "reference_agreement": 0.8077085145837775 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.8994513241595642, |
| "reference_agreement": 0.9422222911112911 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.9577089746675174, |
| "reference_agreement": 0.9715557946667946 |
| } |
| ], |
| "expected_score_mae": 0.2689357269785638, |
| "gold_label_correct": 537, |
| "gold_label_accuracy": 0.67125, |
| "gold_label_ece_15bin_all_decisions": 0.13471575955048368, |
| "full_vector_brier": 0.07159105141244428 |
| } |
| }, |
| "category": { |
| "agent_trace_observability": { |
| "count": 500, |
| "soft_ce": 0.97776378147771, |
| "kl": 0.09929350853248653, |
| "brier": 0.029319985421551766, |
| "argmax_agreement": 0.666, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.3271961322725437, |
| "reference_agreement": 0.333333 |
| }, |
| { |
| "count": 35, |
| "confidence": 0.3798205422614706, |
| "reference_agreement": 0.40077146666672475 |
| }, |
| { |
| "count": 132, |
| "confidence": 0.43641602107268646, |
| "reference_agreement": 0.4047222168687381 |
| }, |
| { |
| "count": 117, |
| "confidence": 0.5006302500044424, |
| "reference_agreement": 0.47911679880348107 |
| }, |
| { |
| "count": 76, |
| "confidence": 0.5690619634091041, |
| "reference_agreement": 0.5512718920614565 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.6338679776529896, |
| "reference_agreement": 0.6111852223703935 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.6938113959359317, |
| "reference_agreement": 0.6235998789334686 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.7711383973508009, |
| "reference_agreement": 0.6864285047620653 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.823789261655095, |
| "reference_agreement": 0.7322917248959386 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.8940903067075735, |
| "reference_agreement": 0.8727536086956522 |
| } |
| ], |
| "expected_score_mae": 0.20094214035170185, |
| "gold_label_correct": 335, |
| "gold_label_accuracy": 0.67, |
| "gold_label_ece_15bin_all_decisions": 0.11802699803796707, |
| "full_vector_brier": 0.05863997084310353 |
| }, |
| "customer_service": { |
| "count": 500, |
| "soft_ce": 0.9356140291801783, |
| "kl": 0.19845939117069544, |
| "brier": 0.054488558914342955, |
| "argmax_agreement": 0.694, |
| "hard_count": 1, |
| "hard_ece": 0.023186746014468573, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9768132539855314, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 10, |
| "confidence": 0.31477128374516633, |
| "reference_agreement": 0.4056665940000813 |
| }, |
| { |
| "count": 39, |
| "confidence": 0.3723859508926986, |
| "reference_agreement": 0.45632472794882123 |
| }, |
| { |
| "count": 73, |
| "confidence": 0.4380748853548461, |
| "reference_agreement": 0.3939339658448251 |
| }, |
| { |
| "count": 81, |
| "confidence": 0.5011240542197034, |
| "reference_agreement": 0.5185185261729083 |
| }, |
| { |
| "count": 80, |
| "confidence": 0.5649588987676203, |
| "reference_agreement": 0.5116249845417362 |
| }, |
| { |
| "count": 69, |
| "confidence": 0.6279948891492104, |
| "reference_agreement": 0.5734783022222739 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.6947774859534929, |
| "reference_agreement": 0.7216593691232294 |
| }, |
| { |
| "count": 33, |
| "confidence": 0.7624282639003352, |
| "reference_agreement": 0.6597979875758164 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.8310417874707298, |
| "reference_agreement": 0.8232055630304752 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.8956622165551549, |
| "reference_agreement": 0.913333472222457 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.9583538062949253, |
| "reference_agreement": 0.955179537115826 |
| } |
| ], |
| "expected_score_mae": 0.3528748230925414, |
| "gold_label_correct": 347, |
| "gold_label_accuracy": 0.694, |
| "gold_label_ece_15bin_all_decisions": 0.09884340864299247, |
| "full_vector_brier": 0.10897711782868591 |
| }, |
| "invoice_processing": { |
| "count": 500, |
| "soft_ce": 0.6654727615892404, |
| "kl": 0.09052528592434582, |
| "brier": 0.02417431701033401, |
| "argmax_agreement": 0.84, |
| "hard_count": 1, |
| "hard_ece": 0.033085978388704196, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9669140216112958, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.3278949544267834, |
| "reference_agreement": 0.5716665 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.36518717835723197, |
| "reference_agreement": 0.3640449954955256 |
| }, |
| { |
| "count": 44, |
| "confidence": 0.43272176571282106, |
| "reference_agreement": 0.4239439440863979 |
| }, |
| { |
| "count": 47, |
| "confidence": 0.4990750370635605, |
| "reference_agreement": 0.4812765950355347 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.5636676327793726, |
| "reference_agreement": 0.5589167148334007 |
| }, |
| { |
| "count": 44, |
| "confidence": 0.6359292707198466, |
| "reference_agreement": 0.618181796969778 |
| }, |
| { |
| "count": 53, |
| "confidence": 0.6993117889365471, |
| "reference_agreement": 0.6881194986793073 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.7613192737634586, |
| "reference_agreement": 0.7343039374609032 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.8299096395763272, |
| "reference_agreement": 0.8130160017462573 |
| }, |
| { |
| "count": 43, |
| "confidence": 0.8979274834748605, |
| "reference_agreement": 0.9520123691472993 |
| }, |
| { |
| "count": 135, |
| "confidence": 0.9695963375504574, |
| "reference_agreement": 0.9691999920000142 |
| } |
| ], |
| "expected_score_mae": 0.21560838991268416, |
| "gold_label_correct": 417, |
| "gold_label_accuracy": 0.834, |
| "gold_label_ece_15bin_all_decisions": 0.11985107384860202, |
| "full_vector_brier": 0.04834863402066802 |
| }, |
| "security_incidents": { |
| "count": 500, |
| "soft_ce": 1.0140889089634417, |
| "kl": 0.13522707308572743, |
| "brier": 0.041357473691987466, |
| "argmax_agreement": 0.622, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 11, |
| "confidence": 0.31220787321098703, |
| "reference_agreement": 0.2290908112120979 |
| }, |
| { |
| "count": 60, |
| "confidence": 0.37884389984455846, |
| "reference_agreement": 0.33113891736116685 |
| }, |
| { |
| "count": 119, |
| "confidence": 0.4319811015831166, |
| "reference_agreement": 0.43378150224093404 |
| }, |
| { |
| "count": 85, |
| "confidence": 0.5008086075853485, |
| "reference_agreement": 0.48219608458828284 |
| }, |
| { |
| "count": 63, |
| "confidence": 0.5657822959404127, |
| "reference_agreement": 0.550846477883684 |
| }, |
| { |
| "count": 50, |
| "confidence": 0.6382982887191337, |
| "reference_agreement": 0.5328665986667214 |
| }, |
| { |
| "count": 52, |
| "confidence": 0.7001826946639205, |
| "reference_agreement": 0.6113462035256458 |
| }, |
| { |
| "count": 36, |
| "confidence": 0.7700350803206342, |
| "reference_agreement": 0.6174074444444444 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.8241586712060176, |
| "reference_agreement": 0.6771668 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.8926362259110124, |
| "reference_agreement": 0.63 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.9362850057480346, |
| "reference_agreement": 0.833333 |
| } |
| ], |
| "expected_score_mae": 0.3063175545573279, |
| "gold_label_correct": 310, |
| "gold_label_accuracy": 0.62, |
| "gold_label_ece_15bin_all_decisions": 0.11599525503936776, |
| "full_vector_brier": 0.08271494738397493 |
| } |
| }, |
| "question": { |
| "agent_trace_observability/action": { |
| "count": 100, |
| "soft_ce": 1.1235396454195041, |
| "kl": 0.08949577063715862, |
| "brier": 0.023420285987952504, |
| "argmax_agreement": 0.66, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.3271961322725437, |
| "reference_agreement": 0.333333 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.37758519376271876, |
| "reference_agreement": 0.4010834071429083 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.42300368726206095, |
| "reference_agreement": 0.40872541411767294 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.508725847017595, |
| "reference_agreement": 0.6019047511905369 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.5700440548082534, |
| "reference_agreement": 0.6007246056522724 |
| } |
| ], |
| "gold_label_correct": 66, |
| "gold_label_accuracy": 0.66, |
| "gold_label_ece_15bin_all_decisions": 0.21139510221170185, |
| "full_vector_brier": 0.04684057197590501 |
| }, |
| "agent_trace_observability/needs_review": { |
| "count": 100, |
| "soft_ce": 0.5772198756410069, |
| "kl": 0.06175400802011981, |
| "brier": 0.026948573243994925, |
| "argmax_agreement": 0.78, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 17, |
| "confidence": 0.522037312516128, |
| "reference_agreement": 0.48039217647058824 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.5740099433811946, |
| "reference_agreement": 0.5532098148148148 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.6307027904464286, |
| "reference_agreement": 0.6412281052631579 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.679178699175393, |
| "reference_agreement": 0.766667 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.831497587641693, |
| "reference_agreement": 0.8541025384615385 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.8940903067075735, |
| "reference_agreement": 0.8727536086956522 |
| } |
| ], |
| "gold_label_correct": 78, |
| "gold_label_accuracy": 0.78, |
| "gold_label_ece_15bin_all_decisions": 0.09591019804659835, |
| "full_vector_brier": 0.05389714648798985 |
| }, |
| "agent_trace_observability/outcome": { |
| "count": 100, |
| "soft_ce": 1.0935064791897204, |
| "kl": 0.16872011458553096, |
| "brier": 0.04551726407182359, |
| "argmax_agreement": 0.62, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.39948630465030277, |
| "reference_agreement": 0.233333 |
| }, |
| { |
| "count": 36, |
| "confidence": 0.44371195558609283, |
| "reference_agreement": 0.36223152500002964 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.4869204497761252, |
| "reference_agreement": 0.42333332505753857 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.5457545962932366, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.6334661880424128, |
| "reference_agreement": 0.8091667499999999 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.68759444755407, |
| "reference_agreement": 0.6505555205556461 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.7680981282943802, |
| "reference_agreement": 0.6516665095834904 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.820754732086627, |
| "reference_agreement": 0.6195556666667333 |
| } |
| ], |
| "gold_label_correct": 60, |
| "gold_label_accuracy": 0.6, |
| "gold_label_ece_15bin_all_decisions": 0.08832517707815522, |
| "full_vector_brier": 0.09103452814364718 |
| }, |
| "agent_trace_observability/risk": { |
| "count": 100, |
| "soft_ce": 0.9829069466566531, |
| "kl": 0.09053683824790124, |
| "brier": 0.027227209135250532, |
| "argmax_agreement": 0.67, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 6, |
| "confidence": 0.3869745415241738, |
| "reference_agreement": 0.4272221555556555 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.4430243807991401, |
| "reference_agreement": 0.4377777200001155 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.5064388102385512, |
| "reference_agreement": 0.552666578333445 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.5633028160410002, |
| "reference_agreement": 0.5042424242424954 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.6350598264314197, |
| "reference_agreement": 0.5333333022222356 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.6937211677482574, |
| "reference_agreement": 0.5712818941026444 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.780801733899054, |
| "reference_agreement": 0.7526666713334713 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.810116688080407, |
| "reference_agreement": 0.7591667991672592 |
| } |
| ], |
| "expected_score_mae": 0.1887612567569762, |
| "gold_label_correct": 69, |
| "gold_label_accuracy": 0.69, |
| "gold_label_ece_15bin_all_decisions": 0.14607924787983295, |
| "full_vector_brier": 0.054454418270501063 |
| }, |
| "agent_trace_observability/urgency": { |
| "count": 100, |
| "soft_ce": 1.111645960481666, |
| "kl": 0.08596081117172201, |
| "brier": 0.023486594668737287, |
| "argmax_agreement": 0.6, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 47, |
| "confidence": 0.43842113407655536, |
| "reference_agreement": 0.42382272014190947 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.49533713057352974, |
| "reference_agreement": 0.4360360527027635 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.5670036460048613, |
| "reference_agreement": 0.43333317777784447 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.6401348324659201, |
| "reference_agreement": 0.5833334961905962 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.7044328666342266, |
| "reference_agreement": 0.6986664453336919 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.7471438670609017, |
| "reference_agreement": 0.6333336333336334 |
| } |
| ], |
| "expected_score_mae": 0.21312302394642757, |
| "gold_label_correct": 62, |
| "gold_label_accuracy": 0.62, |
| "gold_label_ece_15bin_all_decisions": 0.12615469901673237, |
| "full_vector_brier": 0.046973189337474575 |
| }, |
| "customer_service/action": { |
| "count": 100, |
| "soft_ce": 1.241760340801668, |
| "kl": 0.275902444840478, |
| "brier": 0.06755224071529659, |
| "argmax_agreement": 0.6, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 9, |
| "confidence": 0.31273256683149947, |
| "reference_agreement": 0.3525925488889793 |
| }, |
| { |
| "count": 34, |
| "confidence": 0.37221056069924124, |
| "reference_agreement": 0.4573528766667779 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.4324976531209652, |
| "reference_agreement": 0.4009538457334028 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.49707265548386304, |
| "reference_agreement": 0.5388333921667305 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.5563524243492388, |
| "reference_agreement": 0.6407407485186634 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.6089655400408842, |
| "reference_agreement": 0.35 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.705852717684706, |
| "reference_agreement": 0.7800003916668916 |
| } |
| ], |
| "gold_label_correct": 61, |
| "gold_label_accuracy": 0.61, |
| "gold_label_ece_15bin_all_decisions": 0.18966441682569232, |
| "full_vector_brier": 0.13510448143059317 |
| }, |
| "customer_service/category": { |
| "count": 100, |
| "soft_ce": 0.5646570684297757, |
| "kl": 0.15423518549593282, |
| "brier": 0.03216723408432829, |
| "argmax_agreement": 0.93, |
| "hard_count": 1, |
| "hard_ece": 0.023186746014468573, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9768132539855314, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.4430349798534358, |
| "reference_agreement": 0.6166666055560389 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.5046243895746633, |
| "reference_agreement": 0.635333516000116 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.5798365858143383, |
| "reference_agreement": 0.5683333691670058 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.6384560218456447, |
| "reference_agreement": 0.6099999795834395 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.6915068535481631, |
| "reference_agreement": 0.8281108069998411 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.7596657412667888, |
| "reference_agreement": 0.6937035451853623 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.8332816589849125, |
| "reference_agreement": 0.8847014257780302 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.8956622165551549, |
| "reference_agreement": 0.913333472222457 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.9583538062949253, |
| "reference_agreement": 0.955179537115826 |
| } |
| ], |
| "gold_label_correct": 94, |
| "gold_label_accuracy": 0.94, |
| "gold_label_ece_15bin_all_decisions": 0.1407358753162401, |
| "full_vector_brier": 0.06433446816865658 |
| }, |
| "customer_service/churn_risk": { |
| "count": 100, |
| "soft_ce": 1.0723716037197997, |
| "kl": 0.18195307128399665, |
| "brier": 0.05156803196362559, |
| "argmax_agreement": 0.71, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.3331197359681683, |
| "reference_agreement": 0.883333 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.3735786042082082, |
| "reference_agreement": 0.4493333166667167 |
| }, |
| { |
| "count": 31, |
| "confidence": 0.4406830914886085, |
| "reference_agreement": 0.388172083225866 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.5011742997082412, |
| "reference_agreement": 0.578589826794943 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.5633032673902467, |
| "reference_agreement": 0.5179629694445509 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.6284474784336418, |
| "reference_agreement": 0.6143137843137941 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.6870135499018379, |
| "reference_agreement": 0.6716668416668417 |
| } |
| ], |
| "expected_score_mae": 0.3151828379933619, |
| "gold_label_correct": 71, |
| "gold_label_accuracy": 0.71, |
| "gold_label_ece_15bin_all_decisions": 0.19910186568229626, |
| "full_vector_brier": 0.10313606392725118 |
| }, |
| "customer_service/needs_human": { |
| "count": 100, |
| "soft_ce": 0.6761560528212277, |
| "kl": 0.12704364399222592, |
| "brier": 0.05613679930099221, |
| "argmax_agreement": 0.66, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 8, |
| "confidence": 0.517557176152255, |
| "reference_agreement": 0.482499875 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.5656107740630231, |
| "reference_agreement": 0.5101515 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.6310079557436046, |
| "reference_agreement": 0.5293334 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.7003291402466889, |
| "reference_agreement": 0.7098333 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.7643885662708242, |
| "reference_agreement": 0.6455073043478261 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.8262420627974808, |
| "reference_agreement": 0.6914287142857143 |
| } |
| ], |
| "gold_label_correct": 65, |
| "gold_label_accuracy": 0.65, |
| "gold_label_ece_15bin_all_decisions": 0.08754395103445267, |
| "full_vector_brier": 0.11227359860198442 |
| }, |
| "customer_service/urgency": { |
| "count": 100, |
| "soft_ce": 1.1231250801284207, |
| "kl": 0.25316261024084386, |
| "brier": 0.06501848850747206, |
| "argmax_agreement": 0.57, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 14, |
| "confidence": 0.4411960376551762, |
| "reference_agreement": 0.34642849761908573 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.4979765514824134, |
| "reference_agreement": 0.41560594090917574 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.5661962183925248, |
| "reference_agreement": 0.45716044814818446 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.6222290168802378, |
| "reference_agreement": 0.5786956818841255 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.6879913332169492, |
| "reference_agreement": 0.664871788461732 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.7422040130810093, |
| "reference_agreement": 0.6833336833336833 |
| } |
| ], |
| "expected_score_mae": 0.39056680819172096, |
| "gold_label_correct": 56, |
| "gold_label_accuracy": 0.56, |
| "gold_label_ece_15bin_all_decisions": 0.12422167823236911, |
| "full_vector_brier": 0.13003697701494413 |
| }, |
| "invoice_processing/discrepancy_severity": { |
| "count": 100, |
| "soft_ce": 0.7813166885573835, |
| "kl": 0.13707332422163884, |
| "brier": 0.03045788400272809, |
| "argmax_agreement": 0.83, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 10, |
| "confidence": 0.36318909440456576, |
| "reference_agreement": 0.452666601666755 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.41585765000740066, |
| "reference_agreement": 0.4002355722088945 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.5029107001361107, |
| "reference_agreement": 0.46666675555575554 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.554937459938247, |
| "reference_agreement": 0.5833334000001142 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.6405068280987012, |
| "reference_agreement": 0.7933334066669266 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.7045803972407967, |
| "reference_agreement": 0.7338461728206614 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.7675469747476337, |
| "reference_agreement": 0.7801667003890337 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.8164401230878163, |
| "reference_agreement": 0.8336366139397898 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.8994513241595642, |
| "reference_agreement": 0.9422222911112911 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.9577089746675174, |
| "reference_agreement": 0.9715557946667946 |
| } |
| ], |
| "expected_score_mae": 0.27761394944700535, |
| "gold_label_correct": 83, |
| "gold_label_accuracy": 0.83, |
| "gold_label_ece_15bin_all_decisions": 0.1551419518932857, |
| "full_vector_brier": 0.06091576800545618 |
| }, |
| "invoice_processing/disposition": { |
| "count": 100, |
| "soft_ce": 0.9672103205660691, |
| "kl": 0.14375101661084821, |
| "brier": 0.03838214719887722, |
| "argmax_agreement": 0.74, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.3278949544267834, |
| "reference_agreement": 0.5716665 |
| }, |
| { |
| "count": 24, |
| "confidence": 0.3638647878856312, |
| "reference_agreement": 0.3302638513888964 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.4371119558468517, |
| "reference_agreement": 0.42291669833338374 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.48657462908745097, |
| "reference_agreement": 0.5753333066666932 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.5743529947838498, |
| "reference_agreement": 0.39799984333333 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.6335226967478499, |
| "reference_agreement": 0.637592573148225 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.6890930143191779, |
| "reference_agreement": 0.6630302878788333 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.7659479600990471, |
| "reference_agreement": 0.7706664166667834 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.8601290126721075, |
| "reference_agreement": 0.9433335 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.8922016440214707, |
| "reference_agreement": 0.9400165 |
| } |
| ], |
| "gold_label_correct": 74, |
| "gold_label_accuracy": 0.74, |
| "gold_label_ece_15bin_all_decisions": 0.20816706463754298, |
| "full_vector_brier": 0.07676429439775444 |
| }, |
| "invoice_processing/duplicate": { |
| "count": 100, |
| "soft_ce": 0.2334091092156495, |
| "kl": 0.02585605705240466, |
| "brier": 0.007688562652478144, |
| "argmax_agreement": 0.94, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 9, |
| "confidence": 0.5190740869546441, |
| "reference_agreement": 0.5159257777777778 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.5672429607854125, |
| "reference_agreement": 0.5161113333333334 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.6224593312018546, |
| "reference_agreement": 0.046667 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.679178699175393, |
| "reference_agreement": 0.75 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.9192141605617763, |
| "reference_agreement": 0.9795238571428572 |
| }, |
| { |
| "count": 76, |
| "confidence": 0.973002201351445, |
| "reference_agreement": 0.9698990789473684 |
| } |
| ], |
| "gold_label_correct": 94, |
| "gold_label_accuracy": 0.94, |
| "gold_label_ece_15bin_all_decisions": 0.05485489658079937, |
| "full_vector_brier": 0.015377125304956288 |
| }, |
| "invoice_processing/matches_order": { |
| "count": 100, |
| "soft_ce": 0.3306428558838697, |
| "kl": 0.07295218240515576, |
| "brier": 0.025035670669917, |
| "argmax_agreement": 0.91, |
| "hard_count": 1, |
| "hard_ece": 0.033085978388704196, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9669140216112958, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.5, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.5467381519846138, |
| "reference_agreement": 0.05 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.6494972618666175, |
| "reference_agreement": 0.742916625 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.6945122010594892, |
| "reference_agreement": 0.7448718461538462 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7602322944976581, |
| "reference_agreement": 0.49416675 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.8444382467004503, |
| "reference_agreement": 0.7571428571428571 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.8938861411785824, |
| "reference_agreement": 0.951095238095238 |
| }, |
| { |
| "count": 44, |
| "confidence": 0.9677659919679358, |
| "reference_agreement": 0.9671893636363635 |
| } |
| ], |
| "gold_label_correct": 91, |
| "gold_label_accuracy": 0.91, |
| "gold_label_ece_15bin_all_decisions": 0.10098650283026388, |
| "full_vector_brier": 0.050071341339834 |
| }, |
| "invoice_processing/urgency": { |
| "count": 100, |
| "soft_ce": 1.0147848337232301, |
| "kl": 0.07299384933168163, |
| "brier": 0.01930732052766958, |
| "argmax_agreement": 0.78, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.38242658197225876, |
| "reference_agreement": 0.33888879444446113 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.44677704982334543, |
| "reference_agreement": 0.4525640602564577 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.49440195405535564, |
| "reference_agreement": 0.47142861309533574 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.5638182000471827, |
| "reference_agreement": 0.6255555988889798 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.629708983631971, |
| "reference_agreement": 0.4805554763889625 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.7077412786031287, |
| "reference_agreement": 0.6135777344445011 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.7426201120020457, |
| "reference_agreement": 0.7276191690477404 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.8159353248895248, |
| "reference_agreement": 0.7166662833337166 |
| } |
| ], |
| "expected_score_mae": 0.15360283037836298, |
| "gold_label_correct": 75, |
| "gold_label_accuracy": 0.75, |
| "gold_label_ece_15bin_all_decisions": 0.1846883809213806, |
| "full_vector_brier": 0.03861464105533916 |
| }, |
| "security_incidents/credential_compromise": { |
| "count": 100, |
| "soft_ce": 0.7435202424681325, |
| "kl": 0.13032283941349765, |
| "brier": 0.056534706032330906, |
| "argmax_agreement": 0.56, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 13, |
| "confidence": 0.5180136079658565, |
| "reference_agreement": 0.5041024615384615 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.5611483436625656, |
| "reference_agreement": 0.5185416875 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.6424031158653717, |
| "reference_agreement": 0.4575924444444444 |
| }, |
| { |
| "count": 24, |
| "confidence": 0.7033728698546223, |
| "reference_agreement": 0.54125 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.7727795187245017, |
| "reference_agreement": 0.5805127692307692 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.8229402454540159, |
| "reference_agreement": 0.61611125 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.8926362259110124, |
| "reference_agreement": 0.63 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.9362850057480346, |
| "reference_agreement": 0.833333 |
| } |
| ], |
| "gold_label_correct": 56, |
| "gold_label_accuracy": 0.56, |
| "gold_label_ece_15bin_all_decisions": 0.12394694920782158, |
| "full_vector_brier": 0.11306941206466181 |
| }, |
| "security_incidents/disposition": { |
| "count": 100, |
| "soft_ce": 1.1881010051690235, |
| "kl": 0.1639027463248791, |
| "brier": 0.04448559481981512, |
| "argmax_agreement": 0.58, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.32242648663151624, |
| "reference_agreement": 0.4083334083334083 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.3754838182935608, |
| "reference_agreement": 0.32395059913580926 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.43432437938573537, |
| "reference_agreement": 0.46449275565219045 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.4882781369268498, |
| "reference_agreement": 0.49964906052637453 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.5630952682918294, |
| "reference_agreement": 0.5416665000001667 |
| } |
| ], |
| "gold_label_correct": 58, |
| "gold_label_accuracy": 0.58, |
| "gold_label_ece_15bin_all_decisions": 0.2085843245755816, |
| "full_vector_brier": 0.08897118963963024 |
| }, |
| "security_incidents/severity": { |
| "count": 100, |
| "soft_ce": 1.315743899027136, |
| "kl": 0.16019039234495527, |
| "brier": 0.03374204583723859, |
| "argmax_agreement": 0.62, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 9, |
| "confidence": 0.30993707022864725, |
| "reference_agreement": 0.1892591229629178 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.37361903832007515, |
| "reference_agreement": 0.35666686181831636 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.4358135451699136, |
| "reference_agreement": 0.367356292413834 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.4949142252274759, |
| "reference_agreement": 0.4523077201282927 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.5669241476967354, |
| "reference_agreement": 0.5614581968751906 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.629883142448191, |
| "reference_agreement": 0.5296296370372297 |
| } |
| ], |
| "expected_score_mae": 0.3280770725235342, |
| "gold_label_correct": 61, |
| "gold_label_accuracy": 0.61, |
| "gold_label_ece_15bin_all_decisions": 0.17433526899513643, |
| "full_vector_brier": 0.06748409167447718 |
| }, |
| "security_incidents/true_positive": { |
| "count": 100, |
| "soft_ce": 0.6651038558055207, |
| "kl": 0.08578980083661669, |
| "brier": 0.03646839076349738, |
| "argmax_agreement": 0.75, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 10, |
| "confidence": 0.5140548783434865, |
| "reference_agreement": 0.4816668 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.5686844897001768, |
| "reference_agreement": 0.5449999285714285 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.6397580847244457, |
| "reference_agreement": 0.6021052105263158 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.699375893864459, |
| "reference_agreement": 0.6705129230769231 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.7684838760054046, |
| "reference_agreement": 0.6382609565217391 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.8259863098340201, |
| "reference_agreement": 0.768750125 |
| } |
| ], |
| "gold_label_correct": 75, |
| "gold_label_accuracy": 0.75, |
| "gold_label_ece_15bin_all_decisions": 0.0755676945059552, |
| "full_vector_brier": 0.07293678152699476 |
| }, |
| "security_incidents/urgency": { |
| "count": 100, |
| "soft_ce": 1.1579755423473963, |
| "kl": 0.13592958650868842, |
| "brier": 0.035556631007055334, |
| "argmax_agreement": 0.6, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 22, |
| "confidence": 0.3855800670557517, |
| "reference_agreement": 0.327196972045531 |
| }, |
| { |
| "count": 44, |
| "confidence": 0.42700538242544434, |
| "reference_agreement": 0.44545453469702745 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.5028795000140648, |
| "reference_agreement": 0.49196084215690095 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.5686336651767027, |
| "reference_agreement": 0.5948483596970949 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.6318266146454532, |
| "reference_agreement": 0.55000005000025 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.6723890027684984, |
| "reference_agreement": 0.6833332916667916 |
| } |
| ], |
| "expected_score_mae": 0.2845580365911216, |
| "gold_label_correct": 60, |
| "gold_label_accuracy": 0.6, |
| "gold_label_ece_15bin_all_decisions": 0.19018518367205345, |
| "full_vector_brier": 0.07111326201411067 |
| } |
| } |
| }, |
| "calibrated": { |
| "overall": { |
| "count": 2000, |
| "soft_ce": 0.8996057207086053, |
| "kl": 0.13224716508427645, |
| "brier": 0.0371886634007063, |
| "argmax_agreement": 0.7055, |
| "hard_count": 2, |
| "hard_ece": 0.04532655734322799, |
| "hard_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.954673442656772, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.26431766017833624, |
| "reference_agreement": 0.6433336433336433 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.3177876741133899, |
| "reference_agreement": 0.3488332323333685 |
| }, |
| { |
| "count": 214, |
| "confidence": 0.3738574970766125, |
| "reference_agreement": 0.39616281869943265 |
| }, |
| { |
| "count": 379, |
| "confidence": 0.4335478747419467, |
| "reference_agreement": 0.421979419893986 |
| }, |
| { |
| "count": 296, |
| "confidence": 0.5011488512826869, |
| "reference_agreement": 0.5069932456870009 |
| }, |
| { |
| "count": 266, |
| "confidence": 0.5645507022164814, |
| "reference_agreement": 0.5461152577318915 |
| }, |
| { |
| "count": 223, |
| "confidence": 0.633536195077047, |
| "reference_agreement": 0.5981748920314267 |
| }, |
| { |
| "count": 156, |
| "confidence": 0.6977244202652548, |
| "reference_agreement": 0.6617542448932456 |
| }, |
| { |
| "count": 130, |
| "confidence": 0.7643242189776432, |
| "reference_agreement": 0.6995359706949515 |
| }, |
| { |
| "count": 92, |
| "confidence": 0.8331337292292074, |
| "reference_agreement": 0.8546781770653425 |
| }, |
| { |
| "count": 68, |
| "confidence": 0.9005759800850643, |
| "reference_agreement": 0.9235195591667292 |
| }, |
| { |
| "count": 135, |
| "confidence": 0.9612556801725451, |
| "reference_agreement": 0.9694123619383392 |
| } |
| ], |
| "expected_score_mae": 0.26946470447726134, |
| "gold_label_correct": 1409, |
| "gold_label_accuracy": 0.7045, |
| "gold_label_ece_15bin_all_decisions": 0.11873688877234255, |
| "full_vector_brier": 0.0743773268014126 |
| }, |
| "type": { |
| "choice": { |
| "count": 600, |
| "soft_ce": 1.0381506231326942, |
| "kl": 0.17435602628587193, |
| "brier": 0.042938132074843774, |
| "argmax_agreement": 0.6883333333333334, |
| "hard_count": 1, |
| "hard_ece": 0.04531176280353044, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9546882371964696, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.26431766017833624, |
| "reference_agreement": 0.6433336433336433 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.3194104476415337, |
| "reference_agreement": 0.3671427644048106 |
| }, |
| { |
| "count": 151, |
| "confidence": 0.37119215731679844, |
| "reference_agreement": 0.40550005966892083 |
| }, |
| { |
| "count": 171, |
| "confidence": 0.43196031154385833, |
| "reference_agreement": 0.43263354161798406 |
| }, |
| { |
| "count": 56, |
| "confidence": 0.4968234850515549, |
| "reference_agreement": 0.5974404757738895 |
| }, |
| { |
| "count": 41, |
| "confidence": 0.5598861924502044, |
| "reference_agreement": 0.5814633723578526 |
| }, |
| { |
| "count": 38, |
| "confidence": 0.6300906961195497, |
| "reference_agreement": 0.7054297802543781 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.7021830670140048, |
| "reference_agreement": 0.6884998090001994 |
| }, |
| { |
| "count": 30, |
| "confidence": 0.7641036360974187, |
| "reference_agreement": 0.7616667017779263 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.8347770460980283, |
| "reference_agreement": 0.9085641953126645 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.90318181459874, |
| "reference_agreement": 0.9307778106351233 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.949274918346995, |
| "reference_agreement": 0.972878994697626 |
| } |
| ], |
| "gold_label_correct": 413, |
| "gold_label_accuracy": 0.6883333333333334, |
| "gold_label_ece_15bin_all_decisions": 0.1767793119406733, |
| "full_vector_brier": 0.08587626414968755 |
| }, |
| "noul": { |
| "count": 600, |
| "soft_ce": 0.5343593764491412, |
| "kl": 0.08063713309657684, |
| "brier": 0.03351564472605923, |
| "argmax_agreement": 0.7666666666666667, |
| "hard_count": 1, |
| "hard_ece": 0.04534135188292543, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9546586481170746, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 59, |
| "confidence": 0.5162463602685387, |
| "reference_agreement": 0.48372877966101696 |
| }, |
| { |
| "count": 102, |
| "confidence": 0.566982612511053, |
| "reference_agreement": 0.5379411666666667 |
| }, |
| { |
| "count": 92, |
| "confidence": 0.638635665544844, |
| "reference_agreement": 0.5872825978260869 |
| }, |
| { |
| "count": 76, |
| "confidence": 0.6952965001408001, |
| "reference_agreement": 0.6477193421052632 |
| }, |
| { |
| "count": 69, |
| "confidence": 0.7631561753342164, |
| "reference_agreement": 0.6479227681159421 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.8360191336961122, |
| "reference_agreement": 0.8227708958333334 |
| }, |
| { |
| "count": 44, |
| "confidence": 0.8983448122025038, |
| "reference_agreement": 0.91658325 |
| }, |
| { |
| "count": 110, |
| "confidence": 0.9631203976587833, |
| "reference_agreement": 0.9687787545454546 |
| } |
| ], |
| "gold_label_correct": 459, |
| "gold_label_accuracy": 0.765, |
| "gold_label_ece_15bin_all_decisions": 0.03628697257339877, |
| "full_vector_brier": 0.06703128945211846 |
| }, |
| "score": { |
| "count": 800, |
| "soft_ce": 1.0696318020851365, |
| "kl": 0.13937304317385457, |
| "brier": 0.03563132590108849, |
| "argmax_agreement": 0.6725, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 12, |
| "confidence": 0.31400120254772085, |
| "reference_agreement": 0.306110990833337 |
| }, |
| { |
| "count": 63, |
| "confidence": 0.38024585110410336, |
| "reference_agreement": 0.3737830824074847 |
| }, |
| { |
| "count": 208, |
| "confidence": 0.43485303487114435, |
| "reference_agreement": 0.41322050251512227 |
| }, |
| { |
| "count": 181, |
| "confidence": 0.49756579867925116, |
| "reference_agreement": 0.48659301701665447 |
| }, |
| { |
| "count": 123, |
| "confidence": 0.5640888327073031, |
| "reference_agreement": 0.5411110673984649 |
| }, |
| { |
| "count": 93, |
| "confidence": 0.6298993959087413, |
| "reference_agreement": 0.5651254868101266 |
| }, |
| { |
| "count": 60, |
| "confidence": 0.6993135701733142, |
| "reference_agreement": 0.670616600389039 |
| }, |
| { |
| "count": 31, |
| "confidence": 0.7671375253583909, |
| "reference_agreement": 0.7542904560324492 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.8172099330447323, |
| "reference_agreement": 0.8386112533338538 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.9150589341002214, |
| "reference_agreement": 0.9744443333333334 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.9560177842150351, |
| "reference_agreement": 0.9716669228572801 |
| } |
| ], |
| "expected_score_mae": 0.26946470447726134, |
| "gold_label_correct": 537, |
| "gold_label_accuracy": 0.67125, |
| "gold_label_ece_15bin_all_decisions": 0.1383301153500044, |
| "full_vector_brier": 0.07126265180217697 |
| } |
| }, |
| "category": { |
| "agent_trace_observability": { |
| "count": 500, |
| "soft_ce": 0.9784239666090253, |
| "kl": 0.09995369366380179, |
| "brier": 0.02902756320294822, |
| "argmax_agreement": 0.666, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.320696328577158, |
| "reference_agreement": 0.333333 |
| }, |
| { |
| "count": 56, |
| "confidence": 0.3777104214271337, |
| "reference_agreement": 0.4022678604167104 |
| }, |
| { |
| "count": 142, |
| "confidence": 0.4351345221705775, |
| "reference_agreement": 0.4108685441315153 |
| }, |
| { |
| "count": 101, |
| "confidence": 0.5017886292643352, |
| "reference_agreement": 0.5098679828713507 |
| }, |
| { |
| "count": 69, |
| "confidence": 0.5647063184371532, |
| "reference_agreement": 0.5609178477778174 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.6323688610153608, |
| "reference_agreement": 0.6025362419565561 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.7011952059687334, |
| "reference_agreement": 0.6249331570668304 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.7712895410262108, |
| "reference_agreement": 0.684133418933419 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.8322538044396943, |
| "reference_agreement": 0.8557143673016929 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.8840899517188067, |
| "reference_agreement": 0.8664284285714287 |
| } |
| ], |
| "expected_score_mae": 0.20048818990062078, |
| "gold_label_correct": 335, |
| "gold_label_accuracy": 0.67, |
| "gold_label_ece_15bin_all_decisions": 0.13395749510739433, |
| "full_vector_brier": 0.05805512640589644 |
| }, |
| "customer_service": { |
| "count": 500, |
| "soft_ce": 0.9395046654141255, |
| "kl": 0.20235002740464264, |
| "brier": 0.054900227765937, |
| "argmax_agreement": 0.694, |
| "hard_count": 1, |
| "hard_ece": 0.04531176280353044, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9546882371964696, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.26431766017833624, |
| "reference_agreement": 0.6433336433336433 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.3153945397919026, |
| "reference_agreement": 0.4254165366667425 |
| }, |
| { |
| "count": 43, |
| "confidence": 0.3676812200975334, |
| "reference_agreement": 0.4629575986047603 |
| }, |
| { |
| "count": 81, |
| "confidence": 0.43653206317486204, |
| "reference_agreement": 0.40526751485604134 |
| }, |
| { |
| "count": 73, |
| "confidence": 0.5004255499809467, |
| "reference_agreement": 0.5462557171690167 |
| }, |
| { |
| "count": 85, |
| "confidence": 0.5652857404557898, |
| "reference_agreement": 0.5123529096471258 |
| }, |
| { |
| "count": 71, |
| "confidence": 0.6289827242548214, |
| "reference_agreement": 0.5991033191502455 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.6933075112774083, |
| "reference_agreement": 0.6937499738334586 |
| }, |
| { |
| "count": 35, |
| "confidence": 0.757544894325504, |
| "reference_agreement": 0.7550476496191563 |
| }, |
| { |
| "count": 24, |
| "confidence": 0.8298230320463706, |
| "reference_agreement": 0.858494010416886 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.9042490371859675, |
| "reference_agreement": 0.9288167011668795 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.949274918346995, |
| "reference_agreement": 0.972878994697626 |
| } |
| ], |
| "expected_score_mae": 0.3540890427112495, |
| "gold_label_correct": 347, |
| "gold_label_accuracy": 0.694, |
| "gold_label_ece_15bin_all_decisions": 0.11718544849042345, |
| "full_vector_brier": 0.109800455531874 |
| }, |
| "invoice_processing": { |
| "count": 500, |
| "soft_ce": 0.6684132719468862, |
| "kl": 0.09346579628199163, |
| "brier": 0.024514530073744333, |
| "argmax_agreement": 0.84, |
| "hard_count": 1, |
| "hard_ece": 0.04534135188292543, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9546586481170746, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 10, |
| "confidence": 0.3284781191962668, |
| "reference_agreement": 0.39399997833337835 |
| }, |
| { |
| "count": 38, |
| "confidence": 0.3695643829541523, |
| "reference_agreement": 0.35955257149123854 |
| }, |
| { |
| "count": 38, |
| "confidence": 0.43180381052982364, |
| "reference_agreement": 0.45737369490706775 |
| }, |
| { |
| "count": 49, |
| "confidence": 0.4995394302233789, |
| "reference_agreement": 0.4748979573470118 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.5630772128542549, |
| "reference_agreement": 0.5724305864584053 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.633751143134781, |
| "reference_agreement": 0.6486805458333972 |
| }, |
| { |
| "count": 46, |
| "confidence": 0.7006481123389033, |
| "reference_agreement": 0.71297823347834 |
| }, |
| { |
| "count": 30, |
| "confidence": 0.7644638654220844, |
| "reference_agreement": 0.7463223660112589 |
| }, |
| { |
| "count": 37, |
| "confidence": 0.8368844632918799, |
| "reference_agreement": 0.8846810628829918 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.905262180250512, |
| "reference_agreement": 0.961760375 |
| }, |
| { |
| "count": 124, |
| "confidence": 0.9623184896893279, |
| "reference_agreement": 0.9691048380645316 |
| } |
| ], |
| "expected_score_mae": 0.2172820819264362, |
| "gold_label_correct": 417, |
| "gold_label_accuracy": 0.834, |
| "gold_label_ece_15bin_all_decisions": 0.1349665198192152, |
| "full_vector_brier": 0.049029060147488666 |
| }, |
| "security_incidents": { |
| "count": 500, |
| "soft_ce": 1.012080978864384, |
| "kl": 0.13321914298666976, |
| "brier": 0.04031233256019564, |
| "argmax_agreement": 0.622, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 13, |
| "confidence": 0.31228590825579433, |
| "reference_agreement": 0.2210255325640828 |
| }, |
| { |
| "count": 77, |
| "confidence": 0.3766231397145957, |
| "reference_agreement": 0.372489202099624 |
| }, |
| { |
| "count": 118, |
| "confidence": 0.4301516988277852, |
| "reference_agreement": 0.43542370985879325 |
| }, |
| { |
| "count": 73, |
| "confidence": 0.5020672766359284, |
| "reference_agreement": 0.48529681082197323 |
| }, |
| { |
| "count": 64, |
| "confidence": 0.5645118228386581, |
| "reference_agreement": 0.5552603373438221 |
| }, |
| { |
| "count": 58, |
| "confidence": 0.6398582000502253, |
| "reference_agreement": 0.5517815850575183 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.6967336842994563, |
| "reference_agreement": 0.601407457407413 |
| }, |
| { |
| "count": 40, |
| "confidence": 0.7657980669345792, |
| "reference_agreement": 0.6255000500000001 |
| }, |
| { |
| "count": 10, |
| "confidence": 0.8290495284941051, |
| "reference_agreement": 0.7323335 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.9042684049926732, |
| "reference_agreement": 0.6583330000000001 |
| } |
| ], |
| "expected_score_mae": 0.30599950337073883, |
| "gold_label_correct": 310, |
| "gold_label_accuracy": 0.62, |
| "gold_label_ece_15bin_all_decisions": 0.10104896950536107, |
| "full_vector_brier": 0.08062466512039128 |
| } |
| }, |
| "question": { |
| "agent_trace_observability/action": { |
| "count": 100, |
| "soft_ce": 1.1300319363326001, |
| "kl": 0.09598806155025466, |
| "brier": 0.02483876810455199, |
| "argmax_agreement": 0.66, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.320696328577158, |
| "reference_agreement": 0.333333 |
| }, |
| { |
| "count": 45, |
| "confidence": 0.37494667517257485, |
| "reference_agreement": 0.4043777937037419 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.42036053533497303, |
| "reference_agreement": 0.4256666106667256 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.5017129711343226, |
| "reference_agreement": 0.6168181568182417 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.5458754881908042, |
| "reference_agreement": 0.5908332955556567 |
| } |
| ], |
| "gold_label_correct": 66, |
| "gold_label_accuracy": 0.66, |
| "gold_label_ece_15bin_all_decisions": 0.23452694015867082, |
| "full_vector_brier": 0.04967753620910398 |
| }, |
| "agent_trace_observability/needs_review": { |
| "count": 100, |
| "soft_ce": 0.5772502686786852, |
| "kl": 0.061784401057798144, |
| "brier": 0.027046496253035113, |
| "argmax_agreement": 0.78, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 17, |
| "confidence": 0.5199000755681038, |
| "reference_agreement": 0.48039217647058824 |
| }, |
| { |
| "count": 32, |
| "confidence": 0.5717013250980697, |
| "reference_agreement": 0.57124996875 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.6285397565767751, |
| "reference_agreement": 0.6404444666666667 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7878123338263531, |
| "reference_agreement": 0.9075 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.8368480740637315, |
| "reference_agreement": 0.8564815555555555 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.8840899517188067, |
| "reference_agreement": 0.8664284285714287 |
| } |
| ], |
| "gold_label_correct": 78, |
| "gold_label_accuracy": 0.78, |
| "gold_label_ece_15bin_all_decisions": 0.10847385971036504, |
| "full_vector_brier": 0.05409299250607023 |
| }, |
| "agent_trace_observability/outcome": { |
| "count": 100, |
| "soft_ce": 1.0909319011321876, |
| "kl": 0.1661455365279982, |
| "brier": 0.04300572982539011, |
| "argmax_agreement": 0.62, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 5, |
| "confidence": 0.3938691946389984, |
| "reference_agreement": 0.3533333066666934 |
| }, |
| { |
| "count": 58, |
| "confidence": 0.4374378411470648, |
| "reference_agreement": 0.38236207597705485 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.4803700853513563, |
| "reference_agreement": 0.58916684666676 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.5793429761885015, |
| "reference_agreement": 0.7744446666666667 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.635342607275984, |
| "reference_agreement": 0.6880951604762682 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.7121878766387506, |
| "reference_agreement": 0.6516665095834904 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.7635113875345624, |
| "reference_agreement": 0.6195556666667333 |
| } |
| ], |
| "gold_label_correct": 60, |
| "gold_label_accuracy": 0.6, |
| "gold_label_ece_15bin_all_decisions": 0.10083659412693492, |
| "full_vector_brier": 0.08601145965078022 |
| }, |
| "agent_trace_observability/risk": { |
| "count": 100, |
| "soft_ce": 0.9828391693041487, |
| "kl": 0.09046906089539679, |
| "brier": 0.026974569027456388, |
| "argmax_agreement": 0.67, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 6, |
| "confidence": 0.3849728739931042, |
| "reference_agreement": 0.4272221555556555 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.44017390916712973, |
| "reference_agreement": 0.4377777200001155 |
| }, |
| { |
| "count": 22, |
| "confidence": 0.5047424360225681, |
| "reference_agreement": 0.541060543181922 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.5613336073178117, |
| "reference_agreement": 0.5121666475000758 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.6336701385816399, |
| "reference_agreement": 0.5417646784313843 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.6951222762738744, |
| "reference_agreement": 0.5763887186111981 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.7875525659292162, |
| "reference_agreement": 0.7093333680001679 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.8046881866954707, |
| "reference_agreement": 0.8511112377785178 |
| } |
| ], |
| "expected_score_mae": 0.18766299038600726, |
| "gold_label_correct": 69, |
| "gold_label_accuracy": 0.69, |
| "gold_label_ece_15bin_all_decisions": 0.12744205611487303, |
| "full_vector_brier": 0.053949138054912776 |
| }, |
| "agent_trace_observability/urgency": { |
| "count": 100, |
| "soft_ce": 1.1110665575975052, |
| "kl": 0.08538140828756115, |
| "brier": 0.023272252804307492, |
| "argmax_agreement": 0.6, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 49, |
| "confidence": 0.43689567382624167, |
| "reference_agreement": 0.4303333642177499 |
| }, |
| { |
| "count": 36, |
| "confidence": 0.4938569714495298, |
| "reference_agreement": 0.43055556898154956 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5775433179069762, |
| "reference_agreement": 0.38333300000000003 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.6344400930336004, |
| "reference_agreement": 0.5833334961905962 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.6981819641643676, |
| "reference_agreement": 0.6986664453336919 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.740555547685341, |
| "reference_agreement": 0.6333336333336334 |
| } |
| ], |
| "expected_score_mae": 0.2133133894152343, |
| "gold_label_correct": 62, |
| "gold_label_accuracy": 0.62, |
| "gold_label_ece_15bin_all_decisions": 0.13295801626402656, |
| "full_vector_brier": 0.046544505608614985 |
| }, |
| "customer_service/action": { |
| "count": 100, |
| "soft_ce": 1.2561721269415909, |
| "kl": 0.290314230980401, |
| "brier": 0.06932695222424835, |
| "argmax_agreement": 0.6, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.26431766017833624, |
| "reference_agreement": 0.6433336433336433 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.3143181069990199, |
| "reference_agreement": 0.3948887724445253 |
| }, |
| { |
| "count": 38, |
| "confidence": 0.3671960078421094, |
| "reference_agreement": 0.46475026728081864 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.43092514659511294, |
| "reference_agreement": 0.4196297262963243 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.49345382078770295, |
| "reference_agreement": 0.6222222582223413 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.5467833623082197, |
| "reference_agreement": 0.37499980000020006 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.6432513672126315, |
| "reference_agreement": 0.7800003916668916 |
| } |
| ], |
| "gold_label_correct": 61, |
| "gold_label_accuracy": 0.61, |
| "gold_label_ece_15bin_all_decisions": 0.20837740157143791, |
| "full_vector_brier": 0.1386539044484967 |
| }, |
| "customer_service/category": { |
| "count": 100, |
| "soft_ce": 0.5762908772818797, |
| "kl": 0.16586899434803676, |
| "brier": 0.034944317406019984, |
| "argmax_agreement": 0.93, |
| "hard_count": 1, |
| "hard_ece": 0.04531176280353044, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9546882371964696, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 5, |
| "confidence": 0.4322684416135483, |
| "reference_agreement": 0.6660000793337393 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.4905168094943096, |
| "reference_agreement": 0.5655556666666667 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.5612844306710459, |
| "reference_agreement": 0.6383332712502996 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.6268957032862555, |
| "reference_agreement": 0.7177498577498188 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.6937005013196476, |
| "reference_agreement": 0.7020831275002684 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.7640632842535814, |
| "reference_agreement": 0.8997435425643682 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.8328563184532323, |
| "reference_agreement": 0.8911928125002632 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.9042490371859675, |
| "reference_agreement": 0.9288167011668795 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.949274918346995, |
| "reference_agreement": 0.972878994697626 |
| } |
| ], |
| "gold_label_correct": 94, |
| "gold_label_accuracy": 0.94, |
| "gold_label_ece_15bin_all_decisions": 0.1768752555819122, |
| "full_vector_brier": 0.06988863481203997 |
| }, |
| "customer_service/churn_risk": { |
| "count": 100, |
| "soft_ce": 1.0730591179793425, |
| "kl": 0.18264058554353962, |
| "brier": 0.051583714603065454, |
| "argmax_agreement": 0.71, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.3315410316851427, |
| "reference_agreement": 0.883333 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.3713688332387563, |
| "reference_agreement": 0.4493333166667167 |
| }, |
| { |
| "count": 31, |
| "confidence": 0.4375064676665284, |
| "reference_agreement": 0.388172083225866 |
| }, |
| { |
| "count": 27, |
| "confidence": 0.49821156690152874, |
| "reference_agreement": 0.5818519443210562 |
| }, |
| { |
| "count": 19, |
| "confidence": 0.5633747625283594, |
| "reference_agreement": 0.5187718885966044 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.6252985881937501, |
| "reference_agreement": 0.6162223266667289 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.6801812993683634, |
| "reference_agreement": 0.6716668416668417 |
| } |
| ], |
| "expected_score_mae": 0.31676493780927045, |
| "gold_label_correct": 71, |
| "gold_label_accuracy": 0.71, |
| "gold_label_ece_15bin_all_decisions": 0.20353240088435612, |
| "full_vector_brier": 0.10316742920613091 |
| }, |
| "customer_service/needs_human": { |
| "count": 100, |
| "soft_ce": 0.6710381093325837, |
| "kl": 0.12192570050358188, |
| "brier": 0.054199941449838425, |
| "argmax_agreement": 0.66, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 8, |
| "confidence": 0.5158550697220626, |
| "reference_agreement": 0.482499875 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.5674903989336381, |
| "reference_agreement": 0.5160714285714285 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.6357804167556367, |
| "reference_agreement": 0.5612963333333334 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.6961004807233757, |
| "reference_agreement": 0.6815873333333333 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.7545551168362051, |
| "reference_agreement": 0.6688889523809524 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.8146566000120619, |
| "reference_agreement": 0.695 |
| } |
| ], |
| "gold_label_correct": 65, |
| "gold_label_accuracy": 0.65, |
| "gold_label_ece_15bin_all_decisions": 0.058501302775174596, |
| "full_vector_brier": 0.10839988289967685 |
| }, |
| "customer_service/urgency": { |
| "count": 100, |
| "soft_ce": 1.1209630955352308, |
| "kl": 0.251000625647654, |
| "brier": 0.06444621314651275, |
| "argmax_agreement": 0.57, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 18, |
| "confidence": 0.44444863629809206, |
| "reference_agreement": 0.3407406175926695 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.5039577272096432, |
| "reference_agreement": 0.4638332491667292 |
| }, |
| { |
| "count": 28, |
| "confidence": 0.5668426468063085, |
| "reference_agreement": 0.4780952023809791 |
| }, |
| { |
| "count": 24, |
| "confidence": 0.6260414968218451, |
| "reference_agreement": 0.5423611534722869 |
| }, |
| { |
| "count": 9, |
| "confidence": 0.6893581940679483, |
| "reference_agreement": 0.7196295833336129 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.7355911525357738, |
| "reference_agreement": 0.6833336833336833 |
| } |
| ], |
| "expected_score_mae": 0.39141314761322865, |
| "gold_label_correct": 56, |
| "gold_label_accuracy": 0.56, |
| "gold_label_ece_15bin_all_decisions": 0.09986013285263563, |
| "full_vector_brier": 0.1288924262930255 |
| }, |
| "invoice_processing/discrepancy_severity": { |
| "count": 100, |
| "soft_ce": 0.7824188222345687, |
| "kl": 0.13817545789882418, |
| "brier": 0.030670028432809596, |
| "argmax_agreement": 0.83, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.33286669301517746, |
| "reference_agreement": 0.5433333916668917 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.37626617164243364, |
| "reference_agreement": 0.40454538484852426 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.4158789751689152, |
| "reference_agreement": 0.41612779859445137 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.511693219926803, |
| "reference_agreement": 0.43666665333345334 |
| }, |
| { |
| "count": 5, |
| "confidence": 0.5577624108377157, |
| "reference_agreement": 0.66000016000016 |
| }, |
| { |
| "count": 7, |
| "confidence": 0.6424516556670419, |
| "reference_agreement": 0.7361905761906619 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.7078300926213531, |
| "reference_agreement": 0.7558974338463236 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.770513839139187, |
| "reference_agreement": 0.7909834398501792 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.8229931371292696, |
| "reference_agreement": 0.8491668804171221 |
| }, |
| { |
| "count": 3, |
| "confidence": 0.9150589341002214, |
| "reference_agreement": 0.9744443333333334 |
| }, |
| { |
| "count": 14, |
| "confidence": 0.9560177842150351, |
| "reference_agreement": 0.9716669228572801 |
| } |
| ], |
| "expected_score_mae": 0.28116763438747877, |
| "gold_label_correct": 83, |
| "gold_label_accuracy": 0.83, |
| "gold_label_ece_15bin_all_decisions": 0.16034912415177363, |
| "full_vector_brier": 0.06134005686561919 |
| }, |
| "invoice_processing/disposition": { |
| "count": 100, |
| "soft_ce": 0.9782701754641571, |
| "kl": 0.15481087150893616, |
| "brier": 0.03951271603036011, |
| "argmax_agreement": 0.74, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 8, |
| "confidence": 0.3273809757415391, |
| "reference_agreement": 0.356666625 |
| }, |
| { |
| "count": 24, |
| "confidence": 0.3651080883483308, |
| "reference_agreement": 0.3415138374999964 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.43382661018637136, |
| "reference_agreement": 0.5002564646154775 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.4948415192095376, |
| "reference_agreement": 0.5124999458333042 |
| }, |
| { |
| "count": 15, |
| "confidence": 0.5688883847973131, |
| "reference_agreement": 0.5368888311111845 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.6286351192794514, |
| "reference_agreement": 0.6950980264706401 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.6991385791532274, |
| "reference_agreement": 0.7349997708334792 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.7688077873037844, |
| "reference_agreement": 0.9299999999999999 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.837978258839355, |
| "reference_agreement": 0.9375165 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.8818373628541912, |
| "reference_agreement": 0.97 |
| } |
| ], |
| "gold_label_correct": 74, |
| "gold_label_accuracy": 0.74, |
| "gold_label_ece_15bin_all_decisions": 0.21745472516454098, |
| "full_vector_brier": 0.07902543206072021 |
| }, |
| "invoice_processing/duplicate": { |
| "count": 100, |
| "soft_ce": 0.23439542363249471, |
| "kl": 0.02684237146924989, |
| "brier": 0.007817390201271179, |
| "argmax_agreement": 0.94, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 9, |
| "confidence": 0.5172249840934735, |
| "reference_agreement": 0.5159257777777778 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.560787712375488, |
| "reference_agreement": 0.5161113333333334 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.6370394155634268, |
| "reference_agreement": 0.3983335 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.9117582796855447, |
| "reference_agreement": 0.9755277499999999 |
| }, |
| { |
| "count": 71, |
| "confidence": 0.9649128351065898, |
| "reference_agreement": 0.969896676056338 |
| } |
| ], |
| "gold_label_correct": 94, |
| "gold_label_accuracy": 0.94, |
| "gold_label_ece_15bin_all_decisions": 0.05804417051238259, |
| "full_vector_brier": 0.015634780402542358 |
| }, |
| "invoice_processing/matches_order": { |
| "count": 100, |
| "soft_ce": 0.33281042566388214, |
| "kl": 0.0751197521851682, |
| "brier": 0.025450911444578592, |
| "argmax_agreement": 0.91, |
| "hard_count": 1, |
| "hard_ece": 0.04534135188292543, |
| "hard_reliability": [ |
| { |
| "count": 1, |
| "confidence": 0.9546586481170746, |
| "reference_agreement": 1.0 |
| } |
| ], |
| "soft_reliability": [ |
| { |
| "count": 2, |
| "confidence": 0.5, |
| "reference_agreement": 0.25 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.5422205310594118, |
| "reference_agreement": 0.05 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.643209342515234, |
| "reference_agreement": 0.7396969090909091 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.6853901959758365, |
| "reference_agreement": 0.7666667272727273 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7572366579530104, |
| "reference_agreement": 0.50250025 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.8447832124548076, |
| "reference_agreement": 0.8733124999999999 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.900017265414687, |
| "reference_agreement": 0.9485416249999999 |
| }, |
| { |
| "count": 39, |
| "confidence": 0.9598572423050843, |
| "reference_agreement": 0.966743564102564 |
| } |
| ], |
| "gold_label_correct": 91, |
| "gold_label_accuracy": 0.91, |
| "gold_label_ece_15bin_all_decisions": 0.11605332143659476, |
| "full_vector_brier": 0.050901822889157185 |
| }, |
| "invoice_processing/urgency": { |
| "count": 100, |
| "soft_ce": 1.0141715127393283, |
| "kl": 0.07238052834777972, |
| "brier": 0.019121604259702194, |
| "argmax_agreement": 0.78, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 3, |
| "confidence": 0.38064151461035944, |
| "reference_agreement": 0.33888879444446113 |
| }, |
| { |
| "count": 13, |
| "confidence": 0.44448085889873, |
| "reference_agreement": 0.4525640602564577 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.49257155336662545, |
| "reference_agreement": 0.4790805124138998 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.561839123121125, |
| "reference_agreement": 0.617936518254042 |
| }, |
| { |
| "count": 11, |
| "confidence": 0.626064968568645, |
| "reference_agreement": 0.47575751969705 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.7051208606202697, |
| "reference_agreement": 0.6442777231481954 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.7392692433647956, |
| "reference_agreement": 0.6750002958335458 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.8085095394162195, |
| "reference_agreement": 0.7166662833337166 |
| } |
| ], |
| "expected_score_mae": 0.15339652946539362, |
| "gold_label_correct": 75, |
| "gold_label_accuracy": 0.75, |
| "gold_label_ece_15bin_all_decisions": 0.18936000086676474, |
| "full_vector_brier": 0.03824320851940439 |
| }, |
| "security_incidents/credential_compromise": { |
| "count": 100, |
| "soft_ce": 0.7303544474693316, |
| "kl": 0.11715704441469675, |
| "brier": 0.051873219306088056, |
| "argmax_agreement": 0.56, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 13, |
| "confidence": 0.5162662097600973, |
| "reference_agreement": 0.5041024615384615 |
| }, |
| { |
| "count": 17, |
| "confidence": 0.5577534464386459, |
| "reference_agreement": 0.5229411764705882 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.6372227885501143, |
| "reference_agreement": 0.47063480952380954 |
| }, |
| { |
| "count": 21, |
| "confidence": 0.691688303154376, |
| "reference_agreement": 0.5293650952380953 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.7677754511149539, |
| "reference_agreement": 0.5835 |
| }, |
| { |
| "count": 6, |
| "confidence": 0.8359302010758386, |
| "reference_agreement": 0.7183335 |
| }, |
| { |
| "count": 2, |
| "confidence": 0.9042684049926732, |
| "reference_agreement": 0.6583330000000001 |
| } |
| ], |
| "gold_label_correct": 56, |
| "gold_label_accuracy": 0.56, |
| "gold_label_ece_15bin_all_decisions": 0.11893490648195507, |
| "full_vector_brier": 0.10374643861217611 |
| }, |
| "security_incidents/disposition": { |
| "count": 100, |
| "soft_ce": 1.1972067216437494, |
| "kl": 0.17300846279960486, |
| "brier": 0.04600030885849212, |
| "argmax_agreement": 0.58, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 4, |
| "confidence": 0.32224419861704345, |
| "reference_agreement": 0.29249995416670416 |
| }, |
| { |
| "count": 39, |
| "confidence": 0.3715904353978362, |
| "reference_agreement": 0.39512819239318403 |
| }, |
| { |
| "count": 48, |
| "confidence": 0.4302195975460818, |
| "reference_agreement": 0.46097220729168575 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.501278204923741, |
| "reference_agreement": 0.5562498937501396 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.5496311616587236, |
| "reference_agreement": 0.5166664833335167 |
| } |
| ], |
| "gold_label_correct": 58, |
| "gold_label_accuracy": 0.58, |
| "gold_label_ece_15bin_all_decisions": 0.17586552330691987, |
| "full_vector_brier": 0.09200061771698424 |
| }, |
| "security_incidents/severity": { |
| "count": 100, |
| "soft_ce": 1.3148165106977874, |
| "kl": 0.15926300401560653, |
| "brier": 0.033456273290858624, |
| "argmax_agreement": 0.62, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 9, |
| "confidence": 0.3078600014285725, |
| "reference_agreement": 0.1892591229629178 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.3727428250841209, |
| "reference_agreement": 0.36388904000012334 |
| }, |
| { |
| "count": 29, |
| "confidence": 0.433631983234007, |
| "reference_agreement": 0.3698850510345237 |
| }, |
| { |
| "count": 26, |
| "confidence": 0.4919768669037995, |
| "reference_agreement": 0.45679487397444657 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.5652997260041271, |
| "reference_agreement": 0.5520831968751906 |
| }, |
| { |
| "count": 8, |
| "confidence": 0.6261899107804109, |
| "reference_agreement": 0.5354167166668834 |
| } |
| ], |
| "expected_score_mae": 0.32567749172139204, |
| "gold_label_correct": 61, |
| "gold_label_accuracy": 0.61, |
| "gold_label_ece_15bin_all_decisions": 0.1787678515625339, |
| "full_vector_brier": 0.06691254658171725 |
| }, |
| "security_incidents/true_positive": { |
| "count": 100, |
| "soft_ce": 0.6603075839178701, |
| "kl": 0.08099352894896619, |
| "brier": 0.034705909701544004, |
| "argmax_agreement": 0.75, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 10, |
| "confidence": 0.5126907829686985, |
| "reference_agreement": 0.4816668 |
| }, |
| { |
| "count": 18, |
| "confidence": 0.5699609726709991, |
| "reference_agreement": 0.5612962222222222 |
| }, |
| { |
| "count": 25, |
| "confidence": 0.6460510888610294, |
| "reference_agreement": 0.6201333600000001 |
| }, |
| { |
| "count": 23, |
| "confidence": 0.7025946692840792, |
| "reference_agreement": 0.6679710869565217 |
| }, |
| { |
| "count": 20, |
| "confidence": 0.7638206827542047, |
| "reference_agreement": 0.6675001 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.8187285196215046, |
| "reference_agreement": 0.7533335 |
| } |
| ], |
| "gold_label_correct": 75, |
| "gold_label_accuracy": 0.75, |
| "gold_label_ece_15bin_all_decisions": 0.09523922989135299, |
| "full_vector_brier": 0.06941181940308801 |
| }, |
| "security_incidents/urgency": { |
| "count": 100, |
| "soft_ce": 1.1577196305931823, |
| "kl": 0.13567367475447445, |
| "brier": 0.0355259516439954, |
| "argmax_agreement": 0.6, |
| "hard_count": 0, |
| "hard_ece": null, |
| "hard_reliability": [], |
| "soft_reliability": [ |
| { |
| "count": 26, |
| "confidence": 0.38596311063456945, |
| "reference_agreement": 0.34250002224366877 |
| }, |
| { |
| "count": 41, |
| "confidence": 0.42761054306537855, |
| "reference_agreement": 0.45186988617891466 |
| }, |
| { |
| "count": 16, |
| "confidence": 0.5006824036853632, |
| "reference_agreement": 0.4831250822917073 |
| }, |
| { |
| "count": 12, |
| "confidence": 0.5661019822011998, |
| "reference_agreement": 0.599444329722337 |
| }, |
| { |
| "count": 4, |
| "confidence": 0.642325133897913, |
| "reference_agreement": 0.58333330000025 |
| }, |
| { |
| "count": 1, |
| "confidence": 0.6678840336998128, |
| "reference_agreement": 0.5833335833335833 |
| } |
| ], |
| "expected_score_mae": 0.2863215150200857, |
| "gold_label_correct": 60, |
| "gold_label_accuracy": 0.6, |
| "gold_label_ece_15bin_all_decisions": 0.1646168179614661, |
| "full_vector_brier": 0.0710519032879908 |
| } |
| } |
| }, |
| "metric_notes": { |
| "accuracy": "Main accuracy compares original option key with stored gold.label; argmax_agreement separately uses first maximum of rounded target distribution in input option order", |
| "soft_ce": "Mean natural-log cross-entropy against full normalized gold distribution", |
| "kl": "Mean KL(gold || prediction)", |
| "brier": "Existing metric field brier is half-vector Brier; full_vector_brier is twice it", |
| "ece": "gold_label_ece_15bin_all_decisions uses top probability vs stored-label accuracy on all examples; hard_ece uses one-hot references only; exact leaderboard scorer parity is not claimed", |
| "overlap": "Normalized exact state and source-ID checks; paraphrase and pretraining exposure not assessed" |
| } |
| }, |
| "sysone_bench": { |
| "dataset_version": "2.1.0", |
| "upstream_revision": "93afcfe1bd450dc052b21768e2b95b20316ebb68", |
| "manifest_sha256": "0d21a64c6b61b6be7b49050dbd89f89f934912a7b586734e475308e9c9422ea2", |
| "raw": { |
| "evaluation_cases": 952, |
| "evaluation_decisions": 1240, |
| "correct": 953, |
| "accuracy": 0.7685483870967742, |
| "suites": { |
| "triage": { |
| "accuracy": 0.9270833333333334, |
| "cases": 48, |
| "decisions": 192, |
| "latency": { |
| "mean_seconds": 0.03883962416667922, |
| "p50_seconds": 0.03841860600016389, |
| "p95_seconds": 0.04178968690000602 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8541666666666666, |
| "macro_f1": 0.820875420875421, |
| "balanced_accuracy": 0.8444444444444444, |
| "per_class_recall": { |
| "billing_question": 0.2222222222222222, |
| "cancellation": 1.0, |
| "information": 1.0, |
| "refund": 1.0, |
| "technical_help": 1.0 |
| }, |
| "brier": 0.13697967505778072, |
| "nll": 0.2506294304217651, |
| "ece_raw": 0.033931114278253044, |
| "ece_fitted": 0.039914814482762906 |
| }, |
| "noul": { |
| "accuracy": 0.9513888888888888, |
| "balanced_accuracy": 0.912037037037037, |
| "auroc": 0.9890689300411523, |
| "auprc": 0.9701626578233637, |
| "brier": 0.04911263462874029, |
| "nll": 0.1906247610132196, |
| "ece_raw": 0.01181868597485985, |
| "ece_fitted": 0.012443502260767868 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 178 |
| }, |
| "guardrails": { |
| "accuracy": 0.8125, |
| "cases": 48, |
| "decisions": 96, |
| "latency": { |
| "mean_seconds": 0.037130780229148286, |
| "p50_seconds": 0.03704196349997346, |
| "p95_seconds": 0.03867655395000611 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": 0.8125, |
| "balanced_accuracy": 0.8676470588235294, |
| "auroc": 0.9803046218487395, |
| "auprc": 0.9910730108125466, |
| "brier": 0.14491041421218856, |
| "nll": 0.4276831543302518, |
| "ece_raw": 0.02432198575945292, |
| "ece_fitted": 0.026983405418465858 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 78 |
| }, |
| "moderation": { |
| "accuracy": 0.9166666666666666, |
| "cases": 48, |
| "decisions": 144, |
| "latency": { |
| "mean_seconds": 0.03827599806249301, |
| "p50_seconds": 0.037934129999939614, |
| "p95_seconds": 0.041699429449965915 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": 0.9166666666666666, |
| "balanced_accuracy": 0.9106227106227107, |
| "auroc": 0.978021978021978, |
| "auprc": 0.944990138816365, |
| "brier": 0.06061711079669634, |
| "nll": 0.2023230253736832, |
| "ece_raw": 0.010916557094938821, |
| "ece_fitted": 0.011697905217155507 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 132 |
| }, |
| "agnews": { |
| "accuracy": 0.8125, |
| "cases": 160, |
| "decisions": 160, |
| "latency": { |
| "mean_seconds": 0.038758980293738435, |
| "p50_seconds": 0.0382035000001224, |
| "p95_seconds": 0.042714876549950986 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8125, |
| "macro_f1": 0.8064558008777994, |
| "balanced_accuracy": 0.8125, |
| "per_class_recall": { |
| "business": 0.85, |
| "scitech": 0.55, |
| "sports": 0.975, |
| "world": 0.875 |
| }, |
| "brier": 0.30172771335742954, |
| "nll": 0.6319382379388891, |
| "ece_raw": 0.0052000811604517765, |
| "ece_fitted": 0.007738817523861061 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 130 |
| }, |
| "emotion": { |
| "accuracy": 0.7239583333333334, |
| "cases": 192, |
| "decisions": 192, |
| "latency": { |
| "mean_seconds": 0.038396373302075425, |
| "p50_seconds": 0.03797906949989738, |
| "p95_seconds": 0.0415361116498957 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.7239583333333334, |
| "macro_f1": 0.6913006811212603, |
| "balanced_accuracy": 0.6854851048438465, |
| "per_class_recall": { |
| "anger": 0.5714285714285714, |
| "fear": 0.9259259259259259, |
| "joy": 0.7142857142857143, |
| "love": 0.6, |
| "sadness": 0.8275862068965517, |
| "surprise": 0.47368421052631576 |
| }, |
| "brier": 0.39443242180284094, |
| "nll": 0.7806033322263032, |
| "ece_raw": 0.0032102610291140795, |
| "ece_fitted": 0.003416953263655513 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 139 |
| }, |
| "banking77_12": { |
| "accuracy": 0.8020833333333334, |
| "cases": 96, |
| "decisions": 96, |
| "latency": { |
| "mean_seconds": 0.03918704791666509, |
| "p50_seconds": 0.038639419499986616, |
| "p95_seconds": 0.04271133699995744 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8020833333333334, |
| "macro_f1": 0.7940683880389763, |
| "balanced_accuracy": 0.8020833333333334, |
| "per_class_recall": { |
| "card_arrival": 0.625, |
| "card_linking": 0.25, |
| "exchange_rate": 1.0, |
| "lost_or_stolen_card": 1.0, |
| "order_physical_card": 0.875, |
| "pin_blocked": 0.875, |
| "refund_not_showing_up": 0.875, |
| "request_refund": 0.875, |
| "terminate_account": 1.0, |
| "top_up_failed": 0.875, |
| "topping_up_by_card": 0.5, |
| "transaction_charged_twice": 0.875 |
| }, |
| "brier": 0.29183460550386847, |
| "nll": 0.774193924200847, |
| "ece_raw": 0.01661891915597873, |
| "ece_fitted": 0.016552925317361755 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 77 |
| }, |
| "mnli": { |
| "accuracy": 0.5416666666666666, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.0384311384083181, |
| "p50_seconds": 0.03797633499982567, |
| "p95_seconds": 0.04150163254993231 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.5416666666666666, |
| "macro_f1": 0.531705418323941, |
| "balanced_accuracy": 0.5416666666666666, |
| "per_class_recall": { |
| "contradiction": 0.575, |
| "entailment": 0.725, |
| "neutral": 0.325 |
| }, |
| "brier": 0.5619903249840926, |
| "nll": 0.952791383586364, |
| "ece_raw": 0.0067628751799769776, |
| "ece_fitted": 0.00836092388336822 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 65 |
| }, |
| "sst5": { |
| "accuracy": 0.3333333333333333, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.038905105700003634, |
| "p50_seconds": 0.03851513350002733, |
| "p95_seconds": 0.041932194849789536 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": 0.8020032497743765, |
| "rmse": 0.9909589189089723, |
| "nearest_level_accuracy": 0.3333333333333333, |
| "within_one_accuracy": 0.875, |
| "raw_equality_rate": 0.0, |
| "spearman": 0.737594189458718, |
| "ordinal_confusion": { |
| "labels": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4 |
| ], |
| "matrix": [ |
| [ |
| 0, |
| 19, |
| 4, |
| 1, |
| 0 |
| ], |
| [ |
| 2, |
| 10, |
| 12, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 17, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 11, |
| 12, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 8, |
| 14, |
| 1 |
| ] |
| ] |
| } |
| } |
| }, |
| "correct": 40 |
| }, |
| "multilingual_intent": { |
| "accuracy": 0.95, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.039158464716655315, |
| "p50_seconds": 0.03864724799996111, |
| "p95_seconds": 0.04240460730002269 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.95, |
| "macro_f1": 0.8159019343229869, |
| "balanced_accuracy": 0.9650000000000001, |
| "per_class_recall": { |
| "billing_question": 0.95, |
| "cancellation": 1.0, |
| "information": 0.875, |
| "other": null, |
| "refund": 1.0, |
| "technical_help": 1.0 |
| }, |
| "brier": 0.12235859002542863, |
| "nll": 0.24005612544914204, |
| "ece_raw": 0.008839931130203269, |
| "ece_fitted": 0.0071500889173673875 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 114 |
| } |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.7690217391304348, |
| "macro_f1": 0.748753245465977, |
| "balanced_accuracy": 0.7745682853365854, |
| "per_class_recall": { |
| "anger": 0.5714285714285714, |
| "billing_question": 0.7241379310344828, |
| "business": 0.85, |
| "cancellation": 1.0, |
| "card_arrival": 0.625, |
| "card_linking": 0.25, |
| "contradiction": 0.575, |
| "entailment": 0.725, |
| "exchange_rate": 1.0, |
| "fear": 0.9259259259259259, |
| "information": 0.9, |
| "joy": 0.7142857142857143, |
| "lost_or_stolen_card": 1.0, |
| "love": 0.6, |
| "neutral": 0.325, |
| "order_physical_card": 0.875, |
| "other": null, |
| "pin_blocked": 0.875, |
| "refund": 1.0, |
| "refund_not_showing_up": 0.875, |
| "request_refund": 0.875, |
| "sadness": 0.8275862068965517, |
| "scitech": 0.55, |
| "sports": 0.975, |
| "surprise": 0.47368421052631576, |
| "technical_help": 1.0, |
| "terminate_account": 1.0, |
| "top_up_failed": 0.875, |
| "topping_up_by_card": 0.5, |
| "transaction_charged_twice": 0.875, |
| "world": 0.875 |
| }, |
| "brier": 0.3270658633907902, |
| "nll": 0.6528267504422001, |
| "ece_raw": 0.0007309538327833588, |
| "ece_fitted": 0.0004591988587776608 |
| }, |
| "noul": { |
| "accuracy": 0.9036458333333334, |
| "balanced_accuracy": 0.8834257029277777, |
| "auroc": 0.972521254678931, |
| "auprc": 0.9503028848436744, |
| "brier": 0.07737625808758589, |
| "nll": 0.2542762084776515, |
| "ece_raw": 0.004463339661746162, |
| "ece_fitted": 0.0049155860343304296 |
| }, |
| "score": { |
| "mae": 0.8020032497743765, |
| "rmse": 0.9909589189089723, |
| "nearest_level_accuracy": 0.3333333333333333, |
| "within_one_accuracy": 0.875, |
| "raw_equality_rate": 0.0, |
| "spearman": 0.737594189458718, |
| "ordinal_confusion": { |
| "labels": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4 |
| ], |
| "matrix": [ |
| [ |
| 0, |
| 19, |
| 4, |
| 1, |
| 0 |
| ], |
| [ |
| 2, |
| 10, |
| 12, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 17, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 11, |
| 12, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 8, |
| 14, |
| 1 |
| ] |
| ] |
| } |
| } |
| } |
| }, |
| "calibrated": { |
| "evaluation_cases": 952, |
| "evaluation_decisions": 1240, |
| "correct": 953, |
| "accuracy": 0.7685483870967742, |
| "suites": { |
| "triage": { |
| "accuracy": 0.9270833333333334, |
| "cases": 48, |
| "decisions": 192, |
| "latency": { |
| "mean_seconds": 0.03883962416667922, |
| "p50_seconds": 0.03841860600016389, |
| "p95_seconds": 0.04178968690000602 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8541666666666666, |
| "macro_f1": 0.820875420875421, |
| "balanced_accuracy": 0.8444444444444444, |
| "per_class_recall": { |
| "billing_question": 0.2222222222222222, |
| "cancellation": 1.0, |
| "information": 1.0, |
| "refund": 1.0, |
| "technical_help": 1.0 |
| }, |
| "brier": 0.14243087345916075, |
| "nll": 0.2813106462197595, |
| "ece_raw": 0.028952845660412414, |
| "ece_fitted": 0.03991474162123874 |
| }, |
| "noul": { |
| "accuracy": 0.9513888888888888, |
| "balanced_accuracy": 0.912037037037037, |
| "auroc": 0.9886831275720165, |
| "auprc": 0.9700223660164052, |
| "brier": 0.052313952030937215, |
| "nll": 0.20641514576135578, |
| "ece_raw": 0.010566808507195363, |
| "ece_fitted": 0.012443507589047501 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 178 |
| }, |
| "guardrails": { |
| "accuracy": 0.8125, |
| "cases": 48, |
| "decisions": 96, |
| "latency": { |
| "mean_seconds": 0.037130780229148286, |
| "p50_seconds": 0.03704196349997346, |
| "p95_seconds": 0.03867655395000611 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": 0.8125, |
| "balanced_accuracy": 0.8676470588235294, |
| "auroc": 0.9803046218487395, |
| "auprc": 0.9910196723104415, |
| "brier": 0.14399152528197112, |
| "nll": 0.4301690068230142, |
| "ece_raw": 0.027843129969293463, |
| "ece_fitted": 0.02698340407756053 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 78 |
| }, |
| "moderation": { |
| "accuracy": 0.9166666666666666, |
| "cases": 48, |
| "decisions": 144, |
| "latency": { |
| "mean_seconds": 0.03827599806249301, |
| "p50_seconds": 0.037934129999939614, |
| "p95_seconds": 0.041699429449965915 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": 0.9166666666666666, |
| "balanced_accuracy": 0.9106227106227107, |
| "auroc": 0.978021978021978, |
| "auprc": 0.944990138816365, |
| "brier": 0.06164195701639249, |
| "nll": 0.21176174396122044, |
| "ece_raw": 0.014191582921902587, |
| "ece_fitted": 0.01169790659953756 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 132 |
| }, |
| "agnews": { |
| "accuracy": 0.8125, |
| "cases": 160, |
| "decisions": 160, |
| "latency": { |
| "mean_seconds": 0.038758980293738435, |
| "p50_seconds": 0.0382035000001224, |
| "p95_seconds": 0.042714876549950986 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8125, |
| "macro_f1": 0.8064558008777994, |
| "balanced_accuracy": 0.8125, |
| "per_class_recall": { |
| "business": 0.85, |
| "scitech": 0.55, |
| "sports": 0.975, |
| "world": 0.875 |
| }, |
| "brier": 0.2965232296874367, |
| "nll": 0.5908707062914559, |
| "ece_raw": 0.004136447508252946, |
| "ece_fitted": 0.007738817845546403 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 130 |
| }, |
| "emotion": { |
| "accuracy": 0.7239583333333334, |
| "cases": 192, |
| "decisions": 192, |
| "latency": { |
| "mean_seconds": 0.038396373302075425, |
| "p50_seconds": 0.03797906949989738, |
| "p95_seconds": 0.0415361116498957 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.7239583333333334, |
| "macro_f1": 0.6913006811212603, |
| "balanced_accuracy": 0.6854851048438465, |
| "per_class_recall": { |
| "anger": 0.5714285714285714, |
| "fear": 0.9259259259259259, |
| "joy": 0.7142857142857143, |
| "love": 0.6, |
| "sadness": 0.8275862068965517, |
| "surprise": 0.47368421052631576 |
| }, |
| "brier": 0.39805364003393406, |
| "nll": 0.7908805977682863, |
| "ece_raw": 0.004164112978894893, |
| "ece_fitted": 0.0034169522756447193 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 139 |
| }, |
| "banking77_12": { |
| "accuracy": 0.8020833333333334, |
| "cases": 96, |
| "decisions": 96, |
| "latency": { |
| "mean_seconds": 0.03918704791666509, |
| "p50_seconds": 0.038639419499986616, |
| "p95_seconds": 0.04271133699995744 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8020833333333334, |
| "macro_f1": 0.7940683880389763, |
| "balanced_accuracy": 0.8020833333333334, |
| "per_class_recall": { |
| "card_arrival": 0.625, |
| "card_linking": 0.25, |
| "exchange_rate": 1.0, |
| "lost_or_stolen_card": 1.0, |
| "order_physical_card": 0.875, |
| "pin_blocked": 0.875, |
| "refund_not_showing_up": 0.875, |
| "request_refund": 0.875, |
| "terminate_account": 1.0, |
| "top_up_failed": 0.875, |
| "topping_up_by_card": 0.5, |
| "transaction_charged_twice": 0.875 |
| }, |
| "brier": 0.2841311372982186, |
| "nll": 0.7256759563838567, |
| "ece_raw": 0.014651013888121192, |
| "ece_fitted": 0.0165529263424545 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 77 |
| }, |
| "mnli": { |
| "accuracy": 0.5416666666666666, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.0384311384083181, |
| "p50_seconds": 0.03797633499982567, |
| "p95_seconds": 0.04150163254993231 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.5416666666666666, |
| "macro_f1": 0.531705418323941, |
| "balanced_accuracy": 0.5416666666666666, |
| "per_class_recall": { |
| "contradiction": 0.575, |
| "entailment": 0.725, |
| "neutral": 0.325 |
| }, |
| "brier": 0.5534738375592544, |
| "nll": 0.9317788432632266, |
| "ece_raw": 0.005991485962656614, |
| "ece_fitted": 0.008360924461080638 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 65 |
| }, |
| "sst5": { |
| "accuracy": 0.3333333333333333, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.038905105700003634, |
| "p50_seconds": 0.03851513350002733, |
| "p95_seconds": 0.041932194849789536 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": 0.8023576299349467, |
| "rmse": 0.9908613116687078, |
| "nearest_level_accuracy": 0.3333333333333333, |
| "within_one_accuracy": 0.875, |
| "raw_equality_rate": 0.0, |
| "spearman": 0.7382746269028682, |
| "ordinal_confusion": { |
| "labels": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4 |
| ], |
| "matrix": [ |
| [ |
| 0, |
| 19, |
| 4, |
| 1, |
| 0 |
| ], |
| [ |
| 2, |
| 10, |
| 12, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 17, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 11, |
| 12, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 8, |
| 14, |
| 1 |
| ] |
| ] |
| } |
| } |
| }, |
| "correct": 40 |
| }, |
| "multilingual_intent": { |
| "accuracy": 0.95, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.039158464716655315, |
| "p50_seconds": 0.03864724799996111, |
| "p95_seconds": 0.04240460730002269 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.95, |
| "macro_f1": 0.8159019343229869, |
| "balanced_accuracy": 0.9650000000000001, |
| "per_class_recall": { |
| "billing_question": 0.95, |
| "cancellation": 1.0, |
| "information": 0.875, |
| "other": null, |
| "refund": 1.0, |
| "technical_help": 1.0 |
| }, |
| "brier": 0.13506004724959844, |
| "nll": 0.2766243432861846, |
| "ece_raw": 0.01098334737043859, |
| "ece_fitted": 0.007150088729354088 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 114 |
| } |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.7690217391304348, |
| "macro_f1": 0.748753245465977, |
| "balanced_accuracy": 0.7745682853365854, |
| "per_class_recall": { |
| "anger": 0.5714285714285714, |
| "billing_question": 0.7241379310344828, |
| "business": 0.85, |
| "cancellation": 1.0, |
| "card_arrival": 0.625, |
| "card_linking": 0.25, |
| "contradiction": 0.575, |
| "entailment": 0.725, |
| "exchange_rate": 1.0, |
| "fear": 0.9259259259259259, |
| "information": 0.9, |
| "joy": 0.7142857142857143, |
| "lost_or_stolen_card": 1.0, |
| "love": 0.6, |
| "neutral": 0.325, |
| "order_physical_card": 0.875, |
| "other": null, |
| "pin_blocked": 0.875, |
| "refund": 1.0, |
| "refund_not_showing_up": 0.875, |
| "request_refund": 0.875, |
| "sadness": 0.8275862068965517, |
| "scitech": 0.55, |
| "sports": 0.975, |
| "surprise": 0.47368421052631576, |
| "technical_help": 1.0, |
| "terminate_account": 1.0, |
| "top_up_failed": 0.875, |
| "topping_up_by_card": 0.5, |
| "transaction_charged_twice": 0.875, |
| "world": 0.875 |
| }, |
| "brier": 0.32691216429379927, |
| "nll": 0.6447888220047173, |
| "ece_raw": 0.000757314816236097, |
| "ece_fitted": 0.00045919887827611055 |
| }, |
| "noul": { |
| "accuracy": 0.9036458333333334, |
| "balanced_accuracy": 0.8834257029277777, |
| "auroc": 0.972492238052404, |
| "auprc": 0.950340531508563, |
| "brier": 0.07873134721324142, |
| "nll": 0.26435858535171963, |
| "ece_raw": 0.004409246207057182, |
| "ece_fitted": 0.004915585951212646 |
| }, |
| "score": { |
| "mae": 0.8023576299349467, |
| "rmse": 0.9908613116687078, |
| "nearest_level_accuracy": 0.3333333333333333, |
| "within_one_accuracy": 0.875, |
| "raw_equality_rate": 0.0, |
| "spearman": 0.7382746269028682, |
| "ordinal_confusion": { |
| "labels": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4 |
| ], |
| "matrix": [ |
| [ |
| 0, |
| 19, |
| 4, |
| 1, |
| 0 |
| ], |
| [ |
| 2, |
| 10, |
| 12, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 17, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 11, |
| 12, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 8, |
| 14, |
| 1 |
| ] |
| ] |
| } |
| } |
| } |
| }, |
| "flagged_phrase_excluded": { |
| "excluded_cases": [ |
| "banking77_12-0084" |
| ], |
| "raw": { |
| "evaluation_cases": 951, |
| "evaluation_decisions": 1239, |
| "correct": 952, |
| "accuracy": 0.768361581920904, |
| "suites": { |
| "triage": { |
| "accuracy": 0.9270833333333334, |
| "cases": 48, |
| "decisions": 192, |
| "latency": { |
| "mean_seconds": 0.03883962416667922, |
| "p50_seconds": 0.03841860600016389, |
| "p95_seconds": 0.04178968690000602 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8541666666666666, |
| "macro_f1": 0.820875420875421, |
| "balanced_accuracy": 0.8444444444444444, |
| "per_class_recall": { |
| "billing_question": 0.2222222222222222, |
| "cancellation": 1.0, |
| "information": 1.0, |
| "refund": 1.0, |
| "technical_help": 1.0 |
| }, |
| "brier": 0.13697967505778072, |
| "nll": 0.2506294304217651, |
| "ece_raw": 0.033931114278253044, |
| "ece_fitted": 0.039914814482762906 |
| }, |
| "noul": { |
| "accuracy": 0.9513888888888888, |
| "balanced_accuracy": 0.912037037037037, |
| "auroc": 0.9890689300411523, |
| "auprc": 0.9701626578233637, |
| "brier": 0.04911263462874029, |
| "nll": 0.1906247610132196, |
| "ece_raw": 0.01181868597485985, |
| "ece_fitted": 0.012443502260767868 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 178 |
| }, |
| "guardrails": { |
| "accuracy": 0.8125, |
| "cases": 48, |
| "decisions": 96, |
| "latency": { |
| "mean_seconds": 0.037130780229148286, |
| "p50_seconds": 0.03704196349997346, |
| "p95_seconds": 0.03867655395000611 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": 0.8125, |
| "balanced_accuracy": 0.8676470588235294, |
| "auroc": 0.9803046218487395, |
| "auprc": 0.9910730108125466, |
| "brier": 0.14491041421218856, |
| "nll": 0.4276831543302518, |
| "ece_raw": 0.02432198575945292, |
| "ece_fitted": 0.026983405418465858 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 78 |
| }, |
| "moderation": { |
| "accuracy": 0.9166666666666666, |
| "cases": 48, |
| "decisions": 144, |
| "latency": { |
| "mean_seconds": 0.03827599806249301, |
| "p50_seconds": 0.037934129999939614, |
| "p95_seconds": 0.041699429449965915 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": 0.9166666666666666, |
| "balanced_accuracy": 0.9106227106227107, |
| "auroc": 0.978021978021978, |
| "auprc": 0.944990138816365, |
| "brier": 0.06061711079669634, |
| "nll": 0.2023230253736832, |
| "ece_raw": 0.010916557094938821, |
| "ece_fitted": 0.011697905217155507 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 132 |
| }, |
| "agnews": { |
| "accuracy": 0.8125, |
| "cases": 160, |
| "decisions": 160, |
| "latency": { |
| "mean_seconds": 0.038758980293738435, |
| "p50_seconds": 0.0382035000001224, |
| "p95_seconds": 0.042714876549950986 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8125, |
| "macro_f1": 0.8064558008777994, |
| "balanced_accuracy": 0.8125, |
| "per_class_recall": { |
| "business": 0.85, |
| "scitech": 0.55, |
| "sports": 0.975, |
| "world": 0.875 |
| }, |
| "brier": 0.30172771335742954, |
| "nll": 0.6319382379388891, |
| "ece_raw": 0.0052000811604517765, |
| "ece_fitted": 0.007738817523861061 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 130 |
| }, |
| "emotion": { |
| "accuracy": 0.7239583333333334, |
| "cases": 192, |
| "decisions": 192, |
| "latency": { |
| "mean_seconds": 0.038396373302075425, |
| "p50_seconds": 0.03797906949989738, |
| "p95_seconds": 0.0415361116498957 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.7239583333333334, |
| "macro_f1": 0.6913006811212603, |
| "balanced_accuracy": 0.6854851048438465, |
| "per_class_recall": { |
| "anger": 0.5714285714285714, |
| "fear": 0.9259259259259259, |
| "joy": 0.7142857142857143, |
| "love": 0.6, |
| "sadness": 0.8275862068965517, |
| "surprise": 0.47368421052631576 |
| }, |
| "brier": 0.39443242180284094, |
| "nll": 0.7806033322263032, |
| "ece_raw": 0.0032102610291140795, |
| "ece_fitted": 0.003416953263655513 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 139 |
| }, |
| "banking77_12": { |
| "accuracy": 0.8, |
| "cases": 95, |
| "decisions": 95, |
| "latency": { |
| "mean_seconds": 0.03918948851578773, |
| "p50_seconds": 0.03858194800000092, |
| "p95_seconds": 0.04271281379994889 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8, |
| "macro_f1": 0.7934147932677345, |
| "balanced_accuracy": 0.8020833333333334, |
| "per_class_recall": { |
| "card_arrival": 0.625, |
| "card_linking": 0.25, |
| "exchange_rate": 1.0, |
| "lost_or_stolen_card": 1.0, |
| "order_physical_card": 0.875, |
| "pin_blocked": 0.875, |
| "refund_not_showing_up": 0.875, |
| "request_refund": 0.875, |
| "terminate_account": 1.0, |
| "top_up_failed": 0.875, |
| "topping_up_by_card": 0.5, |
| "transaction_charged_twice": 0.875 |
| }, |
| "brier": 0.29490650156145987, |
| "nll": 0.782323559878833, |
| "ece_raw": 0.01680271757873975, |
| "ece_fitted": 0.016734122969269134 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 76 |
| }, |
| "mnli": { |
| "accuracy": 0.5416666666666666, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.0384311384083181, |
| "p50_seconds": 0.03797633499982567, |
| "p95_seconds": 0.04150163254993231 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.5416666666666666, |
| "macro_f1": 0.531705418323941, |
| "balanced_accuracy": 0.5416666666666666, |
| "per_class_recall": { |
| "contradiction": 0.575, |
| "entailment": 0.725, |
| "neutral": 0.325 |
| }, |
| "brier": 0.5619903249840926, |
| "nll": 0.952791383586364, |
| "ece_raw": 0.0067628751799769776, |
| "ece_fitted": 0.00836092388336822 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 65 |
| }, |
| "sst5": { |
| "accuracy": 0.3333333333333333, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.038905105700003634, |
| "p50_seconds": 0.03851513350002733, |
| "p95_seconds": 0.041932194849789536 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": 0.8020032497743765, |
| "rmse": 0.9909589189089723, |
| "nearest_level_accuracy": 0.3333333333333333, |
| "within_one_accuracy": 0.875, |
| "raw_equality_rate": 0.0, |
| "spearman": 0.737594189458718, |
| "ordinal_confusion": { |
| "labels": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4 |
| ], |
| "matrix": [ |
| [ |
| 0, |
| 19, |
| 4, |
| 1, |
| 0 |
| ], |
| [ |
| 2, |
| 10, |
| 12, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 17, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 11, |
| 12, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 8, |
| 14, |
| 1 |
| ] |
| ] |
| } |
| } |
| }, |
| "correct": 40 |
| }, |
| "multilingual_intent": { |
| "accuracy": 0.95, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.039158464716655315, |
| "p50_seconds": 0.03864724799996111, |
| "p95_seconds": 0.04240460730002269 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.95, |
| "macro_f1": 0.8159019343229869, |
| "balanced_accuracy": 0.9650000000000001, |
| "per_class_recall": { |
| "billing_question": 0.95, |
| "cancellation": 1.0, |
| "information": 0.875, |
| "other": null, |
| "refund": 1.0, |
| "technical_help": 1.0 |
| }, |
| "brier": 0.12235859002542863, |
| "nll": 0.24005612544914204, |
| "ece_raw": 0.008839931130203269, |
| "ece_fitted": 0.0071500889173673875 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 114 |
| } |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.7687074829931972, |
| "macro_f1": 0.7485002410383995, |
| "balanced_accuracy": 0.7745682853365854, |
| "per_class_recall": { |
| "anger": 0.5714285714285714, |
| "billing_question": 0.7241379310344828, |
| "business": 0.85, |
| "cancellation": 1.0, |
| "card_arrival": 0.625, |
| "card_linking": 0.25, |
| "contradiction": 0.575, |
| "entailment": 0.725, |
| "exchange_rate": 1.0, |
| "fear": 0.9259259259259259, |
| "information": 0.9, |
| "joy": 0.7142857142857143, |
| "lost_or_stolen_card": 1.0, |
| "love": 0.6, |
| "neutral": 0.325, |
| "order_physical_card": 0.875, |
| "other": null, |
| "pin_blocked": 0.875, |
| "refund": 1.0, |
| "refund_not_showing_up": 0.875, |
| "request_refund": 0.875, |
| "sadness": 0.8275862068965517, |
| "scitech": 0.55, |
| "sports": 0.975, |
| "surprise": 0.47368421052631576, |
| "technical_help": 1.0, |
| "terminate_account": 1.0, |
| "top_up_failed": 0.875, |
| "topping_up_by_card": 0.5, |
| "transaction_charged_twice": 0.875, |
| "world": 0.875 |
| }, |
| "brier": 0.3275108448647468, |
| "nll": 0.6537123942730164, |
| "ece_raw": 0.000732155258879059, |
| "ece_fitted": 0.00045994712310874536 |
| }, |
| "noul": { |
| "accuracy": 0.9036458333333334, |
| "balanced_accuracy": 0.8834257029277777, |
| "auroc": 0.972521254678931, |
| "auprc": 0.9503028848436744, |
| "brier": 0.07737625808758589, |
| "nll": 0.2542762084776515, |
| "ece_raw": 0.004463339661746162, |
| "ece_fitted": 0.0049155860343304296 |
| }, |
| "score": { |
| "mae": 0.8020032497743765, |
| "rmse": 0.9909589189089723, |
| "nearest_level_accuracy": 0.3333333333333333, |
| "within_one_accuracy": 0.875, |
| "raw_equality_rate": 0.0, |
| "spearman": 0.737594189458718, |
| "ordinal_confusion": { |
| "labels": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4 |
| ], |
| "matrix": [ |
| [ |
| 0, |
| 19, |
| 4, |
| 1, |
| 0 |
| ], |
| [ |
| 2, |
| 10, |
| 12, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 17, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 11, |
| 12, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 8, |
| 14, |
| 1 |
| ] |
| ] |
| } |
| } |
| } |
| }, |
| "calibrated": { |
| "evaluation_cases": 951, |
| "evaluation_decisions": 1239, |
| "correct": 952, |
| "accuracy": 0.768361581920904, |
| "suites": { |
| "triage": { |
| "accuracy": 0.9270833333333334, |
| "cases": 48, |
| "decisions": 192, |
| "latency": { |
| "mean_seconds": 0.03883962416667922, |
| "p50_seconds": 0.03841860600016389, |
| "p95_seconds": 0.04178968690000602 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8541666666666666, |
| "macro_f1": 0.820875420875421, |
| "balanced_accuracy": 0.8444444444444444, |
| "per_class_recall": { |
| "billing_question": 0.2222222222222222, |
| "cancellation": 1.0, |
| "information": 1.0, |
| "refund": 1.0, |
| "technical_help": 1.0 |
| }, |
| "brier": 0.14243087345916075, |
| "nll": 0.2813106462197595, |
| "ece_raw": 0.028952845660412414, |
| "ece_fitted": 0.03991474162123874 |
| }, |
| "noul": { |
| "accuracy": 0.9513888888888888, |
| "balanced_accuracy": 0.912037037037037, |
| "auroc": 0.9886831275720165, |
| "auprc": 0.9700223660164052, |
| "brier": 0.052313952030937215, |
| "nll": 0.20641514576135578, |
| "ece_raw": 0.010566808507195363, |
| "ece_fitted": 0.012443507589047501 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 178 |
| }, |
| "guardrails": { |
| "accuracy": 0.8125, |
| "cases": 48, |
| "decisions": 96, |
| "latency": { |
| "mean_seconds": 0.037130780229148286, |
| "p50_seconds": 0.03704196349997346, |
| "p95_seconds": 0.03867655395000611 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": 0.8125, |
| "balanced_accuracy": 0.8676470588235294, |
| "auroc": 0.9803046218487395, |
| "auprc": 0.9910196723104415, |
| "brier": 0.14399152528197112, |
| "nll": 0.4301690068230142, |
| "ece_raw": 0.027843129969293463, |
| "ece_fitted": 0.02698340407756053 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 78 |
| }, |
| "moderation": { |
| "accuracy": 0.9166666666666666, |
| "cases": 48, |
| "decisions": 144, |
| "latency": { |
| "mean_seconds": 0.03827599806249301, |
| "p50_seconds": 0.037934129999939614, |
| "p95_seconds": 0.041699429449965915 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": 0.9166666666666666, |
| "balanced_accuracy": 0.9106227106227107, |
| "auroc": 0.978021978021978, |
| "auprc": 0.944990138816365, |
| "brier": 0.06164195701639249, |
| "nll": 0.21176174396122044, |
| "ece_raw": 0.014191582921902587, |
| "ece_fitted": 0.01169790659953756 |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 132 |
| }, |
| "agnews": { |
| "accuracy": 0.8125, |
| "cases": 160, |
| "decisions": 160, |
| "latency": { |
| "mean_seconds": 0.038758980293738435, |
| "p50_seconds": 0.0382035000001224, |
| "p95_seconds": 0.042714876549950986 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8125, |
| "macro_f1": 0.8064558008777994, |
| "balanced_accuracy": 0.8125, |
| "per_class_recall": { |
| "business": 0.85, |
| "scitech": 0.55, |
| "sports": 0.975, |
| "world": 0.875 |
| }, |
| "brier": 0.2965232296874367, |
| "nll": 0.5908707062914559, |
| "ece_raw": 0.004136447508252946, |
| "ece_fitted": 0.007738817845546403 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 130 |
| }, |
| "emotion": { |
| "accuracy": 0.7239583333333334, |
| "cases": 192, |
| "decisions": 192, |
| "latency": { |
| "mean_seconds": 0.038396373302075425, |
| "p50_seconds": 0.03797906949989738, |
| "p95_seconds": 0.0415361116498957 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.7239583333333334, |
| "macro_f1": 0.6913006811212603, |
| "balanced_accuracy": 0.6854851048438465, |
| "per_class_recall": { |
| "anger": 0.5714285714285714, |
| "fear": 0.9259259259259259, |
| "joy": 0.7142857142857143, |
| "love": 0.6, |
| "sadness": 0.8275862068965517, |
| "surprise": 0.47368421052631576 |
| }, |
| "brier": 0.39805364003393406, |
| "nll": 0.7908805977682863, |
| "ece_raw": 0.004164112978894893, |
| "ece_fitted": 0.0034169522756447193 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 139 |
| }, |
| "banking77_12": { |
| "accuracy": 0.8, |
| "cases": 95, |
| "decisions": 95, |
| "latency": { |
| "mean_seconds": 0.03918948851578773, |
| "p50_seconds": 0.03858194800000092, |
| "p95_seconds": 0.04271281379994889 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.8, |
| "macro_f1": 0.7934147932677345, |
| "balanced_accuracy": 0.8020833333333334, |
| "per_class_recall": { |
| "card_arrival": 0.625, |
| "card_linking": 0.25, |
| "exchange_rate": 1.0, |
| "lost_or_stolen_card": 1.0, |
| "order_physical_card": 0.875, |
| "pin_blocked": 0.875, |
| "refund_not_showing_up": 0.875, |
| "request_refund": 0.875, |
| "terminate_account": 1.0, |
| "top_up_failed": 0.875, |
| "topping_up_by_card": 0.5, |
| "transaction_charged_twice": 0.875 |
| }, |
| "brier": 0.2871215913829621, |
| "nll": 0.7332560464140869, |
| "ece_raw": 0.014809638592014682, |
| "ece_fitted": 0.01673412400480368 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 76 |
| }, |
| "mnli": { |
| "accuracy": 0.5416666666666666, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.0384311384083181, |
| "p50_seconds": 0.03797633499982567, |
| "p95_seconds": 0.04150163254993231 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.5416666666666666, |
| "macro_f1": 0.531705418323941, |
| "balanced_accuracy": 0.5416666666666666, |
| "per_class_recall": { |
| "contradiction": 0.575, |
| "entailment": 0.725, |
| "neutral": 0.325 |
| }, |
| "brier": 0.5534738375592544, |
| "nll": 0.9317788432632266, |
| "ece_raw": 0.005991485962656614, |
| "ece_fitted": 0.008360924461080638 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 65 |
| }, |
| "sst5": { |
| "accuracy": 0.3333333333333333, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.038905105700003634, |
| "p50_seconds": 0.03851513350002733, |
| "p95_seconds": 0.041932194849789536 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": null, |
| "macro_f1": null, |
| "balanced_accuracy": null, |
| "per_class_recall": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": 0.8023576299349467, |
| "rmse": 0.9908613116687078, |
| "nearest_level_accuracy": 0.3333333333333333, |
| "within_one_accuracy": 0.875, |
| "raw_equality_rate": 0.0, |
| "spearman": 0.7382746269028682, |
| "ordinal_confusion": { |
| "labels": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4 |
| ], |
| "matrix": [ |
| [ |
| 0, |
| 19, |
| 4, |
| 1, |
| 0 |
| ], |
| [ |
| 2, |
| 10, |
| 12, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 17, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 11, |
| 12, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 8, |
| 14, |
| 1 |
| ] |
| ] |
| } |
| } |
| }, |
| "correct": 40 |
| }, |
| "multilingual_intent": { |
| "accuracy": 0.95, |
| "cases": 120, |
| "decisions": 120, |
| "latency": { |
| "mean_seconds": 0.039158464716655315, |
| "p50_seconds": 0.03864724799996111, |
| "p95_seconds": 0.04240460730002269 |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.95, |
| "macro_f1": 0.8159019343229869, |
| "balanced_accuracy": 0.9650000000000001, |
| "per_class_recall": { |
| "billing_question": 0.95, |
| "cancellation": 1.0, |
| "information": 0.875, |
| "other": null, |
| "refund": 1.0, |
| "technical_help": 1.0 |
| }, |
| "brier": 0.13506004724959844, |
| "nll": 0.2766243432861846, |
| "ece_raw": 0.01098334737043859, |
| "ece_fitted": 0.007150088729354088 |
| }, |
| "noul": { |
| "accuracy": null, |
| "balanced_accuracy": null, |
| "auroc": null, |
| "auprc": null, |
| "brier": null, |
| "nll": null, |
| "ece_raw": null, |
| "ece_fitted": null |
| }, |
| "score": { |
| "mae": null, |
| "rmse": null, |
| "nearest_level_accuracy": null, |
| "within_one_accuracy": null, |
| "raw_equality_rate": null, |
| "spearman": null, |
| "ordinal_confusion": null |
| } |
| }, |
| "correct": 114 |
| } |
| }, |
| "primitive_metrics": { |
| "choice": { |
| "accuracy": 0.7687074829931972, |
| "macro_f1": 0.7485002410383995, |
| "balanced_accuracy": 0.7745682853365854, |
| "per_class_recall": { |
| "anger": 0.5714285714285714, |
| "billing_question": 0.7241379310344828, |
| "business": 0.85, |
| "cancellation": 1.0, |
| "card_arrival": 0.625, |
| "card_linking": 0.25, |
| "contradiction": 0.575, |
| "entailment": 0.725, |
| "exchange_rate": 1.0, |
| "fear": 0.9259259259259259, |
| "information": 0.9, |
| "joy": 0.7142857142857143, |
| "lost_or_stolen_card": 1.0, |
| "love": 0.6, |
| "neutral": 0.325, |
| "order_physical_card": 0.875, |
| "other": null, |
| "pin_blocked": 0.875, |
| "refund": 1.0, |
| "refund_not_showing_up": 0.875, |
| "request_refund": 0.875, |
| "sadness": 0.8275862068965517, |
| "scitech": 0.55, |
| "sports": 0.975, |
| "surprise": 0.47368421052631576, |
| "technical_help": 1.0, |
| "terminate_account": 1.0, |
| "top_up_failed": 0.875, |
| "topping_up_by_card": 0.5, |
| "transaction_charged_twice": 0.875, |
| "world": 0.875 |
| }, |
| "brier": 0.3273568910489642, |
| "nll": 0.6456585110094694, |
| "ece_raw": 0.0007584697829628141, |
| "ece_fitted": 0.0004599471426753296 |
| }, |
| "noul": { |
| "accuracy": 0.9036458333333334, |
| "balanced_accuracy": 0.8834257029277777, |
| "auroc": 0.972492238052404, |
| "auprc": 0.950340531508563, |
| "brier": 0.07873134721324142, |
| "nll": 0.26435858535171963, |
| "ece_raw": 0.004409246207057182, |
| "ece_fitted": 0.004915585951212646 |
| }, |
| "score": { |
| "mae": 0.8023576299349467, |
| "rmse": 0.9908613116687078, |
| "nearest_level_accuracy": 0.3333333333333333, |
| "within_one_accuracy": 0.875, |
| "raw_equality_rate": 0.0, |
| "spearman": 0.7382746269028682, |
| "ordinal_confusion": { |
| "labels": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4 |
| ], |
| "matrix": [ |
| [ |
| 0, |
| 19, |
| 4, |
| 1, |
| 0 |
| ], |
| [ |
| 2, |
| 10, |
| 12, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 17, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 11, |
| 12, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 8, |
| 14, |
| 1 |
| ] |
| ] |
| } |
| } |
| } |
| } |
| }, |
| "score_rule": "Expected level index passed as scalar score; official floor(score+0.5) correctness, not modal-level accuracy", |
| "label_note": "Original v2 labels were single-review AI-assisted. Corrected v2.1.0 adopts source labels for AGNews, Banking77, MNLI and SST5 and a reviewed emotion decision set. No independent double-review claim is made.", |
| "official_submission": false |
| }, |
| "protocol_and_overlap_notes": "See README.md. The metrics have different denominators and definitions." |
| } |
|
|
