{
  "selected_step": 4153,
  "mixture_test": {
    "test_completed": true,
    "calibration_completed": true,
    "raw": {
      "overall": {
        "count": 3693,
        "soft_ce": 0.7085598693044783,
        "kl": 0.3931482496012985,
        "brier": 0.09769070277607729,
        "argmax_agreement": 0.7733549959382616,
        "hard_count": 2134,
        "hard_ece": 0.042129272019540774,
        "hard_reliability": [
          {
            "count": 1,
            "confidence": 0.1280767528435369,
            "reference_agreement": 0.0
          },
          {
            "count": 18,
            "confidence": 0.18404859928386386,
            "reference_agreement": 0.2222222222222222
          },
          {
            "count": 23,
            "confidence": 0.23661101358223402,
            "reference_agreement": 0.2608695652173913
          },
          {
            "count": 37,
            "confidence": 0.30428753988952256,
            "reference_agreement": 0.1891891891891892
          },
          {
            "count": 36,
            "confidence": 0.36932558263805926,
            "reference_agreement": 0.3888888888888889
          },
          {
            "count": 57,
            "confidence": 0.43265722650360866,
            "reference_agreement": 0.3333333333333333
          },
          {
            "count": 71,
            "confidence": 0.5035452847115158,
            "reference_agreement": 0.5070422535211268
          },
          {
            "count": 93,
            "confidence": 0.5702882440958308,
            "reference_agreement": 0.5053763440860215
          },
          {
            "count": 113,
            "confidence": 0.6289013423061501,
            "reference_agreement": 0.5398230088495575
          },
          {
            "count": 88,
            "confidence": 0.7035225652548067,
            "reference_agreement": 0.5909090909090909
          },
          {
            "count": 97,
            "confidence": 0.7663997989026462,
            "reference_agreement": 0.6804123711340206
          },
          {
            "count": 133,
            "confidence": 0.8331041320580328,
            "reference_agreement": 0.7218045112781954
          },
          {
            "count": 213,
            "confidence": 0.9010564345576307,
            "reference_agreement": 0.812206572769953
          },
          {
            "count": 1154,
            "confidence": 0.9892346259941478,
            "reference_agreement": 0.9809358752166378
          }
        ],
        "soft_reliability": [
          {
            "count": 1,
            "confidence": 0.1280767528435369,
            "reference_agreement": 0.0
          },
          {
            "count": 22,
            "confidence": 0.18189946160649148,
            "reference_agreement": 0.253859649122807
          },
          {
            "count": 52,
            "confidence": 0.24099144225648178,
            "reference_agreement": 0.2604555167055167
          },
          {
            "count": 105,
            "confidence": 0.30435695062773516,
            "reference_agreement": 0.2666363084654525
          },
          {
            "count": 151,
            "confidence": 0.36784610221152053,
            "reference_agreement": 0.38072070275124964
          },
          {
            "count": 219,
            "confidence": 0.4348046806054027,
            "reference_agreement": 0.4029960976805746
          },
          {
            "count": 268,
            "confidence": 0.5012982178117764,
            "reference_agreement": 0.4966141654778031
          },
          {
            "count": 287,
            "confidence": 0.5665963085542071,
            "reference_agreement": 0.5549424985680554
          },
          {
            "count": 284,
            "confidence": 0.6310971171485777,
            "reference_agreement": 0.600541924578161
          },
          {
            "count": 280,
            "confidence": 0.7030994468429518,
            "reference_agreement": 0.6497546304223978
          },
          {
            "count": 218,
            "confidence": 0.7682645359325981,
            "reference_agreement": 0.7226513752826121
          },
          {
            "count": 230,
            "confidence": 0.8322149342184884,
            "reference_agreement": 0.7612036827968656
          },
          {
            "count": 324,
            "confidence": 0.8996811349428281,
            "reference_agreement": 0.8247895057612304
          },
          {
            "count": 1252,
            "confidence": 0.9876585496182678,
            "reference_agreement": 0.9778121922719432
          }
        ],
        "expected_score_mae": 0.4570759707537796
      },
      "type": {
        "choice": {
          "count": 2526,
          "soft_ce": 0.6822833143719118,
          "kl": 0.4189869815862902,
          "brier": 0.10139614023942027,
          "argmax_agreement": 0.7862232779097387,
          "hard_count": 1737,
          "hard_ece": 0.04277428653676444,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.1280767528435369,
              "reference_agreement": 0.0
            },
            {
              "count": 5,
              "confidence": 0.17932821619602474,
              "reference_agreement": 0.4
            },
            {
              "count": 4,
              "confidence": 0.2380902472918557,
              "reference_agreement": 0.25
            },
            {
              "count": 14,
              "confidence": 0.3081997411534075,
              "reference_agreement": 0.2857142857142857
            },
            {
              "count": 24,
              "confidence": 0.3697200232466156,
              "reference_agreement": 0.4166666666666667
            },
            {
              "count": 40,
              "confidence": 0.43214316524587604,
              "reference_agreement": 0.3
            },
            {
              "count": 58,
              "confidence": 0.5047682692169494,
              "reference_agreement": 0.5344827586206896
            },
            {
              "count": 76,
              "confidence": 0.5702475474241634,
              "reference_agreement": 0.5131578947368421
            },
            {
              "count": 85,
              "confidence": 0.6294861640993207,
              "reference_agreement": 0.5411764705882353
            },
            {
              "count": 61,
              "confidence": 0.7034878822257229,
              "reference_agreement": 0.6229508196721312
            },
            {
              "count": 73,
              "confidence": 0.7662886786794736,
              "reference_agreement": 0.6712328767123288
            },
            {
              "count": 109,
              "confidence": 0.8327800034747108,
              "reference_agreement": 0.7155963302752294
            },
            {
              "count": 163,
              "confidence": 0.9005930750387896,
              "reference_agreement": 0.7852760736196319
            },
            {
              "count": 1024,
              "confidence": 0.9905297995078783,
              "reference_agreement": 0.9814453125
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.1280767528435369,
              "reference_agreement": 0.0
            },
            {
              "count": 9,
              "confidence": 0.1761727165792652,
              "reference_agreement": 0.39832358674463936
            },
            {
              "count": 20,
              "confidence": 0.23727536848341457,
              "reference_agreement": 0.23968434343434342
            },
            {
              "count": 56,
              "confidence": 0.30649961126616326,
              "reference_agreement": 0.2928895108131916
            },
            {
              "count": 112,
              "confidence": 0.36821444135421544,
              "reference_agreement": 0.393456101968026
            },
            {
              "count": 148,
              "confidence": 0.43446487817770657,
              "reference_agreement": 0.4016674390496405
            },
            {
              "count": 173,
              "confidence": 0.4989247303895505,
              "reference_agreement": 0.5024153201745094
            },
            {
              "count": 161,
              "confidence": 0.5682084517346793,
              "reference_agreement": 0.5282341373438668
            },
            {
              "count": 170,
              "confidence": 0.6297812369278754,
              "reference_agreement": 0.5908322297566935
            },
            {
              "count": 148,
              "confidence": 0.700524650342172,
              "reference_agreement": 0.6783159155834284
            },
            {
              "count": 123,
              "confidence": 0.7663341138612425,
              "reference_agreement": 0.708765336806816
            },
            {
              "count": 150,
              "confidence": 0.8331362507431996,
              "reference_agreement": 0.7423155758796159
            },
            {
              "count": 207,
              "confidence": 0.8989791587743893,
              "reference_agreement": 0.7958650265103496
            },
            {
              "count": 1048,
              "confidence": 0.9898586616987579,
              "reference_agreement": 0.9796660513168818
            }
          ]
        },
        "noul": {
          "count": 582,
          "soft_ce": 0.485984791408039,
          "kl": 0.1677006591643438,
          "brier": 0.056889637384882515,
          "argmax_agreement": 0.8316151202749141,
          "hard_count": 154,
          "hard_ece": 0.05817309278238439,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.5156046866868782,
              "reference_agreement": 0.5
            },
            {
              "count": 5,
              "confidence": 0.5744020664194579,
              "reference_agreement": 0.8
            },
            {
              "count": 15,
              "confidence": 0.6251505169037502,
              "reference_agreement": 0.4666666666666667
            },
            {
              "count": 15,
              "confidence": 0.7097643525488307,
              "reference_agreement": 0.5333333333333333
            },
            {
              "count": 11,
              "confidence": 0.7837242799754484,
              "reference_agreement": 0.7272727272727273
            },
            {
              "count": 10,
              "confidence": 0.833779143408664,
              "reference_agreement": 0.9
            },
            {
              "count": 22,
              "confidence": 0.9027589227901665,
              "reference_agreement": 0.9090909090909091
            },
            {
              "count": 74,
              "confidence": 0.9777466873130052,
              "reference_agreement": 0.9594594594594594
            }
          ],
          "soft_reliability": [
            {
              "count": 34,
              "confidence": 0.5192844401029223,
              "reference_agreement": 0.502892126954275
            },
            {
              "count": 55,
              "confidence": 0.5665664402627255,
              "reference_agreement": 0.6365357818181818
            },
            {
              "count": 72,
              "confidence": 0.6353817301417451,
              "reference_agreement": 0.617865484172874
            },
            {
              "count": 84,
              "confidence": 0.7075900367197564,
              "reference_agreement": 0.6229389882371539
            },
            {
              "count": 58,
              "confidence": 0.7769928207174944,
              "reference_agreement": 0.7633095172413793
            },
            {
              "count": 55,
              "confidence": 0.829995783859167,
              "reference_agreement": 0.8141728722666914
            },
            {
              "count": 83,
              "confidence": 0.9004798674802651,
              "reference_agreement": 0.8669205460521514
            },
            {
              "count": 141,
              "confidence": 0.975388672803986,
              "reference_agreement": 0.9554196875487998
            }
          ]
        },
        "score": {
          "count": 585,
          "soft_ce": 1.043454353228223,
          "kl": 0.5058693789008174,
          "brier": 0.12228264299021577,
          "argmax_agreement": 0.6598290598290598,
          "hard_count": 243,
          "hard_ece": 0.07115621447701632,
          "hard_reliability": [
            {
              "count": 13,
              "confidence": 0.18586413124072504,
              "reference_agreement": 0.15384615384615385
            },
            {
              "count": 19,
              "confidence": 0.2362995959591558,
              "reference_agreement": 0.2631578947368421
            },
            {
              "count": 23,
              "confidence": 0.3019061999897666,
              "reference_agreement": 0.13043478260869565
            },
            {
              "count": 12,
              "confidence": 0.3685367014209464,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 17,
              "confidence": 0.4338667824041559,
              "reference_agreement": 0.4117647058823529
            },
            {
              "count": 11,
              "confidence": 0.49490420241461824,
              "reference_agreement": 0.36363636363636365
            },
            {
              "count": 12,
              "confidence": 0.5688318970482136,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 13,
              "confidence": 0.6294053829689576,
              "reference_agreement": 0.6153846153846154
            },
            {
              "count": 12,
              "confidence": 0.6958966365351196,
              "reference_agreement": 0.5
            },
            {
              "count": 13,
              "confidence": 0.7523646054019367,
              "reference_agreement": 0.6923076923076923
            },
            {
              "count": 14,
              "confidence": 0.8351455536348746,
              "reference_agreement": 0.6428571428571429
            },
            {
              "count": 28,
              "confidence": 0.9024161795738924,
              "reference_agreement": 0.8928571428571429
            },
            {
              "count": 56,
              "confidence": 0.9807319435717284,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 13,
              "confidence": 0.18586413124072504,
              "reference_agreement": 0.15384615384615385
            },
            {
              "count": 32,
              "confidence": 0.24331398836464874,
              "reference_agreement": 0.2734375
            },
            {
              "count": 49,
              "confidence": 0.3019081956123887,
              "reference_agreement": 0.23663264863946495
            },
            {
              "count": 39,
              "confidence": 0.3667883077504478,
              "reference_agreement": 0.34414724859025103
            },
            {
              "count": 71,
              "confidence": 0.43551300115891045,
              "reference_agreement": 0.4057656959535077
            },
            {
              "count": 61,
              "confidence": 0.49800447627318833,
              "reference_agreement": 0.47666251871173404
            },
            {
              "count": 71,
              "confidence": 0.5629637410045656,
              "reference_agreement": 0.5523004644601318
            },
            {
              "count": 42,
              "confidence": 0.6290782481488474,
              "reference_agreement": 0.610145539550307
            },
            {
              "count": 48,
              "confidence": 0.7031798704359478,
              "reference_agreement": 0.6086180416667305
            },
            {
              "count": 37,
              "confidence": 0.7609997088313216,
              "reference_agreement": 0.7050786860640831
            },
            {
              "count": 25,
              "confidence": 0.8315691658607272,
              "reference_agreement": 0.7580001074667474
            },
            {
              "count": 34,
              "confidence": 0.9020050839504612,
              "reference_agreement": 0.8980392369608144
            },
            {
              "count": 63,
              "confidence": 0.978521012641448,
              "reference_agreement": 0.9970899523809524
            }
          ],
          "expected_score_mae": 0.4570759707537796
        }
      },
      "category": {
        "engineering": {
          "count": 500,
          "soft_ce": 0.0170732927832887,
          "kl": 0.0170732927832887,
          "brier": 0.004504996059548485,
          "argmax_agreement": 0.996,
          "hard_count": 500,
          "hard_ece": 0.005887712376320884,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.48051708584209724,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.5901215718541692,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.659498582567833,
              "reference_agreement": 1.0
            },
            {
              "count": 4,
              "confidence": 0.7592606819949799,
              "reference_agreement": 0.75
            },
            {
              "count": 2,
              "confidence": 0.809986801374795,
              "reference_agreement": 1.0
            },
            {
              "count": 6,
              "confidence": 0.9033714425633134,
              "reference_agreement": 0.8333333333333334
            },
            {
              "count": 485,
              "confidence": 0.9982748543344034,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.48051708584209724,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.5901215718541692,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.659498582567833,
              "reference_agreement": 1.0
            },
            {
              "count": 4,
              "confidence": 0.7592606819949799,
              "reference_agreement": 0.75
            },
            {
              "count": 2,
              "confidence": 0.809986801374795,
              "reference_agreement": 1.0
            },
            {
              "count": 6,
              "confidence": 0.9033714425633134,
              "reference_agreement": 0.8333333333333334
            },
            {
              "count": 485,
              "confidence": 0.9982748543344034,
              "reference_agreement": 1.0
            }
          ],
          "expected_score_mae": 0.36947776287918127
        },
        "data_analytics": {
          "count": 500,
          "soft_ce": 0.63641646085155,
          "kl": 0.6331406095589392,
          "brier": 0.15179326704736298,
          "argmax_agreement": 0.76,
          "hard_count": 497,
          "hard_ece": 0.05635174490474885,
          "hard_reliability": [
            {
              "count": 8,
              "confidence": 0.18512452516415256,
              "reference_agreement": 0.125
            },
            {
              "count": 13,
              "confidence": 0.23685294357226974,
              "reference_agreement": 0.23076923076923078
            },
            {
              "count": 16,
              "confidence": 0.310619358628449,
              "reference_agreement": 0.125
            },
            {
              "count": 15,
              "confidence": 0.36840624755576634,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 13,
              "confidence": 0.4338411624493362,
              "reference_agreement": 0.3076923076923077
            },
            {
              "count": 14,
              "confidence": 0.49562565013910687,
              "reference_agreement": 0.42857142857142855
            },
            {
              "count": 19,
              "confidence": 0.5716282587437927,
              "reference_agreement": 0.47368421052631576
            },
            {
              "count": 18,
              "confidence": 0.6307835105473234,
              "reference_agreement": 0.4444444444444444
            },
            {
              "count": 15,
              "confidence": 0.7034512348597252,
              "reference_agreement": 0.5333333333333333
            },
            {
              "count": 25,
              "confidence": 0.7610956537573322,
              "reference_agreement": 0.64
            },
            {
              "count": 27,
              "confidence": 0.8287917395783113,
              "reference_agreement": 0.7037037037037037
            },
            {
              "count": 60,
              "confidence": 0.9017458338366022,
              "reference_agreement": 0.8
            },
            {
              "count": 254,
              "confidence": 0.9885657799728477,
              "reference_agreement": 0.984251968503937
            }
          ],
          "soft_reliability": [
            {
              "count": 8,
              "confidence": 0.18512452516415256,
              "reference_agreement": 0.125
            },
            {
              "count": 13,
              "confidence": 0.23685294357226974,
              "reference_agreement": 0.23076923076923078
            },
            {
              "count": 16,
              "confidence": 0.310619358628449,
              "reference_agreement": 0.125
            },
            {
              "count": 15,
              "confidence": 0.36840624755576634,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 13,
              "confidence": 0.4338411624493362,
              "reference_agreement": 0.3076923076923077
            },
            {
              "count": 14,
              "confidence": 0.49562565013910687,
              "reference_agreement": 0.42857142857142855
            },
            {
              "count": 22,
              "confidence": 0.5710235531343882,
              "reference_agreement": 0.4762625909090909
            },
            {
              "count": 18,
              "confidence": 0.6307835105473234,
              "reference_agreement": 0.4444444444444444
            },
            {
              "count": 15,
              "confidence": 0.7034512348597252,
              "reference_agreement": 0.5333333333333333
            },
            {
              "count": 25,
              "confidence": 0.7610956537573322,
              "reference_agreement": 0.64
            },
            {
              "count": 27,
              "confidence": 0.8287917395783113,
              "reference_agreement": 0.7037037037037037
            },
            {
              "count": 60,
              "confidence": 0.9017458338366022,
              "reference_agreement": 0.8
            },
            {
              "count": 254,
              "confidence": 0.9885657799728477,
              "reference_agreement": 0.984251968503937
            }
          ],
          "expected_score_mae": 0.8379162320369495
        },
        "finance": {
          "count": 500,
          "soft_ce": 0.8325307227564824,
          "kl": 0.21741036887290283,
          "brier": 0.05805591006171185,
          "argmax_agreement": 0.766,
          "hard_count": 52,
          "hard_ece": 0.12560551004414863,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.42545552219647026,
              "reference_agreement": 0.0
            },
            {
              "count": 3,
              "confidence": 0.5063873523524167,
              "reference_agreement": 1.0
            },
            {
              "count": 3,
              "confidence": 0.5575393887409873,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 3,
              "confidence": 0.6185993089535619,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 4,
              "confidence": 0.7072795840269844,
              "reference_agreement": 0.5
            },
            {
              "count": 2,
              "confidence": 0.7583805784749291,
              "reference_agreement": 0.0
            },
            {
              "count": 5,
              "confidence": 0.8432613843014515,
              "reference_agreement": 0.6
            },
            {
              "count": 10,
              "confidence": 0.8980962253086776,
              "reference_agreement": 0.9
            },
            {
              "count": 21,
              "confidence": 0.9891833973182879,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 11,
              "confidence": 0.32509882113799443,
              "reference_agreement": 0.44772725303034394
            },
            {
              "count": 31,
              "confidence": 0.36165303383874403,
              "reference_agreement": 0.32344085419356344
            },
            {
              "count": 56,
              "confidence": 0.433194585459043,
              "reference_agreement": 0.39091664392857783
            },
            {
              "count": 52,
              "confidence": 0.49251118240718983,
              "reference_agreement": 0.5019289031468974
            },
            {
              "count": 50,
              "confidence": 0.5634722409257146,
              "reference_agreement": 0.6018555810889367
            },
            {
              "count": 65,
              "confidence": 0.6301825914444824,
              "reference_agreement": 0.6561282074359387
            },
            {
              "count": 68,
              "confidence": 0.7056494781490086,
              "reference_agreement": 0.627431364951034
            },
            {
              "count": 37,
              "confidence": 0.7683118932263894,
              "reference_agreement": 0.6799459059550426
            },
            {
              "count": 34,
              "confidence": 0.8290161206378067,
              "reference_agreement": 0.7909373143137849
            },
            {
              "count": 35,
              "confidence": 0.8969496853528575,
              "reference_agreement": 0.9005142873333625
            },
            {
              "count": 61,
              "confidence": 0.9760249311137575,
              "reference_agreement": 0.9830546393442623
            }
          ],
          "expected_score_mae": 0.28921449751489353
        },
        "trust_safety": {
          "count": 500,
          "soft_ce": 0.6965650575760396,
          "kl": 0.3262561593110357,
          "brier": 0.09241847396917179,
          "argmax_agreement": 0.734,
          "hard_count": 222,
          "hard_ece": 0.06416689376484794,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.37375993751238135,
              "reference_agreement": 0.0
            },
            {
              "count": 5,
              "confidence": 0.43472235444626844,
              "reference_agreement": 0.4
            },
            {
              "count": 10,
              "confidence": 0.502873413716155,
              "reference_agreement": 0.5
            },
            {
              "count": 11,
              "confidence": 0.5716600619610657,
              "reference_agreement": 0.8181818181818182
            },
            {
              "count": 14,
              "confidence": 0.6273743231263287,
              "reference_agreement": 0.5714285714285714
            },
            {
              "count": 14,
              "confidence": 0.7048754950230958,
              "reference_agreement": 0.6428571428571429
            },
            {
              "count": 8,
              "confidence": 0.772673527518938,
              "reference_agreement": 0.625
            },
            {
              "count": 23,
              "confidence": 0.8352234500546492,
              "reference_agreement": 0.6956521739130435
            },
            {
              "count": 22,
              "confidence": 0.9039014670840353,
              "reference_agreement": 0.8181818181818182
            },
            {
              "count": 113,
              "confidence": 0.9792441416955855,
              "reference_agreement": 0.9557522123893806
            }
          ],
          "soft_reliability": [
            {
              "count": 6,
              "confidence": 0.23458907659811615,
              "reference_agreement": 0.2075
            },
            {
              "count": 8,
              "confidence": 0.29972115715656333,
              "reference_agreement": 0.255416625
            },
            {
              "count": 17,
              "confidence": 0.36836999088901834,
              "reference_agreement": 0.33265160656977494
            },
            {
              "count": 24,
              "confidence": 0.4353651177707092,
              "reference_agreement": 0.40927912136242695
            },
            {
              "count": 42,
              "confidence": 0.5034541101303811,
              "reference_agreement": 0.4637698182486976
            },
            {
              "count": 44,
              "confidence": 0.5639370342777291,
              "reference_agreement": 0.5822456558441558
            },
            {
              "count": 43,
              "confidence": 0.6301712982399027,
              "reference_agreement": 0.5623823154308876
            },
            {
              "count": 45,
              "confidence": 0.7009662006930144,
              "reference_agreement": 0.625547008811918
            },
            {
              "count": 25,
              "confidence": 0.7682502104322542,
              "reference_agreement": 0.6588631578947368
            },
            {
              "count": 42,
              "confidence": 0.8340236494740164,
              "reference_agreement": 0.7355623967695999
            },
            {
              "count": 53,
              "confidence": 0.9011086418913816,
              "reference_agreement": 0.8135774190795747
            },
            {
              "count": 151,
              "confidence": 0.9780184864854932,
              "reference_agreement": 0.9430664742453473
            }
          ],
          "expected_score_mae": 0.3245526679121535
        },
        "documents_meetings": {
          "count": 500,
          "soft_ce": 0.9479271825455599,
          "kl": 0.27931837942071347,
          "brier": 0.06632900701852151,
          "argmax_agreement": 0.754,
          "hard_count": 111,
          "hard_ece": 0.07985648898540725,
          "hard_reliability": [
            {
              "count": 4,
              "confidence": 0.5112290740361165,
              "reference_agreement": 1.0
            },
            {
              "count": 8,
              "confidence": 0.5755079266886977,
              "reference_agreement": 0.625
            },
            {
              "count": 8,
              "confidence": 0.6215870747073032,
              "reference_agreement": 0.75
            },
            {
              "count": 7,
              "confidence": 0.7132942799377675,
              "reference_agreement": 0.5714285714285714
            },
            {
              "count": 4,
              "confidence": 0.7689218976142511,
              "reference_agreement": 0.5
            },
            {
              "count": 13,
              "confidence": 0.8325792906268336,
              "reference_agreement": 0.7692307692307693
            },
            {
              "count": 13,
              "confidence": 0.894684750793525,
              "reference_agreement": 0.8461538461538461
            },
            {
              "count": 54,
              "confidence": 0.980788268448271,
              "reference_agreement": 0.9444444444444444
            }
          ],
          "soft_reliability": [
            {
              "count": 16,
              "confidence": 0.2529896903034424,
              "reference_agreement": 0.2572032828282828
            },
            {
              "count": 26,
              "confidence": 0.30085803531102356,
              "reference_agreement": 0.2784615384615385
            },
            {
              "count": 37,
              "confidence": 0.3672361246048308,
              "reference_agreement": 0.373009828009828
            },
            {
              "count": 42,
              "confidence": 0.43667169969021885,
              "reference_agreement": 0.49312530062530063
            },
            {
              "count": 64,
              "confidence": 0.5041356739419488,
              "reference_agreement": 0.576640625
            },
            {
              "count": 57,
              "confidence": 0.5696819547588704,
              "reference_agreement": 0.6160136452241716
            },
            {
              "count": 52,
              "confidence": 0.6318176118679493,
              "reference_agreement": 0.6767307692307691
            },
            {
              "count": 51,
              "confidence": 0.7024724227358164,
              "reference_agreement": 0.7055555555555556
            },
            {
              "count": 38,
              "confidence": 0.7733050801429199,
              "reference_agreement": 0.7889473684210526
            },
            {
              "count": 33,
              "confidence": 0.8353726880577852,
              "reference_agreement": 0.8224242424242424
            },
            {
              "count": 30,
              "confidence": 0.8930073021992985,
              "reference_agreement": 0.8578333333333333
            },
            {
              "count": 54,
              "confidence": 0.980788268448271,
              "reference_agreement": 0.9444444444444444
            }
          ],
          "expected_score_mae": 0.3935315866553138
        },
        "customer_support": {
          "count": 500,
          "soft_ce": 1.1372885709258729,
          "kl": 0.5544979078550764,
          "brier": 0.11033544189723725,
          "argmax_agreement": 0.706,
          "hard_count": 142,
          "hard_ece": 0.14171122927377042,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.1280767528435369,
              "reference_agreement": 0.0
            },
            {
              "count": 5,
              "confidence": 0.17932821619602474,
              "reference_agreement": 0.4
            },
            {
              "count": 1,
              "confidence": 0.22071972996391087,
              "reference_agreement": 0.0
            },
            {
              "count": 2,
              "confidence": 0.3112827985653901,
              "reference_agreement": 0.0
            },
            {
              "count": 3,
              "confidence": 0.37505887333787874,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 8,
              "confidence": 0.4293526333096114,
              "reference_agreement": 0.25
            },
            {
              "count": 2,
              "confidence": 0.49253722456655336,
              "reference_agreement": 0.5
            },
            {
              "count": 11,
              "confidence": 0.5647034564825082,
              "reference_agreement": 0.18181818181818182
            },
            {
              "count": 4,
              "confidence": 0.6399711185141979,
              "reference_agreement": 0.75
            },
            {
              "count": 7,
              "confidence": 0.6905290585463988,
              "reference_agreement": 0.5714285714285714
            },
            {
              "count": 10,
              "confidence": 0.7733585610196265,
              "reference_agreement": 0.6
            },
            {
              "count": 17,
              "confidence": 0.8359032378172329,
              "reference_agreement": 0.7647058823529411
            },
            {
              "count": 20,
              "confidence": 0.9086037470245399,
              "reference_agreement": 0.6
            },
            {
              "count": 51,
              "confidence": 0.9778779954883188,
              "reference_agreement": 0.9411764705882353
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.1280767528435369,
              "reference_agreement": 0.0
            },
            {
              "count": 9,
              "confidence": 0.1761727165792652,
              "reference_agreement": 0.39832358674463936
            },
            {
              "count": 8,
              "confidence": 0.23185648880822557,
              "reference_agreement": 0.2729292929292929
            },
            {
              "count": 25,
              "confidence": 0.3010096491709431,
              "reference_agreement": 0.27153918422154916
            },
            {
              "count": 33,
              "confidence": 0.3716936308219563,
              "reference_agreement": 0.43249653598146726
            },
            {
              "count": 46,
              "confidence": 0.43566624910724644,
              "reference_agreement": 0.3784143440302738
            },
            {
              "count": 49,
              "confidence": 0.5004958652365038,
              "reference_agreement": 0.464408579758434
            },
            {
              "count": 64,
              "confidence": 0.5647801490500414,
              "reference_agreement": 0.4939270455676706
            },
            {
              "count": 35,
              "confidence": 0.6338884106064454,
              "reference_agreement": 0.6524762
            },
            {
              "count": 49,
              "confidence": 0.7014723835777643,
              "reference_agreement": 0.6855101955782364
            },
            {
              "count": 34,
              "confidence": 0.7717612201235851,
              "reference_agreement": 0.7027941176470588
            },
            {
              "count": 37,
              "confidence": 0.8313773086286068,
              "reference_agreement": 0.763963945945946
            },
            {
              "count": 47,
              "confidence": 0.9031494358129605,
              "reference_agreement": 0.7396099361702129
            },
            {
              "count": 63,
              "confidence": 0.9737046850235944,
              "reference_agreement": 0.9396507939418092
            }
          ],
          "expected_score_mae": 0.33143847468388904
        },
        "ai_agents_evals": {
          "count": 500,
          "soft_ce": 0.765946732434316,
          "kl": 0.6764210789434673,
          "brier": 0.18685146874516365,
          "argmax_agreement": 0.68,
          "hard_count": 417,
          "hard_ece": 0.06936429979471131,
          "hard_reliability": [
            {
              "count": 5,
              "confidence": 0.187047500963241,
              "reference_agreement": 0.2
            },
            {
              "count": 8,
              "confidence": 0.23585635105072375,
              "reference_agreement": 0.25
            },
            {
              "count": 19,
              "confidence": 0.2982191389855985,
              "reference_agreement": 0.2631578947368421
            },
            {
              "count": 14,
              "confidence": 0.36932960679831744,
              "reference_agreement": 0.5714285714285714
            },
            {
              "count": 27,
              "confidence": 0.43134028318033907,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 32,
              "confidence": 0.5068791830358712,
              "reference_agreement": 0.4375
            },
            {
              "count": 33,
              "confidence": 0.5698085895150488,
              "reference_agreement": 0.5151515151515151
            },
            {
              "count": 53,
              "confidence": 0.6282809191581149,
              "reference_agreement": 0.49056603773584906
            },
            {
              "count": 34,
              "confidence": 0.7055160692185535,
              "reference_agreement": 0.6176470588235294
            },
            {
              "count": 33,
              "confidence": 0.765969600593059,
              "reference_agreement": 0.8181818181818182
            },
            {
              "count": 28,
              "confidence": 0.8332225392168197,
              "reference_agreement": 0.7857142857142857
            },
            {
              "count": 54,
              "confidence": 0.8969179427292381,
              "reference_agreement": 0.8703703703703703
            },
            {
              "count": 77,
              "confidence": 0.9715351814332892,
              "reference_agreement": 0.922077922077922
            }
          ],
          "soft_reliability": [
            {
              "count": 5,
              "confidence": 0.187047500963241,
              "reference_agreement": 0.2
            },
            {
              "count": 8,
              "confidence": 0.23585635105072375,
              "reference_agreement": 0.25
            },
            {
              "count": 19,
              "confidence": 0.2982191389855985,
              "reference_agreement": 0.2631578947368421
            },
            {
              "count": 16,
              "confidence": 0.3728244454729145,
              "reference_agreement": 0.5458333125
            },
            {
              "count": 35,
              "confidence": 0.43292344830072604,
              "reference_agreement": 0.35496548479120027
            },
            {
              "count": 41,
              "confidence": 0.5085266662477907,
              "reference_agreement": 0.45982781975131765
            },
            {
              "count": 42,
              "confidence": 0.5678684243563111,
              "reference_agreement": 0.5322624638888933
            },
            {
              "count": 58,
              "confidence": 0.6299378167372339,
              "reference_agreement": 0.5120080436781644
            },
            {
              "count": 45,
              "confidence": 0.7054146769488767,
              "reference_agreement": 0.6565114530755077
            },
            {
              "count": 44,
              "confidence": 0.7651372249377204,
              "reference_agreement": 0.8189414169069195
            },
            {
              "count": 37,
              "confidence": 0.833874270098686,
              "reference_agreement": 0.8076943700618164
            },
            {
              "count": 65,
              "confidence": 0.8966887344775478,
              "reference_agreement": 0.8630389169038999
            },
            {
              "count": 85,
              "confidence": 0.9707936989016759,
              "reference_agreement": 0.9253352246481347
            }
          ],
          "expected_score_mae": 0.8450261299297381
        },
        "legal": {
          "count": 122,
          "soft_ce": 0.5668266819917146,
          "kl": 0.5668266819917146,
          "brier": 0.13410281322654918,
          "argmax_agreement": 0.819672131147541,
          "hard_count": 122,
          "hard_ece": 0.07370299961080379,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.25539450758217525,
              "reference_agreement": 1.0
            },
            {
              "count": 2,
              "confidence": 0.3631581357093971,
              "reference_agreement": 0.0
            },
            {
              "count": 2,
              "confidence": 0.4544914341376851,
              "reference_agreement": 0.5
            },
            {
              "count": 3,
              "confidence": 0.503085478379179,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 5,
              "confidence": 0.5760999183430778,
              "reference_agreement": 0.4
            },
            {
              "count": 6,
              "confidence": 0.6285356936618459,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 3,
              "confidence": 0.6943533513117489,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 9,
              "confidence": 0.7758294858466038,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 7,
              "confidence": 0.8221451030950648,
              "reference_agreement": 0.42857142857142855
            },
            {
              "count": 12,
              "confidence": 0.9086579271812519,
              "reference_agreement": 0.75
            },
            {
              "count": 72,
              "confidence": 0.9849048761433361,
              "reference_agreement": 0.9861111111111112
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.25539450758217525,
              "reference_agreement": 1.0
            },
            {
              "count": 2,
              "confidence": 0.3631581357093971,
              "reference_agreement": 0.0
            },
            {
              "count": 2,
              "confidence": 0.4544914341376851,
              "reference_agreement": 0.5
            },
            {
              "count": 3,
              "confidence": 0.503085478379179,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 5,
              "confidence": 0.5760999183430778,
              "reference_agreement": 0.4
            },
            {
              "count": 6,
              "confidence": 0.6285356936618459,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 3,
              "confidence": 0.6943533513117489,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 9,
              "confidence": 0.7758294858466038,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 7,
              "confidence": 0.8221451030950648,
              "reference_agreement": 0.42857142857142855
            },
            {
              "count": 12,
              "confidence": 0.9086579271812519,
              "reference_agreement": 0.75
            },
            {
              "count": 72,
              "confidence": 0.9849048761433361,
              "reference_agreement": 0.9861111111111112
            }
          ],
          "expected_score_mae": 0.05627741100384848
        },
        "sales_commerce": {
          "count": 71,
          "soft_ce": 0.43217932678724524,
          "kl": 0.43217932678724524,
          "brier": 0.13052027801486862,
          "argmax_agreement": 0.8169014084507042,
          "hard_count": 71,
          "hard_ece": 0.07210090880407366,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.43246792381509247,
              "reference_agreement": 1.0
            },
            {
              "count": 2,
              "confidence": 0.5125808974731219,
              "reference_agreement": 0.5
            },
            {
              "count": 2,
              "confidence": 0.5624424423010923,
              "reference_agreement": 0.5
            },
            {
              "count": 6,
              "confidence": 0.6350879180799406,
              "reference_agreement": 0.5
            },
            {
              "count": 4,
              "confidence": 0.6908680440851511,
              "reference_agreement": 0.5
            },
            {
              "count": 2,
              "confidence": 0.7547308258492205,
              "reference_agreement": 0.5
            },
            {
              "count": 11,
              "confidence": 0.8418109162556351,
              "reference_agreement": 0.7272727272727273
            },
            {
              "count": 16,
              "confidence": 0.8995504132439553,
              "reference_agreement": 0.875
            },
            {
              "count": 27,
              "confidence": 0.9753556391806691,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.43246792381509247,
              "reference_agreement": 1.0
            },
            {
              "count": 2,
              "confidence": 0.5125808974731219,
              "reference_agreement": 0.5
            },
            {
              "count": 2,
              "confidence": 0.5624424423010923,
              "reference_agreement": 0.5
            },
            {
              "count": 6,
              "confidence": 0.6350879180799406,
              "reference_agreement": 0.5
            },
            {
              "count": 4,
              "confidence": 0.6908680440851511,
              "reference_agreement": 0.5
            },
            {
              "count": 2,
              "confidence": 0.7547308258492205,
              "reference_agreement": 0.5
            },
            {
              "count": 11,
              "confidence": 0.8418109162556351,
              "reference_agreement": 0.7272727272727273
            },
            {
              "count": 16,
              "confidence": 0.8995504132439553,
              "reference_agreement": 0.875
            },
            {
              "count": 27,
              "confidence": 0.9753556391806691,
              "reference_agreement": 1.0
            }
          ],
          "expected_score_mae": 0.3227189172274408
        }
      }
    },
    "calibrated": {
      "overall": {
        "count": 3693,
        "soft_ce": 0.7041168734088186,
        "kl": 0.3887052537056387,
        "brier": 0.09690042536340486,
        "argmax_agreement": 0.7733549959382616,
        "hard_count": 2134,
        "hard_ece": 0.022770969448581948,
        "hard_reliability": [
          {
            "count": 3,
            "confidence": 0.11589315469434829,
            "reference_agreement": 0.3333333333333333
          },
          {
            "count": 17,
            "confidence": 0.17569618187345318,
            "reference_agreement": 0.17647058823529413
          },
          {
            "count": 26,
            "confidence": 0.23574174336686576,
            "reference_agreement": 0.23076923076923078
          },
          {
            "count": 43,
            "confidence": 0.30201610783522903,
            "reference_agreement": 0.23255813953488372
          },
          {
            "count": 51,
            "confidence": 0.3728587574318181,
            "reference_agreement": 0.35294117647058826
          },
          {
            "count": 55,
            "confidence": 0.43468960041354515,
            "reference_agreement": 0.36363636363636365
          },
          {
            "count": 78,
            "confidence": 0.5068967125125762,
            "reference_agreement": 0.47435897435897434
          },
          {
            "count": 116,
            "confidence": 0.5692509240339776,
            "reference_agreement": 0.5689655172413793
          },
          {
            "count": 92,
            "confidence": 0.6274691893699693,
            "reference_agreement": 0.5760869565217391
          },
          {
            "count": 113,
            "confidence": 0.6976255782232683,
            "reference_agreement": 0.584070796460177
          },
          {
            "count": 125,
            "confidence": 0.7655885050647548,
            "reference_agreement": 0.72
          },
          {
            "count": 153,
            "confidence": 0.8348620973385892,
            "reference_agreement": 0.7777777777777778
          },
          {
            "count": 193,
            "confidence": 0.9005488041523733,
            "reference_agreement": 0.8860103626943006
          },
          {
            "count": 1069,
            "confidence": 0.9873898699630395,
            "reference_agreement": 0.9850327408793265
          }
        ],
        "soft_reliability": [
          {
            "count": 3,
            "confidence": 0.11589315469434829,
            "reference_agreement": 0.3333333333333333
          },
          {
            "count": 23,
            "confidence": 0.1712254393277906,
            "reference_agreement": 0.22601507061461293
          },
          {
            "count": 65,
            "confidence": 0.241401549839449,
            "reference_agreement": 0.27101087801087803
          },
          {
            "count": 136,
            "confidence": 0.3047942264115045,
            "reference_agreement": 0.30327285409683113
          },
          {
            "count": 180,
            "confidence": 0.3704360020830201,
            "reference_agreement": 0.3727716890584122
          },
          {
            "count": 239,
            "confidence": 0.4341484282824121,
            "reference_agreement": 0.4200372081322403
          },
          {
            "count": 268,
            "confidence": 0.5028656676116453,
            "reference_agreement": 0.5155282699000787
          },
          {
            "count": 335,
            "confidence": 0.5660902452493901,
            "reference_agreement": 0.600639823533436
          },
          {
            "count": 269,
            "confidence": 0.631949793983103,
            "reference_agreement": 0.6359665011640041
          },
          {
            "count": 261,
            "confidence": 0.6984239785753936,
            "reference_agreement": 0.6471423835974844
          },
          {
            "count": 249,
            "confidence": 0.7657109724234291,
            "reference_agreement": 0.7478915785725628
          },
          {
            "count": 253,
            "confidence": 0.8344639775274048,
            "reference_agreement": 0.8022606606356278
          },
          {
            "count": 261,
            "confidence": 0.8995167793570855,
            "reference_agreement": 0.8815173885724905
          },
          {
            "count": 1151,
            "confidence": 0.9856462873311037,
            "reference_agreement": 0.9822842533704442
          }
        ],
        "expected_score_mae": 0.4584616360574015
      },
      "type": {
        "choice": {
          "count": 2526,
          "soft_ce": 0.6766963060796009,
          "kl": 0.4133999732939792,
          "brier": 0.10042181389472693,
          "argmax_agreement": 0.7862232779097387,
          "hard_count": 1737,
          "hard_ece": 0.024066863233766642,
          "hard_reliability": [
            {
              "count": 3,
              "confidence": 0.11589315469434829,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 4,
              "confidence": 0.1466965963178435,
              "reference_agreement": 0.25
            },
            {
              "count": 7,
              "confidence": 0.23979741242633804,
              "reference_agreement": 0.14285714285714285
            },
            {
              "count": 20,
              "confidence": 0.30543179732871933,
              "reference_agreement": 0.35
            },
            {
              "count": 39,
              "confidence": 0.3752333136919975,
              "reference_agreement": 0.358974358974359
            },
            {
              "count": 37,
              "confidence": 0.43632325318545506,
              "reference_agreement": 0.35135135135135137
            },
            {
              "count": 64,
              "confidence": 0.5078595631087047,
              "reference_agreement": 0.484375
            },
            {
              "count": 91,
              "confidence": 0.5663438578130047,
              "reference_agreement": 0.6043956043956044
            },
            {
              "count": 72,
              "confidence": 0.6259376203313807,
              "reference_agreement": 0.5555555555555556
            },
            {
              "count": 87,
              "confidence": 0.6985076075501826,
              "reference_agreement": 0.6091954022988506
            },
            {
              "count": 98,
              "confidence": 0.7672216909611733,
              "reference_agreement": 0.7142857142857143
            },
            {
              "count": 126,
              "confidence": 0.835107717990711,
              "reference_agreement": 0.7777777777777778
            },
            {
              "count": 140,
              "confidence": 0.8998423906655577,
              "reference_agreement": 0.8857142857142857
            },
            {
              "count": 949,
              "confidence": 0.988617468252103,
              "reference_agreement": 0.9852476290832455
            }
          ],
          "soft_reliability": [
            {
              "count": 3,
              "confidence": 0.11589315469434829,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 10,
              "confidence": 0.15381363979618545,
              "reference_agreement": 0.3198346624136098
            },
            {
              "count": 31,
              "confidence": 0.2398905686918626,
              "reference_agreement": 0.2732486151840991
            },
            {
              "count": 88,
              "confidence": 0.3066439829194413,
              "reference_agreement": 0.33710350424812785
            },
            {
              "count": 140,
              "confidence": 0.37159211827431626,
              "reference_agreement": 0.3838011523963886
            },
            {
              "count": 167,
              "confidence": 0.4341350320260209,
              "reference_agreement": 0.42457801395752337
            },
            {
              "count": 166,
              "confidence": 0.5007513721327872,
              "reference_agreement": 0.5253523924620044
            },
            {
              "count": 197,
              "confidence": 0.565477477433576,
              "reference_agreement": 0.619517122506086
            },
            {
              "count": 158,
              "confidence": 0.6298800441075835,
              "reference_agreement": 0.6306406317730678
            },
            {
              "count": 141,
              "confidence": 0.6982422404036815,
              "reference_agreement": 0.6697869686413617
            },
            {
              "count": 136,
              "confidence": 0.7667611071607225,
              "reference_agreement": 0.7399642556669485
            },
            {
              "count": 165,
              "confidence": 0.8340530433682095,
              "reference_agreement": 0.7888326080745547
            },
            {
              "count": 164,
              "confidence": 0.899220153771901,
              "reference_agreement": 0.8851424152139722
            },
            {
              "count": 960,
              "confidence": 0.988268968121057,
              "reference_agreement": 0.9841545132135424
            }
          ]
        },
        "noul": {
          "count": 582,
          "soft_ce": 0.4827658039249084,
          "kl": 0.16448167168121322,
          "brier": 0.05629811292264301,
          "argmax_agreement": 0.8316151202749141,
          "hard_count": 154,
          "hard_ece": 0.03838079417551199,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.51409214953238,
              "reference_agreement": 0.5
            },
            {
              "count": 12,
              "confidence": 0.5848970391247416,
              "reference_agreement": 0.5833333333333334
            },
            {
              "count": 10,
              "confidence": 0.6345947956878246,
              "reference_agreement": 0.6
            },
            {
              "count": 13,
              "confidence": 0.6959874383006549,
              "reference_agreement": 0.46153846153846156
            },
            {
              "count": 14,
              "confidence": 0.7675695442215852,
              "reference_agreement": 0.7857142857142857
            },
            {
              "count": 13,
              "confidence": 0.8339093290833637,
              "reference_agreement": 0.9230769230769231
            },
            {
              "count": 25,
              "confidence": 0.9051654333633907,
              "reference_agreement": 0.88
            },
            {
              "count": 65,
              "confidence": 0.9758087375812551,
              "reference_agreement": 0.9692307692307692
            }
          ],
          "soft_reliability": [
            {
              "count": 34,
              "confidence": 0.5174138851791591,
              "reference_agreement": 0.502892126954275
            },
            {
              "count": 71,
              "confidence": 0.5685861678053349,
              "reference_agreement": 0.6089361690140845
            },
            {
              "count": 72,
              "confidence": 0.6374148978709203,
              "reference_agreement": 0.6503378173406389
            },
            {
              "count": 72,
              "confidence": 0.6973135426818078,
              "reference_agreement": 0.6177620419978036
            },
            {
              "count": 76,
              "confidence": 0.7668883954141004,
              "reference_agreement": 0.7746528812652387
            },
            {
              "count": 63,
              "confidence": 0.8361374834836659,
              "reference_agreement": 0.8580618729922205
            },
            {
              "count": 65,
              "confidence": 0.899478685155311,
              "reference_agreement": 0.8626053588050548
            },
            {
              "count": 129,
              "confidence": 0.9701079851807889,
              "reference_agreement": 0.9612726817393857
            }
          ]
        },
        "score": {
          "count": 585,
          "soft_ce": 1.0427334127476895,
          "kl": 0.505148438420284,
          "brier": 0.12208934572306966,
          "argmax_agreement": 0.6598290598290598,
          "hard_count": 243,
          "hard_ece": 0.0736454437457963,
          "hard_reliability": [
            {
              "count": 13,
              "confidence": 0.18461913127517923,
              "reference_agreement": 0.15384615384615385
            },
            {
              "count": 19,
              "confidence": 0.23424754950284965,
              "reference_agreement": 0.2631578947368421
            },
            {
              "count": 23,
              "confidence": 0.299045943058281,
              "reference_agreement": 0.13043478260869565
            },
            {
              "count": 12,
              "confidence": 0.365141449586235,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 18,
              "confidence": 0.43133153638239685,
              "reference_agreement": 0.3888888888888889
            },
            {
              "count": 12,
              "confidence": 0.5005622698299229,
              "reference_agreement": 0.4166666666666667
            },
            {
              "count": 13,
              "confidence": 0.5751578198046974,
              "reference_agreement": 0.3076923076923077
            },
            {
              "count": 10,
              "confidence": 0.6313708801299527,
              "reference_agreement": 0.7
            },
            {
              "count": 13,
              "confidence": 0.6933609064965334,
              "reference_agreement": 0.5384615384615384
            },
            {
              "count": 13,
              "confidence": 0.7511433692151679,
              "reference_agreement": 0.6923076923076923
            },
            {
              "count": 14,
              "confidence": 0.8335362248493442,
              "reference_agreement": 0.6428571428571429
            },
            {
              "count": 28,
              "confidence": 0.8999588812194718,
              "reference_agreement": 0.8928571428571429
            },
            {
              "count": 55,
              "confidence": 0.9798950122993066,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 13,
              "confidence": 0.18461913127517923,
              "reference_agreement": 0.15384615384615385
            },
            {
              "count": 34,
              "confidence": 0.24277920912107187,
              "reference_agreement": 0.26897058823529413
            },
            {
              "count": 48,
              "confidence": 0.30140300614695376,
              "reference_agreement": 0.24124999548612047
            },
            {
              "count": 40,
              "confidence": 0.3663895954134836,
              "reference_agreement": 0.3341685673754948
            },
            {
              "count": 72,
              "confidence": 0.4341795001548752,
              "reference_agreement": 0.40950506128748676
            },
            {
              "count": 68,
              "confidence": 0.5007529272027481,
              "reference_agreement": 0.49786392453063255
            },
            {
              "count": 67,
              "confidence": 0.5652470326858571,
              "reference_agreement": 0.5363432798507779
            },
            {
              "count": 39,
              "confidence": 0.6302455119423146,
              "reference_agreement": 0.6310114396011891
            },
            {
              "count": 48,
              "confidence": 0.7006234882951765,
              "reference_agreement": 0.6246944274306163
            },
            {
              "count": 37,
              "confidence": 0.759432527246053,
              "reference_agreement": 0.7220606842622975
            },
            {
              "count": 25,
              "confidence": 0.8329589079683152,
              "reference_agreement": 0.7502667524000977
            },
            {
              "count": 32,
              "confidence": 0.9011143643285106,
              "reference_agreement": 0.9013541875
            },
            {
              "count": 62,
              "confidence": 0.9773667295736088,
              "reference_agreement": 0.9970430161290323
            }
          ],
          "expected_score_mae": 0.4584616360574015
        }
      },
      "category": {
        "engineering": {
          "count": 500,
          "soft_ce": 0.018681013106648606,
          "kl": 0.018681013106648606,
          "brier": 0.004596240452886235,
          "argmax_agreement": 0.996,
          "hard_count": 500,
          "hard_ece": 0.010272557015768891,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.4716738957978698,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.5539038973137703,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.655163400891637,
              "reference_agreement": 1.0
            },
            {
              "count": 2,
              "confidence": 0.7052527420032781,
              "reference_agreement": 0.5
            },
            {
              "count": 4,
              "confidence": 0.7571008671839193,
              "reference_agreement": 1.0
            },
            {
              "count": 3,
              "confidence": 0.8521442972732137,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 9,
              "confidence": 0.9071717975027496,
              "reference_agreement": 1.0
            },
            {
              "count": 479,
              "confidence": 0.9978224823124803,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.4716738957978698,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.5539038973137703,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.655163400891637,
              "reference_agreement": 1.0
            },
            {
              "count": 2,
              "confidence": 0.7052527420032781,
              "reference_agreement": 0.5
            },
            {
              "count": 4,
              "confidence": 0.7571008671839193,
              "reference_agreement": 1.0
            },
            {
              "count": 3,
              "confidence": 0.8521442972732137,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 9,
              "confidence": 0.9071717975027496,
              "reference_agreement": 1.0
            },
            {
              "count": 479,
              "confidence": 0.9978224823124803,
              "reference_agreement": 1.0
            }
          ],
          "expected_score_mae": 0.3754904932299786
        },
        "data_analytics": {
          "count": 500,
          "soft_ce": 0.6222487807440032,
          "kl": 0.6189729294513925,
          "brier": 0.1492023277961967,
          "argmax_agreement": 0.76,
          "hard_count": 497,
          "hard_ece": 0.04324263356987873,
          "hard_reliability": [
            {
              "count": 8,
              "confidence": 0.1836431426040265,
              "reference_agreement": 0.125
            },
            {
              "count": 13,
              "confidence": 0.2326095606931366,
              "reference_agreement": 0.23076923076923078
            },
            {
              "count": 21,
              "confidence": 0.3080856226638128,
              "reference_agreement": 0.19047619047619047
            },
            {
              "count": 11,
              "confidence": 0.37224859620757544,
              "reference_agreement": 0.36363636363636365
            },
            {
              "count": 18,
              "confidence": 0.43216064330256254,
              "reference_agreement": 0.2777777777777778
            },
            {
              "count": 16,
              "confidence": 0.49955516847278475,
              "reference_agreement": 0.375
            },
            {
              "count": 20,
              "confidence": 0.5648218776742052,
              "reference_agreement": 0.5
            },
            {
              "count": 14,
              "confidence": 0.6284579234052629,
              "reference_agreement": 0.5
            },
            {
              "count": 29,
              "confidence": 0.7007100933753049,
              "reference_agreement": 0.5517241379310345
            },
            {
              "count": 25,
              "confidence": 0.7611781134778828,
              "reference_agreement": 0.68
            },
            {
              "count": 42,
              "confidence": 0.8391754153900324,
              "reference_agreement": 0.7857142857142857
            },
            {
              "count": 45,
              "confidence": 0.9013655316115832,
              "reference_agreement": 0.8888888888888888
            },
            {
              "count": 235,
              "confidence": 0.9854872248925048,
              "reference_agreement": 0.9914893617021276
            }
          ],
          "soft_reliability": [
            {
              "count": 8,
              "confidence": 0.1836431426040265,
              "reference_agreement": 0.125
            },
            {
              "count": 13,
              "confidence": 0.2326095606931366,
              "reference_agreement": 0.23076923076923078
            },
            {
              "count": 21,
              "confidence": 0.3080856226638128,
              "reference_agreement": 0.19047619047619047
            },
            {
              "count": 11,
              "confidence": 0.37224859620757544,
              "reference_agreement": 0.36363636363636365
            },
            {
              "count": 18,
              "confidence": 0.43216064330256254,
              "reference_agreement": 0.2777777777777778
            },
            {
              "count": 16,
              "confidence": 0.49955516847278475,
              "reference_agreement": 0.375
            },
            {
              "count": 23,
              "confidence": 0.56429092950471,
              "reference_agreement": 0.49903378260869563
            },
            {
              "count": 14,
              "confidence": 0.6284579234052629,
              "reference_agreement": 0.5
            },
            {
              "count": 29,
              "confidence": 0.7007100933753049,
              "reference_agreement": 0.5517241379310345
            },
            {
              "count": 25,
              "confidence": 0.7611781134778828,
              "reference_agreement": 0.68
            },
            {
              "count": 42,
              "confidence": 0.8391754153900324,
              "reference_agreement": 0.7857142857142857
            },
            {
              "count": 45,
              "confidence": 0.9013655316115832,
              "reference_agreement": 0.8888888888888888
            },
            {
              "count": 235,
              "confidence": 0.9854872248925048,
              "reference_agreement": 0.9914893617021276
            }
          ],
          "expected_score_mae": 0.8394948703521851
        },
        "finance": {
          "count": 500,
          "soft_ce": 0.8372534729882183,
          "kl": 0.22213311910463876,
          "brier": 0.05824555646969622,
          "argmax_agreement": 0.766,
          "hard_count": 52,
          "hard_ece": 0.11310128402928836,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.3723071041294397,
              "reference_agreement": 0.0
            },
            {
              "count": 5,
              "confidence": 0.50016902311118,
              "reference_agreement": 0.6
            },
            {
              "count": 2,
              "confidence": 0.5753408310146032,
              "reference_agreement": 0.5
            },
            {
              "count": 3,
              "confidence": 0.6182583123935925,
              "reference_agreement": 1.0
            },
            {
              "count": 3,
              "confidence": 0.6819092006332838,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 5,
              "confidence": 0.7665259528880326,
              "reference_agreement": 0.4
            },
            {
              "count": 4,
              "confidence": 0.838982843566272,
              "reference_agreement": 0.75
            },
            {
              "count": 8,
              "confidence": 0.881833000901765,
              "reference_agreement": 0.875
            },
            {
              "count": 21,
              "confidence": 0.9797645091294379,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 16,
              "confidence": 0.3226339567135692,
              "reference_agreement": 0.3984374629166975
            },
            {
              "count": 41,
              "confidence": 0.36674248561146494,
              "reference_agreement": 0.33682928918700217
            },
            {
              "count": 62,
              "confidence": 0.4340519346364484,
              "reference_agreement": 0.4111594596595161
            },
            {
              "count": 48,
              "confidence": 0.49924896849347133,
              "reference_agreement": 0.5386290065109846
            },
            {
              "count": 70,
              "confidence": 0.5686484601862719,
              "reference_agreement": 0.655761905714329
            },
            {
              "count": 50,
              "confidence": 0.6308002075172696,
              "reference_agreement": 0.6414333730000463
            },
            {
              "count": 54,
              "confidence": 0.6999272500653496,
              "reference_agreement": 0.6493579925926575
            },
            {
              "count": 46,
              "confidence": 0.7648365922391855,
              "reference_agreement": 0.7116637471812303
            },
            {
              "count": 32,
              "confidence": 0.8315695075811502,
              "reference_agreement": 0.8268750565625763
            },
            {
              "count": 22,
              "confidence": 0.8916310587032179,
              "reference_agreement": 0.9127121363636364
            },
            {
              "count": 59,
              "confidence": 0.9675218038967256,
              "reference_agreement": 0.9836666610169491
            }
          ],
          "expected_score_mae": 0.29112017853517996
        },
        "trust_safety": {
          "count": 500,
          "soft_ce": 0.6856993182154376,
          "kl": 0.31539041995043376,
          "brier": 0.09066598215826954,
          "argmax_agreement": 0.734,
          "hard_count": 222,
          "hard_ece": 0.06329846877237999,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.3683034743483865,
              "reference_agreement": 0.0
            },
            {
              "count": 7,
              "confidence": 0.43137594222756087,
              "reference_agreement": 0.2857142857142857
            },
            {
              "count": 9,
              "confidence": 0.505779039854673,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 16,
              "confidence": 0.569920504948867,
              "reference_agreement": 0.75
            },
            {
              "count": 14,
              "confidence": 0.6316407404385947,
              "reference_agreement": 0.6428571428571429
            },
            {
              "count": 8,
              "confidence": 0.6972827104351492,
              "reference_agreement": 0.5
            },
            {
              "count": 18,
              "confidence": 0.7672982199470547,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 19,
              "confidence": 0.826724451152989,
              "reference_agreement": 0.6842105263157895
            },
            {
              "count": 25,
              "confidence": 0.9010283785718083,
              "reference_agreement": 0.84
            },
            {
              "count": 104,
              "confidence": 0.9729483427736801,
              "reference_agreement": 0.9711538461538461
            }
          ],
          "soft_reliability": [
            {
              "count": 9,
              "confidence": 0.2361890508436565,
              "reference_agreement": 0.27388888888888885
            },
            {
              "count": 7,
              "confidence": 0.30342299595800304,
              "reference_agreement": 0.2961904285714286
            },
            {
              "count": 18,
              "confidence": 0.3674761849264621,
              "reference_agreement": 0.3014857858344078
            },
            {
              "count": 33,
              "confidence": 0.4359383368744807,
              "reference_agreement": 0.3778090549302529
            },
            {
              "count": 35,
              "confidence": 0.5047182741304667,
              "reference_agreement": 0.517333303803201
            },
            {
              "count": 55,
              "confidence": 0.5629326069104805,
              "reference_agreement": 0.5935795651642476
            },
            {
              "count": 49,
              "confidence": 0.6337639434256919,
              "reference_agreement": 0.6102933127493159
            },
            {
              "count": 29,
              "confidence": 0.693334781004934,
              "reference_agreement": 0.5768357275211178
            },
            {
              "count": 40,
              "confidence": 0.7684317409650803,
              "reference_agreement": 0.7117894730881641
            },
            {
              "count": 42,
              "confidence": 0.8344109799487145,
              "reference_agreement": 0.7692592592237804
            },
            {
              "count": 44,
              "confidence": 0.8999478793240552,
              "reference_agreement": 0.8112662345983765
            },
            {
              "count": 139,
              "confidence": 0.9715298963628789,
              "reference_agreement": 0.9553096231010606
            }
          ],
          "expected_score_mae": 0.3249617918461017
        },
        "documents_meetings": {
          "count": 500,
          "soft_ce": 0.9505308054596932,
          "kl": 0.2819220023348468,
          "brier": 0.06778096624550085,
          "argmax_agreement": 0.754,
          "hard_count": 111,
          "hard_ece": 0.0745149993383762,
          "hard_reliability": [
            {
              "count": 7,
              "confidence": 0.5130857536757218,
              "reference_agreement": 0.8571428571428571
            },
            {
              "count": 9,
              "confidence": 0.5759649564141678,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 5,
              "confidence": 0.631365854228681,
              "reference_agreement": 0.8
            },
            {
              "count": 8,
              "confidence": 0.7057283821613531,
              "reference_agreement": 0.375
            },
            {
              "count": 8,
              "confidence": 0.7734135275347481,
              "reference_agreement": 0.75
            },
            {
              "count": 14,
              "confidence": 0.8276341762810223,
              "reference_agreement": 0.7857142857142857
            },
            {
              "count": 11,
              "confidence": 0.9061937668671157,
              "reference_agreement": 0.9090909090909091
            },
            {
              "count": 49,
              "confidence": 0.974518301452977,
              "reference_agreement": 0.9591836734693877
            }
          ],
          "soft_reliability": [
            {
              "count": 20,
              "confidence": 0.2507030605824935,
              "reference_agreement": 0.2815126262626263
            },
            {
              "count": 38,
              "confidence": 0.3027780535641847,
              "reference_agreement": 0.30187799043062197
            },
            {
              "count": 37,
              "confidence": 0.36910116911685614,
              "reference_agreement": 0.4304395304395305
            },
            {
              "count": 46,
              "confidence": 0.43197305120456336,
              "reference_agreement": 0.5305434782608696
            },
            {
              "count": 70,
              "confidence": 0.4998801989425534,
              "reference_agreement": 0.5903253968253969
            },
            {
              "count": 63,
              "confidence": 0.564395576933704,
              "reference_agreement": 0.65
            },
            {
              "count": 52,
              "confidence": 0.6360817173907356,
              "reference_agreement": 0.7249679487179487
            },
            {
              "count": 41,
              "confidence": 0.6999872254915769,
              "reference_agreement": 0.672560975609756
            },
            {
              "count": 34,
              "confidence": 0.769121243337486,
              "reference_agreement": 0.8225
            },
            {
              "count": 34,
              "confidence": 0.8294780729236917,
              "reference_agreement": 0.8451470588235294
            },
            {
              "count": 16,
              "confidence": 0.9032976162782315,
              "reference_agreement": 0.8896875
            },
            {
              "count": 49,
              "confidence": 0.974518301452977,
              "reference_agreement": 0.9591836734693877
            }
          ],
          "expected_score_mae": 0.39489489482681317
        },
        "customer_support": {
          "count": 500,
          "soft_ce": 1.1285135288404535,
          "kl": 0.545722865769657,
          "brier": 0.108362788973766,
          "argmax_agreement": 0.706,
          "hard_count": 142,
          "hard_ece": 0.11241881164831904,
          "hard_reliability": [
            {
              "count": 3,
              "confidence": 0.11589315469434829,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 4,
              "confidence": 0.1466965963178435,
              "reference_agreement": 0.25
            },
            {
              "count": 2,
              "confidence": 0.23363389271036505,
              "reference_agreement": 0.0
            },
            {
              "count": 3,
              "confidence": 0.3018620938804334,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 9,
              "confidence": 0.37797570812443015,
              "reference_agreement": 0.2222222222222222
            },
            {
              "count": 3,
              "confidence": 0.4272157242978048,
              "reference_agreement": 0.0
            },
            {
              "count": 7,
              "confidence": 0.5005016740966861,
              "reference_agreement": 0.14285714285714285
            },
            {
              "count": 7,
              "confidence": 0.5675968327842555,
              "reference_agreement": 0.8571428571428571
            },
            {
              "count": 7,
              "confidence": 0.6248700358768576,
              "reference_agreement": 0.5714285714285714
            },
            {
              "count": 9,
              "confidence": 0.7018524259315515,
              "reference_agreement": 0.5555555555555556
            },
            {
              "count": 18,
              "confidence": 0.763652193538776,
              "reference_agreement": 0.7222222222222222
            },
            {
              "count": 15,
              "confidence": 0.8335790279408464,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 16,
              "confidence": 0.9023386004425975,
              "reference_agreement": 0.8125
            },
            {
              "count": 39,
              "confidence": 0.9678176556415846,
              "reference_agreement": 0.9487179487179487
            }
          ],
          "soft_reliability": [
            {
              "count": 3,
              "confidence": 0.11589315469434829,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 10,
              "confidence": 0.15381363979618545,
              "reference_agreement": 0.3198346624136098
            },
            {
              "count": 12,
              "confidence": 0.24077671068363551,
              "reference_agreement": 0.29337121212121214
            },
            {
              "count": 35,
              "confidence": 0.3022639463465406,
              "reference_agreement": 0.3521546318325211
            },
            {
              "count": 42,
              "confidence": 0.37365140372838623,
              "reference_agreement": 0.35323562863250324
            },
            {
              "count": 45,
              "confidence": 0.4350034398300646,
              "reference_agreement": 0.36927072142944645
            },
            {
              "count": 55,
              "confidence": 0.5024000185598814,
              "reference_agreement": 0.4825757636363636
            },
            {
              "count": 55,
              "confidence": 0.5652814585293939,
              "reference_agreement": 0.631175707569653
            },
            {
              "count": 47,
              "confidence": 0.6319760416224046,
              "reference_agreement": 0.6904964592198635
            },
            {
              "count": 40,
              "confidence": 0.7024876007617367,
              "reference_agreement": 0.669458325
            },
            {
              "count": 43,
              "confidence": 0.766737205353825,
              "reference_agreement": 0.7386433953488373
            },
            {
              "count": 33,
              "confidence": 0.83741488958767,
              "reference_agreement": 0.7505555454545454
            },
            {
              "count": 37,
              "confidence": 0.8973418194210278,
              "reference_agreement": 0.8589099459459459
            },
            {
              "count": 43,
              "confidence": 0.965788577174672,
              "reference_agreement": 0.9501550236821856
            }
          ],
          "expected_score_mae": 0.33319003384682117
        },
        "ai_agents_evals": {
          "count": 500,
          "soft_ce": 0.7625932921761905,
          "kl": 0.6730676386853419,
          "brier": 0.18645192181998924,
          "argmax_agreement": 0.68,
          "hard_count": 417,
          "hard_ece": 0.052463098512104685,
          "hard_reliability": [
            {
              "count": 5,
              "confidence": 0.18618071314902357,
              "reference_agreement": 0.2
            },
            {
              "count": 10,
              "confidence": 0.23838633417547514,
              "reference_agreement": 0.2
            },
            {
              "count": 18,
              "confidence": 0.295781111577869,
              "reference_agreement": 0.2777777777777778
            },
            {
              "count": 26,
              "confidence": 0.37139215520169777,
              "reference_agreement": 0.4230769230769231
            },
            {
              "count": 22,
              "confidence": 0.43633042832674923,
              "reference_agreement": 0.45454545454545453
            },
            {
              "count": 29,
              "confidence": 0.5119118632329336,
              "reference_agreement": 0.4482758620689655
            },
            {
              "count": 51,
              "confidence": 0.5699148465657553,
              "reference_agreement": 0.47058823529411764
            },
            {
              "count": 38,
              "confidence": 0.6258474222785992,
              "reference_agreement": 0.5263157894736842
            },
            {
              "count": 42,
              "confidence": 0.692690594223072,
              "reference_agreement": 0.7380952380952381
            },
            {
              "count": 35,
              "confidence": 0.7681140161954323,
              "reference_agreement": 0.7714285714285715
            },
            {
              "count": 40,
              "confidence": 0.8344200795661016,
              "reference_agreement": 0.85
            },
            {
              "count": 46,
              "confidence": 0.9006389466100536,
              "reference_agreement": 0.9347826086956522
            },
            {
              "count": 55,
              "confidence": 0.9717353590779876,
              "reference_agreement": 0.8909090909090909
            }
          ],
          "soft_reliability": [
            {
              "count": 5,
              "confidence": 0.18618071314902357,
              "reference_agreement": 0.2
            },
            {
              "count": 10,
              "confidence": 0.23838633417547514,
              "reference_agreement": 0.2
            },
            {
              "count": 18,
              "confidence": 0.295781111577869,
              "reference_agreement": 0.2777777777777778
            },
            {
              "count": 29,
              "confidence": 0.3733951639465118,
              "reference_agreement": 0.41724137931034483
            },
            {
              "count": 30,
              "confidence": 0.4336998105448617,
              "reference_agreement": 0.44690416558973367
            },
            {
              "count": 39,
              "confidence": 0.5120708414698647,
              "reference_agreement": 0.4701608617898467
            },
            {
              "count": 58,
              "confidence": 0.5690122989604566,
              "reference_agreement": 0.4917532841954055
            },
            {
              "count": 46,
              "confidence": 0.6283615036372431,
              "reference_agreement": 0.5640713243854648
            },
            {
              "count": 54,
              "confidence": 0.6932351995199476,
              "reference_agreement": 0.7547641003856951
            },
            {
              "count": 45,
              "confidence": 0.7641095557632885,
              "reference_agreement": 0.7842272393489996
            },
            {
              "count": 51,
              "confidence": 0.8318454353186426,
              "reference_agreement": 0.8372494792826007
            },
            {
              "count": 55,
              "confidence": 0.9002777292404353,
              "reference_agreement": 0.9324725290016627
            },
            {
              "count": 60,
              "confidence": 0.9709295090707568,
              "reference_agreement": 0.8988023166666667
            }
          ],
          "expected_score_mae": 0.8456565717140646
        },
        "legal": {
          "count": 122,
          "soft_ce": 0.5436294554804777,
          "kl": 0.5436294554804777,
          "brier": 0.1304449623060641,
          "argmax_agreement": 0.819672131147541,
          "hard_count": 122,
          "hard_ece": 0.08307026110171992,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.2542299113522521,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.28724827093183936,
              "reference_agreement": 0.0
            },
            {
              "count": 2,
              "confidence": 0.377085304774584,
              "reference_agreement": 0.5
            },
            {
              "count": 3,
              "confidence": 0.4441052741464433,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 2,
              "confidence": 0.5174342366971163,
              "reference_agreement": 0.0
            },
            {
              "count": 7,
              "confidence": 0.5644377124854338,
              "reference_agreement": 0.5714285714285714
            },
            {
              "count": 5,
              "confidence": 0.6251477859811808,
              "reference_agreement": 0.8
            },
            {
              "count": 8,
              "confidence": 0.7044969749942575,
              "reference_agreement": 0.25
            },
            {
              "count": 8,
              "confidence": 0.7587080210898536,
              "reference_agreement": 0.75
            },
            {
              "count": 6,
              "confidence": 0.8354103639312468,
              "reference_agreement": 1.0
            },
            {
              "count": 13,
              "confidence": 0.8963652039349991,
              "reference_agreement": 0.7692307692307693
            },
            {
              "count": 66,
              "confidence": 0.9815239804604637,
              "reference_agreement": 0.9848484848484849
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.2542299113522521,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.28724827093183936,
              "reference_agreement": 0.0
            },
            {
              "count": 2,
              "confidence": 0.377085304774584,
              "reference_agreement": 0.5
            },
            {
              "count": 3,
              "confidence": 0.4441052741464433,
              "reference_agreement": 0.3333333333333333
            },
            {
              "count": 2,
              "confidence": 0.5174342366971163,
              "reference_agreement": 0.0
            },
            {
              "count": 7,
              "confidence": 0.5644377124854338,
              "reference_agreement": 0.5714285714285714
            },
            {
              "count": 5,
              "confidence": 0.6251477859811808,
              "reference_agreement": 0.8
            },
            {
              "count": 8,
              "confidence": 0.7044969749942575,
              "reference_agreement": 0.25
            },
            {
              "count": 8,
              "confidence": 0.7587080210898536,
              "reference_agreement": 0.75
            },
            {
              "count": 6,
              "confidence": 0.8354103639312468,
              "reference_agreement": 1.0
            },
            {
              "count": 13,
              "confidence": 0.8963652039349991,
              "reference_agreement": 0.7692307692307693
            },
            {
              "count": 66,
              "confidence": 0.9815239804604637,
              "reference_agreement": 0.9848484848484849
            }
          ],
          "expected_score_mae": 0.05800823336689319
        },
        "sales_commerce": {
          "count": 71,
          "soft_ce": 0.43972836851867453,
          "kl": 0.43972836851867453,
          "brier": 0.13079004940228145,
          "argmax_agreement": 0.8169014084507042,
          "hard_count": 71,
          "hard_ece": 0.08143885493164209,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.4480862145923521,
              "reference_agreement": 1.0
            },
            {
              "count": 2,
              "confidence": 0.5225530044071978,
              "reference_agreement": 0.5
            },
            {
              "count": 3,
              "confidence": 0.5799241321545268,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 5,
              "confidence": 0.6273970582753347,
              "reference_agreement": 0.2
            },
            {
              "count": 4,
              "confidence": 0.6962808034900051,
              "reference_agreement": 0.75
            },
            {
              "count": 4,
              "confidence": 0.7775016660973879,
              "reference_agreement": 0.75
            },
            {
              "count": 10,
              "confidence": 0.8388575355152714,
              "reference_agreement": 0.7
            },
            {
              "count": 20,
              "confidence": 0.9005931196007729,
              "reference_agreement": 0.9
            },
            {
              "count": 21,
              "confidence": 0.9756813796187571,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 2,
              "confidence": 0.4480862145923521,
              "reference_agreement": 1.0
            },
            {
              "count": 2,
              "confidence": 0.5225530044071978,
              "reference_agreement": 0.5
            },
            {
              "count": 3,
              "confidence": 0.5799241321545268,
              "reference_agreement": 0.6666666666666666
            },
            {
              "count": 5,
              "confidence": 0.6273970582753347,
              "reference_agreement": 0.2
            },
            {
              "count": 4,
              "confidence": 0.6962808034900051,
              "reference_agreement": 0.75
            },
            {
              "count": 4,
              "confidence": 0.7775016660973879,
              "reference_agreement": 0.75
            },
            {
              "count": 10,
              "confidence": 0.8388575355152714,
              "reference_agreement": 0.7
            },
            {
              "count": 20,
              "confidence": 0.9005931196007729,
              "reference_agreement": 0.9
            },
            {
              "count": 21,
              "confidence": 0.9756813796187571,
              "reference_agreement": 1.0
            }
          ],
          "expected_score_mae": 0.3234410160549905
        }
      }
    },
    "permutation": {
      "count": 256,
      "completed": true,
      "mean_total_variation": 0.061152504018688186,
      "answer_flip_rate": 0.0625
    }
  },
  "typed_decisions": {
    "dataset": "LocalLLaMA/typed-decisions",
    "revision": "7189f46b02248bd2569df7325c9df8e3b24e90a7",
    "config": "all",
    "split": "test",
    "cases": 400,
    "decisions": 2000,
    "errors": 0,
    "raw": {
      "overall": {
        "count": 2000,
        "soft_ce": 0.8982348703026427,
        "kl": 0.13087631467831382,
        "brier": 0.03733508375955404,
        "argmax_agreement": 0.7055,
        "hard_count": 2,
        "hard_ece": 0.02813636220158644,
        "hard_reliability": [
          {
            "count": 2,
            "confidence": 0.9718636377984136,
            "reference_agreement": 1.0
          }
        ],
        "soft_reliability": [
          {
            "count": 24,
            "confidence": 0.31520772849577633,
            "reference_agreement": 0.3355554526389121
          },
          {
            "count": 171,
            "confidence": 0.3746159687360105,
            "reference_agreement": 0.38106237192013953
          },
          {
            "count": 368,
            "confidence": 0.43486926682677546,
            "reference_agreement": 0.41427729467341934
          },
          {
            "count": 330,
            "confidence": 0.5005758970577506,
            "reference_agreement": 0.4898888885960213
          },
          {
            "count": 259,
            "confidence": 0.5661637493278858,
            "reference_agreement": 0.5401029315058599
          },
          {
            "count": 208,
            "confidence": 0.6334207148716826,
            "reference_agreement": 0.5813301291346679
          },
          {
            "count": 176,
            "confidence": 0.6976026942523854,
            "reference_agreement": 0.6650378634451258
          },
          {
            "count": 117,
            "confidence": 0.765488790351476,
            "reference_agreement": 0.6715926028234322
          },
          {
            "count": 95,
            "confidence": 0.8268994900267613,
            "reference_agreement": 0.7595845223159148
          },
          {
            "count": 90,
            "confidence": 0.8962419341009388,
            "reference_agreement": 0.9119985310000607
          },
          {
            "count": 162,
            "confidence": 0.9675863551773324,
            "reference_agreement": 0.9661111104013173
          }
        ],
        "expected_score_mae": 0.2689357269785638,
        "gold_label_correct": 1409,
        "gold_label_accuracy": 0.7045,
        "gold_label_ece_15bin_all_decisions": 0.10329466050980009,
        "full_vector_brier": 0.07467016751910809
      },
      "type": {
        "choice": {
          "count": 600,
          "soft_ce": 1.0297958099292934,
          "kl": 0.1660012130824713,
          "brier": 0.041920794479682215,
          "argmax_agreement": 0.6883333333333334,
          "hard_count": 1,
          "hard_ece": 0.023186746014468573,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9768132539855314,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 14,
              "confidence": 0.3173165797051884,
              "reference_agreement": 0.39047612547625926
            },
            {
              "count": 114,
              "confidence": 0.3727881473417713,
              "reference_agreement": 0.3832163580409846
            },
            {
              "count": 160,
              "confidence": 0.4341875923805457,
              "reference_agreement": 0.4184011179583752
            },
            {
              "count": 92,
              "confidence": 0.4936694315907819,
              "reference_agreement": 0.511159424456586
            },
            {
              "count": 48,
              "confidence": 0.5673671375615373,
              "reference_agreement": 0.5801388408334563
            },
            {
              "count": 31,
              "confidence": 0.6339963551108706,
              "reference_agreement": 0.6433333275269537
            },
            {
              "count": 28,
              "confidence": 0.6907448914335091,
              "reference_agreement": 0.7217737262976213
            },
            {
              "count": 22,
              "confidence": 0.764159840829608,
              "reference_agreement": 0.6959089121213682
            },
            {
              "count": 32,
              "confidence": 0.8290876216067908,
              "reference_agreement": 0.7640788558334829
            },
            {
              "count": 33,
              "confidence": 0.8944038265429061,
              "reference_agreement": 0.9230363914142908
            },
            {
              "count": 26,
              "confidence": 0.9583538062949253,
              "reference_agreement": 0.955179537115826
            }
          ],
          "gold_label_correct": 413,
          "gold_label_accuracy": 0.6883333333333334,
          "gold_label_ece_15bin_all_decisions": 0.1452815073319075,
          "full_vector_brier": 0.08384158895936443
        },
        "noul": {
          "count": 600,
          "soft_ce": 0.5376753319725679,
          "kl": 0.08395308862000342,
          "brier": 0.034802117110535095,
          "argmax_agreement": 0.7666666666666667,
          "hard_count": 1,
          "hard_ece": 0.033085978388704196,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9669140216112958,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 59,
              "confidence": 0.5179912574843223,
              "reference_agreement": 0.48372877966101696
            },
            {
              "count": 86,
              "confidence": 0.5678122996718532,
              "reference_agreement": 0.525968976744186
            },
            {
              "count": 85,
              "confidence": 0.6369483441744728,
              "reference_agreement": 0.5698431176470588
            },
            {
              "count": 85,
              "confidence": 0.6995096580475858,
              "reference_agreement": 0.6567059176470588
            },
            {
              "count": 63,
              "confidence": 0.7673512554565463,
              "reference_agreement": 0.6198413174603175
            },
            {
              "count": 47,
              "confidence": 0.8295192389350612,
              "reference_agreement": 0.740141914893617
            },
            {
              "count": 54,
              "confidence": 0.8971869226053684,
              "reference_agreement": 0.9035740740740741
            },
            {
              "count": 121,
              "confidence": 0.9707946773144384,
              "reference_agreement": 0.9677850826446281
            }
          ],
          "gold_label_correct": 459,
          "gold_label_accuracy": 0.765,
          "gold_label_ece_15bin_all_decisions": 0.037528508953695645,
          "full_vector_brier": 0.06960423422107019
        },
        "score": {
          "count": 800,
          "soft_ce": 1.0699838193302107,
          "kl": 0.1397250604189285,
          "brier": 0.03579552570622214,
          "argmax_agreement": 0.6725,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 10,
              "confidence": 0.3122553368025993,
              "reference_agreement": 0.25866651066662605
            },
            {
              "count": 57,
              "confidence": 0.3782716115244891,
              "reference_agreement": 0.3767543996784493
            },
            {
              "count": 208,
              "confidence": 0.4353936317854138,
              "reference_agreement": 0.41110512291576096
            },
            {
              "count": 179,
              "confidence": 0.4983853303415126,
              "reference_agreement": 0.4809869731099504
            },
            {
              "count": 125,
              "confidence": 0.564567445609514,
              "reference_agreement": 0.5344533032000944
            },
            {
              "count": 92,
              "confidence": 0.6299675264135088,
              "reference_agreement": 0.5710507468116885
            },
            {
              "count": 63,
              "confidence": 0.6980777189561393,
              "reference_agreement": 0.6510634385715672
            },
            {
              "count": 32,
              "confidence": 0.7627357149721534,
              "reference_agreement": 0.7567604832397332
            },
            {
              "count": 16,
              "confidence": 0.8148277144485708,
              "reference_agreement": 0.8077085145837775
            },
            {
              "count": 3,
              "confidence": 0.8994513241595642,
              "reference_agreement": 0.9422222911112911
            },
            {
              "count": 15,
              "confidence": 0.9577089746675174,
              "reference_agreement": 0.9715557946667946
            }
          ],
          "expected_score_mae": 0.2689357269785638,
          "gold_label_correct": 537,
          "gold_label_accuracy": 0.67125,
          "gold_label_ece_15bin_all_decisions": 0.13471575955048368,
          "full_vector_brier": 0.07159105141244428
        }
      },
      "category": {
        "agent_trace_observability": {
          "count": 500,
          "soft_ce": 0.97776378147771,
          "kl": 0.09929350853248653,
          "brier": 0.029319985421551766,
          "argmax_agreement": 0.666,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.3271961322725437,
              "reference_agreement": 0.333333
            },
            {
              "count": 35,
              "confidence": 0.3798205422614706,
              "reference_agreement": 0.40077146666672475
            },
            {
              "count": 132,
              "confidence": 0.43641602107268646,
              "reference_agreement": 0.4047222168687381
            },
            {
              "count": 117,
              "confidence": 0.5006302500044424,
              "reference_agreement": 0.47911679880348107
            },
            {
              "count": 76,
              "confidence": 0.5690619634091041,
              "reference_agreement": 0.5512718920614565
            },
            {
              "count": 45,
              "confidence": 0.6338679776529896,
              "reference_agreement": 0.6111852223703935
            },
            {
              "count": 25,
              "confidence": 0.6938113959359317,
              "reference_agreement": 0.6235998789334686
            },
            {
              "count": 14,
              "confidence": 0.7711383973508009,
              "reference_agreement": 0.6864285047620653
            },
            {
              "count": 32,
              "confidence": 0.823789261655095,
              "reference_agreement": 0.7322917248959386
            },
            {
              "count": 23,
              "confidence": 0.8940903067075735,
              "reference_agreement": 0.8727536086956522
            }
          ],
          "expected_score_mae": 0.20094214035170185,
          "gold_label_correct": 335,
          "gold_label_accuracy": 0.67,
          "gold_label_ece_15bin_all_decisions": 0.11802699803796707,
          "full_vector_brier": 0.05863997084310353
        },
        "customer_service": {
          "count": 500,
          "soft_ce": 0.9356140291801783,
          "kl": 0.19845939117069544,
          "brier": 0.054488558914342955,
          "argmax_agreement": 0.694,
          "hard_count": 1,
          "hard_ece": 0.023186746014468573,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9768132539855314,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 10,
              "confidence": 0.31477128374516633,
              "reference_agreement": 0.4056665940000813
            },
            {
              "count": 39,
              "confidence": 0.3723859508926986,
              "reference_agreement": 0.45632472794882123
            },
            {
              "count": 73,
              "confidence": 0.4380748853548461,
              "reference_agreement": 0.3939339658448251
            },
            {
              "count": 81,
              "confidence": 0.5011240542197034,
              "reference_agreement": 0.5185185261729083
            },
            {
              "count": 80,
              "confidence": 0.5649588987676203,
              "reference_agreement": 0.5116249845417362
            },
            {
              "count": 69,
              "confidence": 0.6279948891492104,
              "reference_agreement": 0.5734783022222739
            },
            {
              "count": 46,
              "confidence": 0.6947774859534929,
              "reference_agreement": 0.7216593691232294
            },
            {
              "count": 33,
              "confidence": 0.7624282639003352,
              "reference_agreement": 0.6597979875758164
            },
            {
              "count": 22,
              "confidence": 0.8310417874707298,
              "reference_agreement": 0.8232055630304752
            },
            {
              "count": 21,
              "confidence": 0.8956622165551549,
              "reference_agreement": 0.913333472222457
            },
            {
              "count": 26,
              "confidence": 0.9583538062949253,
              "reference_agreement": 0.955179537115826
            }
          ],
          "expected_score_mae": 0.3528748230925414,
          "gold_label_correct": 347,
          "gold_label_accuracy": 0.694,
          "gold_label_ece_15bin_all_decisions": 0.09884340864299247,
          "full_vector_brier": 0.10897711782868591
        },
        "invoice_processing": {
          "count": 500,
          "soft_ce": 0.6654727615892404,
          "kl": 0.09052528592434582,
          "brier": 0.02417431701033401,
          "argmax_agreement": 0.84,
          "hard_count": 1,
          "hard_ece": 0.033085978388704196,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9669140216112958,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 2,
              "confidence": 0.3278949544267834,
              "reference_agreement": 0.5716665
            },
            {
              "count": 37,
              "confidence": 0.36518717835723197,
              "reference_agreement": 0.3640449954955256
            },
            {
              "count": 44,
              "confidence": 0.43272176571282106,
              "reference_agreement": 0.4239439440863979
            },
            {
              "count": 47,
              "confidence": 0.4990750370635605,
              "reference_agreement": 0.4812765950355347
            },
            {
              "count": 40,
              "confidence": 0.5636676327793726,
              "reference_agreement": 0.5589167148334007
            },
            {
              "count": 44,
              "confidence": 0.6359292707198466,
              "reference_agreement": 0.618181796969778
            },
            {
              "count": 53,
              "confidence": 0.6993117889365471,
              "reference_agreement": 0.6881194986793073
            },
            {
              "count": 34,
              "confidence": 0.7613192737634586,
              "reference_agreement": 0.7343039374609032
            },
            {
              "count": 21,
              "confidence": 0.8299096395763272,
              "reference_agreement": 0.8130160017462573
            },
            {
              "count": 43,
              "confidence": 0.8979274834748605,
              "reference_agreement": 0.9520123691472993
            },
            {
              "count": 135,
              "confidence": 0.9695963375504574,
              "reference_agreement": 0.9691999920000142
            }
          ],
          "expected_score_mae": 0.21560838991268416,
          "gold_label_correct": 417,
          "gold_label_accuracy": 0.834,
          "gold_label_ece_15bin_all_decisions": 0.11985107384860202,
          "full_vector_brier": 0.04834863402066802
        },
        "security_incidents": {
          "count": 500,
          "soft_ce": 1.0140889089634417,
          "kl": 0.13522707308572743,
          "brier": 0.041357473691987466,
          "argmax_agreement": 0.622,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 11,
              "confidence": 0.31220787321098703,
              "reference_agreement": 0.2290908112120979
            },
            {
              "count": 60,
              "confidence": 0.37884389984455846,
              "reference_agreement": 0.33113891736116685
            },
            {
              "count": 119,
              "confidence": 0.4319811015831166,
              "reference_agreement": 0.43378150224093404
            },
            {
              "count": 85,
              "confidence": 0.5008086075853485,
              "reference_agreement": 0.48219608458828284
            },
            {
              "count": 63,
              "confidence": 0.5657822959404127,
              "reference_agreement": 0.550846477883684
            },
            {
              "count": 50,
              "confidence": 0.6382982887191337,
              "reference_agreement": 0.5328665986667214
            },
            {
              "count": 52,
              "confidence": 0.7001826946639205,
              "reference_agreement": 0.6113462035256458
            },
            {
              "count": 36,
              "confidence": 0.7700350803206342,
              "reference_agreement": 0.6174074444444444
            },
            {
              "count": 20,
              "confidence": 0.8241586712060176,
              "reference_agreement": 0.6771668
            },
            {
              "count": 3,
              "confidence": 0.8926362259110124,
              "reference_agreement": 0.63
            },
            {
              "count": 1,
              "confidence": 0.9362850057480346,
              "reference_agreement": 0.833333
            }
          ],
          "expected_score_mae": 0.3063175545573279,
          "gold_label_correct": 310,
          "gold_label_accuracy": 0.62,
          "gold_label_ece_15bin_all_decisions": 0.11599525503936776,
          "full_vector_brier": 0.08271494738397493
        }
      },
      "question": {
        "agent_trace_observability/action": {
          "count": 100,
          "soft_ce": 1.1235396454195041,
          "kl": 0.08949577063715862,
          "brier": 0.023420285987952504,
          "argmax_agreement": 0.66,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.3271961322725437,
              "reference_agreement": 0.333333
            },
            {
              "count": 28,
              "confidence": 0.37758519376271876,
              "reference_agreement": 0.4010834071429083
            },
            {
              "count": 34,
              "confidence": 0.42300368726206095,
              "reference_agreement": 0.40872541411767294
            },
            {
              "count": 14,
              "confidence": 0.508725847017595,
              "reference_agreement": 0.6019047511905369
            },
            {
              "count": 23,
              "confidence": 0.5700440548082534,
              "reference_agreement": 0.6007246056522724
            }
          ],
          "gold_label_correct": 66,
          "gold_label_accuracy": 0.66,
          "gold_label_ece_15bin_all_decisions": 0.21139510221170185,
          "full_vector_brier": 0.04684057197590501
        },
        "agent_trace_observability/needs_review": {
          "count": 100,
          "soft_ce": 0.5772198756410069,
          "kl": 0.06175400802011981,
          "brier": 0.026948573243994925,
          "argmax_agreement": 0.78,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 17,
              "confidence": 0.522037312516128,
              "reference_agreement": 0.48039217647058824
            },
            {
              "count": 27,
              "confidence": 0.5740099433811946,
              "reference_agreement": 0.5532098148148148
            },
            {
              "count": 19,
              "confidence": 0.6307027904464286,
              "reference_agreement": 0.6412281052631579
            },
            {
              "count": 1,
              "confidence": 0.679178699175393,
              "reference_agreement": 0.766667
            },
            {
              "count": 13,
              "confidence": 0.831497587641693,
              "reference_agreement": 0.8541025384615385
            },
            {
              "count": 23,
              "confidence": 0.8940903067075735,
              "reference_agreement": 0.8727536086956522
            }
          ],
          "gold_label_correct": 78,
          "gold_label_accuracy": 0.78,
          "gold_label_ece_15bin_all_decisions": 0.09591019804659835,
          "full_vector_brier": 0.05389714648798985
        },
        "agent_trace_observability/outcome": {
          "count": 100,
          "soft_ce": 1.0935064791897204,
          "kl": 0.16872011458553096,
          "brier": 0.04551726407182359,
          "argmax_agreement": 0.62,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.39948630465030277,
              "reference_agreement": 0.233333
            },
            {
              "count": 36,
              "confidence": 0.44371195558609283,
              "reference_agreement": 0.36223152500002964
            },
            {
              "count": 29,
              "confidence": 0.4869204497761252,
              "reference_agreement": 0.42333332505753857
            },
            {
              "count": 1,
              "confidence": 0.5457545962932366,
              "reference_agreement": 0.75
            },
            {
              "count": 4,
              "confidence": 0.6334661880424128,
              "reference_agreement": 0.8091667499999999
            },
            {
              "count": 6,
              "confidence": 0.68759444755407,
              "reference_agreement": 0.6505555205556461
            },
            {
              "count": 8,
              "confidence": 0.7680981282943802,
              "reference_agreement": 0.6516665095834904
            },
            {
              "count": 15,
              "confidence": 0.820754732086627,
              "reference_agreement": 0.6195556666667333
            }
          ],
          "gold_label_correct": 60,
          "gold_label_accuracy": 0.6,
          "gold_label_ece_15bin_all_decisions": 0.08832517707815522,
          "full_vector_brier": 0.09103452814364718
        },
        "agent_trace_observability/risk": {
          "count": 100,
          "soft_ce": 0.9829069466566531,
          "kl": 0.09053683824790124,
          "brier": 0.027227209135250532,
          "argmax_agreement": 0.67,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 6,
              "confidence": 0.3869745415241738,
              "reference_agreement": 0.4272221555556555
            },
            {
              "count": 15,
              "confidence": 0.4430243807991401,
              "reference_agreement": 0.4377777200001155
            },
            {
              "count": 20,
              "confidence": 0.5064388102385512,
              "reference_agreement": 0.552666578333445
            },
            {
              "count": 22,
              "confidence": 0.5633028160410002,
              "reference_agreement": 0.5042424242424954
            },
            {
              "count": 15,
              "confidence": 0.6350598264314197,
              "reference_agreement": 0.5333333022222356
            },
            {
              "count": 13,
              "confidence": 0.6937211677482574,
              "reference_agreement": 0.5712818941026444
            },
            {
              "count": 5,
              "confidence": 0.780801733899054,
              "reference_agreement": 0.7526666713334713
            },
            {
              "count": 4,
              "confidence": 0.810116688080407,
              "reference_agreement": 0.7591667991672592
            }
          ],
          "expected_score_mae": 0.1887612567569762,
          "gold_label_correct": 69,
          "gold_label_accuracy": 0.69,
          "gold_label_ece_15bin_all_decisions": 0.14607924787983295,
          "full_vector_brier": 0.054454418270501063
        },
        "agent_trace_observability/urgency": {
          "count": 100,
          "soft_ce": 1.111645960481666,
          "kl": 0.08596081117172201,
          "brier": 0.023486594668737287,
          "argmax_agreement": 0.6,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 47,
              "confidence": 0.43842113407655536,
              "reference_agreement": 0.42382272014190947
            },
            {
              "count": 37,
              "confidence": 0.49533713057352974,
              "reference_agreement": 0.4360360527027635
            },
            {
              "count": 3,
              "confidence": 0.5670036460048613,
              "reference_agreement": 0.43333317777784447
            },
            {
              "count": 7,
              "confidence": 0.6401348324659201,
              "reference_agreement": 0.5833334961905962
            },
            {
              "count": 5,
              "confidence": 0.7044328666342266,
              "reference_agreement": 0.6986664453336919
            },
            {
              "count": 1,
              "confidence": 0.7471438670609017,
              "reference_agreement": 0.6333336333336334
            }
          ],
          "expected_score_mae": 0.21312302394642757,
          "gold_label_correct": 62,
          "gold_label_accuracy": 0.62,
          "gold_label_ece_15bin_all_decisions": 0.12615469901673237,
          "full_vector_brier": 0.046973189337474575
        },
        "customer_service/action": {
          "count": 100,
          "soft_ce": 1.241760340801668,
          "kl": 0.275902444840478,
          "brier": 0.06755224071529659,
          "argmax_agreement": 0.6,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 9,
              "confidence": 0.31273256683149947,
              "reference_agreement": 0.3525925488889793
            },
            {
              "count": 34,
              "confidence": 0.37221056069924124,
              "reference_agreement": 0.4573528766667779
            },
            {
              "count": 25,
              "confidence": 0.4324976531209652,
              "reference_agreement": 0.4009538457334028
            },
            {
              "count": 20,
              "confidence": 0.49707265548386304,
              "reference_agreement": 0.5388333921667305
            },
            {
              "count": 9,
              "confidence": 0.5563524243492388,
              "reference_agreement": 0.6407407485186634
            },
            {
              "count": 1,
              "confidence": 0.6089655400408842,
              "reference_agreement": 0.35
            },
            {
              "count": 2,
              "confidence": 0.705852717684706,
              "reference_agreement": 0.7800003916668916
            }
          ],
          "gold_label_correct": 61,
          "gold_label_accuracy": 0.61,
          "gold_label_ece_15bin_all_decisions": 0.18966441682569232,
          "full_vector_brier": 0.13510448143059317
        },
        "customer_service/category": {
          "count": 100,
          "soft_ce": 0.5646570684297757,
          "kl": 0.15423518549593282,
          "brier": 0.03216723408432829,
          "argmax_agreement": 0.93,
          "hard_count": 1,
          "hard_ece": 0.023186746014468573,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9768132539855314,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 3,
              "confidence": 0.4430349798534358,
              "reference_agreement": 0.6166666055560389
            },
            {
              "count": 5,
              "confidence": 0.5046243895746633,
              "reference_agreement": 0.635333516000116
            },
            {
              "count": 4,
              "confidence": 0.5798365858143383,
              "reference_agreement": 0.5683333691670058
            },
            {
              "count": 8,
              "confidence": 0.6384560218456447,
              "reference_agreement": 0.6099999795834395
            },
            {
              "count": 9,
              "confidence": 0.6915068535481631,
              "reference_agreement": 0.8281108069998411
            },
            {
              "count": 9,
              "confidence": 0.7596657412667888,
              "reference_agreement": 0.6937035451853623
            },
            {
              "count": 15,
              "confidence": 0.8332816589849125,
              "reference_agreement": 0.8847014257780302
            },
            {
              "count": 21,
              "confidence": 0.8956622165551549,
              "reference_agreement": 0.913333472222457
            },
            {
              "count": 26,
              "confidence": 0.9583538062949253,
              "reference_agreement": 0.955179537115826
            }
          ],
          "gold_label_correct": 94,
          "gold_label_accuracy": 0.94,
          "gold_label_ece_15bin_all_decisions": 0.1407358753162401,
          "full_vector_brier": 0.06433446816865658
        },
        "customer_service/churn_risk": {
          "count": 100,
          "soft_ce": 1.0723716037197997,
          "kl": 0.18195307128399665,
          "brier": 0.05156803196362559,
          "argmax_agreement": 0.71,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.3331197359681683,
              "reference_agreement": 0.883333
            },
            {
              "count": 5,
              "confidence": 0.3735786042082082,
              "reference_agreement": 0.4493333166667167
            },
            {
              "count": 31,
              "confidence": 0.4406830914886085,
              "reference_agreement": 0.388172083225866
            },
            {
              "count": 26,
              "confidence": 0.5011742997082412,
              "reference_agreement": 0.578589826794943
            },
            {
              "count": 18,
              "confidence": 0.5633032673902467,
              "reference_agreement": 0.5179629694445509
            },
            {
              "count": 17,
              "confidence": 0.6284474784336418,
              "reference_agreement": 0.6143137843137941
            },
            {
              "count": 2,
              "confidence": 0.6870135499018379,
              "reference_agreement": 0.6716668416668417
            }
          ],
          "expected_score_mae": 0.3151828379933619,
          "gold_label_correct": 71,
          "gold_label_accuracy": 0.71,
          "gold_label_ece_15bin_all_decisions": 0.19910186568229626,
          "full_vector_brier": 0.10313606392725118
        },
        "customer_service/needs_human": {
          "count": 100,
          "soft_ce": 0.6761560528212277,
          "kl": 0.12704364399222592,
          "brier": 0.05613679930099221,
          "argmax_agreement": 0.66,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 8,
              "confidence": 0.517557176152255,
              "reference_agreement": 0.482499875
            },
            {
              "count": 22,
              "confidence": 0.5656107740630231,
              "reference_agreement": 0.5101515
            },
            {
              "count": 20,
              "confidence": 0.6310079557436046,
              "reference_agreement": 0.5293334
            },
            {
              "count": 20,
              "confidence": 0.7003291402466889,
              "reference_agreement": 0.7098333
            },
            {
              "count": 23,
              "confidence": 0.7643885662708242,
              "reference_agreement": 0.6455073043478261
            },
            {
              "count": 7,
              "confidence": 0.8262420627974808,
              "reference_agreement": 0.6914287142857143
            }
          ],
          "gold_label_correct": 65,
          "gold_label_accuracy": 0.65,
          "gold_label_ece_15bin_all_decisions": 0.08754395103445267,
          "full_vector_brier": 0.11227359860198442
        },
        "customer_service/urgency": {
          "count": 100,
          "soft_ce": 1.1231250801284207,
          "kl": 0.25316261024084386,
          "brier": 0.06501848850747206,
          "argmax_agreement": 0.57,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 14,
              "confidence": 0.4411960376551762,
              "reference_agreement": 0.34642849761908573
            },
            {
              "count": 22,
              "confidence": 0.4979765514824134,
              "reference_agreement": 0.41560594090917574
            },
            {
              "count": 27,
              "confidence": 0.5661962183925248,
              "reference_agreement": 0.45716044814818446
            },
            {
              "count": 23,
              "confidence": 0.6222290168802378,
              "reference_agreement": 0.5786956818841255
            },
            {
              "count": 13,
              "confidence": 0.6879913332169492,
              "reference_agreement": 0.664871788461732
            },
            {
              "count": 1,
              "confidence": 0.7422040130810093,
              "reference_agreement": 0.6833336833336833
            }
          ],
          "expected_score_mae": 0.39056680819172096,
          "gold_label_correct": 56,
          "gold_label_accuracy": 0.56,
          "gold_label_ece_15bin_all_decisions": 0.12422167823236911,
          "full_vector_brier": 0.13003697701494413
        },
        "invoice_processing/discrepancy_severity": {
          "count": 100,
          "soft_ce": 0.7813166885573835,
          "kl": 0.13707332422163884,
          "brier": 0.03045788400272809,
          "argmax_agreement": 0.83,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 10,
              "confidence": 0.36318909440456576,
              "reference_agreement": 0.452666601666755
            },
            {
              "count": 15,
              "confidence": 0.41585765000740066,
              "reference_agreement": 0.4002355722088945
            },
            {
              "count": 3,
              "confidence": 0.5029107001361107,
              "reference_agreement": 0.46666675555575554
            },
            {
              "count": 7,
              "confidence": 0.554937459938247,
              "reference_agreement": 0.5833334000001142
            },
            {
              "count": 5,
              "confidence": 0.6405068280987012,
              "reference_agreement": 0.7933334066669266
            },
            {
              "count": 13,
              "confidence": 0.7045803972407967,
              "reference_agreement": 0.7338461728206614
            },
            {
              "count": 18,
              "confidence": 0.7675469747476337,
              "reference_agreement": 0.7801667003890337
            },
            {
              "count": 11,
              "confidence": 0.8164401230878163,
              "reference_agreement": 0.8336366139397898
            },
            {
              "count": 3,
              "confidence": 0.8994513241595642,
              "reference_agreement": 0.9422222911112911
            },
            {
              "count": 15,
              "confidence": 0.9577089746675174,
              "reference_agreement": 0.9715557946667946
            }
          ],
          "expected_score_mae": 0.27761394944700535,
          "gold_label_correct": 83,
          "gold_label_accuracy": 0.83,
          "gold_label_ece_15bin_all_decisions": 0.1551419518932857,
          "full_vector_brier": 0.06091576800545618
        },
        "invoice_processing/disposition": {
          "count": 100,
          "soft_ce": 0.9672103205660691,
          "kl": 0.14375101661084821,
          "brier": 0.03838214719887722,
          "argmax_agreement": 0.74,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 2,
              "confidence": 0.3278949544267834,
              "reference_agreement": 0.5716665
            },
            {
              "count": 24,
              "confidence": 0.3638647878856312,
              "reference_agreement": 0.3302638513888964
            },
            {
              "count": 16,
              "confidence": 0.4371119558468517,
              "reference_agreement": 0.42291669833338374
            },
            {
              "count": 5,
              "confidence": 0.48657462908745097,
              "reference_agreement": 0.5753333066666932
            },
            {
              "count": 5,
              "confidence": 0.5743529947838498,
              "reference_agreement": 0.39799984333333
            },
            {
              "count": 18,
              "confidence": 0.6335226967478499,
              "reference_agreement": 0.637592573148225
            },
            {
              "count": 11,
              "confidence": 0.6890930143191779,
              "reference_agreement": 0.6630302878788333
            },
            {
              "count": 5,
              "confidence": 0.7659479600990471,
              "reference_agreement": 0.7706664166667834
            },
            {
              "count": 2,
              "confidence": 0.8601290126721075,
              "reference_agreement": 0.9433335
            },
            {
              "count": 12,
              "confidence": 0.8922016440214707,
              "reference_agreement": 0.9400165
            }
          ],
          "gold_label_correct": 74,
          "gold_label_accuracy": 0.74,
          "gold_label_ece_15bin_all_decisions": 0.20816706463754298,
          "full_vector_brier": 0.07676429439775444
        },
        "invoice_processing/duplicate": {
          "count": 100,
          "soft_ce": 0.2334091092156495,
          "kl": 0.02585605705240466,
          "brier": 0.007688562652478144,
          "argmax_agreement": 0.94,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 9,
              "confidence": 0.5190740869546441,
              "reference_agreement": 0.5159257777777778
            },
            {
              "count": 6,
              "confidence": 0.5672429607854125,
              "reference_agreement": 0.5161113333333334
            },
            {
              "count": 1,
              "confidence": 0.6224593312018546,
              "reference_agreement": 0.046667
            },
            {
              "count": 1,
              "confidence": 0.679178699175393,
              "reference_agreement": 0.75
            },
            {
              "count": 7,
              "confidence": 0.9192141605617763,
              "reference_agreement": 0.9795238571428572
            },
            {
              "count": 76,
              "confidence": 0.973002201351445,
              "reference_agreement": 0.9698990789473684
            }
          ],
          "gold_label_correct": 94,
          "gold_label_accuracy": 0.94,
          "gold_label_ece_15bin_all_decisions": 0.05485489658079937,
          "full_vector_brier": 0.015377125304956288
        },
        "invoice_processing/matches_order": {
          "count": 100,
          "soft_ce": 0.3306428558838697,
          "kl": 0.07295218240515576,
          "brier": 0.025035670669917,
          "argmax_agreement": 0.91,
          "hard_count": 1,
          "hard_ece": 0.033085978388704196,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9669140216112958,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 2,
              "confidence": 0.5,
              "reference_agreement": 0.25
            },
            {
              "count": 1,
              "confidence": 0.5467381519846138,
              "reference_agreement": 0.05
            },
            {
              "count": 8,
              "confidence": 0.6494972618666175,
              "reference_agreement": 0.742916625
            },
            {
              "count": 13,
              "confidence": 0.6945122010594892,
              "reference_agreement": 0.7448718461538462
            },
            {
              "count": 4,
              "confidence": 0.7602322944976581,
              "reference_agreement": 0.49416675
            },
            {
              "count": 7,
              "confidence": 0.8444382467004503,
              "reference_agreement": 0.7571428571428571
            },
            {
              "count": 21,
              "confidence": 0.8938861411785824,
              "reference_agreement": 0.951095238095238
            },
            {
              "count": 44,
              "confidence": 0.9677659919679358,
              "reference_agreement": 0.9671893636363635
            }
          ],
          "gold_label_correct": 91,
          "gold_label_accuracy": 0.91,
          "gold_label_ece_15bin_all_decisions": 0.10098650283026388,
          "full_vector_brier": 0.050071341339834
        },
        "invoice_processing/urgency": {
          "count": 100,
          "soft_ce": 1.0147848337232301,
          "kl": 0.07299384933168163,
          "brier": 0.01930732052766958,
          "argmax_agreement": 0.78,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 3,
              "confidence": 0.38242658197225876,
              "reference_agreement": 0.33888879444446113
            },
            {
              "count": 13,
              "confidence": 0.44677704982334543,
              "reference_agreement": 0.4525640602564577
            },
            {
              "count": 28,
              "confidence": 0.49440195405535564,
              "reference_agreement": 0.47142861309533574
            },
            {
              "count": 21,
              "confidence": 0.5638182000471827,
              "reference_agreement": 0.6255555988889798
            },
            {
              "count": 12,
              "confidence": 0.629708983631971,
              "reference_agreement": 0.4805554763889625
            },
            {
              "count": 15,
              "confidence": 0.7077412786031287,
              "reference_agreement": 0.6135777344445011
            },
            {
              "count": 7,
              "confidence": 0.7426201120020457,
              "reference_agreement": 0.7276191690477404
            },
            {
              "count": 1,
              "confidence": 0.8159353248895248,
              "reference_agreement": 0.7166662833337166
            }
          ],
          "expected_score_mae": 0.15360283037836298,
          "gold_label_correct": 75,
          "gold_label_accuracy": 0.75,
          "gold_label_ece_15bin_all_decisions": 0.1846883809213806,
          "full_vector_brier": 0.03861464105533916
        },
        "security_incidents/credential_compromise": {
          "count": 100,
          "soft_ce": 0.7435202424681325,
          "kl": 0.13032283941349765,
          "brier": 0.056534706032330906,
          "argmax_agreement": 0.56,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 13,
              "confidence": 0.5180136079658565,
              "reference_agreement": 0.5041024615384615
            },
            {
              "count": 16,
              "confidence": 0.5611483436625656,
              "reference_agreement": 0.5185416875
            },
            {
              "count": 18,
              "confidence": 0.6424031158653717,
              "reference_agreement": 0.4575924444444444
            },
            {
              "count": 24,
              "confidence": 0.7033728698546223,
              "reference_agreement": 0.54125
            },
            {
              "count": 13,
              "confidence": 0.7727795187245017,
              "reference_agreement": 0.5805127692307692
            },
            {
              "count": 12,
              "confidence": 0.8229402454540159,
              "reference_agreement": 0.61611125
            },
            {
              "count": 3,
              "confidence": 0.8926362259110124,
              "reference_agreement": 0.63
            },
            {
              "count": 1,
              "confidence": 0.9362850057480346,
              "reference_agreement": 0.833333
            }
          ],
          "gold_label_correct": 56,
          "gold_label_accuracy": 0.56,
          "gold_label_ece_15bin_all_decisions": 0.12394694920782158,
          "full_vector_brier": 0.11306941206466181
        },
        "security_incidents/disposition": {
          "count": 100,
          "soft_ce": 1.1881010051690235,
          "kl": 0.1639027463248791,
          "brier": 0.04448559481981512,
          "argmax_agreement": 0.58,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 2,
              "confidence": 0.32242648663151624,
              "reference_agreement": 0.4083334083334083
            },
            {
              "count": 27,
              "confidence": 0.3754838182935608,
              "reference_agreement": 0.32395059913580926
            },
            {
              "count": 46,
              "confidence": 0.43432437938573537,
              "reference_agreement": 0.46449275565219045
            },
            {
              "count": 19,
              "confidence": 0.4882781369268498,
              "reference_agreement": 0.49964906052637453
            },
            {
              "count": 6,
              "confidence": 0.5630952682918294,
              "reference_agreement": 0.5416665000001667
            }
          ],
          "gold_label_correct": 58,
          "gold_label_accuracy": 0.58,
          "gold_label_ece_15bin_all_decisions": 0.2085843245755816,
          "full_vector_brier": 0.08897118963963024
        },
        "security_incidents/severity": {
          "count": 100,
          "soft_ce": 1.315743899027136,
          "kl": 0.16019039234495527,
          "brier": 0.03374204583723859,
          "argmax_agreement": 0.62,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 9,
              "confidence": 0.30993707022864725,
              "reference_agreement": 0.1892591229629178
            },
            {
              "count": 11,
              "confidence": 0.37361903832007515,
              "reference_agreement": 0.35666686181831636
            },
            {
              "count": 29,
              "confidence": 0.4358135451699136,
              "reference_agreement": 0.367356292413834
            },
            {
              "count": 26,
              "confidence": 0.4949142252274759,
              "reference_agreement": 0.4523077201282927
            },
            {
              "count": 16,
              "confidence": 0.5669241476967354,
              "reference_agreement": 0.5614581968751906
            },
            {
              "count": 9,
              "confidence": 0.629883142448191,
              "reference_agreement": 0.5296296370372297
            }
          ],
          "expected_score_mae": 0.3280770725235342,
          "gold_label_correct": 61,
          "gold_label_accuracy": 0.61,
          "gold_label_ece_15bin_all_decisions": 0.17433526899513643,
          "full_vector_brier": 0.06748409167447718
        },
        "security_incidents/true_positive": {
          "count": 100,
          "soft_ce": 0.6651038558055207,
          "kl": 0.08578980083661669,
          "brier": 0.03646839076349738,
          "argmax_agreement": 0.75,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 10,
              "confidence": 0.5140548783434865,
              "reference_agreement": 0.4816668
            },
            {
              "count": 14,
              "confidence": 0.5686844897001768,
              "reference_agreement": 0.5449999285714285
            },
            {
              "count": 19,
              "confidence": 0.6397580847244457,
              "reference_agreement": 0.6021052105263158
            },
            {
              "count": 26,
              "confidence": 0.699375893864459,
              "reference_agreement": 0.6705129230769231
            },
            {
              "count": 23,
              "confidence": 0.7684838760054046,
              "reference_agreement": 0.6382609565217391
            },
            {
              "count": 8,
              "confidence": 0.8259863098340201,
              "reference_agreement": 0.768750125
            }
          ],
          "gold_label_correct": 75,
          "gold_label_accuracy": 0.75,
          "gold_label_ece_15bin_all_decisions": 0.0755676945059552,
          "full_vector_brier": 0.07293678152699476
        },
        "security_incidents/urgency": {
          "count": 100,
          "soft_ce": 1.1579755423473963,
          "kl": 0.13592958650868842,
          "brier": 0.035556631007055334,
          "argmax_agreement": 0.6,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 22,
              "confidence": 0.3855800670557517,
              "reference_agreement": 0.327196972045531
            },
            {
              "count": 44,
              "confidence": 0.42700538242544434,
              "reference_agreement": 0.44545453469702745
            },
            {
              "count": 17,
              "confidence": 0.5028795000140648,
              "reference_agreement": 0.49196084215690095
            },
            {
              "count": 11,
              "confidence": 0.5686336651767027,
              "reference_agreement": 0.5948483596970949
            },
            {
              "count": 4,
              "confidence": 0.6318266146454532,
              "reference_agreement": 0.55000005000025
            },
            {
              "count": 2,
              "confidence": 0.6723890027684984,
              "reference_agreement": 0.6833332916667916
            }
          ],
          "expected_score_mae": 0.2845580365911216,
          "gold_label_correct": 60,
          "gold_label_accuracy": 0.6,
          "gold_label_ece_15bin_all_decisions": 0.19018518367205345,
          "full_vector_brier": 0.07111326201411067
        }
      }
    },
    "calibrated": {
      "overall": {
        "count": 2000,
        "soft_ce": 0.8996057207086053,
        "kl": 0.13224716508427645,
        "brier": 0.0371886634007063,
        "argmax_agreement": 0.7055,
        "hard_count": 2,
        "hard_ece": 0.04532655734322799,
        "hard_reliability": [
          {
            "count": 2,
            "confidence": 0.954673442656772,
            "reference_agreement": 1.0
          }
        ],
        "soft_reliability": [
          {
            "count": 1,
            "confidence": 0.26431766017833624,
            "reference_agreement": 0.6433336433336433
          },
          {
            "count": 40,
            "confidence": 0.3177876741133899,
            "reference_agreement": 0.3488332323333685
          },
          {
            "count": 214,
            "confidence": 0.3738574970766125,
            "reference_agreement": 0.39616281869943265
          },
          {
            "count": 379,
            "confidence": 0.4335478747419467,
            "reference_agreement": 0.421979419893986
          },
          {
            "count": 296,
            "confidence": 0.5011488512826869,
            "reference_agreement": 0.5069932456870009
          },
          {
            "count": 266,
            "confidence": 0.5645507022164814,
            "reference_agreement": 0.5461152577318915
          },
          {
            "count": 223,
            "confidence": 0.633536195077047,
            "reference_agreement": 0.5981748920314267
          },
          {
            "count": 156,
            "confidence": 0.6977244202652548,
            "reference_agreement": 0.6617542448932456
          },
          {
            "count": 130,
            "confidence": 0.7643242189776432,
            "reference_agreement": 0.6995359706949515
          },
          {
            "count": 92,
            "confidence": 0.8331337292292074,
            "reference_agreement": 0.8546781770653425
          },
          {
            "count": 68,
            "confidence": 0.9005759800850643,
            "reference_agreement": 0.9235195591667292
          },
          {
            "count": 135,
            "confidence": 0.9612556801725451,
            "reference_agreement": 0.9694123619383392
          }
        ],
        "expected_score_mae": 0.26946470447726134,
        "gold_label_correct": 1409,
        "gold_label_accuracy": 0.7045,
        "gold_label_ece_15bin_all_decisions": 0.11873688877234255,
        "full_vector_brier": 0.0743773268014126
      },
      "type": {
        "choice": {
          "count": 600,
          "soft_ce": 1.0381506231326942,
          "kl": 0.17435602628587193,
          "brier": 0.042938132074843774,
          "argmax_agreement": 0.6883333333333334,
          "hard_count": 1,
          "hard_ece": 0.04531176280353044,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9546882371964696,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.26431766017833624,
              "reference_agreement": 0.6433336433336433
            },
            {
              "count": 28,
              "confidence": 0.3194104476415337,
              "reference_agreement": 0.3671427644048106
            },
            {
              "count": 151,
              "confidence": 0.37119215731679844,
              "reference_agreement": 0.40550005966892083
            },
            {
              "count": 171,
              "confidence": 0.43196031154385833,
              "reference_agreement": 0.43263354161798406
            },
            {
              "count": 56,
              "confidence": 0.4968234850515549,
              "reference_agreement": 0.5974404757738895
            },
            {
              "count": 41,
              "confidence": 0.5598861924502044,
              "reference_agreement": 0.5814633723578526
            },
            {
              "count": 38,
              "confidence": 0.6300906961195497,
              "reference_agreement": 0.7054297802543781
            },
            {
              "count": 20,
              "confidence": 0.7021830670140048,
              "reference_agreement": 0.6884998090001994
            },
            {
              "count": 30,
              "confidence": 0.7641036360974187,
              "reference_agreement": 0.7616667017779263
            },
            {
              "count": 32,
              "confidence": 0.8347770460980283,
              "reference_agreement": 0.9085641953126645
            },
            {
              "count": 21,
              "confidence": 0.90318181459874,
              "reference_agreement": 0.9307778106351233
            },
            {
              "count": 11,
              "confidence": 0.949274918346995,
              "reference_agreement": 0.972878994697626
            }
          ],
          "gold_label_correct": 413,
          "gold_label_accuracy": 0.6883333333333334,
          "gold_label_ece_15bin_all_decisions": 0.1767793119406733,
          "full_vector_brier": 0.08587626414968755
        },
        "noul": {
          "count": 600,
          "soft_ce": 0.5343593764491412,
          "kl": 0.08063713309657684,
          "brier": 0.03351564472605923,
          "argmax_agreement": 0.7666666666666667,
          "hard_count": 1,
          "hard_ece": 0.04534135188292543,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9546586481170746,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 59,
              "confidence": 0.5162463602685387,
              "reference_agreement": 0.48372877966101696
            },
            {
              "count": 102,
              "confidence": 0.566982612511053,
              "reference_agreement": 0.5379411666666667
            },
            {
              "count": 92,
              "confidence": 0.638635665544844,
              "reference_agreement": 0.5872825978260869
            },
            {
              "count": 76,
              "confidence": 0.6952965001408001,
              "reference_agreement": 0.6477193421052632
            },
            {
              "count": 69,
              "confidence": 0.7631561753342164,
              "reference_agreement": 0.6479227681159421
            },
            {
              "count": 48,
              "confidence": 0.8360191336961122,
              "reference_agreement": 0.8227708958333334
            },
            {
              "count": 44,
              "confidence": 0.8983448122025038,
              "reference_agreement": 0.91658325
            },
            {
              "count": 110,
              "confidence": 0.9631203976587833,
              "reference_agreement": 0.9687787545454546
            }
          ],
          "gold_label_correct": 459,
          "gold_label_accuracy": 0.765,
          "gold_label_ece_15bin_all_decisions": 0.03628697257339877,
          "full_vector_brier": 0.06703128945211846
        },
        "score": {
          "count": 800,
          "soft_ce": 1.0696318020851365,
          "kl": 0.13937304317385457,
          "brier": 0.03563132590108849,
          "argmax_agreement": 0.6725,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 12,
              "confidence": 0.31400120254772085,
              "reference_agreement": 0.306110990833337
            },
            {
              "count": 63,
              "confidence": 0.38024585110410336,
              "reference_agreement": 0.3737830824074847
            },
            {
              "count": 208,
              "confidence": 0.43485303487114435,
              "reference_agreement": 0.41322050251512227
            },
            {
              "count": 181,
              "confidence": 0.49756579867925116,
              "reference_agreement": 0.48659301701665447
            },
            {
              "count": 123,
              "confidence": 0.5640888327073031,
              "reference_agreement": 0.5411110673984649
            },
            {
              "count": 93,
              "confidence": 0.6298993959087413,
              "reference_agreement": 0.5651254868101266
            },
            {
              "count": 60,
              "confidence": 0.6993135701733142,
              "reference_agreement": 0.670616600389039
            },
            {
              "count": 31,
              "confidence": 0.7671375253583909,
              "reference_agreement": 0.7542904560324492
            },
            {
              "count": 12,
              "confidence": 0.8172099330447323,
              "reference_agreement": 0.8386112533338538
            },
            {
              "count": 3,
              "confidence": 0.9150589341002214,
              "reference_agreement": 0.9744443333333334
            },
            {
              "count": 14,
              "confidence": 0.9560177842150351,
              "reference_agreement": 0.9716669228572801
            }
          ],
          "expected_score_mae": 0.26946470447726134,
          "gold_label_correct": 537,
          "gold_label_accuracy": 0.67125,
          "gold_label_ece_15bin_all_decisions": 0.1383301153500044,
          "full_vector_brier": 0.07126265180217697
        }
      },
      "category": {
        "agent_trace_observability": {
          "count": 500,
          "soft_ce": 0.9784239666090253,
          "kl": 0.09995369366380179,
          "brier": 0.02902756320294822,
          "argmax_agreement": 0.666,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.320696328577158,
              "reference_agreement": 0.333333
            },
            {
              "count": 56,
              "confidence": 0.3777104214271337,
              "reference_agreement": 0.4022678604167104
            },
            {
              "count": 142,
              "confidence": 0.4351345221705775,
              "reference_agreement": 0.4108685441315153
            },
            {
              "count": 101,
              "confidence": 0.5017886292643352,
              "reference_agreement": 0.5098679828713507
            },
            {
              "count": 69,
              "confidence": 0.5647063184371532,
              "reference_agreement": 0.5609178477778174
            },
            {
              "count": 46,
              "confidence": 0.6323688610153608,
              "reference_agreement": 0.6025362419565561
            },
            {
              "count": 25,
              "confidence": 0.7011952059687334,
              "reference_agreement": 0.6249331570668304
            },
            {
              "count": 25,
              "confidence": 0.7712895410262108,
              "reference_agreement": 0.684133418933419
            },
            {
              "count": 21,
              "confidence": 0.8322538044396943,
              "reference_agreement": 0.8557143673016929
            },
            {
              "count": 14,
              "confidence": 0.8840899517188067,
              "reference_agreement": 0.8664284285714287
            }
          ],
          "expected_score_mae": 0.20048818990062078,
          "gold_label_correct": 335,
          "gold_label_accuracy": 0.67,
          "gold_label_ece_15bin_all_decisions": 0.13395749510739433,
          "full_vector_brier": 0.05805512640589644
        },
        "customer_service": {
          "count": 500,
          "soft_ce": 0.9395046654141255,
          "kl": 0.20235002740464264,
          "brier": 0.054900227765937,
          "argmax_agreement": 0.694,
          "hard_count": 1,
          "hard_ece": 0.04531176280353044,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9546882371964696,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.26431766017833624,
              "reference_agreement": 0.6433336433336433
            },
            {
              "count": 16,
              "confidence": 0.3153945397919026,
              "reference_agreement": 0.4254165366667425
            },
            {
              "count": 43,
              "confidence": 0.3676812200975334,
              "reference_agreement": 0.4629575986047603
            },
            {
              "count": 81,
              "confidence": 0.43653206317486204,
              "reference_agreement": 0.40526751485604134
            },
            {
              "count": 73,
              "confidence": 0.5004255499809467,
              "reference_agreement": 0.5462557171690167
            },
            {
              "count": 85,
              "confidence": 0.5652857404557898,
              "reference_agreement": 0.5123529096471258
            },
            {
              "count": 71,
              "confidence": 0.6289827242548214,
              "reference_agreement": 0.5991033191502455
            },
            {
              "count": 40,
              "confidence": 0.6933075112774083,
              "reference_agreement": 0.6937499738334586
            },
            {
              "count": 35,
              "confidence": 0.757544894325504,
              "reference_agreement": 0.7550476496191563
            },
            {
              "count": 24,
              "confidence": 0.8298230320463706,
              "reference_agreement": 0.858494010416886
            },
            {
              "count": 20,
              "confidence": 0.9042490371859675,
              "reference_agreement": 0.9288167011668795
            },
            {
              "count": 11,
              "confidence": 0.949274918346995,
              "reference_agreement": 0.972878994697626
            }
          ],
          "expected_score_mae": 0.3540890427112495,
          "gold_label_correct": 347,
          "gold_label_accuracy": 0.694,
          "gold_label_ece_15bin_all_decisions": 0.11718544849042345,
          "full_vector_brier": 0.109800455531874
        },
        "invoice_processing": {
          "count": 500,
          "soft_ce": 0.6684132719468862,
          "kl": 0.09346579628199163,
          "brier": 0.024514530073744333,
          "argmax_agreement": 0.84,
          "hard_count": 1,
          "hard_ece": 0.04534135188292543,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9546586481170746,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 10,
              "confidence": 0.3284781191962668,
              "reference_agreement": 0.39399997833337835
            },
            {
              "count": 38,
              "confidence": 0.3695643829541523,
              "reference_agreement": 0.35955257149123854
            },
            {
              "count": 38,
              "confidence": 0.43180381052982364,
              "reference_agreement": 0.45737369490706775
            },
            {
              "count": 49,
              "confidence": 0.4995394302233789,
              "reference_agreement": 0.4748979573470118
            },
            {
              "count": 48,
              "confidence": 0.5630772128542549,
              "reference_agreement": 0.5724305864584053
            },
            {
              "count": 48,
              "confidence": 0.633751143134781,
              "reference_agreement": 0.6486805458333972
            },
            {
              "count": 46,
              "confidence": 0.7006481123389033,
              "reference_agreement": 0.71297823347834
            },
            {
              "count": 30,
              "confidence": 0.7644638654220844,
              "reference_agreement": 0.7463223660112589
            },
            {
              "count": 37,
              "confidence": 0.8368844632918799,
              "reference_agreement": 0.8846810628829918
            },
            {
              "count": 32,
              "confidence": 0.905262180250512,
              "reference_agreement": 0.961760375
            },
            {
              "count": 124,
              "confidence": 0.9623184896893279,
              "reference_agreement": 0.9691048380645316
            }
          ],
          "expected_score_mae": 0.2172820819264362,
          "gold_label_correct": 417,
          "gold_label_accuracy": 0.834,
          "gold_label_ece_15bin_all_decisions": 0.1349665198192152,
          "full_vector_brier": 0.049029060147488666
        },
        "security_incidents": {
          "count": 500,
          "soft_ce": 1.012080978864384,
          "kl": 0.13321914298666976,
          "brier": 0.04031233256019564,
          "argmax_agreement": 0.622,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 13,
              "confidence": 0.31228590825579433,
              "reference_agreement": 0.2210255325640828
            },
            {
              "count": 77,
              "confidence": 0.3766231397145957,
              "reference_agreement": 0.372489202099624
            },
            {
              "count": 118,
              "confidence": 0.4301516988277852,
              "reference_agreement": 0.43542370985879325
            },
            {
              "count": 73,
              "confidence": 0.5020672766359284,
              "reference_agreement": 0.48529681082197323
            },
            {
              "count": 64,
              "confidence": 0.5645118228386581,
              "reference_agreement": 0.5552603373438221
            },
            {
              "count": 58,
              "confidence": 0.6398582000502253,
              "reference_agreement": 0.5517815850575183
            },
            {
              "count": 45,
              "confidence": 0.6967336842994563,
              "reference_agreement": 0.601407457407413
            },
            {
              "count": 40,
              "confidence": 0.7657980669345792,
              "reference_agreement": 0.6255000500000001
            },
            {
              "count": 10,
              "confidence": 0.8290495284941051,
              "reference_agreement": 0.7323335
            },
            {
              "count": 2,
              "confidence": 0.9042684049926732,
              "reference_agreement": 0.6583330000000001
            }
          ],
          "expected_score_mae": 0.30599950337073883,
          "gold_label_correct": 310,
          "gold_label_accuracy": 0.62,
          "gold_label_ece_15bin_all_decisions": 0.10104896950536107,
          "full_vector_brier": 0.08062466512039128
        }
      },
      "question": {
        "agent_trace_observability/action": {
          "count": 100,
          "soft_ce": 1.1300319363326001,
          "kl": 0.09598806155025466,
          "brier": 0.02483876810455199,
          "argmax_agreement": 0.66,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.320696328577158,
              "reference_agreement": 0.333333
            },
            {
              "count": 45,
              "confidence": 0.37494667517257485,
              "reference_agreement": 0.4043777937037419
            },
            {
              "count": 20,
              "confidence": 0.42036053533497303,
              "reference_agreement": 0.4256666106667256
            },
            {
              "count": 22,
              "confidence": 0.5017129711343226,
              "reference_agreement": 0.6168181568182417
            },
            {
              "count": 12,
              "confidence": 0.5458754881908042,
              "reference_agreement": 0.5908332955556567
            }
          ],
          "gold_label_correct": 66,
          "gold_label_accuracy": 0.66,
          "gold_label_ece_15bin_all_decisions": 0.23452694015867082,
          "full_vector_brier": 0.04967753620910398
        },
        "agent_trace_observability/needs_review": {
          "count": 100,
          "soft_ce": 0.5772502686786852,
          "kl": 0.061784401057798144,
          "brier": 0.027046496253035113,
          "argmax_agreement": 0.78,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 17,
              "confidence": 0.5199000755681038,
              "reference_agreement": 0.48039217647058824
            },
            {
              "count": 32,
              "confidence": 0.5717013250980697,
              "reference_agreement": 0.57124996875
            },
            {
              "count": 15,
              "confidence": 0.6285397565767751,
              "reference_agreement": 0.6404444666666667
            },
            {
              "count": 4,
              "confidence": 0.7878123338263531,
              "reference_agreement": 0.9075
            },
            {
              "count": 18,
              "confidence": 0.8368480740637315,
              "reference_agreement": 0.8564815555555555
            },
            {
              "count": 14,
              "confidence": 0.8840899517188067,
              "reference_agreement": 0.8664284285714287
            }
          ],
          "gold_label_correct": 78,
          "gold_label_accuracy": 0.78,
          "gold_label_ece_15bin_all_decisions": 0.10847385971036504,
          "full_vector_brier": 0.05409299250607023
        },
        "agent_trace_observability/outcome": {
          "count": 100,
          "soft_ce": 1.0909319011321876,
          "kl": 0.1661455365279982,
          "brier": 0.04300572982539011,
          "argmax_agreement": 0.62,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 5,
              "confidence": 0.3938691946389984,
              "reference_agreement": 0.3533333066666934
            },
            {
              "count": 58,
              "confidence": 0.4374378411470648,
              "reference_agreement": 0.38236207597705485
            },
            {
              "count": 4,
              "confidence": 0.4803700853513563,
              "reference_agreement": 0.58916684666676
            },
            {
              "count": 3,
              "confidence": 0.5793429761885015,
              "reference_agreement": 0.7744446666666667
            },
            {
              "count": 7,
              "confidence": 0.635342607275984,
              "reference_agreement": 0.6880951604762682
            },
            {
              "count": 8,
              "confidence": 0.7121878766387506,
              "reference_agreement": 0.6516665095834904
            },
            {
              "count": 15,
              "confidence": 0.7635113875345624,
              "reference_agreement": 0.6195556666667333
            }
          ],
          "gold_label_correct": 60,
          "gold_label_accuracy": 0.6,
          "gold_label_ece_15bin_all_decisions": 0.10083659412693492,
          "full_vector_brier": 0.08601145965078022
        },
        "agent_trace_observability/risk": {
          "count": 100,
          "soft_ce": 0.9828391693041487,
          "kl": 0.09046906089539679,
          "brier": 0.026974569027456388,
          "argmax_agreement": 0.67,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 6,
              "confidence": 0.3849728739931042,
              "reference_agreement": 0.4272221555556555
            },
            {
              "count": 15,
              "confidence": 0.44017390916712973,
              "reference_agreement": 0.4377777200001155
            },
            {
              "count": 22,
              "confidence": 0.5047424360225681,
              "reference_agreement": 0.541060543181922
            },
            {
              "count": 20,
              "confidence": 0.5613336073178117,
              "reference_agreement": 0.5121666475000758
            },
            {
              "count": 17,
              "confidence": 0.6336701385816399,
              "reference_agreement": 0.5417646784313843
            },
            {
              "count": 12,
              "confidence": 0.6951222762738744,
              "reference_agreement": 0.5763887186111981
            },
            {
              "count": 5,
              "confidence": 0.7875525659292162,
              "reference_agreement": 0.7093333680001679
            },
            {
              "count": 3,
              "confidence": 0.8046881866954707,
              "reference_agreement": 0.8511112377785178
            }
          ],
          "expected_score_mae": 0.18766299038600726,
          "gold_label_correct": 69,
          "gold_label_accuracy": 0.69,
          "gold_label_ece_15bin_all_decisions": 0.12744205611487303,
          "full_vector_brier": 0.053949138054912776
        },
        "agent_trace_observability/urgency": {
          "count": 100,
          "soft_ce": 1.1110665575975052,
          "kl": 0.08538140828756115,
          "brier": 0.023272252804307492,
          "argmax_agreement": 0.6,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 49,
              "confidence": 0.43689567382624167,
              "reference_agreement": 0.4303333642177499
            },
            {
              "count": 36,
              "confidence": 0.4938569714495298,
              "reference_agreement": 0.43055556898154956
            },
            {
              "count": 2,
              "confidence": 0.5775433179069762,
              "reference_agreement": 0.38333300000000003
            },
            {
              "count": 7,
              "confidence": 0.6344400930336004,
              "reference_agreement": 0.5833334961905962
            },
            {
              "count": 5,
              "confidence": 0.6981819641643676,
              "reference_agreement": 0.6986664453336919
            },
            {
              "count": 1,
              "confidence": 0.740555547685341,
              "reference_agreement": 0.6333336333336334
            }
          ],
          "expected_score_mae": 0.2133133894152343,
          "gold_label_correct": 62,
          "gold_label_accuracy": 0.62,
          "gold_label_ece_15bin_all_decisions": 0.13295801626402656,
          "full_vector_brier": 0.046544505608614985
        },
        "customer_service/action": {
          "count": 100,
          "soft_ce": 1.2561721269415909,
          "kl": 0.290314230980401,
          "brier": 0.06932695222424835,
          "argmax_agreement": 0.6,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.26431766017833624,
              "reference_agreement": 0.6433336433336433
            },
            {
              "count": 15,
              "confidence": 0.3143181069990199,
              "reference_agreement": 0.3948887724445253
            },
            {
              "count": 38,
              "confidence": 0.3671960078421094,
              "reference_agreement": 0.46475026728081864
            },
            {
              "count": 27,
              "confidence": 0.43092514659511294,
              "reference_agreement": 0.4196297262963243
            },
            {
              "count": 15,
              "confidence": 0.49345382078770295,
              "reference_agreement": 0.6222222582223413
            },
            {
              "count": 2,
              "confidence": 0.5467833623082197,
              "reference_agreement": 0.37499980000020006
            },
            {
              "count": 2,
              "confidence": 0.6432513672126315,
              "reference_agreement": 0.7800003916668916
            }
          ],
          "gold_label_correct": 61,
          "gold_label_accuracy": 0.61,
          "gold_label_ece_15bin_all_decisions": 0.20837740157143791,
          "full_vector_brier": 0.1386539044484967
        },
        "customer_service/category": {
          "count": 100,
          "soft_ce": 0.5762908772818797,
          "kl": 0.16586899434803676,
          "brier": 0.034944317406019984,
          "argmax_agreement": 0.93,
          "hard_count": 1,
          "hard_ece": 0.04531176280353044,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9546882371964696,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 5,
              "confidence": 0.4322684416135483,
              "reference_agreement": 0.6660000793337393
            },
            {
              "count": 3,
              "confidence": 0.4905168094943096,
              "reference_agreement": 0.5655556666666667
            },
            {
              "count": 8,
              "confidence": 0.5612844306710459,
              "reference_agreement": 0.6383332712502996
            },
            {
              "count": 12,
              "confidence": 0.6268957032862555,
              "reference_agreement": 0.7177498577498188
            },
            {
              "count": 8,
              "confidence": 0.6937005013196476,
              "reference_agreement": 0.7020831275002684
            },
            {
              "count": 13,
              "confidence": 0.7640632842535814,
              "reference_agreement": 0.8997435425643682
            },
            {
              "count": 20,
              "confidence": 0.8328563184532323,
              "reference_agreement": 0.8911928125002632
            },
            {
              "count": 20,
              "confidence": 0.9042490371859675,
              "reference_agreement": 0.9288167011668795
            },
            {
              "count": 11,
              "confidence": 0.949274918346995,
              "reference_agreement": 0.972878994697626
            }
          ],
          "gold_label_correct": 94,
          "gold_label_accuracy": 0.94,
          "gold_label_ece_15bin_all_decisions": 0.1768752555819122,
          "full_vector_brier": 0.06988863481203997
        },
        "customer_service/churn_risk": {
          "count": 100,
          "soft_ce": 1.0730591179793425,
          "kl": 0.18264058554353962,
          "brier": 0.051583714603065454,
          "argmax_agreement": 0.71,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.3315410316851427,
              "reference_agreement": 0.883333
            },
            {
              "count": 5,
              "confidence": 0.3713688332387563,
              "reference_agreement": 0.4493333166667167
            },
            {
              "count": 31,
              "confidence": 0.4375064676665284,
              "reference_agreement": 0.388172083225866
            },
            {
              "count": 27,
              "confidence": 0.49821156690152874,
              "reference_agreement": 0.5818519443210562
            },
            {
              "count": 19,
              "confidence": 0.5633747625283594,
              "reference_agreement": 0.5187718885966044
            },
            {
              "count": 15,
              "confidence": 0.6252985881937501,
              "reference_agreement": 0.6162223266667289
            },
            {
              "count": 2,
              "confidence": 0.6801812993683634,
              "reference_agreement": 0.6716668416668417
            }
          ],
          "expected_score_mae": 0.31676493780927045,
          "gold_label_correct": 71,
          "gold_label_accuracy": 0.71,
          "gold_label_ece_15bin_all_decisions": 0.20353240088435612,
          "full_vector_brier": 0.10316742920613091
        },
        "customer_service/needs_human": {
          "count": 100,
          "soft_ce": 0.6710381093325837,
          "kl": 0.12192570050358188,
          "brier": 0.054199941449838425,
          "argmax_agreement": 0.66,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 8,
              "confidence": 0.5158550697220626,
              "reference_agreement": 0.482499875
            },
            {
              "count": 28,
              "confidence": 0.5674903989336381,
              "reference_agreement": 0.5160714285714285
            },
            {
              "count": 18,
              "confidence": 0.6357804167556367,
              "reference_agreement": 0.5612963333333334
            },
            {
              "count": 21,
              "confidence": 0.6961004807233757,
              "reference_agreement": 0.6815873333333333
            },
            {
              "count": 21,
              "confidence": 0.7545551168362051,
              "reference_agreement": 0.6688889523809524
            },
            {
              "count": 4,
              "confidence": 0.8146566000120619,
              "reference_agreement": 0.695
            }
          ],
          "gold_label_correct": 65,
          "gold_label_accuracy": 0.65,
          "gold_label_ece_15bin_all_decisions": 0.058501302775174596,
          "full_vector_brier": 0.10839988289967685
        },
        "customer_service/urgency": {
          "count": 100,
          "soft_ce": 1.1209630955352308,
          "kl": 0.251000625647654,
          "brier": 0.06444621314651275,
          "argmax_agreement": 0.57,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 18,
              "confidence": 0.44444863629809206,
              "reference_agreement": 0.3407406175926695
            },
            {
              "count": 20,
              "confidence": 0.5039577272096432,
              "reference_agreement": 0.4638332491667292
            },
            {
              "count": 28,
              "confidence": 0.5668426468063085,
              "reference_agreement": 0.4780952023809791
            },
            {
              "count": 24,
              "confidence": 0.6260414968218451,
              "reference_agreement": 0.5423611534722869
            },
            {
              "count": 9,
              "confidence": 0.6893581940679483,
              "reference_agreement": 0.7196295833336129
            },
            {
              "count": 1,
              "confidence": 0.7355911525357738,
              "reference_agreement": 0.6833336833336833
            }
          ],
          "expected_score_mae": 0.39141314761322865,
          "gold_label_correct": 56,
          "gold_label_accuracy": 0.56,
          "gold_label_ece_15bin_all_decisions": 0.09986013285263563,
          "full_vector_brier": 0.1288924262930255
        },
        "invoice_processing/discrepancy_severity": {
          "count": 100,
          "soft_ce": 0.7824188222345687,
          "kl": 0.13817545789882418,
          "brier": 0.030670028432809596,
          "argmax_agreement": 0.83,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 2,
              "confidence": 0.33286669301517746,
              "reference_agreement": 0.5433333916668917
            },
            {
              "count": 11,
              "confidence": 0.37626617164243364,
              "reference_agreement": 0.40454538484852426
            },
            {
              "count": 12,
              "confidence": 0.4158789751689152,
              "reference_agreement": 0.41612779859445137
            },
            {
              "count": 5,
              "confidence": 0.511693219926803,
              "reference_agreement": 0.43666665333345334
            },
            {
              "count": 5,
              "confidence": 0.5577624108377157,
              "reference_agreement": 0.66000016000016
            },
            {
              "count": 7,
              "confidence": 0.6424516556670419,
              "reference_agreement": 0.7361905761906619
            },
            {
              "count": 13,
              "confidence": 0.7078300926213531,
              "reference_agreement": 0.7558974338463236
            },
            {
              "count": 20,
              "confidence": 0.770513839139187,
              "reference_agreement": 0.7909834398501792
            },
            {
              "count": 8,
              "confidence": 0.8229931371292696,
              "reference_agreement": 0.8491668804171221
            },
            {
              "count": 3,
              "confidence": 0.9150589341002214,
              "reference_agreement": 0.9744443333333334
            },
            {
              "count": 14,
              "confidence": 0.9560177842150351,
              "reference_agreement": 0.9716669228572801
            }
          ],
          "expected_score_mae": 0.28116763438747877,
          "gold_label_correct": 83,
          "gold_label_accuracy": 0.83,
          "gold_label_ece_15bin_all_decisions": 0.16034912415177363,
          "full_vector_brier": 0.06134005686561919
        },
        "invoice_processing/disposition": {
          "count": 100,
          "soft_ce": 0.9782701754641571,
          "kl": 0.15481087150893616,
          "brier": 0.03951271603036011,
          "argmax_agreement": 0.74,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 8,
              "confidence": 0.3273809757415391,
              "reference_agreement": 0.356666625
            },
            {
              "count": 24,
              "confidence": 0.3651080883483308,
              "reference_agreement": 0.3415138374999964
            },
            {
              "count": 13,
              "confidence": 0.43382661018637136,
              "reference_agreement": 0.5002564646154775
            },
            {
              "count": 4,
              "confidence": 0.4948415192095376,
              "reference_agreement": 0.5124999458333042
            },
            {
              "count": 15,
              "confidence": 0.5688883847973131,
              "reference_agreement": 0.5368888311111845
            },
            {
              "count": 17,
              "confidence": 0.6286351192794514,
              "reference_agreement": 0.6950980264706401
            },
            {
              "count": 4,
              "confidence": 0.6991385791532274,
              "reference_agreement": 0.7349997708334792
            },
            {
              "count": 2,
              "confidence": 0.7688077873037844,
              "reference_agreement": 0.9299999999999999
            },
            {
              "count": 12,
              "confidence": 0.837978258839355,
              "reference_agreement": 0.9375165
            },
            {
              "count": 1,
              "confidence": 0.8818373628541912,
              "reference_agreement": 0.97
            }
          ],
          "gold_label_correct": 74,
          "gold_label_accuracy": 0.74,
          "gold_label_ece_15bin_all_decisions": 0.21745472516454098,
          "full_vector_brier": 0.07902543206072021
        },
        "invoice_processing/duplicate": {
          "count": 100,
          "soft_ce": 0.23439542363249471,
          "kl": 0.02684237146924989,
          "brier": 0.007817390201271179,
          "argmax_agreement": 0.94,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 9,
              "confidence": 0.5172249840934735,
              "reference_agreement": 0.5159257777777778
            },
            {
              "count": 6,
              "confidence": 0.560787712375488,
              "reference_agreement": 0.5161113333333334
            },
            {
              "count": 2,
              "confidence": 0.6370394155634268,
              "reference_agreement": 0.3983335
            },
            {
              "count": 12,
              "confidence": 0.9117582796855447,
              "reference_agreement": 0.9755277499999999
            },
            {
              "count": 71,
              "confidence": 0.9649128351065898,
              "reference_agreement": 0.969896676056338
            }
          ],
          "gold_label_correct": 94,
          "gold_label_accuracy": 0.94,
          "gold_label_ece_15bin_all_decisions": 0.05804417051238259,
          "full_vector_brier": 0.015634780402542358
        },
        "invoice_processing/matches_order": {
          "count": 100,
          "soft_ce": 0.33281042566388214,
          "kl": 0.0751197521851682,
          "brier": 0.025450911444578592,
          "argmax_agreement": 0.91,
          "hard_count": 1,
          "hard_ece": 0.04534135188292543,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.9546586481170746,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 2,
              "confidence": 0.5,
              "reference_agreement": 0.25
            },
            {
              "count": 1,
              "confidence": 0.5422205310594118,
              "reference_agreement": 0.05
            },
            {
              "count": 11,
              "confidence": 0.643209342515234,
              "reference_agreement": 0.7396969090909091
            },
            {
              "count": 11,
              "confidence": 0.6853901959758365,
              "reference_agreement": 0.7666667272727273
            },
            {
              "count": 4,
              "confidence": 0.7572366579530104,
              "reference_agreement": 0.50250025
            },
            {
              "count": 16,
              "confidence": 0.8447832124548076,
              "reference_agreement": 0.8733124999999999
            },
            {
              "count": 16,
              "confidence": 0.900017265414687,
              "reference_agreement": 0.9485416249999999
            },
            {
              "count": 39,
              "confidence": 0.9598572423050843,
              "reference_agreement": 0.966743564102564
            }
          ],
          "gold_label_correct": 91,
          "gold_label_accuracy": 0.91,
          "gold_label_ece_15bin_all_decisions": 0.11605332143659476,
          "full_vector_brier": 0.050901822889157185
        },
        "invoice_processing/urgency": {
          "count": 100,
          "soft_ce": 1.0141715127393283,
          "kl": 0.07238052834777972,
          "brier": 0.019121604259702194,
          "argmax_agreement": 0.78,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 3,
              "confidence": 0.38064151461035944,
              "reference_agreement": 0.33888879444446113
            },
            {
              "count": 13,
              "confidence": 0.44448085889873,
              "reference_agreement": 0.4525640602564577
            },
            {
              "count": 29,
              "confidence": 0.49257155336662545,
              "reference_agreement": 0.4790805124138998
            },
            {
              "count": 21,
              "confidence": 0.561839123121125,
              "reference_agreement": 0.617936518254042
            },
            {
              "count": 11,
              "confidence": 0.626064968568645,
              "reference_agreement": 0.47575751969705
            },
            {
              "count": 18,
              "confidence": 0.7051208606202697,
              "reference_agreement": 0.6442777231481954
            },
            {
              "count": 4,
              "confidence": 0.7392692433647956,
              "reference_agreement": 0.6750002958335458
            },
            {
              "count": 1,
              "confidence": 0.8085095394162195,
              "reference_agreement": 0.7166662833337166
            }
          ],
          "expected_score_mae": 0.15339652946539362,
          "gold_label_correct": 75,
          "gold_label_accuracy": 0.75,
          "gold_label_ece_15bin_all_decisions": 0.18936000086676474,
          "full_vector_brier": 0.03824320851940439
        },
        "security_incidents/credential_compromise": {
          "count": 100,
          "soft_ce": 0.7303544474693316,
          "kl": 0.11715704441469675,
          "brier": 0.051873219306088056,
          "argmax_agreement": 0.56,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 13,
              "confidence": 0.5162662097600973,
              "reference_agreement": 0.5041024615384615
            },
            {
              "count": 17,
              "confidence": 0.5577534464386459,
              "reference_agreement": 0.5229411764705882
            },
            {
              "count": 21,
              "confidence": 0.6372227885501143,
              "reference_agreement": 0.47063480952380954
            },
            {
              "count": 21,
              "confidence": 0.691688303154376,
              "reference_agreement": 0.5293650952380953
            },
            {
              "count": 20,
              "confidence": 0.7677754511149539,
              "reference_agreement": 0.5835
            },
            {
              "count": 6,
              "confidence": 0.8359302010758386,
              "reference_agreement": 0.7183335
            },
            {
              "count": 2,
              "confidence": 0.9042684049926732,
              "reference_agreement": 0.6583330000000001
            }
          ],
          "gold_label_correct": 56,
          "gold_label_accuracy": 0.56,
          "gold_label_ece_15bin_all_decisions": 0.11893490648195507,
          "full_vector_brier": 0.10374643861217611
        },
        "security_incidents/disposition": {
          "count": 100,
          "soft_ce": 1.1972067216437494,
          "kl": 0.17300846279960486,
          "brier": 0.04600030885849212,
          "argmax_agreement": 0.58,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 4,
              "confidence": 0.32224419861704345,
              "reference_agreement": 0.29249995416670416
            },
            {
              "count": 39,
              "confidence": 0.3715904353978362,
              "reference_agreement": 0.39512819239318403
            },
            {
              "count": 48,
              "confidence": 0.4302195975460818,
              "reference_agreement": 0.46097220729168575
            },
            {
              "count": 8,
              "confidence": 0.501278204923741,
              "reference_agreement": 0.5562498937501396
            },
            {
              "count": 1,
              "confidence": 0.5496311616587236,
              "reference_agreement": 0.5166664833335167
            }
          ],
          "gold_label_correct": 58,
          "gold_label_accuracy": 0.58,
          "gold_label_ece_15bin_all_decisions": 0.17586552330691987,
          "full_vector_brier": 0.09200061771698424
        },
        "security_incidents/severity": {
          "count": 100,
          "soft_ce": 1.3148165106977874,
          "kl": 0.15926300401560653,
          "brier": 0.033456273290858624,
          "argmax_agreement": 0.62,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 9,
              "confidence": 0.3078600014285725,
              "reference_agreement": 0.1892591229629178
            },
            {
              "count": 12,
              "confidence": 0.3727428250841209,
              "reference_agreement": 0.36388904000012334
            },
            {
              "count": 29,
              "confidence": 0.433631983234007,
              "reference_agreement": 0.3698850510345237
            },
            {
              "count": 26,
              "confidence": 0.4919768669037995,
              "reference_agreement": 0.45679487397444657
            },
            {
              "count": 16,
              "confidence": 0.5652997260041271,
              "reference_agreement": 0.5520831968751906
            },
            {
              "count": 8,
              "confidence": 0.6261899107804109,
              "reference_agreement": 0.5354167166668834
            }
          ],
          "expected_score_mae": 0.32567749172139204,
          "gold_label_correct": 61,
          "gold_label_accuracy": 0.61,
          "gold_label_ece_15bin_all_decisions": 0.1787678515625339,
          "full_vector_brier": 0.06691254658171725
        },
        "security_incidents/true_positive": {
          "count": 100,
          "soft_ce": 0.6603075839178701,
          "kl": 0.08099352894896619,
          "brier": 0.034705909701544004,
          "argmax_agreement": 0.75,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 10,
              "confidence": 0.5126907829686985,
              "reference_agreement": 0.4816668
            },
            {
              "count": 18,
              "confidence": 0.5699609726709991,
              "reference_agreement": 0.5612962222222222
            },
            {
              "count": 25,
              "confidence": 0.6460510888610294,
              "reference_agreement": 0.6201333600000001
            },
            {
              "count": 23,
              "confidence": 0.7025946692840792,
              "reference_agreement": 0.6679710869565217
            },
            {
              "count": 20,
              "confidence": 0.7638206827542047,
              "reference_agreement": 0.6675001
            },
            {
              "count": 4,
              "confidence": 0.8187285196215046,
              "reference_agreement": 0.7533335
            }
          ],
          "gold_label_correct": 75,
          "gold_label_accuracy": 0.75,
          "gold_label_ece_15bin_all_decisions": 0.09523922989135299,
          "full_vector_brier": 0.06941181940308801
        },
        "security_incidents/urgency": {
          "count": 100,
          "soft_ce": 1.1577196305931823,
          "kl": 0.13567367475447445,
          "brier": 0.0355259516439954,
          "argmax_agreement": 0.6,
          "hard_count": 0,
          "hard_ece": null,
          "hard_reliability": [],
          "soft_reliability": [
            {
              "count": 26,
              "confidence": 0.38596311063456945,
              "reference_agreement": 0.34250002224366877
            },
            {
              "count": 41,
              "confidence": 0.42761054306537855,
              "reference_agreement": 0.45186988617891466
            },
            {
              "count": 16,
              "confidence": 0.5006824036853632,
              "reference_agreement": 0.4831250822917073
            },
            {
              "count": 12,
              "confidence": 0.5661019822011998,
              "reference_agreement": 0.599444329722337
            },
            {
              "count": 4,
              "confidence": 0.642325133897913,
              "reference_agreement": 0.58333330000025
            },
            {
              "count": 1,
              "confidence": 0.6678840336998128,
              "reference_agreement": 0.5833335833335833
            }
          ],
          "expected_score_mae": 0.2863215150200857,
          "gold_label_correct": 60,
          "gold_label_accuracy": 0.6,
          "gold_label_ece_15bin_all_decisions": 0.1646168179614661,
          "full_vector_brier": 0.0710519032879908
        }
      }
    },
    "metric_notes": {
      "accuracy": "Main accuracy compares original option key with stored gold.label; argmax_agreement separately uses first maximum of rounded target distribution in input option order",
      "soft_ce": "Mean natural-log cross-entropy against full normalized gold distribution",
      "kl": "Mean KL(gold || prediction)",
      "brier": "Existing metric field brier is half-vector Brier; full_vector_brier is twice it",
      "ece": "gold_label_ece_15bin_all_decisions uses top probability vs stored-label accuracy on all examples; hard_ece uses one-hot references only; exact leaderboard scorer parity is not claimed",
      "overlap": "Normalized exact state and source-ID checks; paraphrase and pretraining exposure not assessed"
    }
  },
  "sysone_bench": {
    "dataset_version": "2.1.0",
    "upstream_revision": "93afcfe1bd450dc052b21768e2b95b20316ebb68",
    "manifest_sha256": "0d21a64c6b61b6be7b49050dbd89f89f934912a7b586734e475308e9c9422ea2",
    "raw": {
      "evaluation_cases": 952,
      "evaluation_decisions": 1240,
      "correct": 953,
      "accuracy": 0.7685483870967742,
      "suites": {
        "triage": {
          "accuracy": 0.9270833333333334,
          "cases": 48,
          "decisions": 192,
          "latency": {
            "mean_seconds": 0.03883962416667922,
            "p50_seconds": 0.03841860600016389,
            "p95_seconds": 0.04178968690000602
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.8541666666666666,
              "macro_f1": 0.820875420875421,
              "balanced_accuracy": 0.8444444444444444,
              "per_class_recall": {
                "billing_question": 0.2222222222222222,
                "cancellation": 1.0,
                "information": 1.0,
                "refund": 1.0,
                "technical_help": 1.0
              },
              "brier": 0.13697967505778072,
              "nll": 0.2506294304217651,
              "ece_raw": 0.033931114278253044,
              "ece_fitted": 0.039914814482762906
            },
            "noul": {
              "accuracy": 0.9513888888888888,
              "balanced_accuracy": 0.912037037037037,
              "auroc": 0.9890689300411523,
              "auprc": 0.9701626578233637,
              "brier": 0.04911263462874029,
              "nll": 0.1906247610132196,
              "ece_raw": 0.01181868597485985,
              "ece_fitted": 0.012443502260767868
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 178
        },
        "guardrails": {
          "accuracy": 0.8125,
          "cases": 48,
          "decisions": 96,
          "latency": {
            "mean_seconds": 0.037130780229148286,
            "p50_seconds": 0.03704196349997346,
            "p95_seconds": 0.03867655395000611
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": null,
              "macro_f1": null,
              "balanced_accuracy": null,
              "per_class_recall": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "noul": {
              "accuracy": 0.8125,
              "balanced_accuracy": 0.8676470588235294,
              "auroc": 0.9803046218487395,
              "auprc": 0.9910730108125466,
              "brier": 0.14491041421218856,
              "nll": 0.4276831543302518,
              "ece_raw": 0.02432198575945292,
              "ece_fitted": 0.026983405418465858
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 78
        },
        "moderation": {
          "accuracy": 0.9166666666666666,
          "cases": 48,
          "decisions": 144,
          "latency": {
            "mean_seconds": 0.03827599806249301,
            "p50_seconds": 0.037934129999939614,
            "p95_seconds": 0.041699429449965915
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": null,
              "macro_f1": null,
              "balanced_accuracy": null,
              "per_class_recall": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "noul": {
              "accuracy": 0.9166666666666666,
              "balanced_accuracy": 0.9106227106227107,
              "auroc": 0.978021978021978,
              "auprc": 0.944990138816365,
              "brier": 0.06061711079669634,
              "nll": 0.2023230253736832,
              "ece_raw": 0.010916557094938821,
              "ece_fitted": 0.011697905217155507
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 132
        },
        "agnews": {
          "accuracy": 0.8125,
          "cases": 160,
          "decisions": 160,
          "latency": {
            "mean_seconds": 0.038758980293738435,
            "p50_seconds": 0.0382035000001224,
            "p95_seconds": 0.042714876549950986
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.8125,
              "macro_f1": 0.8064558008777994,
              "balanced_accuracy": 0.8125,
              "per_class_recall": {
                "business": 0.85,
                "scitech": 0.55,
                "sports": 0.975,
                "world": 0.875
              },
              "brier": 0.30172771335742954,
              "nll": 0.6319382379388891,
              "ece_raw": 0.0052000811604517765,
              "ece_fitted": 0.007738817523861061
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 130
        },
        "emotion": {
          "accuracy": 0.7239583333333334,
          "cases": 192,
          "decisions": 192,
          "latency": {
            "mean_seconds": 0.038396373302075425,
            "p50_seconds": 0.03797906949989738,
            "p95_seconds": 0.0415361116498957
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.7239583333333334,
              "macro_f1": 0.6913006811212603,
              "balanced_accuracy": 0.6854851048438465,
              "per_class_recall": {
                "anger": 0.5714285714285714,
                "fear": 0.9259259259259259,
                "joy": 0.7142857142857143,
                "love": 0.6,
                "sadness": 0.8275862068965517,
                "surprise": 0.47368421052631576
              },
              "brier": 0.39443242180284094,
              "nll": 0.7806033322263032,
              "ece_raw": 0.0032102610291140795,
              "ece_fitted": 0.003416953263655513
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 139
        },
        "banking77_12": {
          "accuracy": 0.8020833333333334,
          "cases": 96,
          "decisions": 96,
          "latency": {
            "mean_seconds": 0.03918704791666509,
            "p50_seconds": 0.038639419499986616,
            "p95_seconds": 0.04271133699995744
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.8020833333333334,
              "macro_f1": 0.7940683880389763,
              "balanced_accuracy": 0.8020833333333334,
              "per_class_recall": {
                "card_arrival": 0.625,
                "card_linking": 0.25,
                "exchange_rate": 1.0,
                "lost_or_stolen_card": 1.0,
                "order_physical_card": 0.875,
                "pin_blocked": 0.875,
                "refund_not_showing_up": 0.875,
                "request_refund": 0.875,
                "terminate_account": 1.0,
                "top_up_failed": 0.875,
                "topping_up_by_card": 0.5,
                "transaction_charged_twice": 0.875
              },
              "brier": 0.29183460550386847,
              "nll": 0.774193924200847,
              "ece_raw": 0.01661891915597873,
              "ece_fitted": 0.016552925317361755
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 77
        },
        "mnli": {
          "accuracy": 0.5416666666666666,
          "cases": 120,
          "decisions": 120,
          "latency": {
            "mean_seconds": 0.0384311384083181,
            "p50_seconds": 0.03797633499982567,
            "p95_seconds": 0.04150163254993231
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.5416666666666666,
              "macro_f1": 0.531705418323941,
              "balanced_accuracy": 0.5416666666666666,
              "per_class_recall": {
                "contradiction": 0.575,
                "entailment": 0.725,
                "neutral": 0.325
              },
              "brier": 0.5619903249840926,
              "nll": 0.952791383586364,
              "ece_raw": 0.0067628751799769776,
              "ece_fitted": 0.00836092388336822
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 65
        },
        "sst5": {
          "accuracy": 0.3333333333333333,
          "cases": 120,
          "decisions": 120,
          "latency": {
            "mean_seconds": 0.038905105700003634,
            "p50_seconds": 0.03851513350002733,
            "p95_seconds": 0.041932194849789536
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": null,
              "macro_f1": null,
              "balanced_accuracy": null,
              "per_class_recall": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": 0.8020032497743765,
              "rmse": 0.9909589189089723,
              "nearest_level_accuracy": 0.3333333333333333,
              "within_one_accuracy": 0.875,
              "raw_equality_rate": 0.0,
              "spearman": 0.737594189458718,
              "ordinal_confusion": {
                "labels": [
                  0,
                  1,
                  2,
                  3,
                  4
                ],
                "matrix": [
                  [
                    0,
                    19,
                    4,
                    1,
                    0
                  ],
                  [
                    2,
                    10,
                    12,
                    0,
                    0
                  ],
                  [
                    0,
                    6,
                    17,
                    1,
                    0
                  ],
                  [
                    0,
                    1,
                    11,
                    12,
                    0
                  ],
                  [
                    0,
                    1,
                    8,
                    14,
                    1
                  ]
                ]
              }
            }
          },
          "correct": 40
        },
        "multilingual_intent": {
          "accuracy": 0.95,
          "cases": 120,
          "decisions": 120,
          "latency": {
            "mean_seconds": 0.039158464716655315,
            "p50_seconds": 0.03864724799996111,
            "p95_seconds": 0.04240460730002269
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.95,
              "macro_f1": 0.8159019343229869,
              "balanced_accuracy": 0.9650000000000001,
              "per_class_recall": {
                "billing_question": 0.95,
                "cancellation": 1.0,
                "information": 0.875,
                "other": null,
                "refund": 1.0,
                "technical_help": 1.0
              },
              "brier": 0.12235859002542863,
              "nll": 0.24005612544914204,
              "ece_raw": 0.008839931130203269,
              "ece_fitted": 0.0071500889173673875
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 114
        }
      },
      "primitive_metrics": {
        "choice": {
          "accuracy": 0.7690217391304348,
          "macro_f1": 0.748753245465977,
          "balanced_accuracy": 0.7745682853365854,
          "per_class_recall": {
            "anger": 0.5714285714285714,
            "billing_question": 0.7241379310344828,
            "business": 0.85,
            "cancellation": 1.0,
            "card_arrival": 0.625,
            "card_linking": 0.25,
            "contradiction": 0.575,
            "entailment": 0.725,
            "exchange_rate": 1.0,
            "fear": 0.9259259259259259,
            "information": 0.9,
            "joy": 0.7142857142857143,
            "lost_or_stolen_card": 1.0,
            "love": 0.6,
            "neutral": 0.325,
            "order_physical_card": 0.875,
            "other": null,
            "pin_blocked": 0.875,
            "refund": 1.0,
            "refund_not_showing_up": 0.875,
            "request_refund": 0.875,
            "sadness": 0.8275862068965517,
            "scitech": 0.55,
            "sports": 0.975,
            "surprise": 0.47368421052631576,
            "technical_help": 1.0,
            "terminate_account": 1.0,
            "top_up_failed": 0.875,
            "topping_up_by_card": 0.5,
            "transaction_charged_twice": 0.875,
            "world": 0.875
          },
          "brier": 0.3270658633907902,
          "nll": 0.6528267504422001,
          "ece_raw": 0.0007309538327833588,
          "ece_fitted": 0.0004591988587776608
        },
        "noul": {
          "accuracy": 0.9036458333333334,
          "balanced_accuracy": 0.8834257029277777,
          "auroc": 0.972521254678931,
          "auprc": 0.9503028848436744,
          "brier": 0.07737625808758589,
          "nll": 0.2542762084776515,
          "ece_raw": 0.004463339661746162,
          "ece_fitted": 0.0049155860343304296
        },
        "score": {
          "mae": 0.8020032497743765,
          "rmse": 0.9909589189089723,
          "nearest_level_accuracy": 0.3333333333333333,
          "within_one_accuracy": 0.875,
          "raw_equality_rate": 0.0,
          "spearman": 0.737594189458718,
          "ordinal_confusion": {
            "labels": [
              0,
              1,
              2,
              3,
              4
            ],
            "matrix": [
              [
                0,
                19,
                4,
                1,
                0
              ],
              [
                2,
                10,
                12,
                0,
                0
              ],
              [
                0,
                6,
                17,
                1,
                0
              ],
              [
                0,
                1,
                11,
                12,
                0
              ],
              [
                0,
                1,
                8,
                14,
                1
              ]
            ]
          }
        }
      }
    },
    "calibrated": {
      "evaluation_cases": 952,
      "evaluation_decisions": 1240,
      "correct": 953,
      "accuracy": 0.7685483870967742,
      "suites": {
        "triage": {
          "accuracy": 0.9270833333333334,
          "cases": 48,
          "decisions": 192,
          "latency": {
            "mean_seconds": 0.03883962416667922,
            "p50_seconds": 0.03841860600016389,
            "p95_seconds": 0.04178968690000602
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.8541666666666666,
              "macro_f1": 0.820875420875421,
              "balanced_accuracy": 0.8444444444444444,
              "per_class_recall": {
                "billing_question": 0.2222222222222222,
                "cancellation": 1.0,
                "information": 1.0,
                "refund": 1.0,
                "technical_help": 1.0
              },
              "brier": 0.14243087345916075,
              "nll": 0.2813106462197595,
              "ece_raw": 0.028952845660412414,
              "ece_fitted": 0.03991474162123874
            },
            "noul": {
              "accuracy": 0.9513888888888888,
              "balanced_accuracy": 0.912037037037037,
              "auroc": 0.9886831275720165,
              "auprc": 0.9700223660164052,
              "brier": 0.052313952030937215,
              "nll": 0.20641514576135578,
              "ece_raw": 0.010566808507195363,
              "ece_fitted": 0.012443507589047501
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 178
        },
        "guardrails": {
          "accuracy": 0.8125,
          "cases": 48,
          "decisions": 96,
          "latency": {
            "mean_seconds": 0.037130780229148286,
            "p50_seconds": 0.03704196349997346,
            "p95_seconds": 0.03867655395000611
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": null,
              "macro_f1": null,
              "balanced_accuracy": null,
              "per_class_recall": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "noul": {
              "accuracy": 0.8125,
              "balanced_accuracy": 0.8676470588235294,
              "auroc": 0.9803046218487395,
              "auprc": 0.9910196723104415,
              "brier": 0.14399152528197112,
              "nll": 0.4301690068230142,
              "ece_raw": 0.027843129969293463,
              "ece_fitted": 0.02698340407756053
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 78
        },
        "moderation": {
          "accuracy": 0.9166666666666666,
          "cases": 48,
          "decisions": 144,
          "latency": {
            "mean_seconds": 0.03827599806249301,
            "p50_seconds": 0.037934129999939614,
            "p95_seconds": 0.041699429449965915
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": null,
              "macro_f1": null,
              "balanced_accuracy": null,
              "per_class_recall": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "noul": {
              "accuracy": 0.9166666666666666,
              "balanced_accuracy": 0.9106227106227107,
              "auroc": 0.978021978021978,
              "auprc": 0.944990138816365,
              "brier": 0.06164195701639249,
              "nll": 0.21176174396122044,
              "ece_raw": 0.014191582921902587,
              "ece_fitted": 0.01169790659953756
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 132
        },
        "agnews": {
          "accuracy": 0.8125,
          "cases": 160,
          "decisions": 160,
          "latency": {
            "mean_seconds": 0.038758980293738435,
            "p50_seconds": 0.0382035000001224,
            "p95_seconds": 0.042714876549950986
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.8125,
              "macro_f1": 0.8064558008777994,
              "balanced_accuracy": 0.8125,
              "per_class_recall": {
                "business": 0.85,
                "scitech": 0.55,
                "sports": 0.975,
                "world": 0.875
              },
              "brier": 0.2965232296874367,
              "nll": 0.5908707062914559,
              "ece_raw": 0.004136447508252946,
              "ece_fitted": 0.007738817845546403
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 130
        },
        "emotion": {
          "accuracy": 0.7239583333333334,
          "cases": 192,
          "decisions": 192,
          "latency": {
            "mean_seconds": 0.038396373302075425,
            "p50_seconds": 0.03797906949989738,
            "p95_seconds": 0.0415361116498957
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.7239583333333334,
              "macro_f1": 0.6913006811212603,
              "balanced_accuracy": 0.6854851048438465,
              "per_class_recall": {
                "anger": 0.5714285714285714,
                "fear": 0.9259259259259259,
                "joy": 0.7142857142857143,
                "love": 0.6,
                "sadness": 0.8275862068965517,
                "surprise": 0.47368421052631576
              },
              "brier": 0.39805364003393406,
              "nll": 0.7908805977682863,
              "ece_raw": 0.004164112978894893,
              "ece_fitted": 0.0034169522756447193
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 139
        },
        "banking77_12": {
          "accuracy": 0.8020833333333334,
          "cases": 96,
          "decisions": 96,
          "latency": {
            "mean_seconds": 0.03918704791666509,
            "p50_seconds": 0.038639419499986616,
            "p95_seconds": 0.04271133699995744
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.8020833333333334,
              "macro_f1": 0.7940683880389763,
              "balanced_accuracy": 0.8020833333333334,
              "per_class_recall": {
                "card_arrival": 0.625,
                "card_linking": 0.25,
                "exchange_rate": 1.0,
                "lost_or_stolen_card": 1.0,
                "order_physical_card": 0.875,
                "pin_blocked": 0.875,
                "refund_not_showing_up": 0.875,
                "request_refund": 0.875,
                "terminate_account": 1.0,
                "top_up_failed": 0.875,
                "topping_up_by_card": 0.5,
                "transaction_charged_twice": 0.875
              },
              "brier": 0.2841311372982186,
              "nll": 0.7256759563838567,
              "ece_raw": 0.014651013888121192,
              "ece_fitted": 0.0165529263424545
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 77
        },
        "mnli": {
          "accuracy": 0.5416666666666666,
          "cases": 120,
          "decisions": 120,
          "latency": {
            "mean_seconds": 0.0384311384083181,
            "p50_seconds": 0.03797633499982567,
            "p95_seconds": 0.04150163254993231
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.5416666666666666,
              "macro_f1": 0.531705418323941,
              "balanced_accuracy": 0.5416666666666666,
              "per_class_recall": {
                "contradiction": 0.575,
                "entailment": 0.725,
                "neutral": 0.325
              },
              "brier": 0.5534738375592544,
              "nll": 0.9317788432632266,
              "ece_raw": 0.005991485962656614,
              "ece_fitted": 0.008360924461080638
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 65
        },
        "sst5": {
          "accuracy": 0.3333333333333333,
          "cases": 120,
          "decisions": 120,
          "latency": {
            "mean_seconds": 0.038905105700003634,
            "p50_seconds": 0.03851513350002733,
            "p95_seconds": 0.041932194849789536
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": null,
              "macro_f1": null,
              "balanced_accuracy": null,
              "per_class_recall": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": 0.8023576299349467,
              "rmse": 0.9908613116687078,
              "nearest_level_accuracy": 0.3333333333333333,
              "within_one_accuracy": 0.875,
              "raw_equality_rate": 0.0,
              "spearman": 0.7382746269028682,
              "ordinal_confusion": {
                "labels": [
                  0,
                  1,
                  2,
                  3,
                  4
                ],
                "matrix": [
                  [
                    0,
                    19,
                    4,
                    1,
                    0
                  ],
                  [
                    2,
                    10,
                    12,
                    0,
                    0
                  ],
                  [
                    0,
                    6,
                    17,
                    1,
                    0
                  ],
                  [
                    0,
                    1,
                    11,
                    12,
                    0
                  ],
                  [
                    0,
                    1,
                    8,
                    14,
                    1
                  ]
                ]
              }
            }
          },
          "correct": 40
        },
        "multilingual_intent": {
          "accuracy": 0.95,
          "cases": 120,
          "decisions": 120,
          "latency": {
            "mean_seconds": 0.039158464716655315,
            "p50_seconds": 0.03864724799996111,
            "p95_seconds": 0.04240460730002269
          },
          "primitive_metrics": {
            "choice": {
              "accuracy": 0.95,
              "macro_f1": 0.8159019343229869,
              "balanced_accuracy": 0.9650000000000001,
              "per_class_recall": {
                "billing_question": 0.95,
                "cancellation": 1.0,
                "information": 0.875,
                "other": null,
                "refund": 1.0,
                "technical_help": 1.0
              },
              "brier": 0.13506004724959844,
              "nll": 0.2766243432861846,
              "ece_raw": 0.01098334737043859,
              "ece_fitted": 0.007150088729354088
            },
            "noul": {
              "accuracy": null,
              "balanced_accuracy": null,
              "auroc": null,
              "auprc": null,
              "brier": null,
              "nll": null,
              "ece_raw": null,
              "ece_fitted": null
            },
            "score": {
              "mae": null,
              "rmse": null,
              "nearest_level_accuracy": null,
              "within_one_accuracy": null,
              "raw_equality_rate": null,
              "spearman": null,
              "ordinal_confusion": null
            }
          },
          "correct": 114
        }
      },
      "primitive_metrics": {
        "choice": {
          "accuracy": 0.7690217391304348,
          "macro_f1": 0.748753245465977,
          "balanced_accuracy": 0.7745682853365854,
          "per_class_recall": {
            "anger": 0.5714285714285714,
            "billing_question": 0.7241379310344828,
            "business": 0.85,
            "cancellation": 1.0,
            "card_arrival": 0.625,
            "card_linking": 0.25,
            "contradiction": 0.575,
            "entailment": 0.725,
            "exchange_rate": 1.0,
            "fear": 0.9259259259259259,
            "information": 0.9,
            "joy": 0.7142857142857143,
            "lost_or_stolen_card": 1.0,
            "love": 0.6,
            "neutral": 0.325,
            "order_physical_card": 0.875,
            "other": null,
            "pin_blocked": 0.875,
            "refund": 1.0,
            "refund_not_showing_up": 0.875,
            "request_refund": 0.875,
            "sadness": 0.8275862068965517,
            "scitech": 0.55,
            "sports": 0.975,
            "surprise": 0.47368421052631576,
            "technical_help": 1.0,
            "terminate_account": 1.0,
            "top_up_failed": 0.875,
            "topping_up_by_card": 0.5,
            "transaction_charged_twice": 0.875,
            "world": 0.875
          },
          "brier": 0.32691216429379927,
          "nll": 0.6447888220047173,
          "ece_raw": 0.000757314816236097,
          "ece_fitted": 0.00045919887827611055
        },
        "noul": {
          "accuracy": 0.9036458333333334,
          "balanced_accuracy": 0.8834257029277777,
          "auroc": 0.972492238052404,
          "auprc": 0.950340531508563,
          "brier": 0.07873134721324142,
          "nll": 0.26435858535171963,
          "ece_raw": 0.004409246207057182,
          "ece_fitted": 0.004915585951212646
        },
        "score": {
          "mae": 0.8023576299349467,
          "rmse": 0.9908613116687078,
          "nearest_level_accuracy": 0.3333333333333333,
          "within_one_accuracy": 0.875,
          "raw_equality_rate": 0.0,
          "spearman": 0.7382746269028682,
          "ordinal_confusion": {
            "labels": [
              0,
              1,
              2,
              3,
              4
            ],
            "matrix": [
              [
                0,
                19,
                4,
                1,
                0
              ],
              [
                2,
                10,
                12,
                0,
                0
              ],
              [
                0,
                6,
                17,
                1,
                0
              ],
              [
                0,
                1,
                11,
                12,
                0
              ],
              [
                0,
                1,
                8,
                14,
                1
              ]
            ]
          }
        }
      }
    },
    "flagged_phrase_excluded": {
      "excluded_cases": [
        "banking77_12-0084"
      ],
      "raw": {
        "evaluation_cases": 951,
        "evaluation_decisions": 1239,
        "correct": 952,
        "accuracy": 0.768361581920904,
        "suites": {
          "triage": {
            "accuracy": 0.9270833333333334,
            "cases": 48,
            "decisions": 192,
            "latency": {
              "mean_seconds": 0.03883962416667922,
              "p50_seconds": 0.03841860600016389,
              "p95_seconds": 0.04178968690000602
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.8541666666666666,
                "macro_f1": 0.820875420875421,
                "balanced_accuracy": 0.8444444444444444,
                "per_class_recall": {
                  "billing_question": 0.2222222222222222,
                  "cancellation": 1.0,
                  "information": 1.0,
                  "refund": 1.0,
                  "technical_help": 1.0
                },
                "brier": 0.13697967505778072,
                "nll": 0.2506294304217651,
                "ece_raw": 0.033931114278253044,
                "ece_fitted": 0.039914814482762906
              },
              "noul": {
                "accuracy": 0.9513888888888888,
                "balanced_accuracy": 0.912037037037037,
                "auroc": 0.9890689300411523,
                "auprc": 0.9701626578233637,
                "brier": 0.04911263462874029,
                "nll": 0.1906247610132196,
                "ece_raw": 0.01181868597485985,
                "ece_fitted": 0.012443502260767868
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 178
          },
          "guardrails": {
            "accuracy": 0.8125,
            "cases": 48,
            "decisions": 96,
            "latency": {
              "mean_seconds": 0.037130780229148286,
              "p50_seconds": 0.03704196349997346,
              "p95_seconds": 0.03867655395000611
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": null,
                "macro_f1": null,
                "balanced_accuracy": null,
                "per_class_recall": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "noul": {
                "accuracy": 0.8125,
                "balanced_accuracy": 0.8676470588235294,
                "auroc": 0.9803046218487395,
                "auprc": 0.9910730108125466,
                "brier": 0.14491041421218856,
                "nll": 0.4276831543302518,
                "ece_raw": 0.02432198575945292,
                "ece_fitted": 0.026983405418465858
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 78
          },
          "moderation": {
            "accuracy": 0.9166666666666666,
            "cases": 48,
            "decisions": 144,
            "latency": {
              "mean_seconds": 0.03827599806249301,
              "p50_seconds": 0.037934129999939614,
              "p95_seconds": 0.041699429449965915
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": null,
                "macro_f1": null,
                "balanced_accuracy": null,
                "per_class_recall": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "noul": {
                "accuracy": 0.9166666666666666,
                "balanced_accuracy": 0.9106227106227107,
                "auroc": 0.978021978021978,
                "auprc": 0.944990138816365,
                "brier": 0.06061711079669634,
                "nll": 0.2023230253736832,
                "ece_raw": 0.010916557094938821,
                "ece_fitted": 0.011697905217155507
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 132
          },
          "agnews": {
            "accuracy": 0.8125,
            "cases": 160,
            "decisions": 160,
            "latency": {
              "mean_seconds": 0.038758980293738435,
              "p50_seconds": 0.0382035000001224,
              "p95_seconds": 0.042714876549950986
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.8125,
                "macro_f1": 0.8064558008777994,
                "balanced_accuracy": 0.8125,
                "per_class_recall": {
                  "business": 0.85,
                  "scitech": 0.55,
                  "sports": 0.975,
                  "world": 0.875
                },
                "brier": 0.30172771335742954,
                "nll": 0.6319382379388891,
                "ece_raw": 0.0052000811604517765,
                "ece_fitted": 0.007738817523861061
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 130
          },
          "emotion": {
            "accuracy": 0.7239583333333334,
            "cases": 192,
            "decisions": 192,
            "latency": {
              "mean_seconds": 0.038396373302075425,
              "p50_seconds": 0.03797906949989738,
              "p95_seconds": 0.0415361116498957
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.7239583333333334,
                "macro_f1": 0.6913006811212603,
                "balanced_accuracy": 0.6854851048438465,
                "per_class_recall": {
                  "anger": 0.5714285714285714,
                  "fear": 0.9259259259259259,
                  "joy": 0.7142857142857143,
                  "love": 0.6,
                  "sadness": 0.8275862068965517,
                  "surprise": 0.47368421052631576
                },
                "brier": 0.39443242180284094,
                "nll": 0.7806033322263032,
                "ece_raw": 0.0032102610291140795,
                "ece_fitted": 0.003416953263655513
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 139
          },
          "banking77_12": {
            "accuracy": 0.8,
            "cases": 95,
            "decisions": 95,
            "latency": {
              "mean_seconds": 0.03918948851578773,
              "p50_seconds": 0.03858194800000092,
              "p95_seconds": 0.04271281379994889
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.8,
                "macro_f1": 0.7934147932677345,
                "balanced_accuracy": 0.8020833333333334,
                "per_class_recall": {
                  "card_arrival": 0.625,
                  "card_linking": 0.25,
                  "exchange_rate": 1.0,
                  "lost_or_stolen_card": 1.0,
                  "order_physical_card": 0.875,
                  "pin_blocked": 0.875,
                  "refund_not_showing_up": 0.875,
                  "request_refund": 0.875,
                  "terminate_account": 1.0,
                  "top_up_failed": 0.875,
                  "topping_up_by_card": 0.5,
                  "transaction_charged_twice": 0.875
                },
                "brier": 0.29490650156145987,
                "nll": 0.782323559878833,
                "ece_raw": 0.01680271757873975,
                "ece_fitted": 0.016734122969269134
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 76
          },
          "mnli": {
            "accuracy": 0.5416666666666666,
            "cases": 120,
            "decisions": 120,
            "latency": {
              "mean_seconds": 0.0384311384083181,
              "p50_seconds": 0.03797633499982567,
              "p95_seconds": 0.04150163254993231
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.5416666666666666,
                "macro_f1": 0.531705418323941,
                "balanced_accuracy": 0.5416666666666666,
                "per_class_recall": {
                  "contradiction": 0.575,
                  "entailment": 0.725,
                  "neutral": 0.325
                },
                "brier": 0.5619903249840926,
                "nll": 0.952791383586364,
                "ece_raw": 0.0067628751799769776,
                "ece_fitted": 0.00836092388336822
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 65
          },
          "sst5": {
            "accuracy": 0.3333333333333333,
            "cases": 120,
            "decisions": 120,
            "latency": {
              "mean_seconds": 0.038905105700003634,
              "p50_seconds": 0.03851513350002733,
              "p95_seconds": 0.041932194849789536
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": null,
                "macro_f1": null,
                "balanced_accuracy": null,
                "per_class_recall": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": 0.8020032497743765,
                "rmse": 0.9909589189089723,
                "nearest_level_accuracy": 0.3333333333333333,
                "within_one_accuracy": 0.875,
                "raw_equality_rate": 0.0,
                "spearman": 0.737594189458718,
                "ordinal_confusion": {
                  "labels": [
                    0,
                    1,
                    2,
                    3,
                    4
                  ],
                  "matrix": [
                    [
                      0,
                      19,
                      4,
                      1,
                      0
                    ],
                    [
                      2,
                      10,
                      12,
                      0,
                      0
                    ],
                    [
                      0,
                      6,
                      17,
                      1,
                      0
                    ],
                    [
                      0,
                      1,
                      11,
                      12,
                      0
                    ],
                    [
                      0,
                      1,
                      8,
                      14,
                      1
                    ]
                  ]
                }
              }
            },
            "correct": 40
          },
          "multilingual_intent": {
            "accuracy": 0.95,
            "cases": 120,
            "decisions": 120,
            "latency": {
              "mean_seconds": 0.039158464716655315,
              "p50_seconds": 0.03864724799996111,
              "p95_seconds": 0.04240460730002269
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.95,
                "macro_f1": 0.8159019343229869,
                "balanced_accuracy": 0.9650000000000001,
                "per_class_recall": {
                  "billing_question": 0.95,
                  "cancellation": 1.0,
                  "information": 0.875,
                  "other": null,
                  "refund": 1.0,
                  "technical_help": 1.0
                },
                "brier": 0.12235859002542863,
                "nll": 0.24005612544914204,
                "ece_raw": 0.008839931130203269,
                "ece_fitted": 0.0071500889173673875
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 114
          }
        },
        "primitive_metrics": {
          "choice": {
            "accuracy": 0.7687074829931972,
            "macro_f1": 0.7485002410383995,
            "balanced_accuracy": 0.7745682853365854,
            "per_class_recall": {
              "anger": 0.5714285714285714,
              "billing_question": 0.7241379310344828,
              "business": 0.85,
              "cancellation": 1.0,
              "card_arrival": 0.625,
              "card_linking": 0.25,
              "contradiction": 0.575,
              "entailment": 0.725,
              "exchange_rate": 1.0,
              "fear": 0.9259259259259259,
              "information": 0.9,
              "joy": 0.7142857142857143,
              "lost_or_stolen_card": 1.0,
              "love": 0.6,
              "neutral": 0.325,
              "order_physical_card": 0.875,
              "other": null,
              "pin_blocked": 0.875,
              "refund": 1.0,
              "refund_not_showing_up": 0.875,
              "request_refund": 0.875,
              "sadness": 0.8275862068965517,
              "scitech": 0.55,
              "sports": 0.975,
              "surprise": 0.47368421052631576,
              "technical_help": 1.0,
              "terminate_account": 1.0,
              "top_up_failed": 0.875,
              "topping_up_by_card": 0.5,
              "transaction_charged_twice": 0.875,
              "world": 0.875
            },
            "brier": 0.3275108448647468,
            "nll": 0.6537123942730164,
            "ece_raw": 0.000732155258879059,
            "ece_fitted": 0.00045994712310874536
          },
          "noul": {
            "accuracy": 0.9036458333333334,
            "balanced_accuracy": 0.8834257029277777,
            "auroc": 0.972521254678931,
            "auprc": 0.9503028848436744,
            "brier": 0.07737625808758589,
            "nll": 0.2542762084776515,
            "ece_raw": 0.004463339661746162,
            "ece_fitted": 0.0049155860343304296
          },
          "score": {
            "mae": 0.8020032497743765,
            "rmse": 0.9909589189089723,
            "nearest_level_accuracy": 0.3333333333333333,
            "within_one_accuracy": 0.875,
            "raw_equality_rate": 0.0,
            "spearman": 0.737594189458718,
            "ordinal_confusion": {
              "labels": [
                0,
                1,
                2,
                3,
                4
              ],
              "matrix": [
                [
                  0,
                  19,
                  4,
                  1,
                  0
                ],
                [
                  2,
                  10,
                  12,
                  0,
                  0
                ],
                [
                  0,
                  6,
                  17,
                  1,
                  0
                ],
                [
                  0,
                  1,
                  11,
                  12,
                  0
                ],
                [
                  0,
                  1,
                  8,
                  14,
                  1
                ]
              ]
            }
          }
        }
      },
      "calibrated": {
        "evaluation_cases": 951,
        "evaluation_decisions": 1239,
        "correct": 952,
        "accuracy": 0.768361581920904,
        "suites": {
          "triage": {
            "accuracy": 0.9270833333333334,
            "cases": 48,
            "decisions": 192,
            "latency": {
              "mean_seconds": 0.03883962416667922,
              "p50_seconds": 0.03841860600016389,
              "p95_seconds": 0.04178968690000602
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.8541666666666666,
                "macro_f1": 0.820875420875421,
                "balanced_accuracy": 0.8444444444444444,
                "per_class_recall": {
                  "billing_question": 0.2222222222222222,
                  "cancellation": 1.0,
                  "information": 1.0,
                  "refund": 1.0,
                  "technical_help": 1.0
                },
                "brier": 0.14243087345916075,
                "nll": 0.2813106462197595,
                "ece_raw": 0.028952845660412414,
                "ece_fitted": 0.03991474162123874
              },
              "noul": {
                "accuracy": 0.9513888888888888,
                "balanced_accuracy": 0.912037037037037,
                "auroc": 0.9886831275720165,
                "auprc": 0.9700223660164052,
                "brier": 0.052313952030937215,
                "nll": 0.20641514576135578,
                "ece_raw": 0.010566808507195363,
                "ece_fitted": 0.012443507589047501
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 178
          },
          "guardrails": {
            "accuracy": 0.8125,
            "cases": 48,
            "decisions": 96,
            "latency": {
              "mean_seconds": 0.037130780229148286,
              "p50_seconds": 0.03704196349997346,
              "p95_seconds": 0.03867655395000611
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": null,
                "macro_f1": null,
                "balanced_accuracy": null,
                "per_class_recall": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "noul": {
                "accuracy": 0.8125,
                "balanced_accuracy": 0.8676470588235294,
                "auroc": 0.9803046218487395,
                "auprc": 0.9910196723104415,
                "brier": 0.14399152528197112,
                "nll": 0.4301690068230142,
                "ece_raw": 0.027843129969293463,
                "ece_fitted": 0.02698340407756053
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 78
          },
          "moderation": {
            "accuracy": 0.9166666666666666,
            "cases": 48,
            "decisions": 144,
            "latency": {
              "mean_seconds": 0.03827599806249301,
              "p50_seconds": 0.037934129999939614,
              "p95_seconds": 0.041699429449965915
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": null,
                "macro_f1": null,
                "balanced_accuracy": null,
                "per_class_recall": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "noul": {
                "accuracy": 0.9166666666666666,
                "balanced_accuracy": 0.9106227106227107,
                "auroc": 0.978021978021978,
                "auprc": 0.944990138816365,
                "brier": 0.06164195701639249,
                "nll": 0.21176174396122044,
                "ece_raw": 0.014191582921902587,
                "ece_fitted": 0.01169790659953756
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 132
          },
          "agnews": {
            "accuracy": 0.8125,
            "cases": 160,
            "decisions": 160,
            "latency": {
              "mean_seconds": 0.038758980293738435,
              "p50_seconds": 0.0382035000001224,
              "p95_seconds": 0.042714876549950986
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.8125,
                "macro_f1": 0.8064558008777994,
                "balanced_accuracy": 0.8125,
                "per_class_recall": {
                  "business": 0.85,
                  "scitech": 0.55,
                  "sports": 0.975,
                  "world": 0.875
                },
                "brier": 0.2965232296874367,
                "nll": 0.5908707062914559,
                "ece_raw": 0.004136447508252946,
                "ece_fitted": 0.007738817845546403
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 130
          },
          "emotion": {
            "accuracy": 0.7239583333333334,
            "cases": 192,
            "decisions": 192,
            "latency": {
              "mean_seconds": 0.038396373302075425,
              "p50_seconds": 0.03797906949989738,
              "p95_seconds": 0.0415361116498957
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.7239583333333334,
                "macro_f1": 0.6913006811212603,
                "balanced_accuracy": 0.6854851048438465,
                "per_class_recall": {
                  "anger": 0.5714285714285714,
                  "fear": 0.9259259259259259,
                  "joy": 0.7142857142857143,
                  "love": 0.6,
                  "sadness": 0.8275862068965517,
                  "surprise": 0.47368421052631576
                },
                "brier": 0.39805364003393406,
                "nll": 0.7908805977682863,
                "ece_raw": 0.004164112978894893,
                "ece_fitted": 0.0034169522756447193
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 139
          },
          "banking77_12": {
            "accuracy": 0.8,
            "cases": 95,
            "decisions": 95,
            "latency": {
              "mean_seconds": 0.03918948851578773,
              "p50_seconds": 0.03858194800000092,
              "p95_seconds": 0.04271281379994889
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.8,
                "macro_f1": 0.7934147932677345,
                "balanced_accuracy": 0.8020833333333334,
                "per_class_recall": {
                  "card_arrival": 0.625,
                  "card_linking": 0.25,
                  "exchange_rate": 1.0,
                  "lost_or_stolen_card": 1.0,
                  "order_physical_card": 0.875,
                  "pin_blocked": 0.875,
                  "refund_not_showing_up": 0.875,
                  "request_refund": 0.875,
                  "terminate_account": 1.0,
                  "top_up_failed": 0.875,
                  "topping_up_by_card": 0.5,
                  "transaction_charged_twice": 0.875
                },
                "brier": 0.2871215913829621,
                "nll": 0.7332560464140869,
                "ece_raw": 0.014809638592014682,
                "ece_fitted": 0.01673412400480368
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 76
          },
          "mnli": {
            "accuracy": 0.5416666666666666,
            "cases": 120,
            "decisions": 120,
            "latency": {
              "mean_seconds": 0.0384311384083181,
              "p50_seconds": 0.03797633499982567,
              "p95_seconds": 0.04150163254993231
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.5416666666666666,
                "macro_f1": 0.531705418323941,
                "balanced_accuracy": 0.5416666666666666,
                "per_class_recall": {
                  "contradiction": 0.575,
                  "entailment": 0.725,
                  "neutral": 0.325
                },
                "brier": 0.5534738375592544,
                "nll": 0.9317788432632266,
                "ece_raw": 0.005991485962656614,
                "ece_fitted": 0.008360924461080638
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 65
          },
          "sst5": {
            "accuracy": 0.3333333333333333,
            "cases": 120,
            "decisions": 120,
            "latency": {
              "mean_seconds": 0.038905105700003634,
              "p50_seconds": 0.03851513350002733,
              "p95_seconds": 0.041932194849789536
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": null,
                "macro_f1": null,
                "balanced_accuracy": null,
                "per_class_recall": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": 0.8023576299349467,
                "rmse": 0.9908613116687078,
                "nearest_level_accuracy": 0.3333333333333333,
                "within_one_accuracy": 0.875,
                "raw_equality_rate": 0.0,
                "spearman": 0.7382746269028682,
                "ordinal_confusion": {
                  "labels": [
                    0,
                    1,
                    2,
                    3,
                    4
                  ],
                  "matrix": [
                    [
                      0,
                      19,
                      4,
                      1,
                      0
                    ],
                    [
                      2,
                      10,
                      12,
                      0,
                      0
                    ],
                    [
                      0,
                      6,
                      17,
                      1,
                      0
                    ],
                    [
                      0,
                      1,
                      11,
                      12,
                      0
                    ],
                    [
                      0,
                      1,
                      8,
                      14,
                      1
                    ]
                  ]
                }
              }
            },
            "correct": 40
          },
          "multilingual_intent": {
            "accuracy": 0.95,
            "cases": 120,
            "decisions": 120,
            "latency": {
              "mean_seconds": 0.039158464716655315,
              "p50_seconds": 0.03864724799996111,
              "p95_seconds": 0.04240460730002269
            },
            "primitive_metrics": {
              "choice": {
                "accuracy": 0.95,
                "macro_f1": 0.8159019343229869,
                "balanced_accuracy": 0.9650000000000001,
                "per_class_recall": {
                  "billing_question": 0.95,
                  "cancellation": 1.0,
                  "information": 0.875,
                  "other": null,
                  "refund": 1.0,
                  "technical_help": 1.0
                },
                "brier": 0.13506004724959844,
                "nll": 0.2766243432861846,
                "ece_raw": 0.01098334737043859,
                "ece_fitted": 0.007150088729354088
              },
              "noul": {
                "accuracy": null,
                "balanced_accuracy": null,
                "auroc": null,
                "auprc": null,
                "brier": null,
                "nll": null,
                "ece_raw": null,
                "ece_fitted": null
              },
              "score": {
                "mae": null,
                "rmse": null,
                "nearest_level_accuracy": null,
                "within_one_accuracy": null,
                "raw_equality_rate": null,
                "spearman": null,
                "ordinal_confusion": null
              }
            },
            "correct": 114
          }
        },
        "primitive_metrics": {
          "choice": {
            "accuracy": 0.7687074829931972,
            "macro_f1": 0.7485002410383995,
            "balanced_accuracy": 0.7745682853365854,
            "per_class_recall": {
              "anger": 0.5714285714285714,
              "billing_question": 0.7241379310344828,
              "business": 0.85,
              "cancellation": 1.0,
              "card_arrival": 0.625,
              "card_linking": 0.25,
              "contradiction": 0.575,
              "entailment": 0.725,
              "exchange_rate": 1.0,
              "fear": 0.9259259259259259,
              "information": 0.9,
              "joy": 0.7142857142857143,
              "lost_or_stolen_card": 1.0,
              "love": 0.6,
              "neutral": 0.325,
              "order_physical_card": 0.875,
              "other": null,
              "pin_blocked": 0.875,
              "refund": 1.0,
              "refund_not_showing_up": 0.875,
              "request_refund": 0.875,
              "sadness": 0.8275862068965517,
              "scitech": 0.55,
              "sports": 0.975,
              "surprise": 0.47368421052631576,
              "technical_help": 1.0,
              "terminate_account": 1.0,
              "top_up_failed": 0.875,
              "topping_up_by_card": 0.5,
              "transaction_charged_twice": 0.875,
              "world": 0.875
            },
            "brier": 0.3273568910489642,
            "nll": 0.6456585110094694,
            "ece_raw": 0.0007584697829628141,
            "ece_fitted": 0.0004599471426753296
          },
          "noul": {
            "accuracy": 0.9036458333333334,
            "balanced_accuracy": 0.8834257029277777,
            "auroc": 0.972492238052404,
            "auprc": 0.950340531508563,
            "brier": 0.07873134721324142,
            "nll": 0.26435858535171963,
            "ece_raw": 0.004409246207057182,
            "ece_fitted": 0.004915585951212646
          },
          "score": {
            "mae": 0.8023576299349467,
            "rmse": 0.9908613116687078,
            "nearest_level_accuracy": 0.3333333333333333,
            "within_one_accuracy": 0.875,
            "raw_equality_rate": 0.0,
            "spearman": 0.7382746269028682,
            "ordinal_confusion": {
              "labels": [
                0,
                1,
                2,
                3,
                4
              ],
              "matrix": [
                [
                  0,
                  19,
                  4,
                  1,
                  0
                ],
                [
                  2,
                  10,
                  12,
                  0,
                  0
                ],
                [
                  0,
                  6,
                  17,
                  1,
                  0
                ],
                [
                  0,
                  1,
                  11,
                  12,
                  0
                ],
                [
                  0,
                  1,
                  8,
                  14,
                  1
                ]
              ]
            }
          }
        }
      }
    },
    "score_rule": "Expected level index passed as scalar score; official floor(score+0.5) correctness, not modal-level accuracy",
    "label_note": "Original v2 labels were single-review AI-assisted. Corrected v2.1.0 adopts source labels for AGNews, Banking77, MNLI and SST5 and a reviewed emotion decision set. No independent double-review claim is made.",
    "official_submission": false
  },
  "protocol_and_overlap_notes": "See README.md. The metrics have different denominators and definitions."
}