{
  "selected": "head_lora_rlcd",
  "seed_run": "artifacts/runs/e4b-v142",
  "method": "Proper-scoring REINFORCE with Gaussian logit exploration",
  "gates": {
    "seed": {
      "eligible": true,
      "score": 0.0
    },
    "head_rlcd": {
      "eligible": true,
      "preservation_gate": true,
      "probability_gate": true,
      "score": 0.013723702557236433,
      "mean_nll": 0.2912874264480955,
      "mean_brier": 0.17143807150110177,
      "mean_ece": 0.026519458902999775
    },
    "head_lora_rlcd": {
      "eligible": true,
      "preservation_gate": true,
      "probability_gate": true,
      "score": 0.02410640078446494,
      "mean_nll": 0.2851054899756657,
      "mean_brier": 0.1656363697259328,
      "mean_ece": 0.02280542785381634
    }
  },
  "results": {
    "seed": {
      "calibration": {
        "version": 2,
        "type": "primitive_option_temperature",
        "global_temperature": 1.097443699836731,
        "groups": {
          "choice:2": {
            "temperature": 0.7726892229265028,
            "raw_temperature": 0.4564978778362274,
            "n": 80,
            "shrinkage_weight": 0.4
          },
          "choice:3-5": {
            "temperature": 1.0735645605398847,
            "raw_temperature": 1.0707285404205322,
            "n": 998,
            "shrinkage_weight": 0.8926654740608229
          },
          "choice:6-10": {
            "temperature": 1.0213380776161125,
            "raw_temperature": 0.9631407260894775,
            "n": 147,
            "shrinkage_weight": 0.550561797752809
          },
          "noul:*": {
            "temperature": 1.18076753369684,
            "raw_temperature": 1.1976659297943115,
            "n": 618,
            "shrinkage_weight": 0.8373983739837398
          },
          "score:*": {
            "temperature": 1.1118863993359898,
            "raw_temperature": 1.1230534315109253,
            "n": 157,
            "shrinkage_weight": 0.5667870036101083
          }
        },
        "min_group": 50,
        "prior_strength": 120.0,
        "n": 2000
      },
      "sets": {
        "validation": {
          "n": 2000,
          "accuracy": 0.8635,
          "nll": 0.33747788503553516,
          "brier": 0.1907344097228753,
          "ece": 0.016734494030475625,
          "mean_confidence": 0.8650541846752167,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.097443699836731,
            "groups": {
              "choice:2": {
                "temperature": 0.7726892229265028,
                "raw_temperature": 0.4564978778362274,
                "n": 80,
                "shrinkage_weight": 0.4
              },
              "choice:3-5": {
                "temperature": 1.0735645605398847,
                "raw_temperature": 1.0707285404205322,
                "n": 998,
                "shrinkage_weight": 0.8926654740608229
              },
              "choice:6-10": {
                "temperature": 1.0213380776161125,
                "raw_temperature": 0.9631407260894775,
                "n": 147,
                "shrinkage_weight": 0.550561797752809
              },
              "noul:*": {
                "temperature": 1.18076753369684,
                "raw_temperature": 1.1976659297943115,
                "n": 618,
                "shrinkage_weight": 0.8373983739837398
              },
              "score:*": {
                "temperature": 1.1118863993359898,
                "raw_temperature": 1.1230534315109253,
                "n": 157,
                "shrinkage_weight": 0.5667870036101083
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "PolyAI/banking77": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18829089664733842,
              "brier": 0.10488018020936762,
              "ece": 0.05482878689380254,
              "mean_confidence": 0.9316614732584533
            },
            "Rowan/hellaswag": {
              "n": 153,
              "accuracy": 0.934640522875817,
              "nll": 0.20100579900196738,
              "brier": 0.11092448101729271,
              "ece": 0.050551106921987575,
              "mean_confidence": 0.9167824553118812
            },
            "alisawuffles/WANLI": {
              "n": 158,
              "accuracy": 0.7911392405063291,
              "nll": 0.4983649758763517,
              "brier": 0.2937829899183501,
              "ece": 0.06604516147812708,
              "mean_confidence": 0.7893789567147629
            },
            "cais/mmlu": {
              "n": 154,
              "accuracy": 0.8766233766233766,
              "nll": 0.4075193296471239,
              "brier": 0.19814945253592872,
              "ece": 0.07114563940407395,
              "mean_confidence": 0.8730270781300284
            },
            "google/boolq": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.3304804692638724,
              "brier": 0.19049125629416552,
              "ece": 0.04868968210968311,
              "mean_confidence": 0.8886790147014693
            },
            "google/boolq-negation": {
              "n": 153,
              "accuracy": 0.8627450980392157,
              "nll": 0.29803780987980516,
              "brier": 0.1722142930618895,
              "ece": 0.06692373012405595,
              "mean_confidence": 0.8571316631011714
            },
            "nvidia/HelpSteer2": {
              "n": 157,
              "accuracy": 0.535031847133758,
              "nll": 0.9802866589491535,
              "brier": 0.5608620777678718,
              "ece": 0.1083361166677657,
              "mean_confidence": 0.5841244707821281
            },
            "nvidia/HelpSteer2-threshold": {
              "n": 158,
              "accuracy": 0.879746835443038,
              "nll": 0.3300814368538087,
              "brier": 0.19343432728677312,
              "ece": 0.06685320712343043,
              "mean_confidence": 0.832719200019595
            },
            "nyu-mll/multi_nli": {
              "n": 157,
              "accuracy": 0.8662420382165605,
              "nll": 0.3357255889909567,
              "brier": 0.19144842998859854,
              "ece": 0.07937940025025873,
              "mean_confidence": 0.84814400467903
            },
            "product-science/xlam-function-calling-60k-raw": {
              "n": 154,
              "accuracy": 1.0,
              "nll": 0.002737168537177378,
              "brier": 0.0005628128288157573,
              "ece": 0.0025677526151979472,
              "mean_confidence": 0.9974322473848021
            },
            "product-science/xlam-hard": {
              "n": 154,
              "accuracy": 0.961038961038961,
              "nll": 0.105508735974156,
              "brier": 0.06104789821045359,
              "ece": 0.03465073526679697,
              "mean_confidence": 0.9694648733386746
            },
            "stanfordnlp/snli": {
              "n": 159,
              "accuracy": 0.8553459119496856,
              "nll": 0.3588936702368793,
              "brier": 0.2055985772370396,
              "ece": 0.053331098466549334,
              "mean_confidence": 0.8946507621861104
            },
            "synthetic/nli-haystack": {
              "n": 154,
              "accuracy": 0.8766233766233766,
              "nll": 0.31448460003064777,
              "brier": 0.1750220485890215,
              "ece": 0.058161640709096796,
              "mean_confidence": 0.8783779376512998
            }
          },
          "by_kind": {
            "choice": {
              "n": 1225,
              "accuracy": 0.8906122448979592,
              "nll": 0.2910091881993051,
              "brier": 0.16159634624408029,
              "ece": 0.018170039288851744,
              "mean_confidence": 0.8901425058257824
            },
            "noul": {
              "n": 618,
              "accuracy": 0.8932038834951457,
              "nll": 0.2662856133849587,
              "brier": 0.15446270078834373,
              "ece": 0.022636081794319064,
              "mean_confidence": 0.8866929737881164
            },
            "score": {
              "n": 157,
              "accuracy": 0.535031847133758,
              "nll": 0.9802866589491535,
              "brier": 0.5608620777678718,
              "ece": 0.1083361166677657,
              "mean_confidence": 0.5841244707821281
            }
          },
          "by_transform": {
            "agent.invocation.correct.v1": {
              "n": 59,
              "accuracy": 0.9322033898305084,
              "nll": 0.19396388833805667,
              "brier": 0.10943394663022984,
              "ece": 0.06162244788670942,
              "mean_confidence": 0.9474405292737282
            },
            "agent.invocation.corrupt_argument.v1": {
              "n": 47,
              "accuracy": 1.0,
              "nll": 0.007110317709528782,
              "brier": 0.00041410416924144496,
              "ece": 0.007002635205045669,
              "mean_confidence": 0.9929973647949544
            },
            "agent.invocation.wrong_tool.v1": {
              "n": 48,
              "accuracy": 0.9583333333333334,
              "nll": 0.09313106241097557,
              "brier": 0.06094397035983216,
              "ece": 0.048730698724587775,
              "mean_confidence": 0.9734942317008972
            },
            "commonsense.choice.v2": {
              "n": 153,
              "accuracy": 0.934640522875817,
              "nll": 0.20100579900196738,
              "brier": 0.11092448101729271,
              "ece": 0.050551106921987575,
              "mean_confidence": 0.9167824553118812
            },
            "intent.choice.v3": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18829089664733842,
              "brier": 0.10488018020936762,
              "ece": 0.05482878689380254,
              "mean_confidence": 0.9316614732584533
            },
            "knowledge.choice.v1": {
              "n": 154,
              "accuracy": 0.8766233766233766,
              "nll": 0.4075193296471239,
              "brier": 0.19814945253592872,
              "ece": 0.07114563940407395,
              "mean_confidence": 0.8730270781300284
            },
            "nli.choice.v4": {
              "n": 157,
              "accuracy": 0.8662420382165605,
              "nll": 0.3357255889909567,
              "brier": 0.19144842998859854,
              "ece": 0.07937940025025873,
              "mean_confidence": 0.84814400467903
            },
            "nli.hard.v4": {
              "n": 158,
              "accuracy": 0.7911392405063291,
              "nll": 0.4983649758763517,
              "brier": 0.2937829899183501,
              "ece": 0.06604516147812708,
              "mean_confidence": 0.7893789567147629
            },
            "nli.haystack.absent.v1": {
              "n": 9,
              "accuracy": 1.0,
              "nll": 0.05771497719817691,
              "brier": 0.012067315275392806,
              "ece": 0.05367900265587699,
              "mean_confidence": 0.946320997344123
            },
            "nli.haystack.present.v1": {
              "n": 145,
              "accuracy": 0.8689655172413793,
              "nll": 0.33042202489611144,
              "brier": 0.18513648031193639,
              "ece": 0.060484279846322944,
              "mean_confidence": 0.8741607822220901
            },
            "nli.snli.v1": {
              "n": 159,
              "accuracy": 0.8553459119496856,
              "nll": 0.3588936702368793,
              "brier": 0.2055985772370396,
              "ece": 0.053331098466549334,
              "mean_confidence": 0.8946507621861104
            },
            "noul.negated_answer.v1": {
              "n": 153,
              "accuracy": 0.8627450980392157,
              "nll": 0.29803780987980516,
              "brier": 0.1722142930618895,
              "ece": 0.06692373012405595,
              "mean_confidence": 0.8571316631011714
            },
            "noul.v3": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.3304804692638724,
              "brier": 0.19049125629416552,
              "ece": 0.04868968210968311,
              "mean_confidence": 0.8886790147014693
            },
            "score.coherence.threshold.v1": {
              "n": 35,
              "accuracy": 0.9428571428571428,
              "nll": 0.30217631511124116,
              "brier": 0.16118863898768074,
              "ece": 0.13178183862141202,
              "mean_confidence": 0.8401974422591073
            },
            "score.coherence.v3": {
              "n": 32,
              "accuracy": 0.5625,
              "nll": 0.9295596550218761,
              "brier": 0.5722244664793834,
              "ece": 0.20393101125955582,
              "mean_confidence": 0.694971589371562
            },
            "score.complexity.threshold.v1": {
              "n": 30,
              "accuracy": 0.9333333333333333,
              "nll": 0.27167421360500155,
              "brier": 0.14001424941464696,
              "ece": 0.082729438940684,
              "mean_confidence": 0.9072748144467672
            },
            "score.complexity.v3": {
              "n": 27,
              "accuracy": 0.5925925925925926,
              "nll": 0.7247205337992421,
              "brier": 0.4520837900539239,
              "ece": 0.1485392583741082,
              "mean_confidence": 0.6281530835010387
            },
            "score.correctness.threshold.v1": {
              "n": 30,
              "accuracy": 0.7333333333333333,
              "nll": 0.44957111608237027,
              "brier": 0.30260899337008595,
              "ece": 0.13217389583587646,
              "mean_confidence": 0.7230223735173543
            },
            "score.correctness.v3": {
              "n": 30,
              "accuracy": 0.4,
              "nll": 1.2207635447382927,
              "brier": 0.6627602388461431,
              "ece": 0.19082429458697636,
              "mean_confidence": 0.4697169209520022
            },
            "score.helpfulness.threshold.v1": {
              "n": 30,
              "accuracy": 0.8,
              "nll": 0.41629887875169513,
              "brier": 0.27174266742852826,
              "ece": 0.1563175916671753,
              "mean_confidence": 0.7734385410944621
            },
            "score.helpfulness.v3": {
              "n": 36,
              "accuracy": 0.5277777777777778,
              "nll": 1.0197049867775705,
              "brier": 0.5488159826232327,
              "ece": 0.2696710663537184,
              "mean_confidence": 0.44966551951236194
            },
            "score.verbosity.threshold.v1": {
              "n": 33,
              "accuracy": 0.9696969696969697,
              "nll": 0.2257687800253431,
              "brier": 0.10575891031343913,
              "ece": 0.07801289088798288,
              "mean_confidence": 0.9106257955233256
            },
            "score.verbosity.v3": {
              "n": 32,
              "accuracy": 0.59375,
              "nll": 0.9768548817373812,
              "brier": 0.5593037003418431,
              "ece": 0.16308739501982927,
              "mean_confidence": 0.6946516083553433
            },
            "tool.choice.single.v3": {
              "n": 154,
              "accuracy": 1.0,
              "nll": 0.002737168537177378,
              "brier": 0.0005628128288157573,
              "ece": 0.0025677526151979472,
              "mean_confidence": 0.9974322473848021
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 698,
              "accuracy": 0.9054441260744985,
              "nll": 0.23576630212333755,
              "brier": 0.13675923951146768,
              "ece": 0.020041158684345223,
              "mean_confidence": 0.8996789226559309
            },
            "3-5": {
              "n": 1155,
              "accuracy": 0.8285714285714286,
              "nll": 0.41972496986994157,
              "brier": 0.23527893114692047,
              "ece": 0.017945458233614507,
              "mean_confidence": 0.8350021731027794
            },
            "6-10": {
              "n": 147,
              "accuracy": 0.9387755102040817,
              "nll": 0.17420782985849112,
              "brier": 0.09703200538811604,
              "ece": 0.05073268012124669,
              "mean_confidence": 0.9367685129447859
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 1999,
              "accuracy": 0.863431715857929,
              "nll": 0.3375232138199196,
              "brier": 0.19078194724154862,
              "ece": 0.016852297354722528,
              "mean_confidence": 0.865096109905739
            },
            "truncated": {
              "n": 1,
              "accuracy": 1.0,
              "nll": 0.2468656450510025,
              "brier": 0.09570690989494324,
              "ece": 0.2187543511390686,
              "mean_confidence": 0.7812456488609314
            }
          },
          "by_dataset_truncation": {
            "PolyAI/banking77|not_truncated": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18829089664733842,
              "brier": 0.10488018020936762,
              "ece": 0.05482878689380254,
              "mean_confidence": 0.9316614732584533
            },
            "Rowan/hellaswag|not_truncated": {
              "n": 153,
              "accuracy": 0.934640522875817,
              "nll": 0.20100579900196738,
              "brier": 0.11092448101729271,
              "ece": 0.050551106921987575,
              "mean_confidence": 0.9167824553118812
            },
            "alisawuffles/WANLI|not_truncated": {
              "n": 158,
              "accuracy": 0.7911392405063291,
              "nll": 0.4983649758763517,
              "brier": 0.2937829899183501,
              "ece": 0.06604516147812708,
              "mean_confidence": 0.7893789567147629
            },
            "cais/mmlu|not_truncated": {
              "n": 154,
              "accuracy": 0.8766233766233766,
              "nll": 0.4075193296471239,
              "brier": 0.19814945253592872,
              "ece": 0.07114563940407395,
              "mean_confidence": 0.8730270781300284
            },
            "google/boolq-negation|not_truncated": {
              "n": 153,
              "accuracy": 0.8627450980392157,
              "nll": 0.29803780987980516,
              "brier": 0.1722142930618895,
              "ece": 0.06692373012405595,
              "mean_confidence": 0.8571316631011714
            },
            "google/boolq|not_truncated": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.3304804692638724,
              "brier": 0.19049125629416552,
              "ece": 0.04868968210968311,
              "mean_confidence": 0.8886790147014693
            },
            "nvidia/HelpSteer2-threshold|not_truncated": {
              "n": 157,
              "accuracy": 0.8789808917197452,
              "nll": 0.3306114737442724,
              "brier": 0.19405679491347266,
              "ece": 0.06588568391313974,
              "mean_confidence": 0.8330470570333445
            },
            "nvidia/HelpSteer2-threshold|truncated": {
              "n": 1,
              "accuracy": 1.0,
              "nll": 0.2468656450510025,
              "brier": 0.09570690989494324,
              "ece": 0.2187543511390686,
              "mean_confidence": 0.7812456488609314
            },
            "nvidia/HelpSteer2|not_truncated": {
              "n": 157,
              "accuracy": 0.535031847133758,
              "nll": 0.9802866589491535,
              "brier": 0.5608620777678718,
              "ece": 0.1083361166677657,
              "mean_confidence": 0.5841244707821281
            },
            "nyu-mll/multi_nli|not_truncated": {
              "n": 157,
              "accuracy": 0.8662420382165605,
              "nll": 0.3357255889909567,
              "brier": 0.19144842998859854,
              "ece": 0.07937940025025873,
              "mean_confidence": 0.84814400467903
            },
            "product-science/xlam-function-calling-60k-raw|not_truncated": {
              "n": 154,
              "accuracy": 1.0,
              "nll": 0.002737168537177378,
              "brier": 0.0005628128288157573,
              "ece": 0.0025677526151979472,
              "mean_confidence": 0.9974322473848021
            },
            "product-science/xlam-hard|not_truncated": {
              "n": 154,
              "accuracy": 0.961038961038961,
              "nll": 0.105508735974156,
              "brier": 0.06104789821045359,
              "ece": 0.03465073526679697,
              "mean_confidence": 0.9694648733386746
            },
            "stanfordnlp/snli|not_truncated": {
              "n": 159,
              "accuracy": 0.8553459119496856,
              "nll": 0.3588936702368793,
              "brier": 0.2055985772370396,
              "ece": 0.053331098466549334,
              "mean_confidence": 0.8946507621861104
            },
            "synthetic/nli-haystack|not_truncated": {
              "n": 154,
              "accuracy": 0.8766233766233766,
              "nll": 0.31448460003064777,
              "brier": 0.1750220485890215,
              "ece": 0.058161640709096796,
              "mean_confidence": 0.8783779376512998
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1225,
              "accuracy": 0.8906122448979592,
              "nll": 0.2910091881993051,
              "brier": 0.16159634624408029,
              "ece": 0.018170039288851744,
              "mean_confidence": 0.8901425058257824
            },
            "noul|not_truncated": {
              "n": 617,
              "accuracy": 0.893030794165316,
              "nll": 0.26631708821208017,
              "brier": 0.1545579289745567,
              "ece": 0.02302731426260657,
              "mean_confidence": 0.886863877070008
            },
            "noul|truncated": {
              "n": 1,
              "accuracy": 1.0,
              "nll": 0.2468656450510025,
              "brier": 0.09570690989494324,
              "ece": 0.2187543511390686,
              "mean_confidence": 0.7812456488609314
            },
            "score|not_truncated": {
              "n": 157,
              "accuracy": 0.535031847133758,
              "nll": 0.9802866589491535,
              "brier": 0.5608620777678718,
              "ece": 0.1083361166677657,
              "mean_confidence": 0.5841244707821281
            }
          },
          "nli_confusion": {
            "n": 628,
            "labels": [
              "entailment",
              "neutral",
              "contradiction"
            ],
            "matrix": {
              "entailment": {
                "entailment": 173,
                "neutral": 23,
                "contradiction": 3
              },
              "neutral": {
                "entailment": 21,
                "neutral": 203,
                "contradiction": 15
              },
              "contradiction": {
                "entailment": 2,
                "neutral": 32,
                "contradiction": 156
              }
            }
          }
        },
        "schema": {
          "n": 2000,
          "accuracy": 0.758,
          "nll": 0.5204962961420965,
          "brier": 0.32018979337543463,
          "ece": 0.07037245657294991,
          "mean_confidence": 0.8196629058793187,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.097443699836731,
            "groups": {
              "choice:2": {
                "temperature": 0.7726892229265028,
                "raw_temperature": 0.4564978778362274,
                "n": 80,
                "shrinkage_weight": 0.4
              },
              "choice:3-5": {
                "temperature": 1.0735645605398847,
                "raw_temperature": 1.0707285404205322,
                "n": 998,
                "shrinkage_weight": 0.8926654740608229
              },
              "choice:6-10": {
                "temperature": 1.0213380776161125,
                "raw_temperature": 0.9631407260894775,
                "n": 147,
                "shrinkage_weight": 0.550561797752809
              },
              "noul:*": {
                "temperature": 1.18076753369684,
                "raw_temperature": 1.1976659297943115,
                "n": 618,
                "shrinkage_weight": 0.8373983739837398
              },
              "score:*": {
                "temperature": 1.1118863993359898,
                "raw_temperature": 1.1230534315109253,
                "n": 157,
                "shrinkage_weight": 0.5667870036101083
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "schema-firewall/PolyAI/banking77": {
              "n": 46,
              "accuracy": 0.8043478260869565,
              "nll": 0.35612511238073435,
              "brier": 0.22549919875508087,
              "ece": 0.10526247128196384,
              "mean_confidence": 0.8808949822964875
            },
            "schema-firewall/Rowan/hellaswag": {
              "n": 130,
              "accuracy": 0.9,
              "nll": 0.20414897868758125,
              "brier": 0.11925340885983747,
              "ece": 0.05373616493665252,
              "mean_confidence": 0.9159407689021184
            },
            "schema-firewall/alisawuffles/WANLI": {
              "n": 209,
              "accuracy": 0.7655502392344498,
              "nll": 0.5446630402214148,
              "brier": 0.32441898222513327,
              "ece": 0.059890149027536935,
              "mean_confidence": 0.7769627089135385
            },
            "schema-firewall/cais/mmlu": {
              "n": 134,
              "accuracy": 0.917910447761194,
              "nll": 0.22831852445149217,
              "brier": 0.11953794011554492,
              "ece": 0.049322252620511986,
              "mean_confidence": 0.909431281596867
            },
            "schema-firewall/google/boolq": {
              "n": 36,
              "accuracy": 0.8888888888888888,
              "nll": 0.44322766041700057,
              "brier": 0.22839449674166368,
              "ece": 0.1561499618821674,
              "mean_confidence": 0.8758690738015704
            },
            "schema-firewall/google/boolq-negation": {
              "n": 60,
              "accuracy": 0.5166666666666667,
              "nll": 1.2234002001583577,
              "brier": 0.7434715281813321,
              "ece": 0.34772479931513467,
              "mean_confidence": 0.8540057341257731
            },
            "schema-firewall/nvidia/HelpSteer2": {
              "n": 270,
              "accuracy": 0.6296296296296297,
              "nll": 0.8983366615242429,
              "brier": 0.496752599419819,
              "ece": 0.07374763350795817,
              "mean_confidence": 0.5930154732531971
            },
            "schema-firewall/nvidia/HelpSteer2-threshold": {
              "n": 260,
              "accuracy": 0.5230769230769231,
              "nll": 0.8322646450712524,
              "brier": 0.6063095804116249,
              "ece": 0.2711401492357254,
              "mean_confidence": 0.7942170723126485
            },
            "schema-firewall/nyu-mll/multi_nli": {
              "n": 200,
              "accuracy": 0.865,
              "nll": 0.41195048167137427,
              "brier": 0.21928038816042317,
              "ece": 0.061470215469598743,
              "mean_confidence": 0.8334396223723889
            },
            "schema-firewall/product-science/xlam-function-calling-60k-raw": {
              "n": 81,
              "accuracy": 1.0,
              "nll": 0.0050883564066314295,
              "brier": 0.001193973693776232,
              "ece": 0.004606815032017043,
              "mean_confidence": 0.9953931849679829
            },
            "schema-firewall/product-science/xlam-hard": {
              "n": 211,
              "accuracy": 0.6303317535545023,
              "nll": 0.5777726572084735,
              "brier": 0.400591680919437,
              "ece": 0.21581395203468354,
              "mean_confidence": 0.846145705589186
            },
            "schema-firewall/stanfordnlp/snli": {
              "n": 208,
              "accuracy": 0.8846153846153846,
              "nll": 0.26919846923891994,
              "brier": 0.15986069018861843,
              "ece": 0.04227677503457438,
              "mean_confidence": 0.8858483952398484
            },
            "schema-firewall/synthetic/nli-haystack": {
              "n": 155,
              "accuracy": 0.896774193548387,
              "nll": 0.28797892007766474,
              "brier": 0.15715603314421323,
              "ece": 0.06688037726186936,
              "mean_confidence": 0.8773749347656004
            }
          },
          "by_kind": {
            "choice": {
              "n": 1163,
              "accuracy": 0.8718830610490111,
              "nll": 0.3188156900246091,
              "brier": 0.1816513806989808,
              "ece": 0.026625588907830587,
              "mean_confidence": 0.8696533281128671
            },
            "noul": {
              "n": 567,
              "accuracy": 0.5855379188712522,
              "nll": 0.754248758684351,
              "brier": 0.5202748309605,
              "ece": 0.23951466205469096,
              "mean_confidence": 0.8250525809259431
            },
            "score": {
              "n": 270,
              "accuracy": 0.6296296296296297,
              "nll": 0.8983366615242429,
              "brier": 0.496752599419819,
              "ece": 0.07374763350795817,
              "mean_confidence": 0.5930154732531971
            }
          },
          "by_transform": {
            "schema.firewall.distractor_drop.v1": {
              "n": 179,
              "accuracy": 0.9162011173184358,
              "nll": 0.17996553745532837,
              "brier": 0.10203871893360637,
              "ece": 0.041837152822057584,
              "mean_confidence": 0.9357914165411582
            },
            "schema.firewall.instruction.v1": {
              "n": 1395,
              "accuracy": 0.8523297491039427,
              "nll": 0.3699874161727043,
              "brier": 0.21063317768835207,
              "ece": 0.02584742985959546,
              "mean_confidence": 0.8404050774151279
            },
            "schema.firewall.noul_inversion.v1": {
              "n": 292,
              "accuracy": 0.2808219178082192,
              "nll": 1.247713702195005,
              "brier": 0.8843806009206997,
              "ece": 0.47954579167170064,
              "mean_confidence": 0.7603677094799198
            },
            "schema.firewall.score_scale.v1": {
              "n": 134,
              "accuracy": 0.6044776119402985,
              "nll": 0.9575665259539191,
              "brier": 0.5227032665496887,
              "ece": 0.07600586627846333,
              "mean_confidence": 0.5778111495188812
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 594,
              "accuracy": 0.6043771043771043,
              "nll": 0.7199648861602912,
              "brier": 0.4966259750298848,
              "ece": 0.22862747000524092,
              "mean_confidence": 0.8330045743823453
            },
            "3-5": {
              "n": 1375,
              "accuracy": 0.8218181818181818,
              "nll": 0.44010872633975223,
              "brier": 0.24742196818368095,
              "ece": 0.032660349141467676,
              "mean_confidence": 0.8119605057998137
            },
            "6-10": {
              "n": 31,
              "accuracy": 0.8709677419354839,
              "nll": 0.263998425413569,
              "brier": 0.16705004292117379,
              "ece": 0.09422032871553974,
              "mean_confidence": 0.905658035509048
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 1998,
              "accuracy": 0.7587587587587588,
              "nll": 0.5200072703881736,
              "brier": 0.3198700578515258,
              "ece": 0.0704587096193651,
              "mean_confidence": 0.8200187474653289
            },
            "truncated": {
              "n": 2,
              "accuracy": 0.0,
              "nll": 1.0090330243110657,
              "brier": 0.6396055817604065,
              "ece": 0.4641771614551544,
              "mean_confidence": 0.4641771614551544
            }
          },
          "by_dataset_truncation": {
            "schema-firewall/PolyAI/banking77|not_truncated": {
              "n": 46,
              "accuracy": 0.8043478260869565,
              "nll": 0.35612511238073435,
              "brier": 0.22549919875508087,
              "ece": 0.10526247128196384,
              "mean_confidence": 0.8808949822964875
            },
            "schema-firewall/Rowan/hellaswag|not_truncated": {
              "n": 130,
              "accuracy": 0.9,
              "nll": 0.20414897868758125,
              "brier": 0.11925340885983747,
              "ece": 0.05373616493665252,
              "mean_confidence": 0.9159407689021184
            },
            "schema-firewall/alisawuffles/WANLI|not_truncated": {
              "n": 209,
              "accuracy": 0.7655502392344498,
              "nll": 0.5446630402214148,
              "brier": 0.32441898222513327,
              "ece": 0.059890149027536935,
              "mean_confidence": 0.7769627089135385
            },
            "schema-firewall/cais/mmlu|not_truncated": {
              "n": 134,
              "accuracy": 0.917910447761194,
              "nll": 0.22831852445149217,
              "brier": 0.11953794011554492,
              "ece": 0.049322252620511986,
              "mean_confidence": 0.909431281596867
            },
            "schema-firewall/google/boolq-negation|not_truncated": {
              "n": 60,
              "accuracy": 0.5166666666666667,
              "nll": 1.2234002001583577,
              "brier": 0.7434715281813321,
              "ece": 0.34772479931513467,
              "mean_confidence": 0.8540057341257731
            },
            "schema-firewall/google/boolq|not_truncated": {
              "n": 36,
              "accuracy": 0.8888888888888888,
              "nll": 0.44322766041700057,
              "brier": 0.22839449674166368,
              "ece": 0.1561499618821674,
              "mean_confidence": 0.8758690738015704
            },
            "schema-firewall/nvidia/HelpSteer2-threshold|not_truncated": {
              "n": 260,
              "accuracy": 0.5230769230769231,
              "nll": 0.8322646450712524,
              "brier": 0.6063095804116249,
              "ece": 0.2711401492357254,
              "mean_confidence": 0.7942170723126485
            },
            "schema-firewall/nvidia/HelpSteer2|not_truncated": {
              "n": 268,
              "accuracy": 0.6343283582089553,
              "nll": 0.8975105692646397,
              "brier": 0.49568653238742655,
              "ece": 0.07441585716694149,
              "mean_confidence": 0.5939769531919885
            },
            "schema-firewall/nvidia/HelpSteer2|truncated": {
              "n": 2,
              "accuracy": 0.0,
              "nll": 1.0090330243110657,
              "brier": 0.6396055817604065,
              "ece": 0.4641771614551544,
              "mean_confidence": 0.4641771614551544
            },
            "schema-firewall/nyu-mll/multi_nli|not_truncated": {
              "n": 200,
              "accuracy": 0.865,
              "nll": 0.41195048167137427,
              "brier": 0.21928038816042317,
              "ece": 0.061470215469598743,
              "mean_confidence": 0.8334396223723889
            },
            "schema-firewall/product-science/xlam-function-calling-60k-raw|not_truncated": {
              "n": 81,
              "accuracy": 1.0,
              "nll": 0.0050883564066314295,
              "brier": 0.001193973693776232,
              "ece": 0.004606815032017043,
              "mean_confidence": 0.9953931849679829
            },
            "schema-firewall/product-science/xlam-hard|not_truncated": {
              "n": 211,
              "accuracy": 0.6303317535545023,
              "nll": 0.5777726572084735,
              "brier": 0.400591680919437,
              "ece": 0.21581395203468354,
              "mean_confidence": 0.846145705589186
            },
            "schema-firewall/stanfordnlp/snli|not_truncated": {
              "n": 208,
              "accuracy": 0.8846153846153846,
              "nll": 0.26919846923891994,
              "brier": 0.15986069018861843,
              "ece": 0.04227677503457438,
              "mean_confidence": 0.8858483952398484
            },
            "schema-firewall/synthetic/nli-haystack|not_truncated": {
              "n": 155,
              "accuracy": 0.896774193548387,
              "nll": 0.28797892007766474,
              "brier": 0.15715603314421323,
              "ece": 0.06688037726186936,
              "mean_confidence": 0.8773749347656004
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1163,
              "accuracy": 0.8718830610490111,
              "nll": 0.3188156900246091,
              "brier": 0.1816513806989808,
              "ece": 0.026625588907830587,
              "mean_confidence": 0.8696533281128671
            },
            "noul|not_truncated": {
              "n": 567,
              "accuracy": 0.5855379188712522,
              "nll": 0.754248758684351,
              "brier": 0.5202748309605,
              "ece": 0.23951466205469096,
              "mean_confidence": 0.8250525809259431
            },
            "score|not_truncated": {
              "n": 268,
              "accuracy": 0.6343283582089553,
              "nll": 0.8975105692646397,
              "brier": 0.49568653238742655,
              "ece": 0.07441585716694149,
              "mean_confidence": 0.5939769531919885
            },
            "score|truncated": {
              "n": 2,
              "accuracy": 0.0,
              "nll": 1.0090330243110657,
              "brier": 0.6396055817604065,
              "ece": 0.4641771614551544,
              "mean_confidence": 0.4641771614551544
            }
          },
          "nli_confusion": {
            "n": 772,
            "labels": [
              "entailment",
              "neutral",
              "contradiction"
            ],
            "matrix": {
              "entailment": {
                "entailment": 208,
                "neutral": 26,
                "contradiction": 3
              },
              "neutral": {
                "entailment": 40,
                "neutral": 259,
                "contradiction": 19
              },
              "contradiction": {
                "entailment": 4,
                "neutral": 24,
                "contradiction": 189
              }
            }
          }
        },
        "firewall": {
          "n": 2000,
          "accuracy": 0.8725,
          "nll": 0.34271046637929975,
          "brier": 0.2024325927257114,
          "ece": 0.021594500750303322,
          "mean_confidence": 0.8701762627661228,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.097443699836731,
            "groups": {
              "choice:2": {
                "temperature": 0.7726892229265028,
                "raw_temperature": 0.4564978778362274,
                "n": 80,
                "shrinkage_weight": 0.4
              },
              "choice:3-5": {
                "temperature": 1.0735645605398847,
                "raw_temperature": 1.0707285404205322,
                "n": 998,
                "shrinkage_weight": 0.8926654740608229
              },
              "choice:6-10": {
                "temperature": 1.0213380776161125,
                "raw_temperature": 0.9631407260894775,
                "n": 147,
                "shrinkage_weight": 0.550561797752809
              },
              "noul:*": {
                "temperature": 1.18076753369684,
                "raw_temperature": 1.1976659297943115,
                "n": 618,
                "shrinkage_weight": 0.8373983739837398
              },
              "score:*": {
                "temperature": 1.1118863993359898,
                "raw_temperature": 1.1230534315109253,
                "n": 157,
                "shrinkage_weight": 0.5667870036101083
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "firewall/tommccoy1-hans": {
              "n": 2000,
              "accuracy": 0.8725,
              "nll": 0.34271046637929975,
              "brier": 0.2024325927257114,
              "ece": 0.021594500750303322,
              "mean_confidence": 0.8701762627661228
            }
          },
          "by_kind": {
            "noul": {
              "n": 2000,
              "accuracy": 0.8725,
              "nll": 0.34271046637929975,
              "brier": 0.2024325927257114,
              "ece": 0.021594500750303322,
              "mean_confidence": 0.8701762627661228
            }
          },
          "by_transform": {
            "nli.firewall.hans.v1": {
              "n": 2000,
              "accuracy": 0.8725,
              "nll": 0.34271046637929975,
              "brier": 0.2024325927257114,
              "ece": 0.021594500750303322,
              "mean_confidence": 0.8701762627661228
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 2000,
              "accuracy": 0.8725,
              "nll": 0.34271046637929975,
              "brier": 0.2024325927257114,
              "ece": 0.021594500750303322,
              "mean_confidence": 0.8701762627661228
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 2000,
              "accuracy": 0.8725,
              "nll": 0.34271046637929975,
              "brier": 0.2024325927257114,
              "ece": 0.021594500750303322,
              "mean_confidence": 0.8701762627661228
            }
          },
          "by_dataset_truncation": {
            "firewall/tommccoy1-hans|not_truncated": {
              "n": 2000,
              "accuracy": 0.8725,
              "nll": 0.34271046637929975,
              "brier": 0.2024325927257114,
              "ece": 0.021594500750303322,
              "mean_confidence": 0.8701762627661228
            }
          },
          "by_kind_truncation": {
            "noul|not_truncated": {
              "n": 2000,
              "accuracy": 0.8725,
              "nll": 0.34271046637929975,
              "brier": 0.2024325927257114,
              "ece": 0.021594500750303322,
              "mean_confidence": 0.8701762627661228
            }
          }
        },
        "confusability": {
          "n": 1400,
          "accuracy": 1.0,
          "nll": 0.00341335700372578,
          "brier": 7.767048636123954e-05,
          "ece": 0.003391873708793125,
          "mean_confidence": 0.9966081262912069,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.097443699836731,
            "groups": {
              "choice:2": {
                "temperature": 0.7726892229265028,
                "raw_temperature": 0.4564978778362274,
                "n": 80,
                "shrinkage_weight": 0.4
              },
              "choice:3-5": {
                "temperature": 1.0735645605398847,
                "raw_temperature": 1.0707285404205322,
                "n": 998,
                "shrinkage_weight": 0.8926654740608229
              },
              "choice:6-10": {
                "temperature": 1.0213380776161125,
                "raw_temperature": 0.9631407260894775,
                "n": 147,
                "shrinkage_weight": 0.550561797752809
              },
              "noul:*": {
                "temperature": 1.18076753369684,
                "raw_temperature": 1.1976659297943115,
                "n": 618,
                "shrinkage_weight": 0.8373983739837398
              },
              "score:*": {
                "temperature": 1.1118863993359898,
                "raw_temperature": 1.1230534315109253,
                "n": 157,
                "shrinkage_weight": 0.5667870036101083
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "v142-confusability": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.00341335700372578,
              "brier": 7.767048636123954e-05,
              "ece": 0.003391873708793125,
              "mean_confidence": 0.9966081262912069
            }
          },
          "by_kind": {
            "choice": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.0030758743519090784,
              "brier": 8.714231419299279e-05,
              "ece": 0.0030530961751937813,
              "mean_confidence": 0.9969469038248062
            },
            "noul": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0011829274808405899,
              "brier": 3.475759723983174e-06,
              "ece": 0.0011820575594901905,
              "mean_confidence": 0.9988179424405098
            },
            "score": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.00733119978569448,
              "brier": 0.00010450607383972965,
              "ece": 0.007295577526092556,
              "mean_confidence": 0.9927044224739074
            }
          },
          "by_transform": {
            "coreference": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.005743390580028063,
              "brier": 0.00021617284226891798,
              "ece": 0.0056866601109504256,
              "mean_confidence": 0.9943133398890496
            },
            "entity_binding": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.00016332407969457563,
              "brier": 4.9414514606826996e-08,
              "ece": 0.00016330808401110453,
              "mean_confidence": 0.9998366919159889
            },
            "negation": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0011829274808405899,
              "brier": 3.475759723983174e-06,
              "ece": 0.0011820575594901905,
              "mean_confidence": 0.9988179424405098
            },
            "numeric_boundary": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.00014066748185541654,
              "brier": 6.167312968208272e-08,
              "ece": 0.00014065206050872803,
              "mean_confidence": 0.9998593479394913
            },
            "ordinal_boundary": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.00733119978569448,
              "brier": 0.00010450607383972965,
              "ece": 0.007295577526092556,
              "mean_confidence": 0.9927044224739074
            },
            "temporal_state": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.007766418813844211,
              "brier": 0.00015806605823172503,
              "ece": 0.007725143730640371,
              "mean_confidence": 0.9922748562693596
            },
            "tool_argument_validity": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0015655708041231264,
              "brier": 6.1361582820032e-05,
              "ece": 0.001549716889858277,
              "mean_confidence": 0.9984502831101417
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 400,
              "accuracy": 1.0,
              "nll": 0.0006617974813480032,
              "brier": 1.7687164268326283e-06,
              "ece": 0.0006613548099995148,
              "mean_confidence": 0.9993386451900005
            },
            "3-5": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.004513980812676891,
              "brier": 0.0001080311943350023,
              "ece": 0.004484081268310569,
              "mean_confidence": 0.9955159187316894
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.00341335700372578,
              "brier": 7.767048636123954e-05,
              "ece": 0.003391873708793125,
              "mean_confidence": 0.9966081262912069
            }
          },
          "by_dataset_truncation": {
            "v142-confusability|not_truncated": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.00341335700372578,
              "brier": 7.767048636123954e-05,
              "ece": 0.003391873708793125,
              "mean_confidence": 0.9966081262912069
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.0030758743519090784,
              "brier": 8.714231419299279e-05,
              "ece": 0.0030530961751937813,
              "mean_confidence": 0.9969469038248062
            },
            "noul|not_truncated": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0011829274808405899,
              "brier": 3.475759723983174e-06,
              "ece": 0.0011820575594901905,
              "mean_confidence": 0.9988179424405098
            },
            "score|not_truncated": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.00733119978569448,
              "brier": 0.00010450607383972965,
              "ece": 0.007295577526092556,
              "mean_confidence": 0.9927044224739074
            }
          }
        }
      },
      "latency": {
        "validation": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 3647.106461352203,
          "effective_ms_per_example": 455.8883076690254,
          "skipped_unusable": 0
        },
        "schema": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 4148.647082126699,
          "effective_ms_per_example": 518.5808852658374,
          "skipped_unusable": 0
        },
        "firewall": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 335.46002532273997,
          "effective_ms_per_example": 41.932503165342496,
          "skipped_unusable": 0
        },
        "confusability": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 658.9870701618826,
          "effective_ms_per_example": 82.37338377023532,
          "skipped_unusable": 0
        }
      }
    },
    "head_rlcd": {
      "calibration": {
        "version": 2,
        "type": "primitive_option_temperature",
        "global_temperature": 1.0621812343597412,
        "groups": {
          "choice:2": {
            "temperature": 0.7285364649335917,
            "raw_temperature": 0.3852079510688782,
            "n": 71,
            "shrinkage_weight": 0.3717277486910995
          },
          "choice:3-5": {
            "temperature": 1.0312828950285642,
            "raw_temperature": 1.0276613235473633,
            "n": 1007,
            "shrinkage_weight": 0.8935226264418811
          },
          "choice:6-10": {
            "temperature": 1.018117407640952,
            "raw_temperature": 0.9839622974395752,
            "n": 149,
            "shrinkage_weight": 0.5539033457249071
          },
          "noul:*": {
            "temperature": 1.12278263121421,
            "raw_temperature": 1.1349645853042603,
            "n": 617,
            "shrinkage_weight": 0.8371777476255088
          },
          "score:*": {
            "temperature": 1.1063119547040072,
            "raw_temperature": 1.1415023803710938,
            "n": 156,
            "shrinkage_weight": 0.5652173913043478
          }
        },
        "min_group": 50,
        "prior_strength": 120.0,
        "n": 2000
      },
      "sets": {
        "validation": {
          "n": 2000,
          "accuracy": 0.8705,
          "nll": 0.3296415164664375,
          "brier": 0.1840096441006748,
          "ece": 0.015826731696724952,
          "mean_confidence": 0.8667946958690882,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.0621812343597412,
            "groups": {
              "choice:2": {
                "temperature": 0.7285364649335917,
                "raw_temperature": 0.3852079510688782,
                "n": 71,
                "shrinkage_weight": 0.3717277486910995
              },
              "choice:3-5": {
                "temperature": 1.0312828950285642,
                "raw_temperature": 1.0276613235473633,
                "n": 1007,
                "shrinkage_weight": 0.8935226264418811
              },
              "choice:6-10": {
                "temperature": 1.018117407640952,
                "raw_temperature": 0.9839622974395752,
                "n": 149,
                "shrinkage_weight": 0.5539033457249071
              },
              "noul:*": {
                "temperature": 1.12278263121421,
                "raw_temperature": 1.1349645853042603,
                "n": 617,
                "shrinkage_weight": 0.8371777476255088
              },
              "score:*": {
                "temperature": 1.1063119547040072,
                "raw_temperature": 1.1415023803710938,
                "n": 156,
                "shrinkage_weight": 0.5652173913043478
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "PolyAI/banking77": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18870303818857792,
              "brier": 0.10519016720641593,
              "ece": 0.05590041091336926,
              "mean_confidence": 0.9336026931071982
            },
            "Rowan/hellaswag": {
              "n": 153,
              "accuracy": 0.9215686274509803,
              "nll": 0.2399784766869879,
              "brier": 0.11934398814653138,
              "ece": 0.040469242272034245,
              "mean_confidence": 0.9290768474924798
            },
            "alisawuffles/WANLI": {
              "n": 160,
              "accuracy": 0.80625,
              "nll": 0.5381927386275492,
              "brier": 0.30977863866601185,
              "ece": 0.06804261095821856,
              "mean_confidence": 0.7853333316743374
            },
            "cais/mmlu": {
              "n": 154,
              "accuracy": 0.8831168831168831,
              "nll": 0.3485408564913748,
              "brier": 0.17917580716248588,
              "ece": 0.06689213796869503,
              "mean_confidence": 0.8730696820980542
            },
            "google/boolq": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.33914438616278964,
              "brier": 0.19378073975049515,
              "ece": 0.04670275738036708,
              "mean_confidence": 0.8901991384481293
            },
            "google/boolq-negation": {
              "n": 153,
              "accuracy": 0.8627450980392157,
              "nll": 0.29727962399145064,
              "brier": 0.17111000223924158,
              "ece": 0.06959474671120737,
              "mean_confidence": 0.8627709676237667
            },
            "nvidia/HelpSteer2": {
              "n": 156,
              "accuracy": 0.5448717948717948,
              "nll": 1.00863150984813,
              "brier": 0.549988131922407,
              "ece": 0.07374643553525977,
              "mean_confidence": 0.5746784233129941
            },
            "nvidia/HelpSteer2-threshold": {
              "n": 157,
              "accuracy": 0.8853503184713376,
              "nll": 0.31764734924213284,
              "brier": 0.1801272323022331,
              "ece": 0.05705484463151096,
              "mean_confidence": 0.8431880648728389
            },
            "nyu-mll/multi_nli": {
              "n": 157,
              "accuracy": 0.8853503184713376,
              "nll": 0.2977778198298328,
              "brier": 0.17204729962497461,
              "ece": 0.0343798731163049,
              "mean_confidence": 0.8596658488367773
            },
            "product-science/xlam-function-calling-60k-raw": {
              "n": 153,
              "accuracy": 1.0,
              "nll": 0.0033377126531411883,
              "brier": 0.001834733447133328,
              "ece": 0.002718002188439416,
              "mean_confidence": 0.9972819978115606
            },
            "product-science/xlam-hard": {
              "n": 154,
              "accuracy": 0.9935064935064936,
              "nll": 0.0357667315122369,
              "brier": 0.018634809771057987,
              "ece": 0.020999968051910404,
              "mean_confidence": 0.9805593196447794
            },
            "stanfordnlp/snli": {
              "n": 159,
              "accuracy": 0.8805031446540881,
              "nll": 0.3138472866193749,
              "brier": 0.1782226289040226,
              "ece": 0.04512691535289933,
              "mean_confidence": 0.8852375029767834
            },
            "synthetic/nli-haystack": {
              "n": 155,
              "accuracy": 0.864516129032258,
              "nll": 0.32164501180992494,
              "brier": 0.19284149839020814,
              "ece": 0.02968011844542717,
              "mean_confidence": 0.8698015168789894
            }
          },
          "by_kind": {
            "choice": {
              "n": 1227,
              "accuracy": 0.895680521597392,
              "nll": 0.28458429776615213,
              "brier": 0.15912234712374151,
              "ece": 0.016193954776532293,
              "mean_confidence": 0.8902595518027852
            },
            "noul": {
              "n": 617,
              "accuracy": 0.9027552674230146,
              "nll": 0.24757144900729028,
              "brier": 0.14096923776438128,
              "ece": 0.031755420241981885,
              "mean_confidence": 0.8939887968222557
            },
            "score": {
              "n": 156,
              "accuracy": 0.5448717948717948,
              "nll": 1.00863150984813,
              "brier": 0.549988131922407,
              "ece": 0.07374643553525977,
              "mean_confidence": 0.5746784233129941
            }
          },
          "by_transform": {
            "agent.invocation.correct.v1": {
              "n": 50,
              "accuracy": 1.0,
              "nll": 0.04205997154116631,
              "brier": 0.01677228283144359,
              "ece": 0.03617538213729857,
              "mean_confidence": 0.9638246178627015
            },
            "agent.invocation.corrupt_argument.v1": {
              "n": 61,
              "accuracy": 0.9836065573770492,
              "nll": 0.041105691575422335,
              "brier": 0.025920638532433198,
              "ece": 0.02006097699775066,
              "mean_confidence": 0.9925722235538921
            },
            "agent.invocation.wrong_tool.v1": {
              "n": 43,
              "accuracy": 1.0,
              "nll": 0.020875136970358272,
              "brier": 0.010464828202147104,
              "ece": 0.017023286154103805,
              "mean_confidence": 0.9829767138458961
            },
            "commonsense.choice.v2": {
              "n": 153,
              "accuracy": 0.9215686274509803,
              "nll": 0.2399784766869879,
              "brier": 0.11934398814653138,
              "ece": 0.040469242272034245,
              "mean_confidence": 0.9290768474924798
            },
            "intent.choice.v3": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18870303818857792,
              "brier": 0.10519016720641593,
              "ece": 0.05590041091336926,
              "mean_confidence": 0.9336026931071982
            },
            "knowledge.choice.v1": {
              "n": 154,
              "accuracy": 0.8831168831168831,
              "nll": 0.3485408564913748,
              "brier": 0.17917580716248588,
              "ece": 0.06689213796869503,
              "mean_confidence": 0.8730696820980542
            },
            "nli.choice.v4": {
              "n": 157,
              "accuracy": 0.8853503184713376,
              "nll": 0.2977778198298328,
              "brier": 0.17204729962497461,
              "ece": 0.0343798731163049,
              "mean_confidence": 0.8596658488367773
            },
            "nli.hard.v4": {
              "n": 160,
              "accuracy": 0.80625,
              "nll": 0.5381927386275492,
              "brier": 0.30977863866601185,
              "ece": 0.06804261095821856,
              "mean_confidence": 0.7853333316743374
            },
            "nli.haystack.absent.v1": {
              "n": 13,
              "accuracy": 0.9230769230769231,
              "nll": 0.293056732795846,
              "brier": 0.1554580526787434,
              "ece": 0.09880676177831796,
              "mean_confidence": 0.9119220834511977
            },
            "nli.haystack.present.v1": {
              "n": 142,
              "accuracy": 0.8591549295774648,
              "nll": 0.324262248621073,
              "brier": 0.19626392651872251,
              "ece": 0.028329969921582165,
              "mean_confidence": 0.8659454086716746
            },
            "nli.snli.v1": {
              "n": 159,
              "accuracy": 0.8805031446540881,
              "nll": 0.3138472866193749,
              "brier": 0.1782226289040226,
              "ece": 0.04512691535289933,
              "mean_confidence": 0.8852375029767834
            },
            "noul.negated_answer.v1": {
              "n": 153,
              "accuracy": 0.8627450980392157,
              "nll": 0.29727962399145064,
              "brier": 0.17111000223924158,
              "ece": 0.06959474671120737,
              "mean_confidence": 0.8627709676237667
            },
            "noul.v3": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.33914438616278964,
              "brier": 0.19378073975049515,
              "ece": 0.04670275738036708,
              "mean_confidence": 0.8901991384481293
            },
            "score.coherence.threshold.v1": {
              "n": 29,
              "accuracy": 0.9655172413793104,
              "nll": 0.19833977429742186,
              "brier": 0.10644655702463783,
              "ece": 0.1178100828466744,
              "mean_confidence": 0.8477071585326359
            },
            "score.coherence.v3": {
              "n": 27,
              "accuracy": 0.8148148148148148,
              "nll": 0.4967265730654752,
              "brier": 0.2842161387757019,
              "ece": 0.099791513548957,
              "mean_confidence": 0.7278836170832316
            },
            "score.complexity.threshold.v1": {
              "n": 36,
              "accuracy": 0.9166666666666666,
              "nll": 0.32385558721661156,
              "brier": 0.17315642751181157,
              "ece": 0.1148082067569097,
              "mean_confidence": 0.8971044040388532
            },
            "score.complexity.v3": {
              "n": 40,
              "accuracy": 0.6,
              "nll": 0.9197511970996857,
              "brier": 0.5391126235947012,
              "ece": 0.10441315844655036,
              "mean_confidence": 0.6230462364852428
            },
            "score.correctness.threshold.v1": {
              "n": 29,
              "accuracy": 0.7931034482758621,
              "nll": 0.342922273381003,
              "brier": 0.21508417756232465,
              "ece": 0.14295518809351426,
              "mean_confidence": 0.7817615517254534
            },
            "score.correctness.v3": {
              "n": 37,
              "accuracy": 0.35135135135135137,
              "nll": 1.3171428655450408,
              "brier": 0.7058205044752842,
              "ece": 0.20661347017094894,
              "mean_confidence": 0.45046308274204666
            },
            "score.helpfulness.threshold.v1": {
              "n": 25,
              "accuracy": 0.8,
              "nll": 0.44108421333134173,
              "brier": 0.28551679542288183,
              "ece": 0.13267860651016236,
              "mean_confidence": 0.7273315024375916
            },
            "score.helpfulness.v3": {
              "n": 28,
              "accuracy": 0.42857142857142855,
              "nll": 1.2584992029837199,
              "brier": 0.6579760227884565,
              "ece": 0.2361379388187613,
              "mean_confidence": 0.43264825482453617
            },
            "score.verbosity.threshold.v1": {
              "n": 38,
              "accuracy": 0.9210526315789473,
              "nll": 0.30231915724365727,
              "brier": 0.14694823401135326,
              "ece": 0.07115527046354195,
              "mean_confidence": 0.9117601968740162
            },
            "score.verbosity.v3": {
              "n": 24,
              "accuracy": 0.5833333333333334,
              "nll": 0.9655244362850984,
              "brier": 0.5008800243958831,
              "ece": 0.10565355420112608,
              "mean_confidence": 0.6789100716511408
            },
            "tool.choice.single.v3": {
              "n": 153,
              "accuracy": 1.0,
              "nll": 0.0033377126531411883,
              "brier": 0.001834733447133328,
              "ece": 0.002718002188439416,
              "mean_confidence": 0.9972819978115606
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 688,
              "accuracy": 0.9127906976744186,
              "nll": 0.22202356312992652,
              "brier": 0.1264215409403835,
              "ece": 0.02847742297968196,
              "mean_confidence": 0.9049279947093752
            },
            "3-5": {
              "n": 1163,
              "accuracy": 0.8366294067067928,
              "nll": 0.4134522701136551,
              "brier": 0.22935059510468528,
              "ece": 0.013319709643350023,
              "mean_confidence": 0.8349535806058095
            },
            "6-10": {
              "n": 149,
              "accuracy": 0.9395973154362416,
              "nll": 0.17238812991479716,
              "brier": 0.09601695253434102,
              "ece": 0.05117117458541918,
              "mean_confidence": 0.9392477659971122
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 1999,
              "accuracy": 0.8704352176088044,
              "nll": 0.3295081802774739,
              "brier": 0.18389991385929264,
              "ece": 0.015609992585103528,
              "mean_confidence": 0.8669527163351697
            },
            "truncated": {
              "n": 1,
              "accuracy": 1.0,
              "nll": 0.5961805582046509,
              "brier": 0.40336039662361145,
              "ece": 0.4490882158279419,
              "mean_confidence": 0.5509117841720581
            }
          },
          "by_dataset_truncation": {
            "PolyAI/banking77|not_truncated": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18870303818857792,
              "brier": 0.10519016720641593,
              "ece": 0.05590041091336926,
              "mean_confidence": 0.9336026931071982
            },
            "Rowan/hellaswag|not_truncated": {
              "n": 153,
              "accuracy": 0.9215686274509803,
              "nll": 0.2399784766869879,
              "brier": 0.11934398814653138,
              "ece": 0.040469242272034245,
              "mean_confidence": 0.9290768474924798
            },
            "alisawuffles/WANLI|not_truncated": {
              "n": 160,
              "accuracy": 0.80625,
              "nll": 0.5381927386275492,
              "brier": 0.30977863866601185,
              "ece": 0.06804261095821856,
              "mean_confidence": 0.7853333316743374
            },
            "cais/mmlu|not_truncated": {
              "n": 154,
              "accuracy": 0.8831168831168831,
              "nll": 0.3485408564913748,
              "brier": 0.17917580716248588,
              "ece": 0.06689213796869503,
              "mean_confidence": 0.8730696820980542
            },
            "google/boolq-negation|not_truncated": {
              "n": 153,
              "accuracy": 0.8627450980392157,
              "nll": 0.29727962399145064,
              "brier": 0.17111000223924158,
              "ece": 0.06959474671120737,
              "mean_confidence": 0.8627709676237667
            },
            "google/boolq|not_truncated": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.33914438616278964,
              "brier": 0.19378073975049515,
              "ece": 0.04670275738036708,
              "mean_confidence": 0.8901991384481293
            },
            "nvidia/HelpSteer2-threshold|not_truncated": {
              "n": 156,
              "accuracy": 0.8846153846153846,
              "nll": 0.3158618799539116,
              "brier": 0.17869625047966017,
              "ece": 0.0545418102007646,
              "mean_confidence": 0.8450616307747669
            },
            "nvidia/HelpSteer2-threshold|truncated": {
              "n": 1,
              "accuracy": 1.0,
              "nll": 0.5961805582046509,
              "brier": 0.40336039662361145,
              "ece": 0.4490882158279419,
              "mean_confidence": 0.5509117841720581
            },
            "nvidia/HelpSteer2|not_truncated": {
              "n": 156,
              "accuracy": 0.5448717948717948,
              "nll": 1.00863150984813,
              "brier": 0.549988131922407,
              "ece": 0.07374643553525977,
              "mean_confidence": 0.5746784233129941
            },
            "nyu-mll/multi_nli|not_truncated": {
              "n": 157,
              "accuracy": 0.8853503184713376,
              "nll": 0.2977778198298328,
              "brier": 0.17204729962497461,
              "ece": 0.0343798731163049,
              "mean_confidence": 0.8596658488367773
            },
            "product-science/xlam-function-calling-60k-raw|not_truncated": {
              "n": 153,
              "accuracy": 1.0,
              "nll": 0.0033377126531411883,
              "brier": 0.001834733447133328,
              "ece": 0.002718002188439416,
              "mean_confidence": 0.9972819978115606
            },
            "product-science/xlam-hard|not_truncated": {
              "n": 154,
              "accuracy": 0.9935064935064936,
              "nll": 0.0357667315122369,
              "brier": 0.018634809771057987,
              "ece": 0.020999968051910404,
              "mean_confidence": 0.9805593196447794
            },
            "stanfordnlp/snli|not_truncated": {
              "n": 159,
              "accuracy": 0.8805031446540881,
              "nll": 0.3138472866193749,
              "brier": 0.1782226289040226,
              "ece": 0.04512691535289933,
              "mean_confidence": 0.8852375029767834
            },
            "synthetic/nli-haystack|not_truncated": {
              "n": 155,
              "accuracy": 0.864516129032258,
              "nll": 0.32164501180992494,
              "brier": 0.19284149839020814,
              "ece": 0.02968011844542717,
              "mean_confidence": 0.8698015168789894
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1227,
              "accuracy": 0.895680521597392,
              "nll": 0.28458429776615213,
              "brier": 0.15912234712374151,
              "ece": 0.016193954776532293,
              "mean_confidence": 0.8902595518027852
            },
            "noul|not_truncated": {
              "n": 616,
              "accuracy": 0.9025974025974026,
              "nll": 0.24700552512872315,
              "brier": 0.1405432780909085,
              "ece": 0.031077931937459235,
              "mean_confidence": 0.89454574002461
            },
            "noul|truncated": {
              "n": 1,
              "accuracy": 1.0,
              "nll": 0.5961805582046509,
              "brier": 0.40336039662361145,
              "ece": 0.4490882158279419,
              "mean_confidence": 0.5509117841720581
            },
            "score|not_truncated": {
              "n": 156,
              "accuracy": 0.5448717948717948,
              "nll": 1.00863150984813,
              "brier": 0.549988131922407,
              "ece": 0.07374643553525977,
              "mean_confidence": 0.5746784233129941
            }
          },
          "nli_confusion": {
            "n": 631,
            "labels": [
              "entailment",
              "neutral",
              "contradiction"
            ],
            "matrix": {
              "entailment": {
                "entailment": 187,
                "neutral": 24,
                "contradiction": 2
              },
              "neutral": {
                "entailment": 21,
                "neutral": 214,
                "contradiction": 20
              },
              "contradiction": {
                "entailment": 1,
                "neutral": 21,
                "contradiction": 141
              }
            }
          }
        },
        "schema": {
          "n": 2000,
          "accuracy": 0.7645,
          "nll": 0.5092308543568372,
          "brier": 0.31048622908528783,
          "ece": 0.06814040819555524,
          "mean_confidence": 0.8320965484306216,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.0621812343597412,
            "groups": {
              "choice:2": {
                "temperature": 0.7285364649335917,
                "raw_temperature": 0.3852079510688782,
                "n": 71,
                "shrinkage_weight": 0.3717277486910995
              },
              "choice:3-5": {
                "temperature": 1.0312828950285642,
                "raw_temperature": 1.0276613235473633,
                "n": 1007,
                "shrinkage_weight": 0.8935226264418811
              },
              "choice:6-10": {
                "temperature": 1.018117407640952,
                "raw_temperature": 0.9839622974395752,
                "n": 149,
                "shrinkage_weight": 0.5539033457249071
              },
              "noul:*": {
                "temperature": 1.12278263121421,
                "raw_temperature": 1.1349645853042603,
                "n": 617,
                "shrinkage_weight": 0.8371777476255088
              },
              "score:*": {
                "temperature": 1.1063119547040072,
                "raw_temperature": 1.1415023803710938,
                "n": 156,
                "shrinkage_weight": 0.5652173913043478
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "schema-firewall/PolyAI/banking77": {
              "n": 46,
              "accuracy": 0.8043478260869565,
              "nll": 0.35900251572955155,
              "brier": 0.2278477099184452,
              "ece": 0.11447685244290724,
              "mean_confidence": 0.8849650835213454
            },
            "schema-firewall/Rowan/hellaswag": {
              "n": 130,
              "accuracy": 0.9076923076923077,
              "nll": 0.20391861355223787,
              "brier": 0.12032575284317323,
              "ece": 0.05504847719119148,
              "mean_confidence": 0.9189063984614152
            },
            "schema-firewall/alisawuffles/WANLI": {
              "n": 214,
              "accuracy": 0.7570093457943925,
              "nll": 0.5224599649424656,
              "brier": 0.3108746913783881,
              "ece": 0.08684171750166704,
              "mean_confidence": 0.7917580997275415
            },
            "schema-firewall/cais/mmlu": {
              "n": 134,
              "accuracy": 0.917910447761194,
              "nll": 0.2270736379013853,
              "brier": 0.11838710439602669,
              "ece": 0.05173284882929787,
              "mean_confidence": 0.9138704791887483
            },
            "schema-firewall/google/boolq": {
              "n": 36,
              "accuracy": 0.8888888888888888,
              "nll": 0.45656878421868896,
              "brier": 0.2262111558433004,
              "ece": 0.13308524257606935,
              "mean_confidence": 0.8821887258026335
            },
            "schema-firewall/google/boolq-negation": {
              "n": 60,
              "accuracy": 0.5166666666666667,
              "nll": 1.2670575649982008,
              "brier": 0.7512862990629401,
              "ece": 0.35960156718889874,
              "mean_confidence": 0.860178780555725
            },
            "schema-firewall/nvidia/HelpSteer2": {
              "n": 257,
              "accuracy": 0.6108949416342413,
              "nll": 0.8855918734569957,
              "brier": 0.49536697754948056,
              "ece": 0.09132106381863471,
              "mean_confidence": 0.5927287479898808
            },
            "schema-firewall/nvidia/HelpSteer2-threshold": {
              "n": 254,
              "accuracy": 0.5433070866141733,
              "nll": 0.8280847921860258,
              "brier": 0.595998754226859,
              "ece": 0.26783039860838037,
              "mean_confidence": 0.8042075009796563
            },
            "schema-firewall/nyu-mll/multi_nli": {
              "n": 216,
              "accuracy": 0.8703703703703703,
              "nll": 0.35441261314122974,
              "brier": 0.1908318395181865,
              "ece": 0.05234303628956832,
              "mean_confidence": 0.8628430830107795
            },
            "schema-firewall/product-science/xlam-function-calling-60k-raw": {
              "n": 81,
              "accuracy": 1.0,
              "nll": 0.004432377281443713,
              "brier": 0.0009832953881987287,
              "ece": 0.004043651215824124,
              "mean_confidence": 0.9959563487841759
            },
            "schema-firewall/product-science/xlam-hard": {
              "n": 207,
              "accuracy": 0.6521739130434783,
              "nll": 0.5901507024798354,
              "brier": 0.3917848867662268,
              "ece": 0.21952186143340696,
              "mean_confidence": 0.8716957744768852
            },
            "schema-firewall/stanfordnlp/snli": {
              "n": 212,
              "accuracy": 0.8915094339622641,
              "nll": 0.28470200158541425,
              "brier": 0.16288797511131478,
              "ece": 0.05838072018803289,
              "mean_confidence": 0.8921123490018664
            },
            "schema-firewall/synthetic/nli-haystack": {
              "n": 153,
              "accuracy": 0.9019607843137255,
              "nll": 0.28354985410883243,
              "brier": 0.15433488102038004,
              "ece": 0.06596269872453478,
              "mean_confidence": 0.8859285342147927
            }
          },
          "by_kind": {
            "choice": {
              "n": 1186,
              "accuracy": 0.8735244519392917,
              "nll": 0.3085243966904598,
              "brier": 0.17534503393928574,
              "ece": 0.025424960818499803,
              "mean_confidence": 0.8800864731029153
            },
            "noul": {
              "n": 557,
              "accuracy": 0.6032315978456014,
              "nll": 0.762934762585891,
              "brier": 0.5129334554548768,
              "ece": 0.23712612482546905,
              "mean_confidence": 0.8403577226710705
            },
            "score": {
              "n": 257,
              "accuracy": 0.6108949416342413,
              "nll": 0.8855918734569957,
              "brier": 0.49536697754948056,
              "ece": 0.09132106381863471,
              "mean_confidence": 0.5927287479898808
            }
          },
          "by_transform": {
            "schema.firewall.distractor_drop.v1": {
              "n": 179,
              "accuracy": 0.9162011173184358,
              "nll": 0.18095475863384314,
              "brier": 0.10300702612023682,
              "ece": 0.04983094084862223,
              "mean_confidence": 0.9392977393539258
            },
            "schema.firewall.instruction.v1": {
              "n": 1419,
              "accuracy": 0.8611698379140239,
              "nll": 0.34634081345778583,
              "brier": 0.19550318506250666,
              "ece": 0.02325636315421375,
              "mean_confidence": 0.853838113273761
            },
            "schema.firewall.noul_inversion.v1": {
              "n": 276,
              "accuracy": 0.2572463768115942,
              "nll": 1.357916534608365,
              "brier": 0.9378885064367711,
              "ece": 0.5137078725341437,
              "mean_confidence": 0.770954249345738
            },
            "schema.firewall.score_scale.v1": {
              "n": 126,
              "accuracy": 0.5714285714285714,
              "nll": 0.9510176910294427,
              "brier": 0.5258567707524413,
              "ece": 0.10069926556140658,
              "mean_confidence": 0.568882110809523
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 584,
              "accuracy": 0.6215753424657534,
              "nll": 0.7276621770525656,
              "brier": 0.48921906625380607,
              "ece": 0.2261630186479385,
              "mean_confidence": 0.8477383611136919
            },
            "3-5": {
              "n": 1385,
              "accuracy": 0.8223826714801444,
              "nll": 0.42260318807209163,
              "brier": 0.23831324855760574,
              "ece": 0.029846963028184778,
              "mean_confidence": 0.8238253159320742
            },
            "6-10": {
              "n": 31,
              "accuracy": 0.8709677419354839,
              "nll": 0.2645671559719103,
              "brier": 0.16789271696997024,
              "ece": 0.09428581979966932,
              "mean_confidence": 0.9069623033846578
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 2000,
              "accuracy": 0.7645,
              "nll": 0.5092308543568372,
              "brier": 0.31048622908528783,
              "ece": 0.06814040819555524,
              "mean_confidence": 0.8320965484306216
            }
          },
          "by_dataset_truncation": {
            "schema-firewall/PolyAI/banking77|not_truncated": {
              "n": 46,
              "accuracy": 0.8043478260869565,
              "nll": 0.35900251572955155,
              "brier": 0.2278477099184452,
              "ece": 0.11447685244290724,
              "mean_confidence": 0.8849650835213454
            },
            "schema-firewall/Rowan/hellaswag|not_truncated": {
              "n": 130,
              "accuracy": 0.9076923076923077,
              "nll": 0.20391861355223787,
              "brier": 0.12032575284317323,
              "ece": 0.05504847719119148,
              "mean_confidence": 0.9189063984614152
            },
            "schema-firewall/alisawuffles/WANLI|not_truncated": {
              "n": 214,
              "accuracy": 0.7570093457943925,
              "nll": 0.5224599649424656,
              "brier": 0.3108746913783881,
              "ece": 0.08684171750166704,
              "mean_confidence": 0.7917580997275415
            },
            "schema-firewall/cais/mmlu|not_truncated": {
              "n": 134,
              "accuracy": 0.917910447761194,
              "nll": 0.2270736379013853,
              "brier": 0.11838710439602669,
              "ece": 0.05173284882929787,
              "mean_confidence": 0.9138704791887483
            },
            "schema-firewall/google/boolq-negation|not_truncated": {
              "n": 60,
              "accuracy": 0.5166666666666667,
              "nll": 1.2670575649982008,
              "brier": 0.7512862990629401,
              "ece": 0.35960156718889874,
              "mean_confidence": 0.860178780555725
            },
            "schema-firewall/google/boolq|not_truncated": {
              "n": 36,
              "accuracy": 0.8888888888888888,
              "nll": 0.45656878421868896,
              "brier": 0.2262111558433004,
              "ece": 0.13308524257606935,
              "mean_confidence": 0.8821887258026335
            },
            "schema-firewall/nvidia/HelpSteer2-threshold|not_truncated": {
              "n": 254,
              "accuracy": 0.5433070866141733,
              "nll": 0.8280847921860258,
              "brier": 0.595998754226859,
              "ece": 0.26783039860838037,
              "mean_confidence": 0.8042075009796563
            },
            "schema-firewall/nvidia/HelpSteer2|not_truncated": {
              "n": 257,
              "accuracy": 0.6108949416342413,
              "nll": 0.8855918734569957,
              "brier": 0.49536697754948056,
              "ece": 0.09132106381863471,
              "mean_confidence": 0.5927287479898808
            },
            "schema-firewall/nyu-mll/multi_nli|not_truncated": {
              "n": 216,
              "accuracy": 0.8703703703703703,
              "nll": 0.35441261314122974,
              "brier": 0.1908318395181865,
              "ece": 0.05234303628956832,
              "mean_confidence": 0.8628430830107795
            },
            "schema-firewall/product-science/xlam-function-calling-60k-raw|not_truncated": {
              "n": 81,
              "accuracy": 1.0,
              "nll": 0.004432377281443713,
              "brier": 0.0009832953881987287,
              "ece": 0.004043651215824124,
              "mean_confidence": 0.9959563487841759
            },
            "schema-firewall/product-science/xlam-hard|not_truncated": {
              "n": 207,
              "accuracy": 0.6521739130434783,
              "nll": 0.5901507024798354,
              "brier": 0.3917848867662268,
              "ece": 0.21952186143340696,
              "mean_confidence": 0.8716957744768852
            },
            "schema-firewall/stanfordnlp/snli|not_truncated": {
              "n": 212,
              "accuracy": 0.8915094339622641,
              "nll": 0.28470200158541425,
              "brier": 0.16288797511131478,
              "ece": 0.05838072018803289,
              "mean_confidence": 0.8921123490018664
            },
            "schema-firewall/synthetic/nli-haystack|not_truncated": {
              "n": 153,
              "accuracy": 0.9019607843137255,
              "nll": 0.28354985410883243,
              "brier": 0.15433488102038004,
              "ece": 0.06596269872453478,
              "mean_confidence": 0.8859285342147927
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1186,
              "accuracy": 0.8735244519392917,
              "nll": 0.3085243966904598,
              "brier": 0.17534503393928574,
              "ece": 0.025424960818499803,
              "mean_confidence": 0.8800864731029153
            },
            "noul|not_truncated": {
              "n": 557,
              "accuracy": 0.6032315978456014,
              "nll": 0.762934762585891,
              "brier": 0.5129334554548768,
              "ece": 0.23712612482546905,
              "mean_confidence": 0.8403577226710705
            },
            "score|not_truncated": {
              "n": 257,
              "accuracy": 0.6108949416342413,
              "nll": 0.8855918734569957,
              "brier": 0.49536697754948056,
              "ece": 0.09132106381863471,
              "mean_confidence": 0.5927287479898808
            }
          },
          "nli_confusion": {
            "n": 795,
            "labels": [
              "entailment",
              "neutral",
              "contradiction"
            ],
            "matrix": {
              "entailment": {
                "entailment": 222,
                "neutral": 30,
                "contradiction": 4
              },
              "neutral": {
                "entailment": 40,
                "neutral": 255,
                "contradiction": 18
              },
              "contradiction": {
                "entailment": 1,
                "neutral": 25,
                "contradiction": 200
              }
            }
          }
        },
        "firewall": {
          "n": 2000,
          "accuracy": 0.8785,
          "nll": 0.32404716382944027,
          "brier": 0.19121665542999927,
          "ece": 0.019892057061195338,
          "mean_confidence": 0.8801644538640976,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.0621812343597412,
            "groups": {
              "choice:2": {
                "temperature": 0.7285364649335917,
                "raw_temperature": 0.3852079510688782,
                "n": 71,
                "shrinkage_weight": 0.3717277486910995
              },
              "choice:3-5": {
                "temperature": 1.0312828950285642,
                "raw_temperature": 1.0276613235473633,
                "n": 1007,
                "shrinkage_weight": 0.8935226264418811
              },
              "choice:6-10": {
                "temperature": 1.018117407640952,
                "raw_temperature": 0.9839622974395752,
                "n": 149,
                "shrinkage_weight": 0.5539033457249071
              },
              "noul:*": {
                "temperature": 1.12278263121421,
                "raw_temperature": 1.1349645853042603,
                "n": 617,
                "shrinkage_weight": 0.8371777476255088
              },
              "score:*": {
                "temperature": 1.1063119547040072,
                "raw_temperature": 1.1415023803710938,
                "n": 156,
                "shrinkage_weight": 0.5652173913043478
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "firewall/tommccoy1-hans": {
              "n": 2000,
              "accuracy": 0.8785,
              "nll": 0.32404716382944027,
              "brier": 0.19121665542999927,
              "ece": 0.019892057061195338,
              "mean_confidence": 0.8801644538640976
            }
          },
          "by_kind": {
            "noul": {
              "n": 2000,
              "accuracy": 0.8785,
              "nll": 0.32404716382944027,
              "brier": 0.19121665542999927,
              "ece": 0.019892057061195338,
              "mean_confidence": 0.8801644538640976
            }
          },
          "by_transform": {
            "nli.firewall.hans.v1": {
              "n": 2000,
              "accuracy": 0.8785,
              "nll": 0.32404716382944027,
              "brier": 0.19121665542999927,
              "ece": 0.019892057061195338,
              "mean_confidence": 0.8801644538640976
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 2000,
              "accuracy": 0.8785,
              "nll": 0.32404716382944027,
              "brier": 0.19121665542999927,
              "ece": 0.019892057061195338,
              "mean_confidence": 0.8801644538640976
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 2000,
              "accuracy": 0.8785,
              "nll": 0.32404716382944027,
              "brier": 0.19121665542999927,
              "ece": 0.019892057061195338,
              "mean_confidence": 0.8801644538640976
            }
          },
          "by_dataset_truncation": {
            "firewall/tommccoy1-hans|not_truncated": {
              "n": 2000,
              "accuracy": 0.8785,
              "nll": 0.32404716382944027,
              "brier": 0.19121665542999927,
              "ece": 0.019892057061195338,
              "mean_confidence": 0.8801644538640976
            }
          },
          "by_kind_truncation": {
            "noul|not_truncated": {
              "n": 2000,
              "accuracy": 0.8785,
              "nll": 0.32404716382944027,
              "brier": 0.19121665542999927,
              "ece": 0.019892057061195338,
              "mean_confidence": 0.8801644538640976
            }
          }
        },
        "confusability": {
          "n": 1400,
          "accuracy": 1.0,
          "nll": 0.0022301711396669685,
          "brier": 3.975738844519126e-05,
          "ece": 0.002218638658523564,
          "mean_confidence": 0.9977813613414764,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.0621812343597412,
            "groups": {
              "choice:2": {
                "temperature": 0.7285364649335917,
                "raw_temperature": 0.3852079510688782,
                "n": 71,
                "shrinkage_weight": 0.3717277486910995
              },
              "choice:3-5": {
                "temperature": 1.0312828950285642,
                "raw_temperature": 1.0276613235473633,
                "n": 1007,
                "shrinkage_weight": 0.8935226264418811
              },
              "choice:6-10": {
                "temperature": 1.018117407640952,
                "raw_temperature": 0.9839622974395752,
                "n": 149,
                "shrinkage_weight": 0.5539033457249071
              },
              "noul:*": {
                "temperature": 1.12278263121421,
                "raw_temperature": 1.1349645853042603,
                "n": 617,
                "shrinkage_weight": 0.8371777476255088
              },
              "score:*": {
                "temperature": 1.1063119547040072,
                "raw_temperature": 1.1415023803710938,
                "n": 156,
                "shrinkage_weight": 0.5652173913043478
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "v142-confusability": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.0022301711396669685,
              "brier": 3.975738844519126e-05,
              "ece": 0.002218638658523564,
              "mean_confidence": 0.9977813613414764
            }
          },
          "by_kind": {
            "choice": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.001581937825678324,
              "brier": 3.273811125628399e-05,
              "ece": 0.0015734966397285,
              "mean_confidence": 0.9984265033602715
            },
            "noul": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0006024025060469285,
              "brier": 8.392574431326238e-07,
              "ece": 0.000602192580699934,
              "mean_confidence": 0.9993978074193001
            },
            "score": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.007099106343230232,
              "brier": 0.00011377190539178627,
              "ece": 0.007060794830322292,
              "mean_confidence": 0.9929392051696777
            }
          },
          "by_transform": {
            "coreference": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0031666626091100625,
              "brier": 8.986054856441328e-05,
              "ece": 0.00314338117837909,
              "mean_confidence": 0.9968566188216209
            },
            "entity_binding": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 5.6076028463394324e-05,
              "brier": 6.457422713664063e-09,
              "ece": 5.607396364215589e-05,
              "mean_confidence": 0.9999439260363578
            },
            "negation": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0006024025060469285,
              "brier": 8.392574431326238e-07,
              "ece": 0.000602192580699934,
              "mean_confidence": 0.9993978074193001
            },
            "numeric_boundary": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 8.565005251966796e-05,
              "brier": 2.4895733549690215e-08,
              "ece": 8.564382791520497e-05,
              "mean_confidence": 0.9999143561720848
            },
            "ordinal_boundary": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.007099106343230232,
              "brier": 0.00011377190539178627,
              "ece": 0.007060794830322292,
              "mean_confidence": 0.9929392051696777
            },
            "temporal_state": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0034405127598438412,
              "brier": 3.449978205878779e-05,
              "ece": 0.0034316426515579312,
              "mean_confidence": 0.9965683573484421
            },
            "tool_argument_validity": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0011607876784546533,
              "brier": 3.929887250195552e-05,
              "ece": 0.0011507415771484508,
              "mean_confidence": 0.9988492584228515
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 400,
              "accuracy": 1.0,
              "nll": 0.0003440262792832982,
              "brier": 4.32076588341157e-07,
              "ece": 0.0003439182043075695,
              "mean_confidence": 0.9996560817956924
            },
            "3-5": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.0029846290838204366,
              "brier": 5.5487513187931304e-05,
              "ece": 0.002968526840209984,
              "mean_confidence": 0.99703147315979
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.0022301711396669685,
              "brier": 3.975738844519126e-05,
              "ece": 0.002218638658523564,
              "mean_confidence": 0.9977813613414764
            }
          },
          "by_dataset_truncation": {
            "v142-confusability|not_truncated": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.0022301711396669685,
              "brier": 3.975738844519126e-05,
              "ece": 0.002218638658523564,
              "mean_confidence": 0.9977813613414764
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.001581937825678324,
              "brier": 3.273811125628399e-05,
              "ece": 0.0015734966397285,
              "mean_confidence": 0.9984265033602715
            },
            "noul|not_truncated": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0006024025060469285,
              "brier": 8.392574431326238e-07,
              "ece": 0.000602192580699934,
              "mean_confidence": 0.9993978074193001
            },
            "score|not_truncated": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.007099106343230232,
              "brier": 0.00011377190539178627,
              "ece": 0.007060794830322292,
              "mean_confidence": 0.9929392051696777
            }
          }
        }
      },
      "latency": {
        "validation": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 3746.8793283732375,
          "effective_ms_per_example": 468.3599160466547,
          "skipped_unusable": 0
        },
        "schema": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 3996.121177363326,
          "effective_ms_per_example": 499.51514717041573,
          "skipped_unusable": 0
        },
        "firewall": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 335.970186118735,
          "effective_ms_per_example": 41.996273264841875,
          "skipped_unusable": 0
        },
        "confusability": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 658.859060741961,
          "effective_ms_per_example": 82.35738259274513,
          "skipped_unusable": 0
        }
      }
    },
    "head_lora_rlcd": {
      "calibration": {
        "version": 2,
        "type": "primitive_option_temperature",
        "global_temperature": 1.2130241394042969,
        "groups": {
          "choice:2": {
            "temperature": 1.131178691354451,
            "raw_temperature": 0.9999842643737793,
            "n": 68,
            "shrinkage_weight": 0.3617021276595745
          },
          "choice:3-5": {
            "temperature": 1.229406620643328,
            "raw_temperature": 1.2313735485076904,
            "n": 1007,
            "shrinkage_weight": 0.8935226264418811
          },
          "choice:6-10": {
            "temperature": 1.1287873284307708,
            "raw_temperature": 1.0643781423568726,
            "n": 147,
            "shrinkage_weight": 0.550561797752809
          },
          "noul:*": {
            "temperature": 1.2522414435081368,
            "raw_temperature": 1.259989619255066,
            "n": 619,
            "shrinkage_weight": 0.8376184032476319
          },
          "score:*": {
            "temperature": 1.1708337953943981,
            "raw_temperature": 1.1399664878845215,
            "n": 159,
            "shrinkage_weight": 0.5698924731182796
          }
        },
        "min_group": 50,
        "prior_strength": 120.0,
        "n": 2000
      },
      "sets": {
        "validation": {
          "n": 2000,
          "accuracy": 0.874,
          "nll": 0.3413633666240774,
          "brier": 0.18889581276680206,
          "ece": 0.022634778514504437,
          "mean_confidence": 0.8636561000198126,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.2130241394042969,
            "groups": {
              "choice:2": {
                "temperature": 1.131178691354451,
                "raw_temperature": 0.9999842643737793,
                "n": 68,
                "shrinkage_weight": 0.3617021276595745
              },
              "choice:3-5": {
                "temperature": 1.229406620643328,
                "raw_temperature": 1.2313735485076904,
                "n": 1007,
                "shrinkage_weight": 0.8935226264418811
              },
              "choice:6-10": {
                "temperature": 1.1287873284307708,
                "raw_temperature": 1.0643781423568726,
                "n": 147,
                "shrinkage_weight": 0.550561797752809
              },
              "noul:*": {
                "temperature": 1.2522414435081368,
                "raw_temperature": 1.259989619255066,
                "n": 619,
                "shrinkage_weight": 0.8376184032476319
              },
              "score:*": {
                "temperature": 1.1708337953943981,
                "raw_temperature": 1.1399664878845215,
                "n": 159,
                "shrinkage_weight": 0.5698924731182796
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "PolyAI/banking77": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18907353777312358,
              "brier": 0.10504818881286704,
              "ece": 0.06510368646944269,
              "mean_confidence": 0.9324855239075773
            },
            "Rowan/hellaswag": {
              "n": 154,
              "accuracy": 0.922077922077922,
              "nll": 0.21983189224844757,
              "brier": 0.11673031982369635,
              "ece": 0.040538395186523364,
              "mean_confidence": 0.9012584014759435
            },
            "alisawuffles/WANLI": {
              "n": 156,
              "accuracy": 0.7564102564102564,
              "nll": 0.5891167478373226,
              "brier": 0.35234179292866685,
              "ece": 0.06832877985942057,
              "mean_confidence": 0.7601621380219092
            },
            "cais/mmlu": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.40682872855269997,
              "brier": 0.20491553394237635,
              "ece": 0.04900335096845434,
              "mean_confidence": 0.8429947429233127
            },
            "google/boolq": {
              "n": 153,
              "accuracy": 0.8758169934640523,
              "nll": 0.3271605430153393,
              "brier": 0.18819155338973212,
              "ece": 0.05587063856374201,
              "mean_confidence": 0.8991910511372136
            },
            "google/boolq-negation": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.3173225029292547,
              "brier": 0.18408045429418515,
              "ece": 0.0648853922981063,
              "mean_confidence": 0.8740317903312982
            },
            "nvidia/HelpSteer2": {
              "n": 159,
              "accuracy": 0.660377358490566,
              "nll": 0.8923262642614497,
              "brier": 0.4767587937402931,
              "ece": 0.0820022630241682,
              "mean_confidence": 0.6223008767613825
            },
            "nvidia/HelpSteer2-threshold": {
              "n": 158,
              "accuracy": 0.8860759493670886,
              "nll": 0.24275886275381134,
              "brier": 0.14648212109415587,
              "ece": 0.04942092601257034,
              "mean_confidence": 0.8698332471938073
            },
            "nyu-mll/multi_nli": {
              "n": 157,
              "accuracy": 0.910828025477707,
              "nll": 0.3416944928808363,
              "brier": 0.17827376386410335,
              "ece": 0.08725254399001975,
              "mean_confidence": 0.844191101705952
            },
            "product-science/xlam-function-calling-60k-raw": {
              "n": 154,
              "accuracy": 1.0,
              "nll": 0.0060333324190392805,
              "brier": 0.002306750856381662,
              "ece": 0.005282950865757921,
              "mean_confidence": 0.9947170491342421
            },
            "product-science/xlam-hard": {
              "n": 155,
              "accuracy": 0.967741935483871,
              "nll": 0.06760209277854301,
              "brier": 0.040693268881268306,
              "ece": 0.02164438040025776,
              "mean_confidence": 0.9642918621340105
            },
            "stanfordnlp/snli": {
              "n": 158,
              "accuracy": 0.8607594936708861,
              "nll": 0.405171424059103,
              "brier": 0.22694251017077752,
              "ece": 0.06340998675249801,
              "mean_confidence": 0.8710687118995039
            },
            "synthetic/nli-haystack": {
              "n": 154,
              "accuracy": 0.8636363636363636,
              "nll": 0.39674904061384597,
              "brier": 0.21296215491428802,
              "ece": 0.03454946155671952,
              "mean_confidence": 0.8677647721457791
            }
          },
          "by_kind": {
            "choice": {
              "n": 1222,
              "accuracy": 0.8887070376432079,
              "nll": 0.32193676473427585,
              "brier": 0.17641385298182077,
              "ece": 0.02652165657077016,
              "mean_confidence": 0.8757482603948362
            },
            "noul": {
              "n": 619,
              "accuracy": 0.8998384491114702,
              "nll": 0.23819084123634773,
              "brier": 0.13959490950745163,
              "ece": 0.017604535538853695,
              "mean_confidence": 0.9017802688724967
            },
            "score": {
              "n": 159,
              "accuracy": 0.660377358490566,
              "nll": 0.8923262642614497,
              "brier": 0.4767587937402931,
              "ece": 0.0820022630241682,
              "mean_confidence": 0.6223008767613825
            }
          },
          "by_transform": {
            "agent.invocation.correct.v1": {
              "n": 52,
              "accuracy": 0.9230769230769231,
              "nll": 0.1550768981419074,
              "brier": 0.09798712961044377,
              "ece": 0.06195802413500273,
              "mean_confidence": 0.9290768962640029
            },
            "agent.invocation.corrupt_argument.v1": {
              "n": 58,
              "accuracy": 1.0,
              "nll": 0.015422354992740016,
              "brier": 0.007456194708894294,
              "ece": 0.01274871106805474,
              "mean_confidence": 0.9872512889319452
            },
            "agent.invocation.wrong_tool.v1": {
              "n": 45,
              "accuracy": 0.9777777777777777,
              "nll": 0.03377397972702359,
              "brier": 0.017325925416392034,
              "ece": 0.02705154154035782,
              "mean_confidence": 0.9753925614886814
            },
            "commonsense.choice.v2": {
              "n": 154,
              "accuracy": 0.922077922077922,
              "nll": 0.21983189224844757,
              "brier": 0.11673031982369635,
              "ece": 0.040538395186523364,
              "mean_confidence": 0.9012584014759435
            },
            "intent.choice.v3": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18907353777312358,
              "brier": 0.10504818881286704,
              "ece": 0.06510368646944269,
              "mean_confidence": 0.9324855239075773
            },
            "knowledge.choice.v1": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.40682872855269997,
              "brier": 0.20491553394237635,
              "ece": 0.04900335096845434,
              "mean_confidence": 0.8429947429233127
            },
            "nli.choice.v4": {
              "n": 157,
              "accuracy": 0.910828025477707,
              "nll": 0.3416944928808363,
              "brier": 0.17827376386410335,
              "ece": 0.08725254399001975,
              "mean_confidence": 0.844191101705952
            },
            "nli.hard.v4": {
              "n": 156,
              "accuracy": 0.7564102564102564,
              "nll": 0.5891167478373226,
              "brier": 0.35234179292866685,
              "ece": 0.06832877985942057,
              "mean_confidence": 0.7601621380219092
            },
            "nli.haystack.absent.v1": {
              "n": 6,
              "accuracy": 1.0,
              "nll": 0.12007932023455699,
              "brier": 0.042778262247641884,
              "ece": 0.10573691129684447,
              "mean_confidence": 0.8942630887031555
            },
            "nli.haystack.present.v1": {
              "n": 148,
              "accuracy": 0.8581081081081081,
              "nll": 0.40796538062922255,
              "brier": 0.2198615019142872,
              "ece": 0.038296295178903136,
              "mean_confidence": 0.8666905160691287
            },
            "nli.snli.v1": {
              "n": 158,
              "accuracy": 0.8607594936708861,
              "nll": 0.405171424059103,
              "brier": 0.22694251017077752,
              "ece": 0.06340998675249801,
              "mean_confidence": 0.8710687118995039
            },
            "noul.negated_answer.v1": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.3173225029292547,
              "brier": 0.18408045429418515,
              "ece": 0.0648853922981063,
              "mean_confidence": 0.8740317903312982
            },
            "noul.v3": {
              "n": 153,
              "accuracy": 0.8758169934640523,
              "nll": 0.3271605430153393,
              "brier": 0.18819155338973212,
              "ece": 0.05587063856374201,
              "mean_confidence": 0.8991910511372136
            },
            "score.coherence.threshold.v1": {
              "n": 36,
              "accuracy": 0.9444444444444444,
              "nll": 0.15781310862965053,
              "brier": 0.08520178628396025,
              "ece": 0.08752633300092487,
              "mean_confidence": 0.8862901313437356
            },
            "score.coherence.v3": {
              "n": 36,
              "accuracy": 0.8611111111111112,
              "nll": 0.49905938903490704,
              "brier": 0.27527502324018216,
              "ece": 0.14887905451986522,
              "mean_confidence": 0.7211348878012763
            },
            "score.complexity.threshold.v1": {
              "n": 28,
              "accuracy": 0.9285714285714286,
              "nll": 0.19121888622508518,
              "brier": 0.10774182653299899,
              "ece": 0.1085933659757887,
              "mean_confidence": 0.9150230714253017
            },
            "score.complexity.v3": {
              "n": 30,
              "accuracy": 0.8,
              "nll": 0.6180950621763865,
              "brier": 0.36170755798618,
              "ece": 0.1601276993751526,
              "mean_confidence": 0.6420709570248921
            },
            "score.correctness.threshold.v1": {
              "n": 30,
              "accuracy": 0.8,
              "nll": 0.3484654218579332,
              "brier": 0.23496840521499204,
              "ece": 0.13440750638643903,
              "mean_confidence": 0.8265082140763601
            },
            "score.correctness.v3": {
              "n": 22,
              "accuracy": 0.5,
              "nll": 1.0662444193254819,
              "brier": 0.5692707571116361,
              "ece": 0.18789786439050324,
              "mean_confidence": 0.49144711751829495
            },
            "score.helpfulness.threshold.v1": {
              "n": 30,
              "accuracy": 0.8,
              "nll": 0.3833232717588544,
              "brier": 0.2426174452722383,
              "ece": 0.1406406263510386,
              "mean_confidence": 0.7806143343448639
            },
            "score.helpfulness.v3": {
              "n": 32,
              "accuracy": 0.40625,
              "nll": 1.2641449673101306,
              "brier": 0.6717381589114666,
              "ece": 0.10654595633968711,
              "mean_confidence": 0.4571774401701987
            },
            "score.verbosity.threshold.v1": {
              "n": 34,
              "accuracy": 0.9411764705882353,
              "nll": 0.15784819945966935,
              "brier": 0.08036953438568162,
              "ece": 0.047484531122095476,
              "mean_confidence": 0.9321437022265266
            },
            "score.verbosity.v3": {
              "n": 39,
              "accuracy": 0.6666666666666666,
              "nll": 1.0630992299471147,
              "brier": 0.5390754586372238,
              "ece": 0.19610086618325653,
              "mean_confidence": 0.7251625657081604
            },
            "tool.choice.single.v3": {
              "n": 154,
              "accuracy": 1.0,
              "nll": 0.0060333324190392805,
              "brier": 0.002306750856381662,
              "ece": 0.005282950865757921,
              "mean_confidence": 0.9947170491342421
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 687,
              "accuracy": 0.9097525473071325,
              "nll": 0.21461692523512385,
              "brier": 0.1257776561957685,
              "ece": 0.015864489380449164,
              "mean_confidence": 0.9114996939246831
            },
            "3-5": {
              "n": 1166,
              "accuracy": 0.8447684391080618,
              "nll": 0.4370019674408599,
              "brier": 0.2376457747592827,
              "ece": 0.03868178026447169,
              "mean_confidence": 0.8261751901888724
            },
            "6-10": {
              "n": 147,
              "accuracy": 0.9387755102040817,
              "nll": 0.1751062011944359,
              "brier": 0.09719321331828265,
              "ece": 0.0604115242049808,
              "mean_confidence": 0.9373580853955276
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 1999,
              "accuracy": 0.8744372186093047,
              "nll": 0.34017474692719035,
              "brier": 0.18811760026757368,
              "ece": 0.022178890631758277,
              "mean_confidence": 0.8636209331583297
            },
            "truncated": {
              "n": 1,
              "accuracy": 0.0,
              "nll": 2.717414140701294,
              "brier": 1.7445425987243652,
              "ece": 0.933954656124115,
              "mean_confidence": 0.933954656124115
            }
          },
          "by_dataset_truncation": {
            "PolyAI/banking77|not_truncated": {
              "n": 136,
              "accuracy": 0.9338235294117647,
              "nll": 0.18907353777312358,
              "brier": 0.10504818881286704,
              "ece": 0.06510368646944269,
              "mean_confidence": 0.9324855239075773
            },
            "Rowan/hellaswag|not_truncated": {
              "n": 154,
              "accuracy": 0.922077922077922,
              "nll": 0.21983189224844757,
              "brier": 0.11673031982369635,
              "ece": 0.040538395186523364,
              "mean_confidence": 0.9012584014759435
            },
            "alisawuffles/WANLI|not_truncated": {
              "n": 156,
              "accuracy": 0.7564102564102564,
              "nll": 0.5891167478373226,
              "brier": 0.35234179292866685,
              "ece": 0.06832877985942057,
              "mean_confidence": 0.7601621380219092
            },
            "cais/mmlu|not_truncated": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.40682872855269997,
              "brier": 0.20491553394237635,
              "ece": 0.04900335096845434,
              "mean_confidence": 0.8429947429233127
            },
            "google/boolq-negation|not_truncated": {
              "n": 153,
              "accuracy": 0.869281045751634,
              "nll": 0.3173225029292547,
              "brier": 0.18408045429418515,
              "ece": 0.0648853922981063,
              "mean_confidence": 0.8740317903312982
            },
            "google/boolq|not_truncated": {
              "n": 153,
              "accuracy": 0.8758169934640523,
              "nll": 0.3271605430153393,
              "brier": 0.18819155338973212,
              "ece": 0.05587063856374201,
              "mean_confidence": 0.8991910511372136
            },
            "nvidia/HelpSteer2-threshold|not_truncated": {
              "n": 157,
              "accuracy": 0.89171974522293,
              "nll": 0.22699672722548342,
              "brier": 0.13630339193727556,
              "ece": 0.055684464752294466,
              "mean_confidence": 0.8694248305764168
            },
            "nvidia/HelpSteer2-threshold|truncated": {
              "n": 1,
              "accuracy": 0.0,
              "nll": 2.717414140701294,
              "brier": 1.7445425987243652,
              "ece": 0.933954656124115,
              "mean_confidence": 0.933954656124115
            },
            "nvidia/HelpSteer2|not_truncated": {
              "n": 159,
              "accuracy": 0.660377358490566,
              "nll": 0.8923262642614497,
              "brier": 0.4767587937402931,
              "ece": 0.0820022630241682,
              "mean_confidence": 0.6223008767613825
            },
            "nyu-mll/multi_nli|not_truncated": {
              "n": 157,
              "accuracy": 0.910828025477707,
              "nll": 0.3416944928808363,
              "brier": 0.17827376386410335,
              "ece": 0.08725254399001975,
              "mean_confidence": 0.844191101705952
            },
            "product-science/xlam-function-calling-60k-raw|not_truncated": {
              "n": 154,
              "accuracy": 1.0,
              "nll": 0.0060333324190392805,
              "brier": 0.002306750856381662,
              "ece": 0.005282950865757921,
              "mean_confidence": 0.9947170491342421
            },
            "product-science/xlam-hard|not_truncated": {
              "n": 155,
              "accuracy": 0.967741935483871,
              "nll": 0.06760209277854301,
              "brier": 0.040693268881268306,
              "ece": 0.02164438040025776,
              "mean_confidence": 0.9642918621340105
            },
            "stanfordnlp/snli|not_truncated": {
              "n": 158,
              "accuracy": 0.8607594936708861,
              "nll": 0.405171424059103,
              "brier": 0.22694251017077752,
              "ece": 0.06340998675249801,
              "mean_confidence": 0.8710687118995039
            },
            "synthetic/nli-haystack|not_truncated": {
              "n": 154,
              "accuracy": 0.8636363636363636,
              "nll": 0.39674904061384597,
              "brier": 0.21296215491428802,
              "ece": 0.03454946155671952,
              "mean_confidence": 0.8677647721457791
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1222,
              "accuracy": 0.8887070376432079,
              "nll": 0.32193676473427585,
              "brier": 0.17641385298182077,
              "ece": 0.02652165657077016,
              "mean_confidence": 0.8757482603948362
            },
            "noul|not_truncated": {
              "n": 618,
              "accuracy": 0.901294498381877,
              "nll": 0.23417915304951123,
              "brier": 0.13699790677409093,
              "ece": 0.019144275331188614,
              "mean_confidence": 0.9017282067572029
            },
            "noul|truncated": {
              "n": 1,
              "accuracy": 0.0,
              "nll": 2.717414140701294,
              "brier": 1.7445425987243652,
              "ece": 0.933954656124115,
              "mean_confidence": 0.933954656124115
            },
            "score|not_truncated": {
              "n": 159,
              "accuracy": 0.660377358490566,
              "nll": 0.8923262642614497,
              "brier": 0.4767587937402931,
              "ece": 0.0820022630241682,
              "mean_confidence": 0.6223008767613825
            }
          },
          "nli_confusion": {
            "n": 625,
            "labels": [
              "entailment",
              "neutral",
              "contradiction"
            ],
            "matrix": {
              "entailment": {
                "entailment": 179,
                "neutral": 28,
                "contradiction": 6
              },
              "neutral": {
                "entailment": 17,
                "neutral": 178,
                "contradiction": 17
              },
              "contradiction": {
                "entailment": 0,
                "neutral": 27,
                "contradiction": 173
              }
            }
          }
        },
        "schema": {
          "n": 2000,
          "accuracy": 0.777,
          "nll": 0.48716114530798577,
          "brier": 0.2934196735615337,
          "ece": 0.046331627137959026,
          "mean_confidence": 0.8208303308263422,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.2130241394042969,
            "groups": {
              "choice:2": {
                "temperature": 1.131178691354451,
                "raw_temperature": 0.9999842643737793,
                "n": 68,
                "shrinkage_weight": 0.3617021276595745
              },
              "choice:3-5": {
                "temperature": 1.229406620643328,
                "raw_temperature": 1.2313735485076904,
                "n": 1007,
                "shrinkage_weight": 0.8935226264418811
              },
              "choice:6-10": {
                "temperature": 1.1287873284307708,
                "raw_temperature": 1.0643781423568726,
                "n": 147,
                "shrinkage_weight": 0.550561797752809
              },
              "noul:*": {
                "temperature": 1.2522414435081368,
                "raw_temperature": 1.259989619255066,
                "n": 619,
                "shrinkage_weight": 0.8376184032476319
              },
              "score:*": {
                "temperature": 1.1708337953943981,
                "raw_temperature": 1.1399664878845215,
                "n": 159,
                "shrinkage_weight": 0.5698924731182796
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "schema-firewall/PolyAI/banking77": {
              "n": 46,
              "accuracy": 0.8043478260869565,
              "nll": 0.35682467644105403,
              "brier": 0.2239034981356199,
              "ece": 0.12990026823852374,
              "mean_confidence": 0.8811038130003473
            },
            "schema-firewall/Rowan/hellaswag": {
              "n": 130,
              "accuracy": 0.9076923076923077,
              "nll": 0.20287465754243106,
              "brier": 0.11677193438268157,
              "ece": 0.047463341171924855,
              "mean_confidence": 0.9111302653184303
            },
            "schema-firewall/alisawuffles/WANLI": {
              "n": 205,
              "accuracy": 0.7219512195121951,
              "nll": 0.6024273006109203,
              "brier": 0.36376239454186315,
              "ece": 0.06989647847850149,
              "mean_confidence": 0.7581731124622065
            },
            "schema-firewall/cais/mmlu": {
              "n": 134,
              "accuracy": 0.917910447761194,
              "nll": 0.2275545184470434,
              "brier": 0.11969006923772431,
              "ece": 0.05594150722026825,
              "mean_confidence": 0.9056024651474027
            },
            "schema-firewall/google/boolq": {
              "n": 36,
              "accuracy": 0.8888888888888888,
              "nll": 0.42648325838106554,
              "brier": 0.2071607477258826,
              "ece": 0.13994142909844717,
              "mean_confidence": 0.8997285084591972
            },
            "schema-firewall/google/boolq-negation": {
              "n": 60,
              "accuracy": 0.5166666666666667,
              "nll": 1.2815287677299543,
              "brier": 0.7706017639355196,
              "ece": 0.36829513410727177,
              "mean_confidence": 0.8849618007739385
            },
            "schema-firewall/nvidia/HelpSteer2": {
              "n": 273,
              "accuracy": 0.608058608058608,
              "nll": 0.9225735204778748,
              "brier": 0.502879177783053,
              "ece": 0.05189086431330377,
              "mean_confidence": 0.6023068452601904
            },
            "schema-firewall/nvidia/HelpSteer2-threshold": {
              "n": 263,
              "accuracy": 0.6121673003802282,
              "nll": 0.7050396920068494,
              "brier": 0.5057733078411408,
              "ece": 0.20569920426539143,
              "mean_confidence": 0.8013089970037511
            },
            "schema-firewall/nyu-mll/multi_nli": {
              "n": 214,
              "accuracy": 0.8691588785046729,
              "nll": 0.3600308844127701,
              "brier": 0.19274686407703126,
              "ece": 0.06172461933064685,
              "mean_confidence": 0.8438963853867254
            },
            "schema-firewall/product-science/xlam-function-calling-60k-raw": {
              "n": 81,
              "accuracy": 1.0,
              "nll": 0.0060919241069499275,
              "brier": 0.00158877624618902,
              "ece": 0.005427672539228313,
              "mean_confidence": 0.9945723274607717
            },
            "schema-firewall/product-science/xlam-hard": {
              "n": 202,
              "accuracy": 0.7673267326732673,
              "nll": 0.3468692268006905,
              "brier": 0.24214135508483428,
              "ece": 0.09207318767462629,
              "mean_confidence": 0.8529215121623313
            },
            "schema-firewall/stanfordnlp/snli": {
              "n": 202,
              "accuracy": 0.8811881188118812,
              "nll": 0.2818646928258609,
              "brier": 0.16245338230385975,
              "ece": 0.040539493360141704,
              "mean_confidence": 0.8788761552607659
            },
            "schema-firewall/synthetic/nli-haystack": {
              "n": 154,
              "accuracy": 0.8961038961038961,
              "nll": 0.28225524406940616,
              "brier": 0.15355019212951115,
              "ece": 0.0682181323116476,
              "mean_confidence": 0.8718665600210042
            }
          },
          "by_kind": {
            "choice": {
              "n": 1166,
              "accuracy": 0.8653516295025729,
              "nll": 0.3213735620484178,
              "brier": 0.1834721313419164,
              "ece": 0.029624137982700523,
              "mean_confidence": 0.8651016697057524
            },
            "noul": {
              "n": 561,
              "accuracy": 0.6755793226381461,
              "nll": 0.6198540930072308,
              "brier": 0.4200086032863093,
              "ece": 0.15957637436674665,
              "mean_confidence": 0.8351556970048928
            },
            "score": {
              "n": 273,
              "accuracy": 0.608058608058608,
              "nll": 0.9225735204778748,
              "brier": 0.502879177783053,
              "ece": 0.05189086431330377,
              "mean_confidence": 0.6023068452601904
            }
          },
          "by_transform": {
            "schema.firewall.distractor_drop.v1": {
              "n": 179,
              "accuracy": 0.9162011173184358,
              "nll": 0.18068733821303656,
              "brier": 0.1011414483847378,
              "ece": 0.032388108236163664,
              "mean_confidence": 0.9310932723836526
            },
            "schema.firewall.instruction.v1": {
              "n": 1416,
              "accuracy": 0.8509887005649718,
              "nll": 0.36921639851671667,
              "brier": 0.2073376011020019,
              "ece": 0.02585129172535939,
              "mean_confidence": 0.8416002465801388
            },
            "schema.firewall.noul_inversion.v1": {
              "n": 273,
              "accuracy": 0.3882783882783883,
              "nll": 1.0767006237783057,
              "brier": 0.7550764729788241,
              "ece": 0.35771637259822187,
              "mean_confidence": 0.7459947608766102
            },
            "schema.firewall.score_scale.v1": {
              "n": 132,
              "accuracy": 0.5984848484848485,
              "nll": 0.9487088370052251,
              "brier": 0.5227962695344379,
              "ece": 0.10626354102383959,
              "mean_confidence": 0.6032761137819651
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 588,
              "accuracy": 0.6904761904761905,
              "nll": 0.5913940336594421,
              "brier": 0.4007224980486256,
              "ece": 0.15224626021725784,
              "mean_confidence": 0.8427224506934484
            },
            "3-5": {
              "n": 1381,
              "accuracy": 0.8117306299782766,
              "nll": 0.44767054287509334,
              "brier": 0.25053120708128784,
              "ece": 0.027608664106148723,
              "mean_confidence": 0.8095618236397246
            },
            "6-10": {
              "n": 31,
              "accuracy": 0.8709677419354839,
              "nll": 0.26934126173276335,
              "brier": 0.16873939649087583,
              "ece": 0.11212440363822443,
              "mean_confidence": 0.9075800709186062
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 1999,
              "accuracy": 0.7773886943471736,
              "nll": 0.4869243776422231,
              "brier": 0.2932593345994733,
              "ece": 0.04612398467372813,
              "mean_confidence": 0.8210101314354921
            },
            "truncated": {
              "n": 1,
              "accuracy": 0.0,
              "nll": 0.9604597091674805,
              "brier": 0.613937258720398,
              "ece": 0.46140891313552856,
              "mean_confidence": 0.46140891313552856
            }
          },
          "by_dataset_truncation": {
            "schema-firewall/PolyAI/banking77|not_truncated": {
              "n": 46,
              "accuracy": 0.8043478260869565,
              "nll": 0.35682467644105403,
              "brier": 0.2239034981356199,
              "ece": 0.12990026823852374,
              "mean_confidence": 0.8811038130003473
            },
            "schema-firewall/Rowan/hellaswag|not_truncated": {
              "n": 130,
              "accuracy": 0.9076923076923077,
              "nll": 0.20287465754243106,
              "brier": 0.11677193438268157,
              "ece": 0.047463341171924855,
              "mean_confidence": 0.9111302653184303
            },
            "schema-firewall/alisawuffles/WANLI|not_truncated": {
              "n": 205,
              "accuracy": 0.7219512195121951,
              "nll": 0.6024273006109203,
              "brier": 0.36376239454186315,
              "ece": 0.06989647847850149,
              "mean_confidence": 0.7581731124622065
            },
            "schema-firewall/cais/mmlu|not_truncated": {
              "n": 134,
              "accuracy": 0.917910447761194,
              "nll": 0.2275545184470434,
              "brier": 0.11969006923772431,
              "ece": 0.05594150722026825,
              "mean_confidence": 0.9056024651474027
            },
            "schema-firewall/google/boolq-negation|not_truncated": {
              "n": 60,
              "accuracy": 0.5166666666666667,
              "nll": 1.2815287677299543,
              "brier": 0.7706017639355196,
              "ece": 0.36829513410727177,
              "mean_confidence": 0.8849618007739385
            },
            "schema-firewall/google/boolq|not_truncated": {
              "n": 36,
              "accuracy": 0.8888888888888888,
              "nll": 0.42648325838106554,
              "brier": 0.2071607477258826,
              "ece": 0.13994142909844717,
              "mean_confidence": 0.8997285084591972
            },
            "schema-firewall/nvidia/HelpSteer2-threshold|not_truncated": {
              "n": 263,
              "accuracy": 0.6121673003802282,
              "nll": 0.7050396920068494,
              "brier": 0.5057733078411408,
              "ece": 0.20569920426539143,
              "mean_confidence": 0.8013089970037511
            },
            "schema-firewall/nvidia/HelpSteer2|not_truncated": {
              "n": 272,
              "accuracy": 0.6102941176470589,
              "nll": 0.9224342330194572,
              "brier": 0.502470876014901,
              "ece": 0.050385283251457344,
              "mean_confidence": 0.60282485236359
            },
            "schema-firewall/nvidia/HelpSteer2|truncated": {
              "n": 1,
              "accuracy": 0.0,
              "nll": 0.9604597091674805,
              "brier": 0.613937258720398,
              "ece": 0.46140891313552856,
              "mean_confidence": 0.46140891313552856
            },
            "schema-firewall/nyu-mll/multi_nli|not_truncated": {
              "n": 214,
              "accuracy": 0.8691588785046729,
              "nll": 0.3600308844127701,
              "brier": 0.19274686407703126,
              "ece": 0.06172461933064685,
              "mean_confidence": 0.8438963853867254
            },
            "schema-firewall/product-science/xlam-function-calling-60k-raw|not_truncated": {
              "n": 81,
              "accuracy": 1.0,
              "nll": 0.0060919241069499275,
              "brier": 0.00158877624618902,
              "ece": 0.005427672539228313,
              "mean_confidence": 0.9945723274607717
            },
            "schema-firewall/product-science/xlam-hard|not_truncated": {
              "n": 202,
              "accuracy": 0.7673267326732673,
              "nll": 0.3468692268006905,
              "brier": 0.24214135508483428,
              "ece": 0.09207318767462629,
              "mean_confidence": 0.8529215121623313
            },
            "schema-firewall/stanfordnlp/snli|not_truncated": {
              "n": 202,
              "accuracy": 0.8811881188118812,
              "nll": 0.2818646928258609,
              "brier": 0.16245338230385975,
              "ece": 0.040539493360141704,
              "mean_confidence": 0.8788761552607659
            },
            "schema-firewall/synthetic/nli-haystack|not_truncated": {
              "n": 154,
              "accuracy": 0.8961038961038961,
              "nll": 0.28225524406940616,
              "brier": 0.15355019212951115,
              "ece": 0.0682181323116476,
              "mean_confidence": 0.8718665600210042
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1166,
              "accuracy": 0.8653516295025729,
              "nll": 0.3213735620484178,
              "brier": 0.1834721313419164,
              "ece": 0.029624137982700523,
              "mean_confidence": 0.8651016697057524
            },
            "noul|not_truncated": {
              "n": 561,
              "accuracy": 0.6755793226381461,
              "nll": 0.6198540930072308,
              "brier": 0.4200086032863093,
              "ece": 0.15957637436674665,
              "mean_confidence": 0.8351556970048928
            },
            "score|not_truncated": {
              "n": 272,
              "accuracy": 0.6102941176470589,
              "nll": 0.9224342330194572,
              "brier": 0.502470876014901,
              "ece": 0.050385283251457344,
              "mean_confidence": 0.60282485236359
            },
            "score|truncated": {
              "n": 1,
              "accuracy": 0.0,
              "nll": 0.9604597091674805,
              "brier": 0.613937258720398,
              "ece": 0.46140891313552856,
              "mean_confidence": 0.46140891313552856
            }
          },
          "nli_confusion": {
            "n": 775,
            "labels": [
              "entailment",
              "neutral",
              "contradiction"
            ],
            "matrix": {
              "entailment": {
                "entailment": 201,
                "neutral": 38,
                "contradiction": 1
              },
              "neutral": {
                "entailment": 31,
                "neutral": 263,
                "contradiction": 22
              },
              "contradiction": {
                "entailment": 4,
                "neutral": 29,
                "contradiction": 186
              }
            }
          }
        },
        "firewall": {
          "n": 2000,
          "accuracy": 0.889,
          "nll": 0.3097366738639539,
          "brier": 0.1801932942780604,
          "ece": 0.02010454627871515,
          "mean_confidence": 0.8843429988920689,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.2130241394042969,
            "groups": {
              "choice:2": {
                "temperature": 1.131178691354451,
                "raw_temperature": 0.9999842643737793,
                "n": 68,
                "shrinkage_weight": 0.3617021276595745
              },
              "choice:3-5": {
                "temperature": 1.229406620643328,
                "raw_temperature": 1.2313735485076904,
                "n": 1007,
                "shrinkage_weight": 0.8935226264418811
              },
              "choice:6-10": {
                "temperature": 1.1287873284307708,
                "raw_temperature": 1.0643781423568726,
                "n": 147,
                "shrinkage_weight": 0.550561797752809
              },
              "noul:*": {
                "temperature": 1.2522414435081368,
                "raw_temperature": 1.259989619255066,
                "n": 619,
                "shrinkage_weight": 0.8376184032476319
              },
              "score:*": {
                "temperature": 1.1708337953943981,
                "raw_temperature": 1.1399664878845215,
                "n": 159,
                "shrinkage_weight": 0.5698924731182796
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "firewall/tommccoy1-hans": {
              "n": 2000,
              "accuracy": 0.889,
              "nll": 0.3097366738639539,
              "brier": 0.1801932942780604,
              "ece": 0.02010454627871515,
              "mean_confidence": 0.8843429988920689
            }
          },
          "by_kind": {
            "noul": {
              "n": 2000,
              "accuracy": 0.889,
              "nll": 0.3097366738639539,
              "brier": 0.1801932942780604,
              "ece": 0.02010454627871515,
              "mean_confidence": 0.8843429988920689
            }
          },
          "by_transform": {
            "nli.firewall.hans.v1": {
              "n": 2000,
              "accuracy": 0.889,
              "nll": 0.3097366738639539,
              "brier": 0.1801932942780604,
              "ece": 0.02010454627871515,
              "mean_confidence": 0.8843429988920689
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 2000,
              "accuracy": 0.889,
              "nll": 0.3097366738639539,
              "brier": 0.1801932942780604,
              "ece": 0.02010454627871515,
              "mean_confidence": 0.8843429988920689
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 2000,
              "accuracy": 0.889,
              "nll": 0.3097366738639539,
              "brier": 0.1801932942780604,
              "ece": 0.02010454627871515,
              "mean_confidence": 0.8843429988920689
            }
          },
          "by_dataset_truncation": {
            "firewall/tommccoy1-hans|not_truncated": {
              "n": 2000,
              "accuracy": 0.889,
              "nll": 0.3097366738639539,
              "brier": 0.1801932942780604,
              "ece": 0.02010454627871515,
              "mean_confidence": 0.8843429988920689
            }
          },
          "by_kind_truncation": {
            "noul|not_truncated": {
              "n": 2000,
              "accuracy": 0.889,
              "nll": 0.3097366738639539,
              "brier": 0.1801932942780604,
              "ece": 0.02010454627871515,
              "mean_confidence": 0.8843429988920689
            }
          }
        },
        "confusability": {
          "n": 1400,
          "accuracy": 1.0,
          "nll": 0.0021607741066457364,
          "brier": 3.6698297335017013e-05,
          "ece": 0.0021507594840867515,
          "mean_confidence": 0.9978492405159133,
          "calibration": {
            "version": 2,
            "type": "primitive_option_temperature",
            "global_temperature": 1.2130241394042969,
            "groups": {
              "choice:2": {
                "temperature": 1.131178691354451,
                "raw_temperature": 0.9999842643737793,
                "n": 68,
                "shrinkage_weight": 0.3617021276595745
              },
              "choice:3-5": {
                "temperature": 1.229406620643328,
                "raw_temperature": 1.2313735485076904,
                "n": 1007,
                "shrinkage_weight": 0.8935226264418811
              },
              "choice:6-10": {
                "temperature": 1.1287873284307708,
                "raw_temperature": 1.0643781423568726,
                "n": 147,
                "shrinkage_weight": 0.550561797752809
              },
              "noul:*": {
                "temperature": 1.2522414435081368,
                "raw_temperature": 1.259989619255066,
                "n": 619,
                "shrinkage_weight": 0.8376184032476319
              },
              "score:*": {
                "temperature": 1.1708337953943981,
                "raw_temperature": 1.1399664878845215,
                "n": 159,
                "shrinkage_weight": 0.5698924731182796
              }
            },
            "min_group": 50,
            "prior_strength": 120.0,
            "n": 2000
          },
          "by_dataset": {
            "v142-confusability": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.0021607741066457364,
              "brier": 3.6698297335017013e-05,
              "ece": 0.0021507594840867515,
              "mean_confidence": 0.9978492405159133
            }
          },
          "by_kind": {
            "choice": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.001977618149085174,
              "brier": 4.1289017551536136e-05,
              "ece": 0.001966866195201821,
              "mean_confidence": 0.9980331338047981
            },
            "noul": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0006506792725122068,
              "brier": 9.928545260606826e-07,
              "ece": 0.0006504309177398637,
              "mean_confidence": 0.9993495690822601
            },
            "score": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.004586648728582077,
              "brier": 4.945013906137774e-05,
              "ece": 0.00457055449485777,
              "mean_confidence": 0.9954294455051422
            }
          },
          "by_transform": {
            "coreference": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.00249042555813503,
              "brier": 4.369134036505073e-05,
              "ece": 0.002478986680507611,
              "mean_confidence": 0.9975210133194924
            },
            "entity_binding": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0001403224484056409,
              "brier": 3.782463963375449e-08,
              "ece": 0.0001403102278709767,
              "mean_confidence": 0.999859689772129
            },
            "negation": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0006506792725122068,
              "brier": 9.928545260606826e-07,
              "ece": 0.0006504309177398637,
              "mean_confidence": 0.9993495690822601
            },
            "numeric_boundary": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0012902074388694018,
              "brier": 4.61289219387595e-06,
              "ece": 0.0012890523672104415,
              "mean_confidence": 0.9987109476327896
            },
            "ordinal_boundary": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.004586648728582077,
              "brier": 4.945013906137774e-05,
              "ece": 0.00457055449485777,
              "mean_confidence": 0.9954294455051422
            },
            "temporal_state": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.003792114706011489,
              "brier": 3.8238258427440995e-05,
              "ece": 0.0037821048498153376,
              "mean_confidence": 0.9962178951501847
            },
            "tool_argument_validity": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.002175020594004309,
              "brier": 0.00011986477213167924,
              "ece": 0.0021438768506050277,
              "mean_confidence": 0.997856123149395
            }
          },
          "by_option_bucket": {
            "2": {
              "n": 400,
              "accuracy": 1.0,
              "nll": 0.0009704433556908043,
              "brier": 2.8028733599683165e-06,
              "ece": 0.0009697416424750971,
              "mean_confidence": 0.9990302583575249
            },
            "3-5": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.002636906407027709,
              "brier": 5.025646692503649e-05,
              "ece": 0.0026231666207314017,
              "mean_confidence": 0.9973768333792686
            }
          },
          "by_truncation": {
            "not_truncated": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.0021607741066457364,
              "brier": 3.6698297335017013e-05,
              "ece": 0.0021507594840867515,
              "mean_confidence": 0.9978492405159133
            }
          },
          "by_dataset_truncation": {
            "v142-confusability|not_truncated": {
              "n": 1400,
              "accuracy": 1.0,
              "nll": 0.0021607741066457364,
              "brier": 3.6698297335017013e-05,
              "ece": 0.0021507594840867515,
              "mean_confidence": 0.9978492405159133
            }
          },
          "by_kind_truncation": {
            "choice|not_truncated": {
              "n": 1000,
              "accuracy": 1.0,
              "nll": 0.001977618149085174,
              "brier": 4.1289017551536136e-05,
              "ece": 0.001966866195201821,
              "mean_confidence": 0.9980331338047981
            },
            "noul|not_truncated": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.0006506792725122068,
              "brier": 9.928545260606826e-07,
              "ece": 0.0006504309177398637,
              "mean_confidence": 0.9993495690822601
            },
            "score|not_truncated": {
              "n": 200,
              "accuracy": 1.0,
              "nll": 0.004586648728582077,
              "brier": 4.945013906137774e-05,
              "ece": 0.00457055449485777,
              "mean_confidence": 0.9954294455051422
            }
          }
        }
      },
      "latency": {
        "validation": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 3659.3674976518378,
          "effective_ms_per_example": 457.4209372064797,
          "skipped_unusable": 0
        },
        "schema": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 4174.043139163288,
          "effective_ms_per_example": 521.755392395411,
          "skipped_unusable": 0
        },
        "firewall": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 336.11437101254705,
          "effective_ms_per_example": 42.01429637656838,
          "skipped_unusable": 0
        },
        "confusability": {
          "eval_batch_size": 8,
          "batch_latency_ms_mean": 658.7701784420226,
          "effective_ms_per_example": 82.34627230525282,
          "skipped_unusable": 0
        }
      }
    }
  },
  "lora_mode": "auto"
}