Upload folder using huggingface_hub

README.md

@@ -88,8 +88,9 @@ SHP, HH-RLHF) and tool selection (Glaive, ToolACE).
 v4 adds next-action choice from agent trajectories (AgentGym AgentTraj-L), web element
 choice (Mind2Web), 1.5k synthetic situations written by Qwen3.5-27B, and teacher-labelled
 states from Pong, Breakout, CliffWalking, MiniGrid and Super Mario Bros.
-Abstention augmentation: in 10% of questions with three or more options the
-gold option is removed and "none of the above" becomes the answer.
+Abstention augmentation: in 10% of questions with three or more options an abstain
+option with one of twelve wordings is added; in a quarter of those the whole option list is
+replaced by labels from an unrelated task, making the abstain option correct.
 
 ## Evaluation
 
@@ -99,8 +100,8 @@ gold option is removed and "none of the above" becomes the answer.
 | Qwen3.5-2B-Base, zero-shot | held-out (23) | 0.642 | 0.853 | 0.460 | 0.105 | 0.242 | 0.685 |
 | Qwen3.5-4B-Base, zero-shot | in-task (64) | 0.695 | 0.768 | 0.405 | 0.090 | 0.206 | 0.742 |
 | Qwen3.5-4B-Base, zero-shot | held-out (23) | 0.711 | 0.734 | 0.390 | 0.089 | 0.169 | 0.761 |
-| **this model (v4)** | in-task (69) | 0.813 | 0.450 | 0.250 | 0.033 | 0.093 | 0.864 |
-| **this model (v4)** | held-out (23) | 0.748 | 0.634 | 0.345 | 0.076 | 0.137 | 0.803 |
+| **this model (v5)** | in-task (69) | 0.815 | 0.445 | 0.248 | 0.028 | 0.093 | 0.866 |
+| **this model (v5)** | held-out (24) | 0.738 | 0.678 | 0.360 | 0.084 | 0.145 | 0.793 |
 
 
 Per-task accuracy / ECE on the held-out datasets:
@@ -196,7 +197,7 @@ Per-task accuracy / ECE on the held-out datasets:
 | tweet_irony | 0.511 / 0.158 | 0.676 / 0.073 | 0.769 / 0.048 |
 
 
-Probabilities use a temperature of 1.15 fitted on in-task data (stored in `decider_config.json`, applied by the helper).
+Probabilities use a temperature of 1.05 fitted on in-task data (stored in `decider_config.json`, applied by the helper).
 *In-task* = test splits of the training datasets (in-task rows for the zero-shot baselines cover the original 64). *Held-out* = 23 datasets never
 seen in training: TREC, BBC news, PAWS, SciQ, Social IQa, StrategyQA, PubMedQA,
 TruthfulQA, tweet irony, financial sentiment, ADE, MASSIVE scenario, student
@@ -240,11 +241,12 @@ less than the evaluation noise (18-task check: accuracy 0.833 vs 0.835, ECE equa
   raised held-out accuracy but lowered two held-out tasks: Hermes tool selection
   (0.80 to 0.74) and TruthfulQA (0.54 to 0.50).
 * Scale fields are the least trained type; expect wider distributions there.
-* The abstention augmentation used the literal option text "none of the above", and the
-  model learned that exact string as an abstain signal: offered verbatim, it abstains even on
-  clear cases. The bundled helper rewrites such an option to a neutral phrasing internally
-  ("not listed here") and maps it back, which restores correct behaviour (8/8 on a routing
-  battery). If you call the model without the helper, avoid the literal phrase.
+* Abstention (v5): an option such as "none of the above", "other" or "unsure" is chosen when
+  nothing on offer fits the situation, not when the exact fine-grained label is merely absent
+  (it then takes the best available option). Trained with twelve abstain wordings and
+  off-topic option lists; on a held-out probe with off-topic option lists it scores 0.83
+  (v4: 0.68). Earlier versions (v4 and before) had learned the literal phrase as an abstain
+  signal; the bundled helper's rewrite for that is disabled for v5 via `decider_config.json`.
 * One in-task dataset, `tweet_hate` (SemEval-2019 HatEval), stays near chance on its
   test split. That split is known to differ from its training split in collection
   and label definition; the number is reported as measured.

decider/infer.py

@@ -44,14 +44,17 @@ class Decider:
     """use_graphs=True (default on CUDA) routes scoring through decider.engine.Engine: shape-bucketed
     CUDA graphs, ~7x lower single-request latency than eager. Set False for CPU or debugging."""
     def __init__(self, path, device="cuda", dtype=torch.bfloat16, temperature=None, abstain_below=0.0, use_graphs=None):
-        if temperature is None:                      # model folder may carry a fitted temperature (decider_config.json)
-            import json, os
-            try:
-                from huggingface_hub import hf_hub_download
-                cfg_path = os.path.join(path, "decider_config.json") if os.path.isdir(path) else hf_hub_download(path, "decider_config.json")
-                temperature = float(json.load(open(cfg_path)).get("temperature", 1.0))
-            except Exception:
-                temperature = 1.0
+        import json, os
+        cfg = {}
+        try:                                          # model folder may carry decider_config.json (temperature, flags)
+            from huggingface_hub import hf_hub_download
+            cfg_path = os.path.join(path, "decider_config.json") if os.path.isdir(path) else hf_hub_download(path, "decider_config.json")
+            cfg = json.load(open(cfg_path))
+        except Exception:
+            pass
+        if temperature is None:
+            temperature = float(cfg.get("temperature", 1.0))
+        self.neutralize_none = bool(cfg.get("neutralize_none", True))   # v4 and earlier learned the literal string as an abstain signal
         if use_graphs is None:
             use_graphs = str(device).startswith("cuda")
         if use_graphs:
@@ -65,7 +68,8 @@ class Decider:
     def decide_batch(self, requests, max_ctx_tokens=1536):
         """requests: list of (context:str, questions:list[dict(question, options)]). One forward pass for everything."""
         exs, meta = [], []
-        requests = [(context, [dict(q, options=neutralize_options(q["options"])[0], _back=neutralize_options(q["options"])[1]) for q in qs]) for context, qs in requests]
+        if self.neutralize_none:
+            requests = [(context, [dict(q, options=neutralize_options(q["options"])[0], _back=neutralize_options(q["options"])[1]) for q in qs]) for context, qs in requests]
         for context, qs in requests:
             for q in qs:
                 assert 2 <= len(q["options"]) <= MAX_OPTIONS, f"2..{MAX_OPTIONS} options required"

decider/prompt.py

@@ -9,6 +9,13 @@ import random
 
 LETTERS = "ABCDEFGHIJ"
 MAX_OPTIONS = len(LETTERS)
+ABSTAIN_PREFIXES = ("none of the above", "none of these", "not listed", "no suitable", "does not apply", "cannot tell")
+ABSTAIN_EXACT = ("other", "unsure", "something else", "neither of these", "other / not covered")
+
+
+def is_abstain_option(o):
+    o = o.strip().lower()
+    return o.startswith(ABSTAIN_PREFIXES) or o in ABSTAIN_EXACT
 
 
 def build(example, tok, rng=None, max_options=MAX_OPTIONS, max_ctx_tokens=1536):
@@ -21,8 +28,10 @@ def build(example, tok, rng=None, max_options=MAX_OPTIONS, max_ctx_tokens=1536):
     for k, q in enumerate(example.qs):
         opts = list(range(len(q.options)))
         if len(opts) > max_options:
-            others = [i for i in opts if i != q.gold]
-            keep = rng.sample(others, max_options - 1) + [q.gold]
+            # always keep the gold and any abstain-style option (its mere presence must not carry information)
+            forced = {q.gold} | {i for i, o in enumerate(q.options) if is_abstain_option(o)}
+            others = [i for i in opts if i not in forced]
+            keep = rng.sample(others, max_options - len(forced)) + list(forced)
             opts = keep
         rng.shuffle(opts)
         lines = [f"\n\nQuestion{' ' + str(k + 1) if multi else ''}: {q.text}\nOptions:"]

decider_config.json

@@ -1 +1 @@
-{"temperature": 1.15, "version": "v4", "base": "Qwen/Qwen3.5-2B-Base"}
+{"temperature": 1.05, "neutralize_none": false, "version": "v5", "base": "Qwen/Qwen3.5-2B-Base"}

eval_results.json

@@ -2,1361 +2,1375 @@
  "results": {
   "clinc_oos": {
    "n": 1500,
-   "acc": 0.978,
-   "nll": 0.06232544407248497,
-   "brier": 0.033815909177064896,
-   "ece": 0.008496353924274395,
-   "aurc": 0.0008730387999259201,
-   "acc_at_80": 0.9991666666666666,
-   "acc_at_50": 1.0,
+   "acc": 0.9613333333333334,
+   "nll": 0.1350308209657669,
+   "brier": 0.05826561525464058,
+   "ece": 0.01548567197720209,
+   "aurc": 0.0036533645736265955,
+   "acc_at_80": 0.9958333333333333,
+   "acc_at_50": 0.9986666666666667,
    "chance": 0.1,
-   "mean_conf": 0.9839518070220947,
-   "sec": 13.8,
+   "mean_conf": 0.971788763999939,
+   "sec": 14.3,
    "heldout": false
   },
   "massive_intent": {
    "n": 1500,
-   "acc": 0.9486666666666667,
-   "nll": 0.15551087260246277,
-   "brier": 0.0762505829334259,
-   "ece": 0.012290600876013396,
-   "aurc": 0.005614506940959013,
-   "acc_at_80": 0.9933333333333333,
-   "acc_at_50": 0.9973333333333333,
+   "acc": 0.9506666666666667,
+   "nll": 0.14456188678741455,
+   "brier": 0.07078584283590317,
+   "ece": 0.0097324760754903,
+   "aurc": 0.004686208748570398,
+   "acc_at_80": 0.9941666666666666,
+   "acc_at_50": 0.9986666666666667,
    "chance": 0.1,
-   "mean_conf": 0.953382670879364,
-   "sec": 4.0,
+   "mean_conf": 0.9509131908416748,
+   "sec": 3.9,
    "heldout": false
   },
   "massive_scenario": {
    "n": 1500,
-   "acc": 0.8033333333333333,
-   "nll": 0.5681269764900208,
-   "brier": 0.2758673429489136,
-   "ece": 0.03523811907569568,
-   "aurc": 0.05579813486552701,
-   "acc_at_80": 0.8858333333333334,
-   "acc_at_50": 0.968,
+   "acc": 0.7866666666666666,
+   "nll": 0.6393386125564575,
+   "brier": 0.29952406883239746,
+   "ece": 0.04220357580979665,
+   "aurc": 0.062301248078291196,
+   "acc_at_80": 0.8675,
+   "acc_at_50": 0.9706666666666667,
    "chance": 0.1,
-   "mean_conf": 0.8278322219848633,
+   "mean_conf": 0.8156749606132507,
    "sec": 1.5,
    "heldout": true
   },
   "bitext_support": {
    "n": 3000,
-   "acc": 0.998,
-   "nll": 0.007169861346483231,
-   "brier": 0.0035593616776168346,
-   "ece": 0.0009388573169708132,
-   "aurc": 1.788459220568468e-05,
-   "acc_at_80": 1.0,
+   "acc": 0.9976666666666667,
+   "nll": 0.010479671880602837,
+   "brier": 0.0043285866267979145,
+   "ece": 0.0017144261995951603,
+   "aurc": 0.0001253647229479039,
+   "acc_at_80": 0.9995833333333334,
    "acc_at_50": 1.0,
    "chance": 0.09999999999999998,
-   "mean_conf": 0.998285710811615,
+   "mean_conf": 0.9976046681404114,
    "sec": 2.1,
    "heldout": false,
    "per_q": [
     0.9986666666666667,
-    0.9973333333333333
+    0.9966666666666667
    ]
   },
   "support_tickets": {
    "n": 4500,
-   "acc": 0.5737777777777778,
-   "nll": 0.9863417148590088,
-   "brier": 0.5184141993522644,
-   "ece": 0.0138452862434917,
-   "aurc": 0.21858182796967993,
-   "acc_at_80": 0.6361111111111111,
-   "acc_at_50": 0.7497777777777778,
+   "acc": 0.5755555555555556,
+   "nll": 0.9841309785842896,
+   "brier": 0.5168883204460144,
+   "ece": 0.014961424820952942,
+   "aurc": 0.21558989155560015,
+   "acc_at_80": 0.6402777777777777,
+   "acc_at_50": 0.7546666666666667,
    "chance": 0.22777777777777777,
-   "mean_conf": 0.5744046568870544,
-   "sec": 10.3,
+   "mean_conf": 0.5705822706222534,
+   "sec": 10.1,
    "heldout": false,
    "per_q": [
-    0.8113333333333334,
-    0.43133333333333335,
-    0.4786666666666667
+    0.812,
+    0.424,
+    0.49066666666666664
    ]
   },
   "ag_news": {
    "n": 1500,
-   "acc": 0.9173333333333333,
-   "nll": 0.24114005267620087,
-   "brier": 0.12489213049411774,
-   "ece": 0.016586370726426467,
-   "aurc": 0.018462071969618875,
-   "acc_at_80": 0.9716666666666667,
-   "acc_at_50": 0.9853333333333333,
+   "acc": 0.9186666666666666,
+   "nll": 0.2343619018793106,
+   "brier": 0.12385667860507965,
+   "ece": 0.011591263870398227,
+   "aurc": 0.017780534818882282,
+   "acc_at_80": 0.9691666666666666,
+   "acc_at_50": 0.9906666666666667,
    "chance": 0.25,
-   "mean_conf": 0.9118769764900208,
+   "mean_conf": 0.9164977073669434,
    "sec": 1.5,
    "heldout": false
   },
   "dbpedia": {
    "n": 1500,
-   "acc": 0.9913333333333333,
-   "nll": 0.024175753816962242,
-   "brier": 0.013113866560161114,
-   "ece": 0.005916546404361727,
-   "aurc": 0.00010674119407793753,
+   "acc": 0.992,
+   "nll": 0.024847088381648064,
+   "brier": 0.012629971839487553,
+   "ece": 0.004867519597212441,
+   "aurc": 0.00010065472533696882,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.1,
-   "mean_conf": 0.9923719763755798,
+   "mean_conf": 0.9911227822303772,
    "sec": 2.0,
    "heldout": false
   },
   "yahoo_topics": {
    "n": 1500,
-   "acc": 0.76,
-   "nll": 0.7366493940353394,
-   "brier": 0.33801746368408203,
-   "ece": 0.028587430814902,
-   "aurc": 0.08282859840202575,
-   "acc_at_80": 0.855,
-   "acc_at_50": 0.9453333333333334,
+   "acc": 0.766,
+   "nll": 0.7322157621383667,
+   "brier": 0.3357123136520386,
+   "ece": 0.02381875579555828,
+   "aurc": 0.08019171930336874,
+   "acc_at_80": 0.8508333333333333,
+   "acc_at_50": 0.9413333333333334,
    "chance": 0.1,
-   "mean_conf": 0.7742340564727783,
+   "mean_conf": 0.7529987096786499,
    "sec": 1.8,
    "heldout": false
   },
   "newsgroups": {
    "n": 1462,
-   "acc": 0.8125854993160054,
-   "nll": 0.5523399710655212,
-   "brier": 0.26114463806152344,
-   "ece": 0.044583431815807525,
-   "aurc": 0.04188347828916838,
-   "acc_at_80": 0.917094017094017,
-   "acc_at_50": 0.9835841313269493,
+   "acc": 0.79890560875513,
+   "nll": 0.5369898676872253,
+   "brier": 0.2600807249546051,
+   "ece": 0.040705675404473,
+   "aurc": 0.042875743065342066,
+   "acc_at_80": 0.9128205128205128,
+   "acc_at_50": 0.987688098495212,
    "chance": 0.09999999999999999,
-   "mean_conf": 0.8485810160636902,
-   "sec": 14.4,
+   "mean_conf": 0.8376736044883728,
+   "sec": 14.0,
    "heldout": false
   },
   "bbc_news": {
    "n": 1000,
-   "acc": 0.939,
-   "nll": 0.16493572294712067,
-   "brier": 0.09448844194412231,
-   "ece": 0.01869928520917896,
-   "aurc": 0.008020306999373308,
-   "acc_at_80": 0.9825,
+   "acc": 0.937,
+   "nll": 0.15569916367530823,
+   "brier": 0.09091777354478836,
+   "ece": 0.015425997406244282,
+   "aurc": 0.007425713401158614,
+   "acc_at_80": 0.98625,
    "acc_at_50": 1.0,
    "chance": 0.20000000000000004,
-   "mean_conf": 0.9388766884803772,
+   "mean_conf": 0.9392964839935303,
    "sec": 5.8,
    "heldout": true
   },
   "trec": {
    "n": 500,
-   "acc": 0.754,
-   "nll": 0.6227552890777588,
-   "brier": 0.33451518416404724,
-   "ece": 0.041722716212272644,
-   "aurc": 0.10665603039652902,
-   "acc_at_80": 0.82,
-   "acc_at_50": 0.896,
+   "acc": 0.8,
+   "nll": 0.6379921436309814,
+   "brier": 0.3222799003124237,
+   "ece": 0.06791729182004927,
+   "aurc": 0.10405531504977719,
+   "acc_at_80": 0.8425,
+   "acc_at_50": 0.892,
    "chance": 0.16666666666666663,
-   "mean_conf": 0.7837464213371277,
-   "sec": 3.3,
+   "mean_conf": 0.8069738745689392,
+   "sec": 2.8,
    "heldout": true
   },
   "student_questions": {
    "n": 1500,
-   "acc": 0.9246666666666666,
-   "nll": 0.22754858434200287,
-   "brier": 0.11876419931650162,
-   "ece": 0.022739037255446105,
-   "aurc": 0.016019787677674244,
-   "acc_at_80": 0.97,
-   "acc_at_50": 0.992,
+   "acc": 0.93,
+   "nll": 0.21689555048942566,
+   "brier": 0.11190912872552872,
+   "ece": 0.020858444054921495,
+   "aurc": 0.015722714843614413,
+   "acc_at_80": 0.9758333333333333,
+   "acc_at_50": 0.9893333333333333,
    "chance": 0.25,
-   "mean_conf": 0.9101096987724304,
-   "sec": 3.4,
+   "mean_conf": 0.9092566967010498,
+   "sec": 1.8,
    "heldout": true
   },
   "dolly_category": {
    "n": 1500,
-   "acc": 0.33266666666666667,
-   "nll": 2.009742021560669,
-   "brier": 0.8325088620185852,
-   "ece": 0.22487777996063235,
-   "aurc": 0.47846646274987925,
-   "acc_at_80": 0.3725,
-   "acc_at_50": 0.4493333333333333,
+   "acc": 0.322,
+   "nll": 2.024930238723755,
+   "brier": 0.8547297120094299,
+   "ece": 0.242468265692393,
+   "aurc": 0.5258592960293854,
+   "acc_at_80": 0.3491666666666667,
+   "acc_at_50": 0.428,
    "chance": 0.125,
-   "mean_conf": 0.5575444102287292,
-   "sec": 3.8,
+   "mean_conf": 0.564468264579773,
+   "sec": 2.2,
    "heldout": true
   },
   "imdb": {
    "n": 1500,
    "acc": 0.9613333333333334,
-   "nll": 0.09619174897670746,
-   "brier": 0.057166311889886856,
-   "ece": 0.010577520211537642,
-   "aurc": 0.002844706294540048,
-   "acc_at_80": 0.9983333333333333,
+   "nll": 0.09483654797077179,
+   "brier": 0.05605172738432884,
+   "ece": 0.01115041021505996,
+   "aurc": 0.0028939979083350527,
+   "acc_at_80": 0.9975,
    "acc_at_50": 1.0,
    "chance": 0.5,
-   "mean_conf": 0.9708292484283447,
-   "sec": 5.5,
+   "mean_conf": 0.9652077555656433,
+   "sec": 3.6,
    "heldout": false
   },
   "sst2": {
    "n": 872,
-   "acc": 0.9426605504587156,
-   "nll": 0.15219736099243164,
-   "brier": 0.08451657742261887,
-   "ece": 0.018561122545955384,
-   "aurc": 0.0094376600127498,
-   "acc_at_80": 0.9899713467048711,
+   "acc": 0.9438073394495413,
+   "nll": 0.15489669144153595,
+   "brier": 0.08650284260511398,
+   "ece": 0.022252202990951892,
+   "aurc": 0.01050770504038895,
+   "acc_at_80": 0.9813753581661891,
    "acc_at_50": 0.9931192660550459,
    "chance": 0.5,
-   "mean_conf": 0.9517649412155151,
-   "sec": 3.8,
+   "mean_conf": 0.9430466294288635,
+   "sec": 3.0,
    "heldout": false
   },
   "sst5": {
    "n": 1500,
-   "acc": 0.6073333333333333,
-   "nll": 0.9326017498970032,
-   "brier": 0.5300925970077515,
-   "ece": 0.02625397855043411,
-   "aurc": 0.3098600910602178,
-   "acc_at_80": 0.6466666666666666,
-   "acc_at_50": 0.68,
+   "acc": 0.6013333333333334,
+   "nll": 0.9259645342826843,
+   "brier": 0.5310559868812561,
+   "ece": 0.02794535907109577,
+   "aurc": 0.3189110639202516,
+   "acc_at_80": 0.63,
+   "acc_at_50": 0.6746666666666666,
    "chance": 0.2,
-   "mean_conf": 0.5991732478141785,
-   "sec": 2.6,
+   "mean_conf": 0.5887239575386047,
+   "sec": 1.2,
    "heldout": false
   },
   "yelp": {
    "n": 1500,
-   "acc": 0.688,
-   "nll": 0.7123478055000305,
-   "brier": 0.41503453254699707,
-   "ece": 0.030766211390495284,
-   "aurc": 0.16381766100924522,
-   "acc_at_80": 0.7466666666666667,
-   "acc_at_50": 0.8146666666666667,
+   "acc": 0.6973333333333334,
+   "nll": 0.6948801279067993,
+   "brier": 0.4055171310901642,
+   "ece": 0.023447723527749403,
+   "aurc": 0.1553584409152187,
+   "acc_at_80": 0.7533333333333333,
+   "acc_at_50": 0.8333333333333334,
    "chance": 0.2,
-   "mean_conf": 0.7086488604545593,
-   "sec": 4.6,
+   "mean_conf": 0.6991026997566223,
+   "sec": 2.9,
    "heldout": false
   },
   "amazon_stars": {
    "n": 1500,
-   "acc": 0.6073333333333333,
-   "nll": 0.9104115962982178,
-   "brier": 0.508173942565918,
-   "ece": 0.024688152035077397,
-   "aurc": 0.23917450560661985,
-   "acc_at_80": 0.6625,
-   "acc_at_50": 0.748,
+   "acc": 0.61,
+   "nll": 0.903746485710144,
+   "brier": 0.501812756061554,
+   "ece": 0.02551889550685881,
+   "aurc": 0.22928301992632438,
+   "acc_at_80": 0.6608333333333334,
+   "acc_at_50": 0.756,
    "chance": 0.2,
-   "mean_conf": 0.6209068894386292,
-   "sec": 3.1,
+   "mean_conf": 0.6187715530395508,
+   "sec": 1.6,
    "heldout": false
   },
   "emotion": {
    "n": 1500,
-   "acc": 0.8526666666666667,
-   "nll": 0.3817766606807709,
-   "brier": 0.20716996490955353,
-   "ece": 0.028686144828796382,
-   "aurc": 0.036016887247968345,
-   "acc_at_80": 0.9241666666666667,
-   "acc_at_50": 0.9813333333333333,
+   "acc": 0.8546666666666667,
+   "nll": 0.3901277184486389,
+   "brier": 0.20813561975955963,
+   "ece": 0.012389385322729728,
+   "aurc": 0.035741524192542756,
+   "acc_at_80": 0.9275,
+   "acc_at_50": 0.98,
    "chance": 0.16666666666666666,
-   "mean_conf": 0.8751572370529175,
-   "sec": 2.6,
+   "mean_conf": 0.8557289838790894,
+   "sec": 1.2,
    "heldout": false
   },
   "go_emotions": {
    "n": 1500,
-   "acc": 0.77,
-   "nll": 0.6605949997901917,
-   "brier": 0.3219051659107208,
-   "ece": 0.02531181337436041,
-   "aurc": 0.07909983628498353,
-   "acc_at_80": 0.8525,
-   "acc_at_50": 0.9413333333333334,
+   "acc": 0.7866666666666666,
+   "nll": 0.633638322353363,
+   "brier": 0.30786433815956116,
+   "ece": 0.023466816226641336,
+   "aurc": 0.07477865198611486,
+   "acc_at_80": 0.8633333333333333,
+   "acc_at_50": 0.94,
    "chance": 0.1,
-   "mean_conf": 0.7867252826690674,
-   "sec": 3.0,
+   "mean_conf": 0.7958080172538757,
+   "sec": 1.5,
    "heldout": false
   },
   "tweet_sentiment": {
    "n": 1500,
-   "acc": 0.732,
-   "nll": 0.5844112038612366,
-   "brier": 0.35898950695991516,
-   "ece": 0.023715959330399827,
-   "aurc": 0.1277814283266402,
-   "acc_at_80": 0.7908333333333334,
-   "acc_at_50": 0.868,
+   "acc": 0.7306666666666667,
+   "nll": 0.589668869972229,
+   "brier": 0.36195889115333557,
+   "ece": 0.03478943075736365,
+   "aurc": 0.13245546777874462,
+   "acc_at_80": 0.7883333333333333,
+   "acc_at_50": 0.8653333333333333,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.7393827438354492,
-   "sec": 2.4,
+   "mean_conf": 0.7084402441978455,
+   "sec": 1.0,
    "heldout": false
   },
   "tweet_emotion": {
    "n": 1421,
-   "acc": 0.8332160450387052,
-   "nll": 0.4378463625907898,
-   "brier": 0.23256292939186096,
-   "ece": 0.029261003112725864,
-   "aurc": 0.0408950375447702,
-   "acc_at_80": 0.9129287598944591,
-   "acc_at_50": 0.9845070422535211,
+   "acc": 0.8465869106263195,
+   "nll": 0.4326176941394806,
+   "brier": 0.2285701334476471,
+   "ece": 0.0309200050501988,
+   "aurc": 0.04118810674643607,
+   "acc_at_80": 0.9226033421284081,
+   "acc_at_50": 0.9816901408450704,
    "chance": 0.25,
-   "mean_conf": 0.8131034970283508,
-   "sec": 4.8,
+   "mean_conf": 0.819736659526825,
+   "sec": 3.5,
    "heldout": false
   },
   "tweet_irony": {
    "n": 784,
-   "acc": 0.7933673469387755,
-   "nll": 0.4786851406097412,
-   "brier": 0.3143369257450104,
-   "ece": 0.05477343446442059,
-   "aurc": 0.11260140253357594,
-   "acc_at_80": 0.8261562998405104,
-   "acc_at_50": 0.8826530612244898,
+   "acc": 0.7882653061224489,
+   "nll": 0.49027082324028015,
+   "brier": 0.32242321968078613,
+   "ece": 0.08050694453472992,
+   "aurc": 0.11250390021122361,
+   "acc_at_80": 0.8373205741626795,
+   "acc_at_50": 0.875,
    "chance": 0.5,
-   "mean_conf": 0.7504966855049133,
-   "sec": 1.2,
+   "mean_conf": 0.707758367061615,
+   "sec": 0.5,
    "heldout": true
   },
   "fin_sentiment": {
    "n": 1500,
-   "acc": 0.784,
-   "nll": 0.4931422472000122,
-   "brier": 0.29916250705718994,
-   "ece": 0.02881191293398539,
-   "aurc": 0.09077011331787095,
-   "acc_at_80": 0.8375,
-   "acc_at_50": 0.9266666666666666,
+   "acc": 0.764,
+   "nll": 0.5191769003868103,
+   "brier": 0.3192051649093628,
+   "ece": 0.038631361166636145,
+   "aurc": 0.10554895981253572,
+   "acc_at_80": 0.82,
+   "acc_at_50": 0.92,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.7918360233306885,
-   "sec": 2.5,
+   "mean_conf": 0.7789527773857117,
+   "sec": 1.1,
    "heldout": true
   },
   "cr_reviews": {
    "n": 753,
-   "acc": 0.9070385126162018,
-   "nll": 0.23999790847301483,
-   "brier": 0.13726605474948883,
-   "ece": 0.02803241042976838,
-   "aurc": 0.019896790015697566,
-   "acc_at_80": 0.9617940199335548,
-   "acc_at_50": 0.9920212765957447,
+   "acc": 0.8937583001328021,
+   "nll": 0.24244317412376404,
+   "brier": 0.1483953297138214,
+   "ece": 0.023112064576243988,
+   "aurc": 0.02109664626036971,
+   "acc_at_80": 0.9551495016611296,
+   "acc_at_50": 0.9946808510638298,
    "chance": 0.5,
-   "mean_conf": 0.9328383803367615,
-   "sec": 1.2,
+   "mean_conf": 0.9141256213188171,
+   "sec": 0.5,
    "heldout": true
   },
   "counterfactual": {
    "n": 1500,
-   "acc": 0.9546666666666667,
-   "nll": 0.1323074996471405,
-   "brier": 0.07490427792072296,
-   "ece": 0.014571970144907676,
-   "aurc": 0.0058697408516071295,
+   "acc": 0.9526666666666667,
+   "nll": 0.13047298789024353,
+   "brier": 0.07368191331624985,
+   "ece": 0.007460888942082723,
+   "aurc": 0.006147812477564298,
    "acc_at_80": 0.99,
    "acc_at_50": 0.9973333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9637404084205627,
-   "sec": 2.4,
+   "mean_conf": 0.9580413699150085,
+   "sec": 1.1,
    "heldout": false
   },
   "subjectivity": {
    "n": 1500,
-   "acc": 0.9633333333333334,
-   "nll": 0.11715030670166016,
-   "brier": 0.06305266171693802,
-   "ece": 0.011594313383102427,
-   "aurc": 0.005805298576335658,
-   "acc_at_80": 0.9916666666666667,
+   "acc": 0.9586666666666667,
+   "nll": 0.1124299094080925,
+   "brier": 0.06167744845151901,
+   "ece": 0.010298379937807677,
+   "aurc": 0.004914430563202389,
+   "acc_at_80": 0.9925,
    "acc_at_50": 0.9973333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9517436623573303,
-   "sec": 2.4,
+   "mean_conf": 0.9552116990089417,
+   "sec": 1.1,
    "heldout": false
   },
   "tweet_offensive": {
    "n": 860,
-   "acc": 0.8604651162790697,
-   "nll": 0.3307216167449951,
-   "brier": 0.2062027007341385,
-   "ece": 0.03277220109174419,
-   "aurc": 0.043970018789679495,
-   "acc_at_80": 0.9186046511627907,
-   "acc_at_50": 0.9627906976744186,
+   "acc": 0.858139534883721,
+   "nll": 0.3267144560813904,
+   "brier": 0.20206692814826965,
+   "ece": 0.03781627153241357,
+   "aurc": 0.04183193769642952,
+   "acc_at_80": 0.9229651162790697,
+   "acc_at_50": 0.9744186046511628,
    "chance": 0.5,
-   "mean_conf": 0.8332623243331909,
-   "sec": 1.5,
+   "mean_conf": 0.8322479128837585,
+   "sec": 0.7,
    "heldout": false
   },
   "tweet_hate": {
    "n": 1500,
-   "acc": 0.5106666666666667,
-   "nll": 0.951435387134552,
-   "brier": 0.6554798483848572,
-   "ece": 0.27647322726249696,
-   "aurc": 0.421070973829564,
-   "acc_at_80": 0.535,
-   "acc_at_50": 0.556,
+   "acc": 0.5226666666666666,
+   "nll": 0.9635146260261536,
+   "brier": 0.6607663035392761,
+   "ece": 0.26240865075588227,
+   "aurc": 0.4248328286104504,
+   "acc_at_80": 0.53,
+   "acc_at_50": 0.5386666666666666,
    "chance": 0.5,
-   "mean_conf": 0.7855966687202454,
-   "sec": 2.6,
+   "mean_conf": 0.7850753664970398,
+   "sec": 1.2,
    "heldout": false
   },
   "hate_offensive": {
    "n": 1500,
-   "acc": 0.9206666666666666,
-   "nll": 0.2384883612394333,
-   "brier": 0.1258707195520401,
-   "ece": 0.013242818454901406,
-   "aurc": 0.018655782825341898,
-   "acc_at_80": 0.9716666666666667,
-   "acc_at_50": 0.9853333333333333,
+   "acc": 0.918,
+   "nll": 0.23910340666770935,
+   "brier": 0.1262882500886917,
+   "ece": 0.01397482017676035,
+   "aurc": 0.020077507534176645,
+   "acc_at_80": 0.9691666666666666,
+   "acc_at_50": 0.9866666666666667,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.9204931855201721,
-   "sec": 2.6,
+   "mean_conf": 0.9220314025878906,
+   "sec": 1.2,
    "heldout": false
   },
   "civil_comments": {
    "n": 7500,
-   "acc": 0.9356,
-   "nll": 0.15526148676872253,
-   "brier": 0.09336414933204651,
-   "ece": 0.013219891810417186,
-   "aurc": 0.007861904760099968,
-   "acc_at_80": 0.9856666666666667,
-   "acc_at_50": 0.9997333333333334,
+   "acc": 0.9346666666666666,
+   "nll": 0.14896972477436066,
+   "brier": 0.09157063812017441,
+   "ece": 0.005636482063929209,
+   "aurc": 0.007494801475142145,
+   "acc_at_80": 0.9886666666666667,
+   "acc_at_50": 1.0,
    "chance": 0.5,
-   "mean_conf": 0.9488199353218079,
-   "sec": 4.4,
+   "mean_conf": 0.9398742914199829,
+   "sec": 2.6,
    "heldout": false,
    "per_q": [
-    0.8753333333333333,
-    0.9786666666666667,
-    0.9913333333333333,
-    0.8633333333333333,
-    0.9693333333333334
+    0.87,
+    0.9773333333333334,
+    0.992,
+    0.8593333333333333,
+    0.9746666666666667
    ]
   },
   "toxic_chat": {
    "n": 3000,
-   "acc": 0.9793333333333333,
-   "nll": 0.05689281225204468,
-   "brier": 0.029941946268081665,
-   "ece": 0.008714853028456338,
-   "aurc": 0.00120893581584804,
+   "acc": 0.9796666666666667,
+   "nll": 0.05411078408360481,
+   "brier": 0.030054867267608643,
+   "ece": 0.004126144587993601,
+   "aurc": 0.0010637207148584903,
    "acc_at_80": 0.9991666666666666,
-   "acc_at_50": 0.9993333333333333,
+   "acc_at_50": 1.0,
    "chance": 0.5,
-   "mean_conf": 0.985617458820343,
-   "sec": 3.3,
+   "mean_conf": 0.980755627155304,
+   "sec": 1.8,
    "heldout": false,
    "per_q": [
-    0.964,
-    0.9946666666666667
+    0.9666666666666667,
+    0.9926666666666667
    ]
   },
   "sms_spam": {
    "n": 1000,
-   "acc": 0.984,
-   "nll": 0.054738420993089676,
-   "brier": 0.023743100464344025,
-   "ece": 0.009705231130123096,
-   "aurc": 0.002162345807605099,
-   "acc_at_80": 0.99625,
+   "acc": 0.981,
+   "nll": 0.05420927703380585,
+   "brier": 0.02655675821006298,
+   "ece": 0.010960762321948961,
+   "aurc": 0.0016593188515115816,
+   "acc_at_80": 0.9975,
    "acc_at_50": 0.998,
    "chance": 0.5,
-   "mean_conf": 0.9895715117454529,
-   "sec": 4.2,
+   "mean_conf": 0.989021360874176,
+   "sec": 3.2,
    "heldout": false
   },
   "enron_spam": {
    "n": 1500,
-   "acc": 0.9866666666666667,
-   "nll": 0.02601483091711998,
-   "brier": 0.016239631921052933,
-   "ece": 0.004016611814498852,
-   "aurc": 0.0002021199142868427,
+   "acc": 0.9893333333333333,
+   "nll": 0.0317784920334816,
+   "brier": 0.01731153391301632,
+   "ece": 0.005647406299908973,
+   "aurc": 0.00032969116371446284,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.5,
-   "mean_conf": 0.9905825257301331,
-   "sec": 9.2,
+   "mean_conf": 0.9897382855415344,
+   "sec": 7.0,
    "heldout": false
   },
   "insincere_questions": {
    "n": 1500,
-   "acc": 0.9553333333333334,
-   "nll": 0.10723885893821716,
-   "brier": 0.06371062248945236,
-   "ece": 0.011764580051104229,
-   "aurc": 0.004172164123818539,
+   "acc": 0.9533333333333334,
+   "nll": 0.108225516974926,
+   "brier": 0.06507585942745209,
+   "ece": 0.010330648342768334,
+   "aurc": 0.004540149557653359,
    "acc_at_80": 0.9975,
-   "acc_at_50": 0.9986666666666667,
+   "acc_at_50": 0.9973333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9626001119613647,
-   "sec": 2.4,
+   "mean_conf": 0.956480860710144,
+   "sec": 1.0,
    "heldout": false
   },
   "ade": {
    "n": 1500,
-   "acc": 0.7893333333333333,
-   "nll": 0.4881707429885864,
-   "brier": 0.3113546073436737,
-   "ece": 0.0730705860455831,
-   "aurc": 0.10379213741472014,
-   "acc_at_80": 0.8325,
-   "acc_at_50": 0.896,
+   "acc": 0.8013333333333333,
+   "nll": 0.4438433349132538,
+   "brier": 0.28561800718307495,
+   "ece": 0.030588836590449038,
+   "aurc": 0.08975771135749887,
+   "acc_at_80": 0.855,
+   "acc_at_50": 0.9026666666666666,
    "chance": 0.5,
-   "mean_conf": 0.8475932478904724,
-   "sec": 2.5,
+   "mean_conf": 0.8299436569213867,
+   "sec": 1.1,
    "heldout": true
   },
   "snli": {
    "n": 1476,
-   "acc": 0.9234417344173442,
-   "nll": 0.22889889776706696,
-   "brier": 0.1168968454003334,
-   "ece": 0.011546386673360035,
-   "aurc": 0.017872040290664873,
-   "acc_at_80": 0.9729043183742591,
-   "acc_at_50": 0.9850948509485095,
+   "acc": 0.9281842818428184,
+   "nll": 0.22167986631393433,
+   "brier": 0.11423322558403015,
+   "ece": 0.015901014794340612,
+   "aurc": 0.016313831745876334,
+   "acc_at_80": 0.9720575783234547,
+   "acc_at_50": 0.986449864498645,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.9265872240066528,
-   "sec": 11.6,
+   "mean_conf": 0.9138893485069275,
+   "sec": 3.9,
    "heldout": false
   },
   "mnli": {
    "n": 1500,
-   "acc": 0.878,
-   "nll": 0.29665282368659973,
-   "brier": 0.16776761412620544,
-   "ece": 0.028746872107187925,
-   "aurc": 0.024781012578722568,
-   "acc_at_80": 0.9558333333333333,
-   "acc_at_50": 0.9893333333333333,
+   "acc": 0.8906666666666667,
+   "nll": 0.2871558368206024,
+   "brier": 0.16026780009269714,
+   "ece": 0.019413771212100986,
+   "aurc": 0.023009362411430274,
+   "acc_at_80": 0.955,
+   "acc_at_50": 0.992,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.8953821659088135,
-   "sec": 3.1,
+   "mean_conf": 0.8811207413673401,
+   "sec": 1.6,
    "heldout": false
   },
   "rte": {
    "n": 277,
-   "acc": 0.8736462093862816,
-   "nll": 0.3355311453342438,
-   "brier": 0.20029526948928833,
-   "ece": 0.049293557659383265,
-   "aurc": 0.04237874308319795,
-   "acc_at_80": 0.9144144144144144,
-   "acc_at_50": 0.9637681159420289,
+   "acc": 0.8664259927797834,
+   "nll": 0.34656208753585815,
+   "brier": 0.21244081854820251,
+   "ece": 0.05030047463165724,
+   "aurc": 0.04784538059954504,
+   "acc_at_80": 0.918918918918919,
+   "acc_at_50": 0.9710144927536232,
    "chance": 0.5,
-   "mean_conf": 0.9088274240493774,
-   "sec": 0.7,
+   "mean_conf": 0.8952643275260925,
+   "sec": 0.4,
    "heldout": false
   },
   "qnli": {
    "n": 1500,
-   "acc": 0.9386666666666666,
-   "nll": 0.15850575268268585,
-   "brier": 0.0912380963563919,
-   "ece": 0.008112372795740749,
-   "aurc": 0.009528591919053303,
-   "acc_at_80": 0.9841666666666666,
-   "acc_at_50": 0.996,
+   "acc": 0.9433333333333334,
+   "nll": 0.15965798497200012,
+   "brier": 0.08883056789636612,
+   "ece": 0.019613722920417815,
+   "aurc": 0.009650867829334674,
+   "acc_at_80": 0.985,
+   "acc_at_50": 0.9973333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9442304968833923,
-   "sec": 3.0,
+   "mean_conf": 0.9275038242340088,
+   "sec": 1.5,
    "heldout": false
   },
   "qqp": {
    "n": 1500,
-   "acc": 0.8546666666666667,
-   "nll": 0.3271322548389435,
-   "brier": 0.20945638418197632,
-   "ece": 0.026240944465001415,
-   "aurc": 0.04136483929310011,
-   "acc_at_80": 0.9116666666666666,
-   "acc_at_50": 0.976,
+   "acc": 0.856,
+   "nll": 0.3196488618850708,
+   "brier": 0.2046487182378769,
+   "ece": 0.01062200272083283,
+   "aurc": 0.040487616797730115,
+   "acc_at_80": 0.9166666666666666,
+   "acc_at_50": 0.9733333333333334,
    "chance": 0.5,
-   "mean_conf": 0.8754271864891052,
-   "sec": 2.6,
+   "mean_conf": 0.8592682480812073,
+   "sec": 1.2,
    "heldout": false
   },
   "mrpc": {
    "n": 408,
-   "acc": 0.8455882352941176,
-   "nll": 0.33147454261779785,
-   "brier": 0.2129886895418167,
-   "ece": 0.053440071058039586,
-   "aurc": 0.04242757999322468,
-   "acc_at_80": 0.9141104294478528,
-   "acc_at_50": 0.9803921568627451,
+   "acc": 0.8480392156862745,
+   "nll": 0.3168051540851593,
+   "brier": 0.20282277464866638,
+   "ece": 0.03800497072584488,
+   "aurc": 0.03737257985377352,
+   "acc_at_80": 0.9171779141104295,
+   "acc_at_50": 0.9754901960784313,
    "chance": 0.5,
-   "mean_conf": 0.8646318912506104,
-   "sec": 0.8,
+   "mean_conf": 0.8617921471595764,
+   "sec": 0.4,
    "heldout": false
   },
   "paws": {
    "n": 1500,
-   "acc": 0.7013333333333334,
-   "nll": 0.8306891918182373,
-   "brier": 0.4740028381347656,
-   "ece": 0.22124919974803925,
-   "aurc": 0.13745002888739216,
-   "acc_at_80": 0.77,
-   "acc_at_50": 0.888,
+   "acc": 0.7086666666666667,
+   "nll": 0.747458815574646,
+   "brier": 0.44605201482772827,
+   "ece": 0.19753822358449297,
+   "aurc": 0.13740906017361146,
+   "acc_at_80": 0.7716666666666666,
+   "acc_at_50": 0.884,
    "chance": 0.5,
-   "mean_conf": 0.9219720959663391,
-   "sec": 3.0,
+   "mean_conf": 0.9062049388885498,
+   "sec": 1.6,
    "heldout": true
   },
   "cola": {
    "n": 1043,
-   "acc": 0.8024928092042186,
-   "nll": 0.4121968150138855,
-   "brier": 0.26549777388572693,
-   "ece": 0.03725165488735943,
-   "aurc": 0.07404416991022049,
-   "acc_at_80": 0.8741007194244604,
-   "acc_at_50": 0.9367816091954023,
+   "acc": 0.8053691275167785,
+   "nll": 0.4110952615737915,
+   "brier": 0.263612300157547,
+   "ece": 0.026030545552419217,
+   "aurc": 0.0753830017250337,
+   "acc_at_80": 0.86810551558753,
+   "acc_at_50": 0.9310344827586207,
    "chance": 0.5,
-   "mean_conf": 0.8369820713996887,
-   "sec": 1.6,
+   "mean_conf": 0.8122093081474304,
+   "sec": 0.6,
    "heldout": false
   },
   "boolq": {
    "n": 1500,
-   "acc": 0.8853333333333333,
-   "nll": 0.2897628843784332,
-   "brier": 0.17195458710193634,
-   "ece": 0.028059083223342877,
-   "aurc": 0.033273033407070704,
-   "acc_at_80": 0.9441666666666667,
-   "acc_at_50": 0.9746666666666667,
+   "acc": 0.89,
+   "nll": 0.2775375545024872,
+   "brier": 0.16621354222297668,
+   "ece": 0.015124458909034735,
+   "aurc": 0.03136212917899602,
+   "acc_at_80": 0.9408333333333333,
+   "acc_at_50": 0.9813333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9083826541900635,
-   "sec": 4.0,
+   "mean_conf": 0.8884056210517883,
+   "sec": 2.3,
    "heldout": false
   },
   "strategyqa": {
    "n": 687,
-   "acc": 0.6142649199417758,
-   "nll": 0.6972374320030212,
-   "brier": 0.4870849847793579,
-   "ece": 0.10019853498564363,
-   "aurc": 0.31452270422741296,
-   "acc_at_80": 0.6327272727272727,
-   "acc_at_50": 0.6569767441860465,
+   "acc": 0.6055312954876274,
+   "nll": 0.6860260963439941,
+   "brier": 0.4836777150630951,
+   "ece": 0.07135456410037377,
+   "aurc": 0.33034478489678043,
+   "acc_at_80": 0.6145454545454545,
+   "acc_at_50": 0.6686046511627907,
    "chance": 0.5,
-   "mean_conf": 0.7026323080062866,
-   "sec": 7.9,
+   "mean_conf": 0.6736506819725037,
+   "sec": 2.8,
    "heldout": true
   },
   "pubmedqa": {
    "n": 500,
-   "acc": 0.798,
-   "nll": 0.5936447381973267,
-   "brier": 0.31999242305755615,
-   "ece": 0.06877634388208391,
-   "aurc": 0.10006724050038115,
-   "acc_at_80": 0.845,
-   "acc_at_50": 0.928,
+   "acc": 0.78,
+   "nll": 0.604637086391449,
+   "brier": 0.3268354535102844,
+   "ece": 0.04217318385839462,
+   "aurc": 0.1023213883692996,
+   "acc_at_80": 0.8525,
+   "acc_at_50": 0.912,
    "chance": 0.33333333333333326,
-   "mean_conf": 0.8425376415252686,
-   "sec": 2.3,
+   "mean_conf": 0.8035740852355957,
+   "sec": 1.6,
    "heldout": true
   },
   "arc": {
    "n": 1172,
-   "acc": 0.856655290102389,
-   "nll": 0.3848453760147095,
-   "brier": 0.20024119317531586,
-   "ece": 0.03268225236760882,
-   "aurc": 0.028683364797018013,
-   "acc_at_80": 0.9466950959488273,
-   "acc_at_50": 0.9897610921501706,
+   "acc": 0.8617747440273038,
+   "nll": 0.3778131902217865,
+   "brier": 0.19557851552963257,
+   "ece": 0.025041668768018607,
+   "aurc": 0.027995019899211025,
+   "acc_at_80": 0.9477611940298507,
+   "acc_at_50": 0.9880546075085325,
    "chance": 0.25015642775881686,
-   "mean_conf": 0.8850091695785522,
-   "sec": 2.3,
+   "mean_conf": 0.8545714616775513,
+   "sec": 1.2,
    "heldout": false
   },
   "commonsense_qa": {
    "n": 1221,
-   "acc": 0.7641277641277642,
-   "nll": 0.6035410165786743,
-   "brier": 0.32016345858573914,
-   "ece": 0.024896302787521474,
-   "aurc": 0.07891618232675111,
-   "acc_at_80": 0.8454452405322416,
-   "acc_at_50": 0.9459016393442623,
+   "acc": 0.7682227682227682,
+   "nll": 0.596526026725769,
+   "brier": 0.3169134259223938,
+   "ece": 0.024408742675617233,
+   "aurc": 0.07639980329813019,
+   "acc_at_80": 0.8505629477993859,
+   "acc_at_50": 0.9442622950819672,
    "chance": 0.19999999999999998,
-   "mean_conf": 0.7828702926635742,
-   "sec": 2.0,
+   "mean_conf": 0.7619057297706604,
+   "sec": 0.8,
    "heldout": false
   },
   "qasc": {
    "n": 926,
-   "acc": 0.8142548596112311,
-   "nll": 0.5944124460220337,
-   "brier": 0.27712827920913696,
-   "ece": 0.05084534546211523,
-   "aurc": 0.04955932347383356,
-   "acc_at_80": 0.8960863697705803,
-   "acc_at_50": 0.9676025917926566,
+   "acc": 0.8088552915766739,
+   "nll": 0.5853570699691772,
+   "brier": 0.2697433531284332,
+   "ece": 0.06495514207923903,
+   "aurc": 0.046915675887241735,
+   "acc_at_80": 0.9041835357624831,
+   "acc_at_50": 0.978401727861771,
    "chance": 0.125,
-   "mean_conf": 0.7678154706954956,
-   "sec": 1.8,
+   "mean_conf": 0.7473546862602234,
+   "sec": 0.9,
    "heldout": false
   },
   "openbookqa": {
    "n": 500,
-   "acc": 0.808,
-   "nll": 0.5015584826469421,
-   "brier": 0.25752735137939453,
-   "ece": 0.044371710240840904,
-   "aurc": 0.057444096145339726,
-   "acc_at_80": 0.905,
-   "acc_at_50": 0.968,
+   "acc": 0.834,
+   "nll": 0.4812253713607788,
+   "brier": 0.24745029211044312,
+   "ece": 0.04192176961898805,
+   "aurc": 0.05150388916107792,
+   "acc_at_80": 0.9075,
+   "acc_at_50": 0.972,
    "chance": 0.25,
-   "mean_conf": 0.8454900979995728,
-   "sec": 0.9,
+   "mean_conf": 0.8025239109992981,
+   "sec": 0.4,
    "heldout": false
   },
   "sciq": {
    "n": 1000,
-   "acc": 0.983,
-   "nll": 0.0540383905172348,
-   "brier": 0.026997487992048264,
-   "ece": 0.015873215794563272,
-   "aurc": 0.0004725583644347805,
+   "acc": 0.987,
+   "nll": 0.051316503435373306,
+   "brier": 0.02284967713057995,
+   "ece": 0.018773878514766716,
+   "aurc": 0.0003853215476759977,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.25,
-   "mean_conf": 0.9763537645339966,
-   "sec": 4.9,
+   "mean_conf": 0.9696812629699707,
+   "sec": 3.8,
    "heldout": true
   },
   "hellaswag": {
    "n": 1500,
-   "acc": 0.8726666666666667,
-   "nll": 0.35282617807388306,
-   "brier": 0.1844407618045807,
-   "ece": 0.05096171965201693,
-   "aurc": 0.022184705622736417,
-   "acc_at_80": 0.9541666666666667,
-   "acc_at_50": 0.9986666666666667,
+   "acc": 0.892,
+   "nll": 0.29868125915527344,
+   "brier": 0.16158466041088104,
+   "ece": 0.01794782030582424,
+   "aurc": 0.019411743923739497,
+   "acc_at_80": 0.9608333333333333,
+   "acc_at_50": 0.9973333333333333,
    "chance": 0.25,
-   "mean_conf": 0.9218340516090393,
-   "sec": 4.4,
+   "mean_conf": 0.89520263671875,
+   "sec": 2.7,
    "heldout": false
   },
   "piqa": {
    "n": 1500,
-   "acc": 0.8353333333333334,
-   "nll": 0.3836018443107605,
-   "brier": 0.2365727722644806,
-   "ece": 0.04386041740576427,
-   "aurc": 0.05569590131269642,
-   "acc_at_80": 0.8966666666666666,
+   "acc": 0.842,
+   "nll": 0.35798120498657227,
+   "brier": 0.22625073790550232,
+   "ece": 0.02936798457304638,
+   "aurc": 0.05338353492667896,
+   "acc_at_80": 0.9008333333333334,
    "acc_at_50": 0.9506666666666667,
    "chance": 0.5,
-   "mean_conf": 0.8791936635971069,
-   "sec": 2.9,
+   "mean_conf": 0.837594747543335,
+   "sec": 1.5,
    "heldout": false
   },
   "social_iqa": {
    "n": 1500,
-   "acc": 0.7186666666666667,
-   "nll": 0.7424599528312683,
-   "brier": 0.41328999400138855,
-   "ece": 0.09968751738468809,
-   "aurc": 0.1436145844762169,
-   "acc_at_80": 0.775,
-   "acc_at_50": 0.8546666666666667,
+   "acc": 0.71,
+   "nll": 0.695899486541748,
+   "brier": 0.3991619348526001,
+   "ece": 0.060694456895192456,
+   "aurc": 0.14411079894615622,
+   "acc_at_80": 0.7683333333333333,
+   "acc_at_50": 0.8613333333333333,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.8163115382194519,
-   "sec": 2.7,
+   "mean_conf": 0.7695742845535278,
+   "sec": 1.2,
    "heldout": true
   },
   "winogrande": {
    "n": 1267,
-   "acc": 0.681136543014996,
-   "nll": 0.7226526737213135,
-   "brier": 0.45336562395095825,
-   "ece": 0.16130592141832623,
-   "aurc": 0.19273822608019725,
-   "acc_at_80": 0.7209072978303748,
-   "acc_at_50": 0.8123028391167192,
+   "acc": 0.6803472770323599,
+   "nll": 0.6707175374031067,
+   "brier": 0.43641534447669983,
+   "ece": 0.13178394578438438,
+   "aurc": 0.19499483213034147,
+   "acc_at_80": 0.7248520710059172,
+   "acc_at_50": 0.7902208201892744,
    "chance": 0.5,
-   "mean_conf": 0.8380995988845825,
-   "sec": 1.9,
+   "mean_conf": 0.8093914985656738,
+   "sec": 0.8,
    "heldout": false
   },
   "race": {
    "n": 1500,
-   "acc": 0.8326666666666667,
-   "nll": 0.48354849219322205,
-   "brier": 0.24252651631832123,
-   "ece": 0.05754320712884267,
-   "aurc": 0.04422304284234363,
-   "acc_at_80": 0.9175,
-   "acc_at_50": 0.9733333333333334,
+   "acc": 0.8426666666666667,
+   "nll": 0.4418315887451172,
+   "brier": 0.22411726415157318,
+   "ece": 0.024219590028127014,
+   "aurc": 0.0420007716587064,
+   "acc_at_80": 0.9241666666666667,
+   "acc_at_50": 0.9746666666666667,
    "chance": 0.25,
-   "mean_conf": 0.8896796703338623,
-   "sec": 7.3,
+   "mean_conf": 0.8613157272338867,
+   "sec": 5.1,
    "heldout": false
   },
   "mmlu": {
    "n": 1500,
-   "acc": 0.6333333333333333,
-   "nll": 0.9389562010765076,
-   "brier": 0.47845685482025146,
-   "ece": 0.10005126521984736,
-   "aurc": 0.16047564173919576,
-   "acc_at_80": 0.71,
-   "acc_at_50": 0.8533333333333334,
+   "acc": 0.6313333333333333,
+   "nll": 0.890999436378479,
+   "brier": 0.46188458800315857,
+   "ece": 0.0515740847190221,
+   "aurc": 0.15556783894771342,
+   "acc_at_80": 0.7191666666666666,
+   "acc_at_50": 0.8613333333333333,
    "chance": 0.25,
-   "mean_conf": 0.7325466871261597,
-   "sec": 3.4,
+   "mean_conf": 0.6770187020301819,
+   "sec": 1.8,
    "heldout": false
   },
   "medqa": {
    "n": 1273,
-   "acc": 0.5632364493322859,
-   "nll": 1.079268455505371,
-   "brier": 0.575121283531189,
-   "ece": 0.1046450646112776,
-   "aurc": 0.2684613401796164,
-   "acc_at_80": 0.6129666011787819,
-   "acc_at_50": 0.710691823899371,
+   "acc": 0.5577376276512176,
+   "nll": 1.0580443143844604,
+   "brier": 0.5702889561653137,
+   "ece": 0.029768652458970035,
+   "aurc": 0.2773348347125782,
+   "acc_at_80": 0.611984282907662,
+   "acc_at_50": 0.6981132075471698,
    "chance": 0.25,
-   "mean_conf": 0.6660441756248474,
-   "sec": 4.1,
+   "mean_conf": 0.5789350271224976,
+   "sec": 2.5,
    "heldout": false
   },
   "truthfulqa": {
    "n": 817,
-   "acc": 0.5018359853121175,
-   "nll": 1.5319370031356812,
-   "brier": 0.6671698689460754,
-   "ece": 0.1319449910919115,
-   "aurc": 0.2963526198417848,
-   "acc_at_80": 0.5581039755351682,
-   "acc_at_50": 0.678921568627451,
+   "acc": 0.5128518971848225,
+   "nll": 1.4920024871826172,
+   "brier": 0.6377546191215515,
+   "ece": 0.08506929861652954,
+   "aurc": 0.2737489336879717,
+   "acc_at_80": 0.581039755351682,
+   "acc_at_50": 0.7181372549019608,
    "chance": 0.22622350449767833,
-   "mean_conf": 0.6299614310264587,
-   "sec": 1.7,
+   "mean_conf": 0.5959253907203674,
+   "sec": 0.9,
    "heldout": true
   },
   "fin_phrasebank": {
    "n": 970,
-   "acc": 0.6989690721649484,
-   "nll": 0.6254894733428955,
-   "brier": 0.39401963353157043,
-   "ece": 0.058691233703770586,
-   "aurc": 0.16686274774956414,
-   "acc_at_80": 0.7577319587628866,
-   "acc_at_50": 0.8329896907216495,
+   "acc": 0.634020618556701,
+   "nll": 0.708746612071991,
+   "brier": 0.4601970314979553,
+   "ece": 0.11248690885366852,
+   "aurc": 0.22041065209169486,
+   "acc_at_80": 0.6868556701030928,
+   "acc_at_50": 0.7731958762886598,
    "chance": 0.33333333333333326,
-   "mean_conf": 0.7559286952018738,
-   "sec": 6.3,
+   "mean_conf": 0.7462515234947205,
+   "sec": 5.1,
    "heldout": true
   },
   "banking77": {
    "n": 1500,
-   "acc": 0.9626666666666667,
-   "nll": 0.12322639673948288,
-   "brier": 0.05743316188454628,
-   "ece": 0.007931326011816654,
-   "aurc": 0.002740366914790048,
-   "acc_at_80": 0.9966666666666667,
+   "acc": 0.962,
+   "nll": 0.1172320544719696,
+   "brier": 0.056729208678007126,
+   "ece": 0.00940998808542892,
+   "aurc": 0.002673441928791999,
+   "acc_at_80": 0.9983333333333333,
    "acc_at_50": 1.0,
    "chance": 0.1,
-   "mean_conf": 0.9672797918319702,
-   "sec": 3.0,
+   "mean_conf": 0.960239589214325,
+   "sec": 1.5,
    "heldout": false
   },
   "bias_in_bios": {
    "n": 3000,
-   "acc": 0.9516666666666667,
-   "nll": 0.13784465193748474,
-   "brier": 0.06864367425441742,
-   "ece": 0.006546956946452431,
-   "aurc": 0.0033626200148913327,
-   "acc_at_80": 0.9975,
+   "acc": 0.9513333333333334,
+   "nll": 0.1394692361354828,
+   "brier": 0.06979931145906448,
+   "ece": 0.007795257161060937,
+   "aurc": 0.0035514571665865736,
+   "acc_at_80": 0.9970833333333333,
    "acc_at_50": 1.0,
    "chance": 0.30000000000000004,
-   "mean_conf": 0.953985333442688,
-   "sec": 4.2,
+   "mean_conf": 0.9533326625823975,
+   "sec": 2.5,
    "heldout": false,
    "per_q": [
-    0.906,
+    0.9053333333333333,
     0.9973333333333333
    ]
   },
   "helpsteer2": {
    "n": 5190,
-   "acc": 0.6028901734104046,
-   "nll": 0.9502481818199158,
-   "brier": 0.5227839350700378,
-   "ece": 0.02052123054201663,
-   "aurc": 0.2681509263336261,
-   "acc_at_80": 0.6623314065510597,
-   "acc_at_50": 0.7294797687861272,
+   "acc": 0.6013487475915221,
+   "nll": 0.934554398059845,
+   "brier": 0.5175615549087524,
+   "ece": 0.010821180559536854,
+   "aurc": 0.2635892747983117,
+   "acc_at_80": 0.6592003853564548,
+   "acc_at_50": 0.7391136801541426,
    "chance": 0.2,
-   "mean_conf": 0.6144348978996277,
-   "sec": 23.2,
+   "mean_conf": 0.5985665917396545,
+   "sec": 12.5,
    "heldout": false,
    "per_q": [
-    0.45664739884393063,
-    0.5067437379576107,
-    0.7225433526011561,
-    0.6358381502890174,
-    0.6926782273603083
+    0.45472061657032753,
+    0.5086705202312138,
+    0.7273603082851637,
+    0.6213872832369942,
+    0.6946050096339114
    ]
   },
   "helpsteer3_pref": {
    "n": 1176,
-   "acc": 0.4005102040816326,
-   "nll": 1.457674503326416,
-   "brier": 0.7135226726531982,
-   "ece": 0.05034625236274434,
-   "aurc": 0.49111859183201545,
-   "acc_at_80": 0.4293304994686504,
-   "acc_at_50": 0.49489795918367346,
+   "acc": 0.40561224489795916,
+   "nll": 1.4524648189544678,
+   "brier": 0.710177481174469,
+   "ece": 0.03173913492434691,
+   "aurc": 0.49141414112121073,
+   "acc_at_80": 0.4410201912858661,
+   "acc_at_50": 0.5017006802721088,
    "chance": 0.14285714285714282,
-   "mean_conf": 0.44308316707611084,
-   "sec": 24.6,
+   "mean_conf": 0.4128914773464203,
+   "sec": 12.8,
    "heldout": false
   },
   "hate_speech_scales": {
    "n": 7500,
-   "acc": 0.5605333333333333,
-   "nll": 0.9819027781486511,
-   "brier": 0.5431735515594482,
-   "ece": 0.0195502276579539,
-   "aurc": 0.267479078510345,
-   "acc_at_80": 0.6098333333333333,
-   "acc_at_50": 0.7061333333333333,
+   "acc": 0.5636,
+   "nll": 0.9782006740570068,
+   "brier": 0.5410642027854919,
+   "ece": 0.011817029543717711,
+   "aurc": 0.2650615424796364,
+   "acc_at_80": 0.6103333333333333,
+   "acc_at_50": 0.7074666666666667,
    "chance": 0.22666666666666674,
-   "mean_conf": 0.5698475241661072,
-   "sec": 4.9,
+   "mean_conf": 0.5654744505882263,
+   "sec": 3.7,
    "heldout": false,
    "per_q": [
-    0.706,
-    0.4826666666666667,
-    0.43533333333333335,
+    0.7046666666666667,
+    0.4866666666666667,
+    0.4493333333333333,
     0.578,
-    0.6006666666666667
+    0.5993333333333334
    ]
   },
   "liar2": {
    "n": 1500,
-   "acc": 0.37466666666666665,
-   "nll": 1.465606451034546,
-   "brier": 0.7157273888587952,
-   "ece": 0.028325774570306134,
-   "aurc": 0.47234860740182666,
-   "acc_at_80": 0.4075,
+   "acc": 0.37066666666666664,
+   "nll": 1.4561086893081665,
+   "brier": 0.7133269906044006,
+   "ece": 0.02327027595043182,
+   "aurc": 0.4659199761493187,
+   "acc_at_80": 0.41083333333333333,
    "acc_at_50": 0.48933333333333334,
    "chance": 0.16666666666666666,
-   "mean_conf": 0.39819955825805664,
-   "sec": 3.7,
+   "mean_conf": 0.3748621940612793,
+   "sec": 2.4,
    "heldout": false
   },
   "prosocial_safety": {
    "n": 1500,
-   "acc": 0.524,
-   "nll": 1.1723508834838867,
-   "brier": 0.592208206653595,
-   "ece": 0.02242480476697286,
-   "aurc": 0.2745796786723848,
-   "acc_at_80": 0.5883333333333334,
-   "acc_at_50": 0.6906666666666667,
+   "acc": 0.532,
+   "nll": 1.1633950471878052,
+   "brier": 0.5889540314674377,
+   "ece": 0.03297985295454661,
+   "aurc": 0.2684920476580785,
+   "acc_at_80": 0.5925,
+   "acc_at_50": 0.7053333333333334,
    "chance": 0.2,
-   "mean_conf": 0.5037593841552734,
-   "sec": 3.0,
+   "mean_conf": 0.5102941989898682,
+   "sec": 1.5,
    "heldout": false
   },
   "ultrafeedback_pref": {
    "n": 1500,
-   "acc": 0.762,
-   "nll": 0.5141094923019409,
-   "brier": 0.33274728059768677,
-   "ece": 0.03840167435010275,
-   "aurc": 0.1266040181966966,
-   "acc_at_80": 0.81,
-   "acc_at_50": 0.864,
+   "acc": 0.7573333333333333,
+   "nll": 0.48778781294822693,
+   "brier": 0.32244637608528137,
+   "ece": 0.038332425316174826,
+   "aurc": 0.11649624186832548,
+   "acc_at_80": 0.815,
+   "acc_at_50": 0.884,
    "chance": 0.5,
-   "mean_conf": 0.794404923915863,
-   "sec": 9.8,
+   "mean_conf": 0.7588924169540405,
+   "sec": 7.1,
    "heldout": false
   },
   "shp": {
    "n": 1500,
-   "acc": 0.744,
-   "nll": 0.5253914594650269,
-   "brier": 0.35134565830230713,
-   "ece": 0.029734440008799243,
-   "aurc": 0.14422793871032347,
-   "acc_at_80": 0.7808333333333334,
-   "acc_at_50": 0.848,
+   "acc": 0.728,
+   "nll": 0.5200306177139282,
+   "brier": 0.34814339876174927,
+   "ece": 0.039244854211807255,
+   "aurc": 0.13985241632778875,
+   "acc_at_80": 0.7916666666666666,
+   "acc_at_50": 0.8546666666666667,
    "chance": 0.5,
-   "mean_conf": 0.752452552318573,
-   "sec": 7.2,
+   "mean_conf": 0.7454078197479248,
+   "sec": 5.0,
    "heldout": false
   },
   "hh_rlhf": {
    "n": 1488,
-   "acc": 0.6478494623655914,
-   "nll": 0.6344334483146667,
-   "brier": 0.44163978099823,
-   "ece": 0.06129897718307793,
-   "aurc": 0.24894185193228557,
-   "acc_at_80": 0.6705882352941176,
-   "acc_at_50": 0.7338709677419355,
+   "acc": 0.6384408602150538,
+   "nll": 0.6243960857391357,
+   "brier": 0.43749359250068665,
+   "ece": 0.025252133768091923,
+   "aurc": 0.254013929940744,
+   "acc_at_80": 0.6638655462184874,
+   "acc_at_50": 0.7190860215053764,
    "chance": 0.5,
-   "mean_conf": 0.6988418698310852,
-   "sec": 12.3,
+   "mean_conf": 0.6413496136665344,
+   "sec": 6.6,
    "heldout": false
   },
   "arena_pref": {
    "n": 1500,
-   "acc": 0.47533333333333333,
-   "nll": 1.2214066982269287,
-   "brier": 0.6920773386955261,
-   "ece": 0.1775780066649119,
-   "aurc": 0.4264101430176788,
-   "acc_at_80": 0.5025,
-   "acc_at_50": 0.5493333333333333,
+   "acc": 0.4786666666666667,
+   "nll": 1.2097605466842651,
+   "brier": 0.6734665632247925,
+   "ece": 0.1468491670290629,
+   "aurc": 0.41109683289171867,
+   "acc_at_80": 0.5108333333333334,
+   "acc_at_50": 0.5733333333333334,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.652911365032196,
-   "sec": 9.3,
+   "mean_conf": 0.6255158185958862,
+   "sec": 6.7,
    "heldout": true
   },
   "reward_bench": {
    "n": 1500,
-   "acc": 0.794,
-   "nll": 0.45838916301727295,
-   "brier": 0.28988850116729736,
-   "ece": 0.04195846394697825,
-   "aurc": 0.09336704492407273,
-   "acc_at_80": 0.8533333333333334,
-   "acc_at_50": 0.912,
+   "acc": 0.8046666666666666,
+   "nll": 0.4391920268535614,
+   "brier": 0.2816815972328186,
+   "ece": 0.030526579976081835,
+   "aurc": 0.09005779253848542,
+   "acc_at_80": 0.8616666666666667,
+   "acc_at_50": 0.92,
    "chance": 0.5,
-   "mean_conf": 0.8290508389472961,
-   "sec": 7.7,
+   "mean_conf": 0.7789213061332703,
+   "sec": 5.3,
    "heldout": true
   },
   "glaive_tools": {
    "n": 1348,
    "acc": 0.9473293768545994,
-   "nll": 0.1643306165933609,
-   "brier": 0.087461918592453,
-   "ece": 0.007846767320477323,
-   "aurc": 0.011936188106662745,
-   "acc_at_80": 0.9795918367346939,
-   "acc_at_50": 0.9925816023738873,
+   "nll": 0.16986463963985443,
+   "brier": 0.09119585901498795,
+   "ece": 0.01463318938936606,
+   "aurc": 0.012421285819056932,
+   "acc_at_80": 0.9786641929499073,
+   "acc_at_50": 0.9896142433234422,
    "chance": 0.16666666666666666,
-   "mean_conf": 0.9517403841018677,
-   "sec": 9.1,
+   "mean_conf": 0.950721263885498,
+   "sec": 4.4,
    "heldout": false
   },
   "hermes_tools": {
    "n": 1500,
-   "acc": 0.7273333333333334,
-   "nll": 0.5851268768310547,
-   "brier": 0.3813258111476898,
-   "ece": 0.19555236707131068,
-   "aurc": 0.06225953895023327,
-   "acc_at_80": 0.8558333333333333,
-   "acc_at_50": 0.988,
+   "acc": 0.7113333333333334,
+   "nll": 0.6692715883255005,
+   "brier": 0.43301042914390564,
+   "ece": 0.21088105460007986,
+   "aurc": 0.08194760887819506,
+   "acc_at_80": 0.8108333333333333,
+   "acc_at_50": 0.98,
    "chance": 0.125,
-   "mean_conf": 0.92147296667099,
-   "sec": 8.8,
+   "mean_conf": 0.9190990328788757,
+   "sec": 6.3,
    "heldout": true
   },
   "copa": {
    "n": 100,
-   "acc": 0.94,
-   "nll": 0.1961805373430252,
-   "brier": 0.09868918359279633,
-   "ece": 0.011551505327224751,
-   "aurc": 0.021329015107119983,
-   "acc_at_80": 0.975,
+   "acc": 0.93,
+   "nll": 0.21066521108150482,
+   "brier": 0.11394889652729034,
+   "ece": 0.040256121754646275,
+   "aurc": 0.021342808523311715,
+   "acc_at_80": 0.9625,
    "acc_at_50": 0.98,
    "chance": 0.5,
-   "mean_conf": 0.9395062327384949,
-   "sec": 6.0,
+   "mean_conf": 0.904087245464325,
+   "sec": 2.6,
    "heldout": false
   },
   "wic": {
    "n": 638,
-   "acc": 0.7163009404388715,
-   "nll": 0.5828382968902588,
-   "brier": 0.38676366209983826,
-   "ece": 0.05568265924259413,
-   "aurc": 0.20020452896287377,
-   "acc_at_80": 0.7647058823529411,
-   "acc_at_50": 0.8119122257053292,
+   "acc": 0.6990595611285266,
+   "nll": 0.6114251017570496,
+   "brier": 0.40890803933143616,
+   "ece": 0.057320481370608994,
+   "aurc": 0.22488178917696844,
+   "acc_at_80": 0.7392156862745098,
+   "acc_at_50": 0.7836990595611285,
    "chance": 0.5,
-   "mean_conf": 0.7616857290267944,
-   "sec": 1.1,
+   "mean_conf": 0.753724217414856,
+   "sec": 0.5,
    "heldout": false
   },
   "multirc": {
    "n": 1500,
-   "acc": 0.8913333333333333,
-   "nll": 0.300986111164093,
-   "brier": 0.16819369792938232,
-   "ece": 0.03357027542591094,
-   "aurc": 0.03652944216687471,
-   "acc_at_80": 0.9408333333333333,
-   "acc_at_50": 0.9693333333333334,
+   "acc": 0.8953333333333333,
+   "nll": 0.2838270962238312,
+   "brier": 0.1647527664899826,
+   "ece": 0.015789239207903517,
+   "aurc": 0.03669604113839991,
+   "acc_at_80": 0.9425,
+   "acc_at_50": 0.9706666666666667,
    "chance": 0.5,
-   "mean_conf": 0.9222791194915771,
-   "sec": 6.7,
+   "mean_conf": 0.8994993567466736,
+   "sec": 4.6,
    "heldout": false
   },
   "cb": {
    "n": 56,
    "acc": 0.8928571428571429,
-   "nll": 0.3820924460887909,
-   "brier": 0.17802833020687103,
-   "ece": 0.08807407692074777,
-   "aurc": 0.023497414016688704,
+   "nll": 0.36988380551338196,
+   "brier": 0.17701731622219086,
+   "ece": 0.10170899544443401,
+   "aurc": 0.022862006902143423,
    "acc_at_80": 0.9555555555555556,
    "acc_at_50": 0.9642857142857143,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.8515821695327759,
-   "sec": 0.2,
+   "mean_conf": 0.8231589198112488,
+   "sec": 0.1,
    "heldout": true
   },
   "fever": {
    "n": 1500,
-   "acc": 0.8866666666666667,
-   "nll": 0.3317285180091858,
-   "brier": 0.1768377274274826,
-   "ece": 0.017198302884896575,
-   "aurc": 0.03783623559244957,
-   "acc_at_80": 0.9383333333333334,
+   "acc": 0.894,
+   "nll": 0.3236566185951233,
+   "brier": 0.17617946863174438,
+   "ece": 0.0255238478779793,
+   "aurc": 0.034462965066177104,
+   "acc_at_80": 0.9391666666666667,
    "acc_at_50": 0.972,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.8916481137275696,
-   "sec": 3.3,
+   "mean_conf": 0.8865981698036194,
+   "sec": 1.7,
    "heldout": false
   },
   "wiki_qa": {
    "n": 879,
-   "acc": 0.8987485779294653,
-   "nll": 0.2437155544757843,
-   "brier": 0.1463335007429123,
-   "ece": 0.012018481686931658,
-   "aurc": 0.02256072022253832,
-   "acc_at_80": 0.9587482219061166,
-   "acc_at_50": 0.9863636363636363,
+   "acc": 0.9078498293515358,
+   "nll": 0.25518864393234253,
+   "brier": 0.15072549879550934,
+   "ece": 0.01926714921024741,
+   "aurc": 0.025838258571267432,
+   "acc_at_80": 0.9516358463726885,
+   "acc_at_50": 0.9840909090909091,
    "chance": 0.5,
-   "mean_conf": 0.8944457769393921,
-   "sec": 4.3,
+   "mean_conf": 0.8916476964950562,
+   "sec": 3.4,
    "heldout": false
   },
   "msmarco_rel": {
    "n": 1446,
-   "acc": 0.6763485477178424,
-   "nll": 0.6001450419425964,
-   "brier": 0.4146299362182617,
-   "ece": 0.05068599555337411,
-   "aurc": 0.2088099598857701,
-   "acc_at_80": 0.7113223854796888,
-   "acc_at_50": 0.7745504840940526,
+   "acc": 0.686030428769018,
+   "nll": 0.5892097353935242,
+   "brier": 0.4052460789680481,
+   "ece": 0.017308449885327464,
+   "aurc": 0.2062339594393076,
+   "acc_at_80": 0.7208297320656871,
+   "acc_at_50": 0.7786998616874136,
    "chance": 0.5,
-   "mean_conf": 0.724265456199646,
-   "sec": 3.4,
+   "mean_conf": 0.6871178150177002,
+   "sec": 1.9,
    "heldout": false
   },
   "medmcqa": {
    "n": 1500,
-   "acc": 0.5006666666666667,
-   "nll": 1.1471084356307983,
-   "brier": 0.6143077611923218,
-   "ece": 0.11058435388406118,
-   "aurc": 0.30678103490481,
-   "acc_at_80": 0.5591666666666667,
-   "acc_at_50": 0.6693333333333333,
+   "acc": 0.516,
+   "nll": 1.0883218050003052,
+   "brier": 0.5872463583946228,
+   "ece": 0.036159954885641735,
+   "aurc": 0.28812445893217514,
+   "acc_at_80": 0.5775,
+   "acc_at_50": 0.692,
    "chance": 0.25,
-   "mean_conf": 0.6112509965896606,
-   "sec": 2.7,
+   "mean_conf": 0.5479114055633545,
+   "sec": 1.3,
    "heldout": false
   },
   "quality": {
    "n": 1500,
-   "acc": 0.5146666666666667,
-   "nll": 1.3762184381484985,
-   "brier": 0.6870999932289124,
-   "ece": 0.2046667002439499,
-   "aurc": 0.34077759893205944,
-   "acc_at_80": 0.5633333333333334,
-   "acc_at_50": 0.6173333333333333,
+   "acc": 0.51,
+   "nll": 1.2439022064208984,
+   "brier": 0.6477080583572388,
+   "ece": 0.15170047092437744,
+   "aurc": 0.329958192571683,
+   "acc_at_80": 0.5591666666666667,
+   "acc_at_50": 0.6386666666666667,
    "chance": 0.25,
-   "mean_conf": 0.7156844139099121,
-   "sec": 18.7,
+   "mean_conf": 0.6539396047592163,
+   "sec": 14.5,
    "heldout": true
   },
   "xstory_cloze": {
    "n": 1500,
-   "acc": 0.9613333333333334,
-   "nll": 0.1043626070022583,
-   "brier": 0.05890524387359619,
-   "ece": 0.006435122688611336,
-   "aurc": 0.004630966744998978,
+   "acc": 0.966,
+   "nll": 0.10946577042341232,
+   "brier": 0.057800862938165665,
+   "ece": 0.031253365834554,
+   "aurc": 0.0037750691801176164,
    "acc_at_80": 0.9966666666666667,
-   "acc_at_50": 0.9986666666666667,
+   "acc_at_50": 0.9973333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9595363140106201,
-   "sec": 3.0,
+   "mean_conf": 0.9347465634346008,
+   "sec": 1.5,
    "heldout": true
   },
   "abstain_probe": {
    "n": 1500,
-   "acc": 0.794,
-   "nll": 0.6064885258674622,
-   "brier": 0.2841270864009857,
-   "ece": 0.037370719770590435,
-   "aurc": 0.054226749633954824,
-   "acc_at_80": 0.8966666666666666,
-   "acc_at_50": 0.968,
+   "acc": 0.57,
+   "nll": 1.453281283378601,
+   "brier": 0.6186018586158752,
+   "ece": 0.20962640981872876,
+   "aurc": 0.2494339184298514,
+   "acc_at_80": 0.6541666666666667,
+   "acc_at_50": 0.756,
    "chance": 0.17950317460317464,
-   "mean_conf": 0.8279028534889221,
-   "sec": 3.8,
+   "mean_conf": 0.7796264886856079,
+   "sec": 2.2,
    "heldout": true
   },
   "toolace": {
    "n": 1000,
-   "acc": 0.924,
-   "nll": 0.24141927063465118,
-   "brier": 0.11758895963430405,
-   "ece": 0.025210315197706203,
-   "aurc": 0.02069434380751835,
-   "acc_at_80": 0.97125,
-   "acc_at_50": 0.982,
+   "acc": 0.93,
+   "nll": 0.2150496393442154,
+   "brier": 0.10874852538108826,
+   "ece": 0.010460366845130945,
+   "aurc": 0.01925174173374489,
+   "acc_at_80": 0.97875,
+   "acc_at_50": 0.986,
    "chance": 0.12470555555555554,
-   "mean_conf": 0.9345914125442505,
-   "sec": 9.7,
+   "mean_conf": 0.9300835132598877,
+   "sec": 4.9,
    "heldout": false
   },
   "agenttraj": {
    "n": 1500,
-   "acc": 0.916,
-   "nll": 0.20774629712104797,
-   "brier": 0.10922007262706757,
-   "ece": 0.0139970025122166,
-   "aurc": 0.00845390614589135,
+   "acc": 0.9133333333333333,
+   "nll": 0.20159707963466644,
+   "brier": 0.11264414340257645,
+   "ece": 0.012760755966107066,
+   "aurc": 0.008911506957123411,
    "acc_at_80": 0.9891666666666666,
-   "acc_at_50": 0.9986666666666667,
+   "acc_at_50": 1.0,
    "chance": 0.2,
-   "mean_conf": 0.9180088043212891,
-   "sec": 6.6,
+   "mean_conf": 0.9171887040138245,
+   "sec": 4.4,
    "heldout": false
   },
   "mind2web": {
    "n": 1500,
-   "acc": 0.8086666666666666,
-   "nll": 0.5181217789649963,
-   "brier": 0.2681422531604767,
-   "ece": 0.025145165185133628,
-   "aurc": 0.05308884826749579,
-   "acc_at_80": 0.8883333333333333,
-   "acc_at_50": 0.9666666666666667,
+   "acc": 0.8273333333333334,
+   "nll": 0.48024988174438477,
+   "brier": 0.24723227322101593,
+   "ece": 0.02287068088849383,
+   "aurc": 0.04421884694063308,
+   "acc_at_80": 0.9066666666666666,
+   "acc_at_50": 0.9813333333333333,
    "chance": 0.16666666666666666,
-   "mean_conf": 0.8295513391494751,
-   "sec": 4.2,
+   "mean_conf": 0.8286987543106079,
+   "sec": 2.5,
    "heldout": false
   },
   "synth": {
    "n": 300,
-   "acc": 0.7566666666666667,
-   "nll": 0.542948305606842,
-   "brier": 0.3258218467235565,
-   "ece": 0.06296401838461561,
-   "aurc": 0.09320432439081776,
-   "acc_at_80": 0.825,
-   "acc_at_50": 0.9266666666666666,
+   "acc": 0.7866666666666666,
+   "nll": 0.5258040428161621,
+   "brier": 0.3138435482978821,
+   "ece": 0.050415676832199115,
+   "aurc": 0.08665319404930355,
+   "acc_at_80": 0.8416666666666667,
+   "acc_at_50": 0.9,
    "chance": 0.37149999999999994,
-   "mean_conf": 0.7900449633598328,
-   "sec": 0.5,
+   "mean_conf": 0.7729342579841614,
+   "sec": 0.3,
    "heldout": false,
    "per_q": [
-    0.7133333333333334,
-    0.8
+    0.76,
+    0.8133333333333334
    ]
   },
   "games": {
    "n": 500,
-   "acc": 0.998,
-   "nll": 0.0028993848245590925,
-   "brier": 0.0012032075319439173,
-   "ece": 0.002253464698791504,
-   "aurc": 4e-06,
+   "acc": 1.0,
+   "nll": 0.0030352056492120028,
+   "brier": 0.001155107282102108,
+   "ece": 0.002569102644920349,
+   "aurc": 0.0,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.2885,
-   "mean_conf": 0.9976871609687805,
-   "sec": 1.2,
+   "mean_conf": 0.9974308609962463,
+   "sec": 0.6,
    "heldout": false
   },
   "mario": {
    "n": 276,
-   "acc": 0.9927536231884058,
-   "nll": 0.013057849369943142,
-   "brier": 0.006284656934440136,
-   "ece": 0.010539730821830639,
-   "aurc": 3.9430219014339996e-05,
+   "acc": 1.0,
+   "nll": 0.009527844376862049,
+   "brier": 0.004120680037885904,
+   "ece": 0.00811098947905112,
+   "aurc": 0.0,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.9905721545219421,
-   "sec": 6.4,
+   "mean_conf": 0.9918889403343201,
+   "sec": 2.8,
    "heldout": false,
    "per_q": [
-    0.9855072463768116,
+    1.0,
     1.0
    ]
+  },
+  "offtopic_probe": {
+   "n": 1600,
+   "acc": 0.82875,
+   "nll": 0.6285893321037292,
+   "brier": 0.25918903946876526,
+   "ece": 0.07270433625206353,
+   "aurc": 0.04255102774957042,
+   "acc_at_80": 0.91171875,
+   "acc_at_50": 0.975,
+   "chance": 0.18774627976190478,
+   "mean_conf": 0.9010530114173889,
+   "sec": 2.3,
+   "heldout": true
   }
  },
  "agg": {
   "in_task": {
-   "acc": 0.8126446732755706,
-   "nll": 0.4534154008986239,
-   "brier": 0.25158945492306806,
-   "ece": 0.03454096135309228,
-   "aurc": 0.09352685127729836,
-   "acc_at_80": 0.8639655870629418,
+   "acc": 0.8149500819963449,
+   "nll": 0.44540443098968896,
+   "brier": 0.2484059167917872,
+   "ece": 0.027587339109589562,
+   "aurc": 0.09255141592820487,
+   "acc_at_80": 0.8653429127460076,
    "chance": 0.32498304691713953
   },
   "heldout": {
-   "acc": 0.7479565063694622,
-   "nll": 0.656638503074646,
-   "brier": 0.35096842001961626,
-   "ece": 0.08591399023846887,
-   "aurc": 0.13724056983642266,
-   "acc_at_80": 0.8022276412618095,
-   "chance": 0.32053884112032693
+   "acc": 0.7384736622364532,
+   "nll": 0.6866676495410502,
+   "brier": 0.361666936504965,
+   "ece": 0.0877105698314277,
+   "aurc": 0.14519520391245042,
+   "acc_at_80": 0.7926778581130387,
+   "chance": 0.31500581773039266
   },
   "n_in": 69,
-  "n_heldout": 23
+  "n_heldout": 24
  },
- "model": "runs/r7_v4/model",
+ "model": "runs/r10_v5/model",
  "engine": "compile"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:005b182713b64d6a37f37b30740cd36e46225217c0183798fbc3f278bc7d7da3
+oid sha256:5d8271b508f71697fd0a55e15fc98e1e894b93c0c37d97cbe5cc1111f81b2dca
 size 3763692048

tokenizer_config.json

@@ -9,7 +9,7 @@
   "eos_token": "<|endoftext|>",
   "errors": "replace",
   "image_token": "<|image_pad|>",
-  "is_local": true,
+  "is_local": false,
   "local_files_only": false,
   "model_max_length": 262144,
   "model_specific_special_tokens": {