Upload folder using huggingface_hub

README.md

@@ -17,7 +17,9 @@ schema violations. It is meant to be called from software, not chatted with.
 Base model: [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base) (1.9B parameters),
 fully fine-tuned for one epoch (942k examples, 183M tokens, 2.5 hours on one
 NVIDIA GH200) with cross-entropy, a proper scoring rule, on a mixture of 64
-public decision datasets.
+public decision datasets, then continued for one epoch on 45k situation-to-action
+examples (agent trajectories, web element choice, synthetic situations, game states)
+with a replay of the general mixture (v4).
 
 ## Usage
 
@@ -76,13 +78,16 @@ the supplied candidates rather than a fixed head.
 
 ## Training data
 
-64 public datasets, up to 20k examples each (`decider/data.py`, `decider/data2.py`):
+69 public datasets plus synthetic situations, up to 20k examples each (`decider/data.py`, `decider/data2.py`, `decider/data3.py`):
 intent detection, ticket routing, topic classification, sentiment, emotion,
 moderation (toxicity, hate, spam, jailbreak, safety), NLI, paraphrase, fact
 verification, passage relevance, reading comprehension, multiple-choice QA,
 ordinal rating scales (HelpSteer2 attributes, STS-B, hate-speech intensity,
 LIAR2 truthfulness), pairwise response preference (HelpSteer3, UltraFeedback,
 SHP, HH-RLHF) and tool selection (Glaive, ToolACE).
+v4 adds next-action choice from agent trajectories (AgentGym AgentTraj-L), web element
+choice (Mind2Web), 1.5k synthetic situations written by Qwen3.5-27B, and teacher-labelled
+states from Pong, Breakout, CliffWalking, MiniGrid and Super Mario Bros.
 Abstention augmentation: in 10% of questions with three or more options the
 gold option is removed and "none of the above" becomes the answer.
 
@@ -94,8 +99,8 @@ gold option is removed and "none of the above" becomes the answer.
 | Qwen3.5-2B-Base, zero-shot | held-out (23) | 0.642 | 0.853 | 0.460 | 0.105 | 0.242 | 0.685 |
 | Qwen3.5-4B-Base, zero-shot | in-task (64) | 0.695 | 0.768 | 0.405 | 0.090 | 0.206 | 0.742 |
 | Qwen3.5-4B-Base, zero-shot | held-out (23) | 0.711 | 0.734 | 0.390 | 0.089 | 0.169 | 0.761 |
-| **this model** | in-task (64) | 0.807 | 0.460 | 0.257 | 0.029 | 0.097 | 0.859 |
-| **this model** | held-out (23) | 0.745 | 0.634 | 0.343 | 0.075 | 0.138 | 0.802 |
+| **this model (v4)** | in-task (69) | 0.813 | 0.450 | 0.250 | 0.033 | 0.093 | 0.864 |
+| **this model (v4)** | held-out (23) | 0.748 | 0.634 | 0.345 | 0.076 | 0.137 | 0.803 |
 
 
 Per-task accuracy / ECE on the held-out datasets:
@@ -191,7 +196,8 @@ Per-task accuracy / ECE on the held-out datasets:
 | tweet_irony | 0.511 / 0.158 | 0.676 / 0.073 | 0.769 / 0.048 |
 
 
-*In-task* = test splits of the 64 training datasets. *Held-out* = 23 datasets never
+Probabilities use a temperature of 1.15 fitted on in-task data (stored in `decider_config.json`, applied by the helper).
+*In-task* = test splits of the training datasets (in-task rows for the zero-shot baselines cover the original 64). *Held-out* = 23 datasets never
 seen in training: TREC, BBC news, PAWS, SciQ, Social IQa, StrategyQA, PubMedQA,
 TruthfulQA, tweet irony, financial sentiment, ADE, MASSIVE scenario, student
 question categories, Dolly categories, CR reviews, Financial PhraseBank,
@@ -212,10 +218,14 @@ questions each:
 | setting | p50 latency | throughput |
 |---|---|---|
 | single request, eager PyTorch | 49 ms | |
-| single request, CUDA-graph engine | 6.6 ms | |
-| batch of 32, in-process | 115 ms | ~840 decisions/s |
-| HTTP server, 1 client | 10 ms | 97 req/s |
-| HTTP server, 64 clients | 231 ms | 263 req/s, 1314 decisions/s |
+| single request, CUDA graphs + torch.compile (helper default) | 4.0 ms | |
+| batch of 32, in-process, bf16 | 70 ms | ~1370 decisions/s |
+| batch of 32, in-process, FP8 linears | 58 ms | ~1670 decisions/s |
+| HTTP server (FP8), 1 client | 6.8 ms | 134 req/s |
+| HTTP server (FP8), 64 clients | 126 ms | 431 req/s, 2152 decisions/s |
+
+FP8 (e4m3 weights, per-token activation scales) changes accuracy and calibration by
+less than the evaluation noise (18-task check: accuracy 0.833 vs 0.835, ECE equal).
 
 ## Limitations
 

decider/engine.py

@@ -4,7 +4,7 @@ Right padding + causal layers => pad positions never influence earlier slots, so
 mask is needed and every (B, T) bucket can be captured once and replayed.  The graph outputs
 option-letter logits for all positions [B, T, K]; slots are gathered outside.
 """
-import time, torch, torch.nn.functional as F
+import time, torch, torch._dynamo, torch.nn.functional as F
 from .model import DecisionModel, collate
 from .prompt import build, MAX_OPTIONS
 
@@ -58,7 +58,6 @@ class Engine:
             from .fp8 import convert_to_fp8
             self.cfg["fp8_layers"] = convert_to_fp8(self.core)
         if compile:
-            import torch._dynamo
             torch._dynamo.config.cache_size_limit = 128
             self._fwd_impl = torch.compile(self._fwd_eager, dynamic=False)
         else:

decider/infer.py

@@ -26,7 +26,15 @@ class Example:
 class Decider:
     """use_graphs=True (default on CUDA) routes scoring through decider.engine.Engine: shape-bucketed
     CUDA graphs, ~7x lower single-request latency than eager. Set False for CPU or debugging."""
-    def __init__(self, path, device="cuda", dtype=torch.bfloat16, temperature=1.0, abstain_below=0.0, use_graphs=None):
+    def __init__(self, path, device="cuda", dtype=torch.bfloat16, temperature=None, abstain_below=0.0, use_graphs=None):
+        if temperature is None:                      # model folder may carry a fitted temperature (decider_config.json)
+            import json, os
+            try:
+                from huggingface_hub import hf_hub_download
+                cfg_path = os.path.join(path, "decider_config.json") if os.path.isdir(path) else hf_hub_download(path, "decider_config.json")
+                temperature = float(json.load(open(cfg_path)).get("temperature", 1.0))
+            except Exception:
+                temperature = 1.0
         if use_graphs is None:
             use_graphs = str(device).startswith("cuda")
         if use_graphs:

decider_config.json

@@ -0,0 +1 @@
+{"temperature": 1.15, "version": "v4", "base": "Qwen/Qwen3.5-2B-Base"}

eval_results.json

@@ -2,1282 +2,1361 @@
  "results": {
   "clinc_oos": {
    "n": 1500,
-   "acc": 0.9766666666666667,
-   "nll": 0.07261181622743607,
-   "brier": 0.03852468356490135,
-   "ece": 0.010659157812595393,
-   "aurc": 0.0013287162793807251,
+   "acc": 0.978,
+   "nll": 0.06232544407248497,
+   "brier": 0.033815909177064896,
+   "ece": 0.008496353924274395,
+   "aurc": 0.0008730387999259201,
    "acc_at_80": 0.9991666666666666,
    "acc_at_50": 1.0,
    "chance": 0.1,
-   "mean_conf": 0.9788497686386108,
-   "sec": 3.5,
+   "mean_conf": 0.9839518070220947,
+   "sec": 13.8,
    "heldout": false
   },
   "massive_intent": {
    "n": 1500,
-   "acc": 0.9493333333333334,
-   "nll": 0.15807873010635376,
-   "brier": 0.07667268067598343,
-   "ece": 0.012617475867271442,
-   "aurc": 0.005890076997496865,
+   "acc": 0.9486666666666667,
+   "nll": 0.15551087260246277,
+   "brier": 0.0762505829334259,
+   "ece": 0.012290600876013396,
+   "aurc": 0.005614506940959013,
    "acc_at_80": 0.9933333333333333,
-   "acc_at_50": 0.9986666666666667,
+   "acc_at_50": 0.9973333333333333,
    "chance": 0.1,
-   "mean_conf": 0.9502438306808472,
-   "sec": 2.5,
+   "mean_conf": 0.953382670879364,
+   "sec": 4.0,
    "heldout": false
   },
   "massive_scenario": {
    "n": 1500,
-   "acc": 0.8146666666666667,
-   "nll": 0.5371937155723572,
-   "brier": 0.2577172517776489,
-   "ece": 0.023450730780760452,
-   "aurc": 0.049948004214958666,
-   "acc_at_80": 0.9033333333333333,
-   "acc_at_50": 0.9706666666666667,
+   "acc": 0.8033333333333333,
+   "nll": 0.5681269764900208,
+   "brier": 0.2758673429489136,
+   "ece": 0.03523811907569568,
+   "aurc": 0.05579813486552701,
+   "acc_at_80": 0.8858333333333334,
+   "acc_at_50": 0.968,
    "chance": 0.1,
-   "mean_conf": 0.8208412528038025,
-   "sec": 2.4,
+   "mean_conf": 0.8278322219848633,
+   "sec": 1.5,
    "heldout": true
   },
   "bitext_support": {
    "n": 3000,
-   "acc": 0.9973333333333333,
-   "nll": 0.006749222986400127,
-   "brier": 0.0037496667355298996,
-   "ece": 0.0017460443178812545,
-   "aurc": 1.1261882856520142e-05,
+   "acc": 0.998,
+   "nll": 0.007169861346483231,
+   "brier": 0.0035593616776168346,
+   "ece": 0.0009388573169708132,
+   "aurc": 1.788459220568468e-05,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.09999999999999998,
-   "mean_conf": 0.9978217482566833,
-   "sec": 3.4,
+   "mean_conf": 0.998285710811615,
+   "sec": 2.1,
    "heldout": false,
    "per_q": [
-    0.998,
-    0.9966666666666667
+    0.9986666666666667,
+    0.9973333333333333
    ]
   },
   "support_tickets": {
    "n": 4500,
-   "acc": 0.5791111111111111,
-   "nll": 0.9825455546379089,
-   "brier": 0.5169190764427185,
-   "ece": 0.016227112455500483,
-   "aurc": 0.21513268536739544,
-   "acc_at_80": 0.6405555555555555,
-   "acc_at_50": 0.7631111111111111,
+   "acc": 0.5737777777777778,
+   "nll": 0.9863417148590088,
+   "brier": 0.5184141993522644,
+   "ece": 0.0138452862434917,
+   "aurc": 0.21858182796967993,
+   "acc_at_80": 0.6361111111111111,
+   "acc_at_50": 0.7497777777777778,
    "chance": 0.22777777777777777,
-   "mean_conf": 0.573243260383606,
-   "sec": 4.7,
+   "mean_conf": 0.5744046568870544,
+   "sec": 10.3,
    "heldout": false,
    "per_q": [
-    0.8153333333333334,
-    0.44,
-    0.482
+    0.8113333333333334,
+    0.43133333333333335,
+    0.4786666666666667
    ]
   },
   "ag_news": {
    "n": 1500,
    "acc": 0.9173333333333333,
-   "nll": 0.24047400057315826,
-   "brier": 0.1257070153951645,
-   "ece": 0.010078805923461939,
-   "aurc": 0.018073040554543826,
+   "nll": 0.24114005267620087,
+   "brier": 0.12489213049411774,
+   "ece": 0.016586370726426467,
+   "aurc": 0.018462071969618875,
    "acc_at_80": 0.9716666666666667,
-   "acc_at_50": 0.988,
+   "acc_at_50": 0.9853333333333333,
    "chance": 0.25,
-   "mean_conf": 0.914076030254364,
-   "sec": 2.5,
+   "mean_conf": 0.9118769764900208,
+   "sec": 1.5,
    "heldout": false
   },
   "dbpedia": {
    "n": 1500,
    "acc": 0.9913333333333333,
-   "nll": 0.0235174261033535,
-   "brier": 0.012497784569859505,
-   "ece": 0.0035786836942037336,
-   "aurc": 0.00010865530954364588,
+   "nll": 0.024175753816962242,
+   "brier": 0.013113866560161114,
+   "ece": 0.005916546404361727,
+   "aurc": 0.00010674119407793753,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.1,
-   "mean_conf": 0.9918569326400757,
-   "sec": 3.2,
+   "mean_conf": 0.9923719763755798,
+   "sec": 2.0,
    "heldout": false
   },
   "yahoo_topics": {
    "n": 1500,
-   "acc": 0.762,
-   "nll": 0.7322853207588196,
-   "brier": 0.33554312586784363,
-   "ece": 0.024258904794851946,
-   "aurc": 0.08055392547857332,
-   "acc_at_80": 0.8533333333333334,
-   "acc_at_50": 0.9426666666666667,
+   "acc": 0.76,
+   "nll": 0.7366493940353394,
+   "brier": 0.33801746368408203,
+   "ece": 0.028587430814902,
+   "aurc": 0.08282859840202575,
+   "acc_at_80": 0.855,
+   "acc_at_50": 0.9453333333333334,
    "chance": 0.1,
-   "mean_conf": 0.7577335834503174,
-   "sec": 2.9,
+   "mean_conf": 0.7742340564727783,
+   "sec": 1.8,
    "heldout": false
   },
   "newsgroups": {
    "n": 1462,
-   "acc": 0.8146374829001368,
-   "nll": 0.5313005447387695,
-   "brier": 0.25277355313301086,
-   "ece": 0.02700514194495698,
-   "aurc": 0.03996642898945821,
+   "acc": 0.8125854993160054,
+   "nll": 0.5523399710655212,
+   "brier": 0.26114463806152344,
+   "ece": 0.044583431815807525,
+   "aurc": 0.04188347828916838,
    "acc_at_80": 0.917094017094017,
    "acc_at_50": 0.9835841313269493,
    "chance": 0.09999999999999999,
-   "mean_conf": 0.8385738730430603,
-   "sec": 5.3,
+   "mean_conf": 0.8485810160636902,
+   "sec": 14.4,
    "heldout": false
   },
   "bbc_news": {
    "n": 1000,
-   "acc": 0.941,
-   "nll": 0.15981632471084595,
-   "brier": 0.09309598803520203,
-   "ece": 0.01646793046593665,
-   "aurc": 0.007680621955810168,
-   "acc_at_80": 0.98375,
+   "acc": 0.939,
+   "nll": 0.16493572294712067,
+   "brier": 0.09448844194412231,
+   "ece": 0.01869928520917896,
+   "aurc": 0.008020306999373308,
+   "acc_at_80": 0.9825,
    "acc_at_50": 1.0,
    "chance": 0.20000000000000004,
-   "mean_conf": 0.936293363571167,
-   "sec": 5.3,
+   "mean_conf": 0.9388766884803772,
+   "sec": 5.8,
    "heldout": true
   },
   "trec": {
    "n": 500,
-   "acc": 0.766,
-   "nll": 0.5928767323493958,
-   "brier": 0.31943121552467346,
-   "ece": 0.03574929744005202,
-   "aurc": 0.09966853443922356,
-   "acc_at_80": 0.8275,
-   "acc_at_50": 0.916,
+   "acc": 0.754,
+   "nll": 0.6227552890777588,
+   "brier": 0.33451518416404724,
+   "ece": 0.041722716212272644,
+   "aurc": 0.10665603039652902,
+   "acc_at_80": 0.82,
+   "acc_at_50": 0.896,
    "chance": 0.16666666666666663,
-   "mean_conf": 0.769515872001648,
-   "sec": 0.9,
+   "mean_conf": 0.7837464213371277,
+   "sec": 3.3,
    "heldout": true
   },
   "student_questions": {
    "n": 1500,
-   "acc": 0.9273333333333333,
-   "nll": 0.22138677537441254,
-   "brier": 0.11496378481388092,
-   "ece": 0.029807024538516993,
-   "aurc": 0.015550148490194371,
-   "acc_at_80": 0.9716666666666667,
+   "acc": 0.9246666666666666,
+   "nll": 0.22754858434200287,
+   "brier": 0.11876419931650162,
+   "ece": 0.022739037255446105,
+   "aurc": 0.016019787677674244,
+   "acc_at_80": 0.97,
    "acc_at_50": 0.992,
    "chance": 0.25,
-   "mean_conf": 0.9056621193885803,
-   "sec": 3.0,
+   "mean_conf": 0.9101096987724304,
+   "sec": 3.4,
    "heldout": true
   },
   "dolly_category": {
    "n": 1500,
-   "acc": 0.318,
-   "nll": 2.0031864643096924,
-   "brier": 0.8271570801734924,
-   "ece": 0.21146952118476237,
-   "aurc": 0.48668464321903865,
-   "acc_at_80": 0.3575,
-   "acc_at_50": 0.44,
+   "acc": 0.33266666666666667,
+   "nll": 2.009742021560669,
+   "brier": 0.8325088620185852,
+   "ece": 0.22487777996063235,
+   "aurc": 0.47846646274987925,
+   "acc_at_80": 0.3725,
+   "acc_at_50": 0.4493333333333333,
    "chance": 0.125,
-   "mean_conf": 0.5294612646102905,
-   "sec": 3.5,
+   "mean_conf": 0.5575444102287292,
+   "sec": 3.8,
    "heldout": true
   },
   "imdb": {
    "n": 1500,
-   "acc": 0.9633333333333334,
-   "nll": 0.0927785336971283,
-   "brier": 0.053890664130449295,
-   "ece": 0.007267733812332158,
-   "aurc": 0.0027718147959179054,
+   "acc": 0.9613333333333334,
+   "nll": 0.09619174897670746,
+   "brier": 0.057166311889886856,
+   "ece": 0.010577520211537642,
+   "aurc": 0.002844706294540048,
    "acc_at_80": 0.9983333333333333,
    "acc_at_50": 1.0,
    "chance": 0.5,
-   "mean_conf": 0.9614710211753845,
-   "sec": 5.6,
+   "mean_conf": 0.9708292484283447,
+   "sec": 5.5,
    "heldout": false
   },
   "sst2": {
    "n": 872,
-   "acc": 0.9415137614678899,
-   "nll": 0.15022407472133636,
-   "brier": 0.08405344188213348,
-   "ece": 0.014767689784185644,
-   "aurc": 0.009848323778768498,
+   "acc": 0.9426605504587156,
+   "nll": 0.15219736099243164,
+   "brier": 0.08451657742261887,
+   "ece": 0.018561122545955384,
+   "aurc": 0.0094376600127498,
    "acc_at_80": 0.9899713467048711,
    "acc_at_50": 0.9931192660550459,
    "chance": 0.5,
-   "mean_conf": 0.9442880749702454,
-   "sec": 1.5,
+   "mean_conf": 0.9517649412155151,
+   "sec": 3.8,
    "heldout": false
   },
   "sst5": {
    "n": 1500,
-   "acc": 0.6133333333333333,
-   "nll": 0.9277896881103516,
-   "brier": 0.5273030996322632,
-   "ece": 0.03280426164468128,
-   "aurc": 0.30489681515108674,
+   "acc": 0.6073333333333333,
+   "nll": 0.9326017498970032,
+   "brier": 0.5300925970077515,
+   "ece": 0.02625397855043411,
+   "aurc": 0.3098600910602178,
    "acc_at_80": 0.6466666666666666,
-   "acc_at_50": 0.684,
+   "acc_at_50": 0.68,
    "chance": 0.2,
-   "mean_conf": 0.58907151222229,
-   "sec": 2.4,
+   "mean_conf": 0.5991732478141785,
+   "sec": 2.6,
    "heldout": false
   },
   "yelp": {
    "n": 1500,
-   "acc": 0.69,
-   "nll": 0.7099072337150574,
-   "brier": 0.41327351331710815,
-   "ece": 0.023333197732766472,
-   "aurc": 0.16233091407117872,
-   "acc_at_80": 0.7483333333333333,
+   "acc": 0.688,
+   "nll": 0.7123478055000305,
+   "brier": 0.41503453254699707,
+   "ece": 0.030766211390495284,
+   "aurc": 0.16381766100924522,
+   "acc_at_80": 0.7466666666666667,
    "acc_at_50": 0.8146666666666667,
    "chance": 0.2,
-   "mean_conf": 0.7003123164176941,
-   "sec": 4.5,
+   "mean_conf": 0.7086488604545593,
+   "sec": 4.6,
    "heldout": false
   },
   "amazon_stars": {
    "n": 1500,
-   "acc": 0.6066666666666667,
-   "nll": 0.9025667309761047,
-   "brier": 0.5039957761764526,
-   "ece": 0.030678236802419018,
-   "aurc": 0.2318602277252905,
-   "acc_at_80": 0.6533333333333333,
-   "acc_at_50": 0.7546666666666667,
+   "acc": 0.6073333333333333,
+   "nll": 0.9104115962982178,
+   "brier": 0.508173942565918,
+   "ece": 0.024688152035077397,
+   "aurc": 0.23917450560661985,
+   "acc_at_80": 0.6625,
+   "acc_at_50": 0.748,
    "chance": 0.2,
-   "mean_conf": 0.6180839538574219,
-   "sec": 2.7,
+   "mean_conf": 0.6209068894386292,
+   "sec": 3.1,
    "heldout": false
   },
   "emotion": {
    "n": 1500,
-   "acc": 0.854,
-   "nll": 0.3842936158180237,
-   "brier": 0.20495271682739258,
-   "ece": 0.014472351928551988,
-   "aurc": 0.03513619107373314,
-   "acc_at_80": 0.9308333333333333,
+   "acc": 0.8526666666666667,
+   "nll": 0.3817766606807709,
+   "brier": 0.20716996490955353,
+   "ece": 0.028686144828796382,
+   "aurc": 0.036016887247968345,
+   "acc_at_80": 0.9241666666666667,
    "acc_at_50": 0.9813333333333333,
    "chance": 0.16666666666666666,
-   "mean_conf": 0.8586899638175964,
-   "sec": 2.4,
+   "mean_conf": 0.8751572370529175,
+   "sec": 2.6,
    "heldout": false
   },
   "go_emotions": {
    "n": 1500,
-   "acc": 0.7766666666666666,
-   "nll": 0.6451207399368286,
-   "brier": 0.3155723512172699,
-   "ece": 0.01979676757256191,
-   "aurc": 0.07662852818492483,
-   "acc_at_80": 0.8558333333333333,
-   "acc_at_50": 0.9453333333333334,
+   "acc": 0.77,
+   "nll": 0.6605949997901917,
+   "brier": 0.3219051659107208,
+   "ece": 0.02531181337436041,
+   "aurc": 0.07909983628498353,
+   "acc_at_80": 0.8525,
+   "acc_at_50": 0.9413333333333334,
    "chance": 0.1,
-   "mean_conf": 0.787257730960846,
-   "sec": 2.4,
+   "mean_conf": 0.7867252826690674,
+   "sec": 3.0,
    "heldout": false
   },
   "tweet_sentiment": {
    "n": 1500,
-   "acc": 0.7353333333333333,
-   "nll": 0.5786967873573303,
-   "brier": 0.35522446036338806,
-   "ece": 0.02083511827389399,
-   "aurc": 0.12570788661149288,
-   "acc_at_80": 0.7933333333333333,
+   "acc": 0.732,
+   "nll": 0.5844112038612366,
+   "brier": 0.35898950695991516,
+   "ece": 0.023715959330399827,
+   "aurc": 0.1277814283266402,
+   "acc_at_80": 0.7908333333333334,
    "acc_at_50": 0.868,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.7195278406143188,
+   "mean_conf": 0.7393827438354492,
    "sec": 2.4,
    "heldout": false
   },
   "tweet_emotion": {
    "n": 1421,
-   "acc": 0.8395496129486277,
-   "nll": 0.4268813133239746,
-   "brier": 0.22563911974430084,
-   "ece": 0.03929315151923309,
-   "aurc": 0.03901553887763414,
-   "acc_at_80": 0.9217238346525946,
-   "acc_at_50": 0.9830985915492958,
+   "acc": 0.8332160450387052,
+   "nll": 0.4378463625907898,
+   "brier": 0.23256292939186096,
+   "ece": 0.029261003112725864,
+   "aurc": 0.0408950375447702,
+   "acc_at_80": 0.9129287598944591,
+   "acc_at_50": 0.9845070422535211,
    "chance": 0.25,
-   "mean_conf": 0.8121002912521362,
-   "sec": 2.3,
+   "mean_conf": 0.8131034970283508,
+   "sec": 4.8,
    "heldout": false
   },
   "tweet_irony": {
    "n": 784,
-   "acc": 0.7793367346938775,
-   "nll": 0.4821619391441345,
-   "brier": 0.3177129328250885,
-   "ece": 0.05867012658593607,
-   "aurc": 0.10753610876267371,
-   "acc_at_80": 0.8373205741626795,
-   "acc_at_50": 0.8928571428571429,
+   "acc": 0.7933673469387755,
+   "nll": 0.4786851406097412,
+   "brier": 0.3143369257450104,
+   "ece": 0.05477343446442059,
+   "aurc": 0.11260140253357594,
+   "acc_at_80": 0.8261562998405104,
+   "acc_at_50": 0.8826530612244898,
    "chance": 0.5,
-   "mean_conf": 0.7206665277481079,
-   "sec": 1.3,
+   "mean_conf": 0.7504966855049133,
+   "sec": 1.2,
    "heldout": true
   },
   "fin_sentiment": {
    "n": 1500,
-   "acc": 0.7693333333333333,
-   "nll": 0.5099801421165466,
-   "brier": 0.313610315322876,
-   "ece": 0.03050452242294948,
-   "aurc": 0.0985651541028985,
-   "acc_at_80": 0.8283333333333334,
-   "acc_at_50": 0.9186666666666666,
+   "acc": 0.784,
+   "nll": 0.4931422472000122,
+   "brier": 0.29916250705718994,
+   "ece": 0.02881191293398539,
+   "aurc": 0.09077011331787095,
+   "acc_at_80": 0.8375,
+   "acc_at_50": 0.9266666666666666,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.7750865817070007,
-   "sec": 2.4,
+   "mean_conf": 0.7918360233306885,
+   "sec": 2.5,
    "heldout": true
   },
   "cr_reviews": {
    "n": 753,
-   "acc": 0.9017264276228419,
-   "nll": 0.23763678967952728,
-   "brier": 0.14010722935199738,
-   "ece": 0.02007227114947191,
-   "aurc": 0.02070501416560806,
-   "acc_at_80": 0.9634551495016611,
-   "acc_at_50": 0.9946808510638298,
+   "acc": 0.9070385126162018,
+   "nll": 0.23999790847301483,
+   "brier": 0.13726605474948883,
+   "ece": 0.02803241042976838,
+   "aurc": 0.019896790015697566,
+   "acc_at_80": 0.9617940199335548,
+   "acc_at_50": 0.9920212765957447,
    "chance": 0.5,
-   "mean_conf": 0.9198644161224365,
-   "sec": 1.3,
+   "mean_conf": 0.9328383803367615,
+   "sec": 1.2,
    "heldout": true
   },
   "counterfactual": {
    "n": 1500,
-   "acc": 0.952,
-   "nll": 0.12973414361476898,
-   "brier": 0.07501313090324402,
-   "ece": 0.009895903388659118,
-   "aurc": 0.0057674469796164165,
-   "acc_at_80": 0.9891666666666666,
-   "acc_at_50": 1.0,
+   "acc": 0.9546666666666667,
+   "nll": 0.1323074996471405,
+   "brier": 0.07490427792072296,
+   "ece": 0.014571970144907676,
+   "aurc": 0.0058697408516071295,
+   "acc_at_80": 0.99,
+   "acc_at_50": 0.9973333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9529626369476318,
-   "sec": 2.5,
+   "mean_conf": 0.9637404084205627,
+   "sec": 2.4,
    "heldout": false
   },
   "subjectivity": {
    "n": 1500,
-   "acc": 0.9653333333333334,
-   "nll": 0.1150687038898468,
-   "brier": 0.06236080080270767,
-   "ece": 0.014521381457646683,
-   "aurc": 0.005428128396492908,
+   "acc": 0.9633333333333334,
+   "nll": 0.11715030670166016,
+   "brier": 0.06305266171693802,
+   "ece": 0.011594313383102427,
+   "aurc": 0.005805298576335658,
    "acc_at_80": 0.9916666666666667,
    "acc_at_50": 0.9973333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9508118629455566,
+   "mean_conf": 0.9517436623573303,
    "sec": 2.4,
    "heldout": false
   },
   "tweet_offensive": {
    "n": 860,
-   "acc": 0.8569767441860465,
-   "nll": 0.3296823799610138,
-   "brier": 0.2049010545015335,
-   "ece": 0.048828685491584076,
-   "aurc": 0.04312619584445423,
-   "acc_at_80": 0.9229651162790697,
-   "acc_at_50": 0.9674418604651163,
+   "acc": 0.8604651162790697,
+   "nll": 0.3307216167449951,
+   "brier": 0.2062027007341385,
+   "ece": 0.03277220109174419,
+   "aurc": 0.043970018789679495,
+   "acc_at_80": 0.9186046511627907,
+   "acc_at_50": 0.9627906976744186,
    "chance": 0.5,
-   "mean_conf": 0.8316621780395508,
-   "sec": 1.4,
+   "mean_conf": 0.8332623243331909,
+   "sec": 1.5,
    "heldout": false
   },
   "tweet_hate": {
    "n": 1500,
-   "acc": 0.5126666666666667,
-   "nll": 0.8970687389373779,
-   "brier": 0.634926974773407,
-   "ece": 0.25461129041512803,
-   "aurc": 0.41152814993800513,
+   "acc": 0.5106666666666667,
+   "nll": 0.951435387134552,
+   "brier": 0.6554798483848572,
+   "ece": 0.27647322726249696,
+   "aurc": 0.421070973829564,
    "acc_at_80": 0.535,
-   "acc_at_50": 0.5426666666666666,
+   "acc_at_50": 0.556,
    "chance": 0.5,
-   "mean_conf": 0.7672780156135559,
-   "sec": 2.4,
+   "mean_conf": 0.7855966687202454,
+   "sec": 2.6,
    "heldout": false
   },
   "hate_offensive": {
    "n": 1500,
-   "acc": 0.9213333333333333,
-   "nll": 0.2372422218322754,
-   "brier": 0.12591436505317688,
-   "ece": 0.012463796754678066,
-   "aurc": 0.01799362902327372,
+   "acc": 0.9206666666666666,
+   "nll": 0.2384883612394333,
+   "brier": 0.1258707195520401,
+   "ece": 0.013242818454901406,
+   "aurc": 0.018655782825341898,
    "acc_at_80": 0.9716666666666667,
-   "acc_at_50": 0.988,
+   "acc_at_50": 0.9853333333333333,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.9196624159812927,
-   "sec": 2.4,
+   "mean_conf": 0.9204931855201721,
+   "sec": 2.6,
    "heldout": false
   },
   "civil_comments": {
    "n": 7500,
-   "acc": 0.9353333333333333,
-   "nll": 0.1511416733264923,
-   "brier": 0.09208963811397552,
-   "ece": 0.004735005791982015,
-   "aurc": 0.007750731444351848,
-   "acc_at_80": 0.9876666666666667,
+   "acc": 0.9356,
+   "nll": 0.15526148676872253,
+   "brier": 0.09336414933204651,
+   "ece": 0.013219891810417186,
+   "aurc": 0.007861904760099968,
+   "acc_at_80": 0.9856666666666667,
    "acc_at_50": 0.9997333333333334,
    "chance": 0.5,
-   "mean_conf": 0.9398914575576782,
-   "sec": 4.2,
+   "mean_conf": 0.9488199353218079,
+   "sec": 4.4,
    "heldout": false,
    "per_q": [
-    0.872,
-    0.978,
-    0.9906666666666667,
-    0.8666666666666667,
+    0.8753333333333333,
+    0.9786666666666667,
+    0.9913333333333333,
+    0.8633333333333333,
     0.9693333333333334
    ]
   },
   "toxic_chat": {
    "n": 3000,
-   "acc": 0.979,
-   "nll": 0.055397775024175644,
-   "brier": 0.03014867566525936,
-   "ece": 0.004773223658402735,
-   "aurc": 0.0011957276641795629,
-   "acc_at_80": 0.99875,
-   "acc_at_50": 1.0,
+   "acc": 0.9793333333333333,
+   "nll": 0.05689281225204468,
+   "brier": 0.029941946268081665,
+   "ece": 0.008714853028456338,
+   "aurc": 0.00120893581584804,
+   "acc_at_80": 0.9991666666666666,
+   "acc_at_50": 0.9993333333333333,
    "chance": 0.5,
-   "mean_conf": 0.9801945090293884,
-   "sec": 2.8,
+   "mean_conf": 0.985617458820343,
+   "sec": 3.3,
    "heldout": false,
    "per_q": [
-    0.9646666666666667,
-    0.9933333333333333
+    0.964,
+    0.9946666666666667
    ]
   },
   "sms_spam": {
    "n": 1000,
-   "acc": 0.986,
-   "nll": 0.05686425045132637,
-   "brier": 0.025409400463104248,
-   "ece": 0.009181047439575173,
-   "aurc": 0.002507962440114501,
+   "acc": 0.984,
+   "nll": 0.054738420993089676,
+   "brier": 0.023743100464344025,
+   "ece": 0.009705231130123096,
+   "aurc": 0.002162345807605099,
    "acc_at_80": 0.99625,
    "acc_at_50": 0.998,
    "chance": 0.5,
-   "mean_conf": 0.989531397819519,
-   "sec": 1.7,
+   "mean_conf": 0.9895715117454529,
+   "sec": 4.2,
    "heldout": false
   },
   "enron_spam": {
    "n": 1500,
-   "acc": 0.9873333333333333,
-   "nll": 0.027067676186561584,
-   "brier": 0.016892286017537117,
-   "ece": 0.004601201574007648,
-   "aurc": 0.00021669720532009002,
+   "acc": 0.9866666666666667,
+   "nll": 0.02601483091711998,
+   "brier": 0.016239631921052933,
+   "ece": 0.004016611814498852,
+   "aurc": 0.0002021199142868427,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.5,
-   "mean_conf": 0.9897943735122681,
-   "sec": 5.8,
+   "mean_conf": 0.9905825257301331,
+   "sec": 9.2,
    "heldout": false
   },
   "insincere_questions": {
    "n": 1500,
    "acc": 0.9553333333333334,
-   "nll": 0.10724982619285583,
-   "brier": 0.06384952366352081,
-   "ece": 0.01187876927852626,
-   "aurc": 0.0043015761367758325,
+   "nll": 0.10723885893821716,
+   "brier": 0.06371062248945236,
+   "ece": 0.011764580051104229,
+   "aurc": 0.004172164123818539,
    "acc_at_80": 0.9975,
    "acc_at_50": 0.9986666666666667,
    "chance": 0.5,
-   "mean_conf": 0.9536961317062378,
+   "mean_conf": 0.9626001119613647,
    "sec": 2.4,
    "heldout": false
   },
   "ade": {
    "n": 1500,
-   "acc": 0.808,
-   "nll": 0.44578930735588074,
-   "brier": 0.2858363091945648,
-   "ece": 0.03843762199083964,
-   "aurc": 0.09344845724457498,
-   "acc_at_80": 0.8516666666666667,
-   "acc_at_50": 0.9133333333333333,
+   "acc": 0.7893333333333333,
+   "nll": 0.4881707429885864,
+   "brier": 0.3113546073436737,
+   "ece": 0.0730705860455831,
+   "aurc": 0.10379213741472014,
+   "acc_at_80": 0.8325,
+   "acc_at_50": 0.896,
    "chance": 0.5,
-   "mean_conf": 0.8236058950424194,
-   "sec": 2.4,
+   "mean_conf": 0.8475932478904724,
+   "sec": 2.5,
    "heldout": true
   },
   "snli": {
    "n": 1476,
-   "acc": 0.9295392953929539,
-   "nll": 0.22098393738269806,
-   "brier": 0.11396971344947815,
-   "ece": 0.02313795467702355,
-   "aurc": 0.01651608891549282,
+   "acc": 0.9234417344173442,
+   "nll": 0.22889889776706696,
+   "brier": 0.1168968454003334,
+   "ece": 0.011546386673360035,
+   "aurc": 0.017872040290664873,
    "acc_at_80": 0.9729043183742591,
-   "acc_at_50": 0.986449864498645,
+   "acc_at_50": 0.9850948509485095,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.9117720127105713,
-   "sec": 2.4,
+   "mean_conf": 0.9265872240066528,
+   "sec": 11.6,
    "heldout": false
   },
   "mnli": {
    "n": 1500,
-   "acc": 0.8846666666666667,
-   "nll": 0.29387927055358887,
-   "brier": 0.16493351757526398,
-   "ece": 0.030576728582382185,
-   "aurc": 0.023789420553220787,
-   "acc_at_80": 0.9616666666666667,
-   "acc_at_50": 0.9906666666666667,
+   "acc": 0.878,
+   "nll": 0.29665282368659973,
+   "brier": 0.16776761412620544,
+   "ece": 0.028746872107187925,
+   "aurc": 0.024781012578722568,
+   "acc_at_80": 0.9558333333333333,
+   "acc_at_50": 0.9893333333333333,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.8777571320533752,
-   "sec": 2.5,
+   "mean_conf": 0.8953821659088135,
+   "sec": 3.1,
    "heldout": false
   },
   "rte": {
    "n": 277,
-   "acc": 0.8592057761732852,
-   "nll": 0.3232092559337616,
-   "brier": 0.198373481631279,
-   "ece": 0.03932678484314188,
-   "aurc": 0.0427986576794893,
-   "acc_at_80": 0.918918918918919,
-   "acc_at_50": 0.9710144927536232,
+   "acc": 0.8736462093862816,
+   "nll": 0.3355311453342438,
+   "brier": 0.20029526948928833,
+   "ece": 0.049293557659383265,
+   "aurc": 0.04237874308319795,
+   "acc_at_80": 0.9144144144144144,
+   "acc_at_50": 0.9637681159420289,
    "chance": 0.5,
-   "mean_conf": 0.8895930647850037,
-   "sec": 0.6,
+   "mean_conf": 0.9088274240493774,
+   "sec": 0.7,
    "heldout": false
   },
   "qnli": {
    "n": 1500,
-   "acc": 0.9393333333333334,
-   "nll": 0.15822429955005646,
-   "brier": 0.08866540342569351,
-   "ece": 0.01913584089279174,
-   "aurc": 0.009447146399598809,
+   "acc": 0.9386666666666666,
+   "nll": 0.15850575268268585,
+   "brier": 0.0912380963563919,
+   "ece": 0.008112372795740749,
+   "aurc": 0.009528591919053303,
    "acc_at_80": 0.9841666666666666,
-   "acc_at_50": 0.9973333333333333,
+   "acc_at_50": 0.996,
    "chance": 0.5,
-   "mean_conf": 0.9270872473716736,
-   "sec": 2.4,
+   "mean_conf": 0.9442304968833923,
+   "sec": 3.0,
    "heldout": false
   },
   "qqp": {
    "n": 1500,
-   "acc": 0.8533333333333334,
-   "nll": 0.3225935399532318,
-   "brier": 0.2073184996843338,
-   "ece": 0.010325272003809595,
-   "aurc": 0.04126678230090233,
-   "acc_at_80": 0.9141666666666667,
-   "acc_at_50": 0.9773333333333334,
+   "acc": 0.8546666666666667,
+   "nll": 0.3271322548389435,
+   "brier": 0.20945638418197632,
+   "ece": 0.026240944465001415,
+   "aurc": 0.04136483929310011,
+   "acc_at_80": 0.9116666666666666,
+   "acc_at_50": 0.976,
    "chance": 0.5,
-   "mean_conf": 0.8565818667411804,
-   "sec": 2.4,
+   "mean_conf": 0.8754271864891052,
+   "sec": 2.6,
    "heldout": false
   },
   "mrpc": {
    "n": 408,
    "acc": 0.8455882352941176,
-   "nll": 0.33268094062805176,
-   "brier": 0.21494312584400177,
-   "ece": 0.052983781724583866,
-   "aurc": 0.04073823335434742,
-   "acc_at_80": 0.9079754601226994,
+   "nll": 0.33147454261779785,
+   "brier": 0.2129886895418167,
+   "ece": 0.053440071058039586,
+   "aurc": 0.04242757999322468,
+   "acc_at_80": 0.9141104294478528,
    "acc_at_50": 0.9803921568627451,
    "chance": 0.5,
-   "mean_conf": 0.8508865237236023,
-   "sec": 0.7,
+   "mean_conf": 0.8646318912506104,
+   "sec": 0.8,
    "heldout": false
   },
   "paws": {
    "n": 1500,
-   "acc": 0.6906666666666667,
-   "nll": 0.8107166290283203,
-   "brier": 0.4746285080909729,
-   "ece": 0.22073563802242277,
-   "aurc": 0.14589154286965242,
-   "acc_at_80": 0.765,
-   "acc_at_50": 0.8693333333333333,
+   "acc": 0.7013333333333334,
+   "nll": 0.8306891918182373,
+   "brier": 0.4740028381347656,
+   "ece": 0.22124919974803925,
+   "aurc": 0.13745002888739216,
+   "acc_at_80": 0.77,
+   "acc_at_50": 0.888,
    "chance": 0.5,
-   "mean_conf": 0.9114023447036743,
-   "sec": 2.4,
+   "mean_conf": 0.9219720959663391,
+   "sec": 3.0,
    "heldout": true
   },
   "cola": {
    "n": 1043,
-   "acc": 0.8082454458293384,
-   "nll": 0.40937936305999756,
-   "brier": 0.2624674439430237,
-   "ece": 0.03032427043265601,
-   "aurc": 0.07498094574041442,
-   "acc_at_80": 0.8669064748201439,
+   "acc": 0.8024928092042186,
+   "nll": 0.4121968150138855,
+   "brier": 0.26549777388572693,
+   "ece": 0.03725165488735943,
+   "aurc": 0.07404416991022049,
+   "acc_at_80": 0.8741007194244604,
    "acc_at_50": 0.9367816091954023,
    "chance": 0.5,
-   "mean_conf": 0.8143807053565979,
-   "sec": 1.7,
+   "mean_conf": 0.8369820713996887,
+   "sec": 1.6,
    "heldout": false
   },
   "boolq": {
    "n": 1500,
-   "acc": 0.884,
-   "nll": 0.28092503547668457,
-   "brier": 0.16896213591098785,
-   "ece": 0.013350882569948819,
-   "aurc": 0.03244594523564409,
-   "acc_at_80": 0.9425,
-   "acc_at_50": 0.9733333333333334,
+   "acc": 0.8853333333333333,
+   "nll": 0.2897628843784332,
+   "brier": 0.17195458710193634,
+   "ece": 0.028059083223342877,
+   "aurc": 0.033273033407070704,
+   "acc_at_80": 0.9441666666666667,
+   "acc_at_50": 0.9746666666666667,
    "chance": 0.5,
-   "mean_conf": 0.8836175799369812,
-   "sec": 3.7,
+   "mean_conf": 0.9083826541900635,
+   "sec": 4.0,
    "heldout": false
   },
   "strategyqa": {
    "n": 687,
-   "acc": 0.6128093158660844,
-   "nll": 0.6776027679443359,
-   "brier": 0.4776630997657776,
-   "ece": 0.0688689137650369,
-   "aurc": 0.31974077928206984,
-   "acc_at_80": 0.6272727272727273,
-   "acc_at_50": 0.6656976744186046,
+   "acc": 0.6142649199417758,
+   "nll": 0.6972374320030212,
+   "brier": 0.4870849847793579,
+   "ece": 0.10019853498564363,
+   "aurc": 0.31452270422741296,
+   "acc_at_80": 0.6327272727272727,
+   "acc_at_50": 0.6569767441860465,
    "chance": 0.5,
-   "mean_conf": 0.6726659536361694,
-   "sec": 1.2,
+   "mean_conf": 0.7026323080062866,
+   "sec": 7.9,
    "heldout": true
   },
   "pubmedqa": {
    "n": 500,
-   "acc": 0.79,
-   "nll": 0.5823958516120911,
-   "brier": 0.3185318112373352,
-   "ece": 0.06283757454156877,
-   "aurc": 0.09994473800171019,
-   "acc_at_80": 0.8525,
-   "acc_at_50": 0.924,
+   "acc": 0.798,
+   "nll": 0.5936447381973267,
+   "brier": 0.31999242305755615,
+   "ece": 0.06877634388208391,
+   "aurc": 0.10006724050038115,
+   "acc_at_80": 0.845,
+   "acc_at_50": 0.928,
    "chance": 0.33333333333333326,
-   "mean_conf": 0.7943322658538818,
-   "sec": 2.4,
+   "mean_conf": 0.8425376415252686,
+   "sec": 2.3,
    "heldout": true
   },
   "arc": {
    "n": 1172,
-   "acc": 0.8575085324232082,
-   "nll": 0.37918952107429504,
-   "brier": 0.19906748831272125,
-   "ece": 0.029848500348805575,
-   "aurc": 0.028284304075720537,
-   "acc_at_80": 0.9477611940298507,
-   "acc_at_50": 0.9880546075085325,
+   "acc": 0.856655290102389,
+   "nll": 0.3848453760147095,
+   "brier": 0.20024119317531586,
+   "ece": 0.03268225236760882,
+   "aurc": 0.028683364797018013,
+   "acc_at_80": 0.9466950959488273,
+   "acc_at_50": 0.9897610921501706,
    "chance": 0.25015642775881686,
-   "mean_conf": 0.8538938164710999,
-   "sec": 1.9,
+   "mean_conf": 0.8850091695785522,
+   "sec": 2.3,
    "heldout": false
   },
   "commonsense_qa": {
    "n": 1221,
-   "acc": 0.7592137592137592,
-   "nll": 0.5952430367469788,
-   "brier": 0.31855452060699463,
-   "ece": 0.030964655577404226,
-   "aurc": 0.07921845119739539,
-   "acc_at_80": 0.8382804503582395,
-   "acc_at_50": 0.9475409836065574,
+   "acc": 0.7641277641277642,
+   "nll": 0.6035410165786743,
+   "brier": 0.32016345858573914,
+   "ece": 0.024896302787521474,
+   "aurc": 0.07891618232675111,
+   "acc_at_80": 0.8454452405322416,
+   "acc_at_50": 0.9459016393442623,
    "chance": 0.19999999999999998,
-   "mean_conf": 0.7593477368354797,
+   "mean_conf": 0.7828702926635742,
    "sec": 2.0,
    "heldout": false
   },
   "qasc": {
    "n": 926,
-   "acc": 0.816414686825054,
-   "nll": 0.5963356494903564,
-   "brier": 0.27422401309013367,
-   "ece": 0.07903238865456365,
-   "aurc": 0.04646154243439387,
-   "acc_at_80": 0.9149797570850202,
-   "acc_at_50": 0.978401727861771,
+   "acc": 0.8142548596112311,
+   "nll": 0.5944124460220337,
+   "brier": 0.27712827920913696,
+   "ece": 0.05084534546211523,
+   "aurc": 0.04955932347383356,
+   "acc_at_80": 0.8960863697705803,
+   "acc_at_50": 0.9676025917926566,
    "chance": 0.125,
-   "mean_conf": 0.7387062311172485,
-   "sec": 1.5,
+   "mean_conf": 0.7678154706954956,
+   "sec": 1.8,
    "heldout": false
   },
   "openbookqa": {
    "n": 500,
-   "acc": 0.824,
-   "nll": 0.4846096932888031,
-   "brier": 0.250370055437088,
-   "ece": 0.04813598054647445,
-   "aurc": 0.05417227716258954,
+   "acc": 0.808,
+   "nll": 0.5015584826469421,
+   "brier": 0.25752735137939453,
+   "ece": 0.044371710240840904,
+   "aurc": 0.057444096145339726,
    "acc_at_80": 0.905,
-   "acc_at_50": 0.964,
+   "acc_at_50": 0.968,
    "chance": 0.25,
-   "mean_conf": 0.7986846566200256,
-   "sec": 0.8,
+   "mean_conf": 0.8454900979995728,
+   "sec": 0.9,
    "heldout": false
   },
   "sciq": {
    "n": 1000,
-   "acc": 0.982,
-   "nll": 0.05202531814575195,
-   "brier": 0.025178927928209305,
-   "ece": 0.017304384350776688,
-   "aurc": 0.000406837269511718,
+   "acc": 0.983,
+   "nll": 0.0540383905172348,
+   "brier": 0.026997487992048264,
+   "ece": 0.015873215794563272,
+   "aurc": 0.0004725583644347805,
    "acc_at_80": 1.0,
    "acc_at_50": 1.0,
    "chance": 0.25,
-   "mean_conf": 0.9719231128692627,
-   "sec": 2.3,
+   "mean_conf": 0.9763537645339966,
+   "sec": 4.9,
    "heldout": true
   },
   "hellaswag": {
    "n": 1500,
-   "acc": 0.8733333333333333,
-   "nll": 0.3213328719139099,
-   "brier": 0.17381860315799713,
-   "ece": 0.027953611870606753,
-   "aurc": 0.021264232574898293,
-   "acc_at_80": 0.9566666666666667,
+   "acc": 0.8726666666666667,
+   "nll": 0.35282617807388306,
+   "brier": 0.1844407618045807,
+   "ece": 0.05096171965201693,
+   "aurc": 0.022184705622736417,
+   "acc_at_80": 0.9541666666666667,
    "acc_at_50": 0.9986666666666667,
    "chance": 0.25,
-   "mean_conf": 0.8975942134857178,
-   "sec": 4.3,
+   "mean_conf": 0.9218340516090393,
+   "sec": 4.4,
    "heldout": false
   },
   "piqa": {
    "n": 1500,
-   "acc": 0.8366666666666667,
-   "nll": 0.35584479570388794,
-   "brier": 0.22590981423854828,
-   "ece": 0.021878054340680464,
-   "aurc": 0.052599671905179834,
-   "acc_at_80": 0.9025,
-   "acc_at_50": 0.956,
+   "acc": 0.8353333333333334,
+   "nll": 0.3836018443107605,
+   "brier": 0.2365727722644806,
+   "ece": 0.04386041740576427,
+   "aurc": 0.05569590131269642,
+   "acc_at_80": 0.8966666666666666,
+   "acc_at_50": 0.9506666666666667,
    "chance": 0.5,
-   "mean_conf": 0.8382094502449036,
-   "sec": 2.6,
+   "mean_conf": 0.8791936635971069,
+   "sec": 2.9,
    "heldout": false
   },
   "social_iqa": {
    "n": 1500,
-   "acc": 0.712,
-   "nll": 0.6847297549247742,
-   "brier": 0.3952922224998474,
-   "ece": 0.060257167994976046,
-   "aurc": 0.14267195194658805,
-   "acc_at_80": 0.7783333333333333,
-   "acc_at_50": 0.8493333333333334,
+   "acc": 0.7186666666666667,
+   "nll": 0.7424599528312683,
+   "brier": 0.41328999400138855,
+   "ece": 0.09968751738468809,
+   "aurc": 0.1436145844762169,
+   "acc_at_80": 0.775,
+   "acc_at_50": 0.8546666666666667,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.7657021284103394,
-   "sec": 2.4,
+   "mean_conf": 0.8163115382194519,
+   "sec": 2.7,
    "heldout": true
   },
   "winogrande": {
    "n": 1267,
-   "acc": 0.6748224151539068,
-   "nll": 0.6599352359771729,
-   "brier": 0.43405067920684814,
-   "ece": 0.12888923459101967,
-   "aurc": 0.19936165776962148,
-   "acc_at_80": 0.7199211045364892,
-   "acc_at_50": 0.8012618296529969,
+   "acc": 0.681136543014996,
+   "nll": 0.7226526737213135,
+   "brier": 0.45336562395095825,
+   "ece": 0.16130592141832623,
+   "aurc": 0.19273822608019725,
+   "acc_at_80": 0.7209072978303748,
+   "acc_at_50": 0.8123028391167192,
    "chance": 0.5,
-   "mean_conf": 0.8037116527557373,
-   "sec": 2.1,
+   "mean_conf": 0.8380995988845825,
+   "sec": 1.9,
    "heldout": false
   },
   "race": {
    "n": 1500,
-   "acc": 0.8406666666666667,
-   "nll": 0.45064714550971985,
-   "brier": 0.22838135063648224,
-   "ece": 0.024594745457172388,
-   "aurc": 0.04331003810864532,
-   "acc_at_80": 0.92,
-   "acc_at_50": 0.9773333333333334,
+   "acc": 0.8326666666666667,
+   "nll": 0.48354849219322205,
+   "brier": 0.24252651631832123,
+   "ece": 0.05754320712884267,
+   "aurc": 0.04422304284234363,
+   "acc_at_80": 0.9175,
+   "acc_at_50": 0.9733333333333334,
    "chance": 0.25,
-   "mean_conf": 0.8616733551025391,
-   "sec": 7.7,
+   "mean_conf": 0.8896796703338623,
+   "sec": 7.3,
    "heldout": false
   },
   "mmlu": {
    "n": 1500,
-   "acc": 0.632,
-   "nll": 0.9076293110847473,
-   "brier": 0.46681055426597595,
-   "ece": 0.047448410332202914,
-   "aurc": 0.1664288415331668,
-   "acc_at_80": 0.7158333333333333,
-   "acc_at_50": 0.8466666666666667,
+   "acc": 0.6333333333333333,
+   "nll": 0.9389562010765076,
+   "brier": 0.47845685482025146,
+   "ece": 0.10005126521984736,
+   "aurc": 0.16047564173919576,
+   "acc_at_80": 0.71,
+   "acc_at_50": 0.8533333333333334,
    "chance": 0.25,
-   "mean_conf": 0.6779200434684753,
-   "sec": 3.1,
+   "mean_conf": 0.7325466871261597,
+   "sec": 3.4,
    "heldout": false
   },
   "medqa": {
    "n": 1273,
-   "acc": 0.5640219952867243,
-   "nll": 1.0263904333114624,
-   "brier": 0.5558311939239502,
-   "ece": 0.024355607152640676,
-   "aurc": 0.26470923464312796,
-   "acc_at_80": 0.6198428290766208,
-   "acc_at_50": 0.7044025157232704,
+   "acc": 0.5632364493322859,
+   "nll": 1.079268455505371,
+   "brier": 0.575121283531189,
+   "ece": 0.1046450646112776,
+   "aurc": 0.2684613401796164,
+   "acc_at_80": 0.6129666011787819,
+   "acc_at_50": 0.710691823899371,
    "chance": 0.25,
-   "mean_conf": 0.5750094056129456,
-   "sec": 4.0,
+   "mean_conf": 0.6660441756248474,
+   "sec": 4.1,
    "heldout": false
   },
   "truthfulqa": {
    "n": 817,
-   "acc": 0.4969400244798042,
-   "nll": 1.4452694654464722,
-   "brier": 0.6520512104034424,
-   "ece": 0.09137629450388902,
-   "aurc": 0.29796415629912126,
-   "acc_at_80": 0.5565749235474006,
-   "acc_at_50": 0.6764705882352942,
+   "acc": 0.5018359853121175,
+   "nll": 1.5319370031356812,
+   "brier": 0.6671698689460754,
+   "ece": 0.1319449910919115,
+   "aurc": 0.2963526198417848,
+   "acc_at_80": 0.5581039755351682,
+   "acc_at_50": 0.678921568627451,
    "chance": 0.22622350449767833,
-   "mean_conf": 0.5813891291618347,
-   "sec": 1.4,
+   "mean_conf": 0.6299614310264587,
+   "sec": 1.7,
    "heldout": true
   },
   "fin_phrasebank": {
    "n": 970,
-   "acc": 0.6597938144329897,
-   "nll": 0.6683916449546814,
-   "brier": 0.42705729603767395,
-   "ece": 0.08229427663321345,
-   "aurc": 0.1944303228748502,
-   "acc_at_80": 0.7190721649484536,
-   "acc_at_50": 0.8123711340206186,
+   "acc": 0.6989690721649484,
+   "nll": 0.6254894733428955,
+   "brier": 0.39401963353157043,
+   "ece": 0.058691233703770586,
+   "aurc": 0.16686274774956414,
+   "acc_at_80": 0.7577319587628866,
+   "acc_at_50": 0.8329896907216495,
    "chance": 0.33333333333333326,
-   "mean_conf": 0.7379865050315857,
-   "sec": 4.0,
+   "mean_conf": 0.7559286952018738,
+   "sec": 6.3,
    "heldout": true
   },
   "banking77": {
    "n": 1500,
-   "acc": 0.96,
-   "nll": 0.12018896639347076,
-   "brier": 0.058943431824445724,
-   "ece": 0.010675735374291784,
-   "aurc": 0.0028384511661530853,
+   "acc": 0.9626666666666667,
+   "nll": 0.12322639673948288,
+   "brier": 0.05743316188454628,
+   "ece": 0.007931326011816654,
+   "aurc": 0.002740366914790048,
    "acc_at_80": 0.9966666666666667,
    "acc_at_50": 1.0,
    "chance": 0.1,
-   "mean_conf": 0.9642454981803894,
-   "sec": 2.4,
+   "mean_conf": 0.9672797918319702,
+   "sec": 3.0,
    "heldout": false
   },
   "bias_in_bios": {
    "n": 3000,
-   "acc": 0.953,
-   "nll": 0.13723737001419067,
-   "brier": 0.06831751763820648,
-   "ece": 0.007673392459750155,
-   "aurc": 0.003360504536259007,
-   "acc_at_80": 0.9979166666666667,
+   "acc": 0.9516666666666667,
+   "nll": 0.13784465193748474,
+   "brier": 0.06864367425441742,
+   "ece": 0.006546956946452431,
+   "aurc": 0.0033626200148913327,
+   "acc_at_80": 0.9975,
    "acc_at_50": 1.0,
    "chance": 0.30000000000000004,
-   "mean_conf": 0.9521626234054565,
-   "sec": 3.9,
+   "mean_conf": 0.953985333442688,
+   "sec": 4.2,
    "heldout": false,
    "per_q": [
-    0.9086666666666666,
+    0.906,
     0.9973333333333333
    ]
   },
   "helpsteer2": {
    "n": 5190,
-   "acc": 0.6021194605009634,
-   "nll": 0.9371625185012817,
-   "brier": 0.5185062289237976,
-   "ece": 0.018472507194057828,
-   "aurc": 0.26482025743101484,
-   "acc_at_80": 0.6572736030828517,
-   "acc_at_50": 0.735645472061657,
+   "acc": 0.6028901734104046,
+   "nll": 0.9502481818199158,
+   "brier": 0.5227839350700378,
+   "ece": 0.02052123054201663,
+   "aurc": 0.2681509263336261,
+   "acc_at_80": 0.6623314065510597,
+   "acc_at_50": 0.7294797687861272,
    "chance": 0.2,
-   "mean_conf": 0.6043042540550232,
-   "sec": 9.6,
+   "mean_conf": 0.6144348978996277,
+   "sec": 23.2,
    "heldout": false,
    "per_q": [
-    0.45857418111753373,
-    0.5173410404624278,
-    0.7244701348747592,
-    0.6242774566473989,
-    0.6859344894026975
+    0.45664739884393063,
+    0.5067437379576107,
+    0.7225433526011561,
+    0.6358381502890174,
+    0.6926782273603083
    ]
   },
   "helpsteer3_pref": {
    "n": 1176,
-   "acc": 0.41241496598639454,
-   "nll": 1.4535510540008545,
-   "brier": 0.707725465297699,
-   "ece": 0.025285047713388412,
-   "aurc": 0.48559289517381626,
-   "acc_at_80": 0.45483528161530284,
-   "acc_at_50": 0.5102040816326531,
+   "acc": 0.4005102040816326,
+   "nll": 1.457674503326416,
+   "brier": 0.7135226726531982,
+   "ece": 0.05034625236274434,
+   "aurc": 0.49111859183201545,
+   "acc_at_80": 0.4293304994686504,
+   "acc_at_50": 0.49489795918367346,
    "chance": 0.14285714285714282,
-   "mean_conf": 0.4194914400577545,
-   "sec": 13.0,
+   "mean_conf": 0.44308316707611084,
+   "sec": 24.6,
    "heldout": false
   },
   "hate_speech_scales": {
    "n": 7500,
-   "acc": 0.5602666666666667,
-   "nll": 0.9815105199813843,
-   "brier": 0.5425266623497009,
-   "ece": 0.008547326453526816,
-   "aurc": 0.2667196222552012,
-   "acc_at_80": 0.608,
-   "acc_at_50": 0.7093333333333334,
+   "acc": 0.5605333333333333,
+   "nll": 0.9819027781486511,
+   "brier": 0.5431735515594482,
+   "ece": 0.0195502276579539,
+   "aurc": 0.267479078510345,
+   "acc_at_80": 0.6098333333333333,
+   "acc_at_50": 0.7061333333333333,
    "chance": 0.22666666666666674,
-   "mean_conf": 0.5625489354133606,
-   "sec": 5.9,
+   "mean_conf": 0.5698475241661072,
+   "sec": 4.9,
    "heldout": false,
    "per_q": [
     0.706,
-    0.47733333333333333,
-    0.44666666666666666,
-    0.5793333333333334,
-    0.592
+    0.4826666666666667,
+    0.43533333333333335,
+    0.578,
+    0.6006666666666667
    ]
   },
   "liar2": {
    "n": 1500,
-   "acc": 0.37133333333333335,
-   "nll": 1.4664359092712402,
-   "brier": 0.717261552810669,
-   "ece": 0.010819464365641272,
-   "aurc": 0.47753927157183623,
-   "acc_at_80": 0.4066666666666667,
-   "acc_at_50": 0.4786666666666667,
+   "acc": 0.37466666666666665,
+   "nll": 1.465606451034546,
+   "brier": 0.7157273888587952,
+   "ece": 0.028325774570306134,
+   "aurc": 0.47234860740182666,
+   "acc_at_80": 0.4075,
+   "acc_at_50": 0.48933333333333334,
    "chance": 0.16666666666666666,
-   "mean_conf": 0.3717862069606781,
-   "sec": 3.8,
+   "mean_conf": 0.39819955825805664,
+   "sec": 3.7,
    "heldout": false
   },
   "prosocial_safety": {
    "n": 1500,
-   "acc": 0.5246666666666666,
-   "nll": 1.1752827167510986,
-   "brier": 0.5942376852035522,
-   "ece": 0.033485592842102035,
-   "aurc": 0.2778985096517121,
-   "acc_at_80": 0.5908333333333333,
-   "acc_at_50": 0.7,
+   "acc": 0.524,
+   "nll": 1.1723508834838867,
+   "brier": 0.592208206653595,
+   "ece": 0.02242480476697286,
+   "aurc": 0.2745796786723848,
+   "acc_at_80": 0.5883333333333334,
+   "acc_at_50": 0.6906666666666667,
    "chance": 0.2,
-   "mean_conf": 0.5058215260505676,
-   "sec": 2.4,
+   "mean_conf": 0.5037593841552734,
+   "sec": 3.0,
    "heldout": false
   },
   "ultrafeedback_pref": {
    "n": 1500,
-   "acc": 0.758,
-   "nll": 0.49144116044044495,
-   "brier": 0.32426393032073975,
-   "ece": 0.015855272690455117,
-   "aurc": 0.11929765138761142,
-   "acc_at_80": 0.8133333333333334,
-   "acc_at_50": 0.88,
+   "acc": 0.762,
+   "nll": 0.5141094923019409,
+   "brier": 0.33274728059768677,
+   "ece": 0.03840167435010275,
+   "aurc": 0.1266040181966966,
+   "acc_at_80": 0.81,
+   "acc_at_50": 0.864,
    "chance": 0.5,
-   "mean_conf": 0.7586716413497925,
-   "sec": 10.6,
+   "mean_conf": 0.794404923915863,
+   "sec": 9.8,
    "heldout": false
   },
   "shp": {
    "n": 1500,
-   "acc": 0.746,
-   "nll": 0.517654299736023,
-   "brier": 0.34469103813171387,
-   "ece": 0.018088009436925262,
-   "aurc": 0.13967686763081924,
-   "acc_at_80": 0.7875,
-   "acc_at_50": 0.8626666666666667,
+   "acc": 0.744,
+   "nll": 0.5253914594650269,
+   "brier": 0.35134565830230713,
+   "ece": 0.029734440008799243,
+   "aurc": 0.14422793871032347,
+   "acc_at_80": 0.7808333333333334,
+   "acc_at_50": 0.848,
    "chance": 0.5,
-   "mean_conf": 0.7509077191352844,
-   "sec": 7.7,
+   "mean_conf": 0.752452552318573,
+   "sec": 7.2,
    "heldout": false
   },
   "hh_rlhf": {
    "n": 1488,
-   "acc": 0.6404569892473119,
-   "nll": 0.6205067038536072,
-   "brier": 0.4332164227962494,
-   "ece": 0.02331842826579209,
-   "aurc": 0.24516950655920614,
-   "acc_at_80": 0.6714285714285714,
-   "acc_at_50": 0.7271505376344086,
+   "acc": 0.6478494623655914,
+   "nll": 0.6344334483146667,
+   "brier": 0.44163978099823,
+   "ece": 0.06129897718307793,
+   "aurc": 0.24894185193228557,
+   "acc_at_80": 0.6705882352941176,
+   "acc_at_50": 0.7338709677419355,
    "chance": 0.5,
-   "mean_conf": 0.6637664437294006,
-   "sec": 6.1,
+   "mean_conf": 0.6988418698310852,
+   "sec": 12.3,
    "heldout": false
   },
   "arena_pref": {
    "n": 1500,
-   "acc": 0.474,
-   "nll": 1.1644456386566162,
-   "brier": 0.6687793731689453,
-   "ece": 0.14372908343871435,
-   "aurc": 0.4160631567711759,
-   "acc_at_80": 0.5008333333333334,
-   "acc_at_50": 0.5506666666666666,
+   "acc": 0.47533333333333333,
+   "nll": 1.2214066982269287,
+   "brier": 0.6920773386955261,
+   "ece": 0.1775780066649119,
+   "aurc": 0.4264101430176788,
+   "acc_at_80": 0.5025,
+   "acc_at_50": 0.5493333333333333,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.6177290678024292,
-   "sec": 10.0,
+   "mean_conf": 0.652911365032196,
+   "sec": 9.3,
    "heldout": true
   },
   "reward_bench": {
    "n": 1500,
-   "acc": 0.7993333333333333,
-   "nll": 0.4464208483695984,
-   "brier": 0.28693464398384094,
-   "ece": 0.04333787786960602,
-   "aurc": 0.09299298896270519,
-   "acc_at_80": 0.8541666666666666,
-   "acc_at_50": 0.9053333333333333,
+   "acc": 0.794,
+   "nll": 0.45838916301727295,
+   "brier": 0.28988850116729736,
+   "ece": 0.04195846394697825,
+   "aurc": 0.09336704492407273,
+   "acc_at_80": 0.8533333333333334,
+   "acc_at_50": 0.912,
    "chance": 0.5,
-   "mean_conf": 0.7756828665733337,
-   "sec": 8.0,
+   "mean_conf": 0.8290508389472961,
+   "sec": 7.7,
    "heldout": true
   },
   "glaive_tools": {
    "n": 1348,
-   "acc": 0.9458456973293768,
-   "nll": 0.1641840934753418,
-   "brier": 0.08934415131807327,
-   "ece": 0.00926415945019139,
-   "aurc": 0.010922496406982686,
-   "acc_at_80": 0.9768089053803339,
-   "acc_at_50": 0.9955489614243324,
+   "acc": 0.9473293768545994,
+   "nll": 0.1643306165933609,
+   "brier": 0.087461918592453,
+   "ece": 0.007846767320477323,
+   "aurc": 0.011936188106662745,
+   "acc_at_80": 0.9795918367346939,
+   "acc_at_50": 0.9925816023738873,
    "chance": 0.16666666666666666,
-   "mean_conf": 0.945382297039032,
-   "sec": 3.1,
+   "mean_conf": 0.9517403841018677,
+   "sec": 9.1,
    "heldout": false
   },
   "hermes_tools": {
    "n": 1500,
-   "acc": 0.7373333333333333,
-   "nll": 0.46730339527130127,
-   "brier": 0.3166651129722595,
-   "ece": 0.15524024434884393,
-   "aurc": 0.05736832196708593,
-   "acc_at_80": 0.8666666666666667,
-   "acc_at_50": 0.9893333333333333,
+   "acc": 0.7273333333333334,
+   "nll": 0.5851268768310547,
+   "brier": 0.3813258111476898,
+   "ece": 0.19555236707131068,
+   "aurc": 0.06225953895023327,
+   "acc_at_80": 0.8558333333333333,
+   "acc_at_50": 0.988,
    "chance": 0.125,
-   "mean_conf": 0.8914145231246948,
-   "sec": 9.5,
+   "mean_conf": 0.92147296667099,
+   "sec": 8.8,
    "heldout": true
   },
   "copa": {
    "n": 100,
-   "acc": 0.93,
-   "nll": 0.21767663955688477,
-   "brier": 0.11604493111371994,
-   "ece": 0.04409654974937439,
-   "aurc": 0.024093551673163044,
+   "acc": 0.94,
+   "nll": 0.1961805373430252,
+   "brier": 0.09868918359279633,
+   "ece": 0.011551505327224751,
+   "aurc": 0.021329015107119983,
    "acc_at_80": 0.975,
    "acc_at_50": 0.98,
    "chance": 0.5,
-   "mean_conf": 0.9102426767349243,
-   "sec": 0.2,
+   "mean_conf": 0.9395062327384949,
+   "sec": 6.0,
    "heldout": false
   },
   "wic": {
    "n": 638,
-   "acc": 0.713166144200627,
-   "nll": 0.5775361657142639,
-   "brier": 0.3854121267795563,
-   "ece": 0.051907789651129334,
-   "aurc": 0.20507789115696606,
-   "acc_at_80": 0.7666666666666667,
-   "acc_at_50": 0.8087774294670846,
+   "acc": 0.7163009404388715,
+   "nll": 0.5828382968902588,
+   "brier": 0.38676366209983826,
+   "ece": 0.05568265924259413,
+   "aurc": 0.20020452896287377,
+   "acc_at_80": 0.7647058823529411,
+   "acc_at_50": 0.8119122257053292,
    "chance": 0.5,
-   "mean_conf": 0.7423800826072693,
-   "sec": 1.0,
+   "mean_conf": 0.7616857290267944,
+   "sec": 1.1,
    "heldout": false
   },
   "multirc": {
    "n": 1500,
-   "acc": 0.892,
-   "nll": 0.28638261556625366,
-   "brier": 0.1648022085428238,
-   "ece": 0.025759932279586766,
-   "aurc": 0.03694649471413434,
-   "acc_at_80": 0.9441666666666667,
-   "acc_at_50": 0.964,
+   "acc": 0.8913333333333333,
+   "nll": 0.300986111164093,
+   "brier": 0.16819369792938232,
+   "ece": 0.03357027542591094,
+   "aurc": 0.03652944216687471,
+   "acc_at_80": 0.9408333333333333,
+   "acc_at_50": 0.9693333333333334,
    "chance": 0.5,
-   "mean_conf": 0.8986733555793762,
-   "sec": 7.1,
+   "mean_conf": 0.9222791194915771,
+   "sec": 6.7,
    "heldout": false
   },
   "cb": {
    "n": 56,
    "acc": 0.8928571428571429,
-   "nll": 0.39255955815315247,
-   "brier": 0.18121369183063507,
-   "ece": 0.11490496354443687,
-   "aurc": 0.02600997889991392,
-   "acc_at_80": 0.9777777777777777,
+   "nll": 0.3820924460887909,
+   "brier": 0.17802833020687103,
+   "ece": 0.08807407692074777,
+   "aurc": 0.023497414016688704,
+   "acc_at_80": 0.9555555555555556,
    "acc_at_50": 0.9642857142857143,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.8255382180213928,
+   "mean_conf": 0.8515821695327759,
    "sec": 0.2,
    "heldout": true
   },
   "fever": {
    "n": 1500,
-   "acc": 0.8813333333333333,
-   "nll": 0.33560705184936523,
-   "brier": 0.17902693152427673,
-   "ece": 0.021976249794165285,
-   "aurc": 0.03778744252180358,
-   "acc_at_80": 0.9416666666666667,
-   "acc_at_50": 0.9693333333333334,
+   "acc": 0.8866666666666667,
+   "nll": 0.3317285180091858,
+   "brier": 0.1768377274274826,
+   "ece": 0.017198302884896575,
+   "aurc": 0.03783623559244957,
+   "acc_at_80": 0.9383333333333334,
+   "acc_at_50": 0.972,
    "chance": 0.3333333333333333,
-   "mean_conf": 0.8861227035522461,
-   "sec": 2.7,
+   "mean_conf": 0.8916481137275696,
+   "sec": 3.3,
    "heldout": false
   },
   "wiki_qa": {
    "n": 879,
-   "acc": 0.906712172923777,
-   "nll": 0.2421109974384308,
-   "brier": 0.14265964925289154,
-   "ece": 0.031170608783066635,
-   "aurc": 0.02204155557599466,
-   "acc_at_80": 0.9559032716927454,
-   "acc_at_50": 0.9818181818181818,
+   "acc": 0.8987485779294653,
+   "nll": 0.2437155544757843,
+   "brier": 0.1463335007429123,
+   "ece": 0.012018481686931658,
+   "aurc": 0.02256072022253832,
+   "acc_at_80": 0.9587482219061166,
+   "acc_at_50": 0.9863636363636363,
    "chance": 0.5,
-   "mean_conf": 0.8812997341156006,
-   "sec": 1.5,
+   "mean_conf": 0.8944457769393921,
+   "sec": 4.3,
    "heldout": false
   },
   "msmarco_rel": {
    "n": 1446,
-   "acc": 0.6708160442600276,
-   "nll": 0.5872405171394348,
-   "brier": 0.40445321798324585,
-   "ece": 0.030953758072886067,
-   "aurc": 0.2003357991113142,
-   "acc_at_80": 0.7147796024200519,
+   "acc": 0.6763485477178424,
+   "nll": 0.6001450419425964,
+   "brier": 0.4146299362182617,
+   "ece": 0.05068599555337411,
+   "aurc": 0.2088099598857701,
+   "acc_at_80": 0.7113223854796888,
    "acc_at_50": 0.7745504840940526,
    "chance": 0.5,
-   "mean_conf": 0.68407142162323,
-   "sec": 3.0,
+   "mean_conf": 0.724265456199646,
+   "sec": 3.4,
    "heldout": false
   },
   "medmcqa": {
    "n": 1500,
-   "acc": 0.5046666666666667,
-   "nll": 1.1041309833526611,
-   "brier": 0.597272515296936,
-   "ece": 0.045152134160200745,
-   "aurc": 0.3042279038689118,
-   "acc_at_80": 0.5558333333333333,
-   "acc_at_50": 0.6573333333333333,
+   "acc": 0.5006666666666667,
+   "nll": 1.1471084356307983,
+   "brier": 0.6143077611923218,
+   "ece": 0.11058435388406118,
+   "aurc": 0.30678103490481,
+   "acc_at_80": 0.5591666666666667,
+   "acc_at_50": 0.6693333333333333,
    "chance": 0.25,
-   "mean_conf": 0.5472774505615234,
-   "sec": 2.4,
+   "mean_conf": 0.6112509965896606,
+   "sec": 2.7,
    "heldout": false
   },
   "quality": {
    "n": 1500,
-   "acc": 0.5133333333333333,
-   "nll": 1.2461479902267456,
-   "brier": 0.646840512752533,
-   "ece": 0.13738841543594996,
-   "aurc": 0.33496979709462643,
-   "acc_at_80": 0.5583333333333333,
-   "acc_at_50": 0.6373333333333333,
+   "acc": 0.5146666666666667,
+   "nll": 1.3762184381484985,
+   "brier": 0.6870999932289124,
+   "ece": 0.2046667002439499,
+   "aurc": 0.34077759893205944,
+   "acc_at_80": 0.5633333333333334,
+   "acc_at_50": 0.6173333333333333,
    "chance": 0.25,
-   "mean_conf": 0.6507217884063721,
-   "sec": 21.8,
+   "mean_conf": 0.7156844139099121,
+   "sec": 18.7,
    "heldout": true
   },
   "xstory_cloze": {
    "n": 1500,
-   "acc": 0.9646666666666667,
-   "nll": 0.11040383577346802,
-   "brier": 0.05835951864719391,
-   "ece": 0.02761857227484382,
-   "aurc": 0.004502607034060362,
-   "acc_at_80": 0.9941666666666666,
+   "acc": 0.9613333333333334,
+   "nll": 0.1043626070022583,
+   "brier": 0.05890524387359619,
+   "ece": 0.006435122688611336,
+   "aurc": 0.004630966744998978,
+   "acc_at_80": 0.9966666666666667,
    "acc_at_50": 0.9986666666666667,
    "chance": 0.5,
-   "mean_conf": 0.9391650557518005,
-   "sec": 2.4,
+   "mean_conf": 0.9595363140106201,
+   "sec": 3.0,
    "heldout": true
   },
   "abstain_probe": {
    "n": 1500,
-   "acc": 0.7846666666666666,
-   "nll": 0.6353165507316589,
-   "brier": 0.2965729832649231,
-   "ece": 0.03740532821416854,
-   "aurc": 0.057853419152916634,
-   "acc_at_80": 0.8808333333333334,
-   "acc_at_50": 0.9626666666666667,
+   "acc": 0.794,
+   "nll": 0.6064885258674622,
+   "brier": 0.2841270864009857,
+   "ece": 0.037370719770590435,
+   "aurc": 0.054226749633954824,
+   "acc_at_80": 0.8966666666666666,
+   "acc_at_50": 0.968,
    "chance": 0.17950317460317464,
-   "mean_conf": 0.8122005462646484,
-   "sec": 3.5,
+   "mean_conf": 0.8279028534889221,
+   "sec": 3.8,
    "heldout": true
   },
   "toolace": {
    "n": 1000,
-   "acc": 0.932,
-   "nll": 0.22606024146080017,
-   "brier": 0.10948573052883148,
-   "ece": 0.03118352198600772,
-   "aurc": 0.019889396033981634,
+   "acc": 0.924,
+   "nll": 0.24141927063465118,
+   "brier": 0.11758895963430405,
+   "ece": 0.025210315197706203,
+   "aurc": 0.02069434380751835,
    "acc_at_80": 0.97125,
-   "acc_at_50": 0.99,
+   "acc_at_50": 0.982,
    "chance": 0.12470555555555554,
-   "mean_conf": 0.9266018867492676,
-   "sec": 4.0,
+   "mean_conf": 0.9345914125442505,
+   "sec": 9.7,
+   "heldout": false
+  },
+  "agenttraj": {
+   "n": 1500,
+   "acc": 0.916,
+   "nll": 0.20774629712104797,
+   "brier": 0.10922007262706757,
+   "ece": 0.0139970025122166,
+   "aurc": 0.00845390614589135,
+   "acc_at_80": 0.9891666666666666,
+   "acc_at_50": 0.9986666666666667,
+   "chance": 0.2,
+   "mean_conf": 0.9180088043212891,
+   "sec": 6.6,
    "heldout": false
+  },
+  "mind2web": {
+   "n": 1500,
+   "acc": 0.8086666666666666,
+   "nll": 0.5181217789649963,
+   "brier": 0.2681422531604767,
+   "ece": 0.025145165185133628,
+   "aurc": 0.05308884826749579,
+   "acc_at_80": 0.8883333333333333,
+   "acc_at_50": 0.9666666666666667,
+   "chance": 0.16666666666666666,
+   "mean_conf": 0.8295513391494751,
+   "sec": 4.2,
+   "heldout": false
+  },
+  "synth": {
+   "n": 300,
+   "acc": 0.7566666666666667,
+   "nll": 0.542948305606842,
+   "brier": 0.3258218467235565,
+   "ece": 0.06296401838461561,
+   "aurc": 0.09320432439081776,
+   "acc_at_80": 0.825,
+   "acc_at_50": 0.9266666666666666,
+   "chance": 0.37149999999999994,
+   "mean_conf": 0.7900449633598328,
+   "sec": 0.5,
+   "heldout": false,
+   "per_q": [
+    0.7133333333333334,
+    0.8
+   ]
+  },
+  "games": {
+   "n": 500,
+   "acc": 0.998,
+   "nll": 0.0028993848245590925,
+   "brier": 0.0012032075319439173,
+   "ece": 0.002253464698791504,
+   "aurc": 4e-06,
+   "acc_at_80": 1.0,
+   "acc_at_50": 1.0,
+   "chance": 0.2885,
+   "mean_conf": 0.9976871609687805,
+   "sec": 1.2,
+   "heldout": false
+  },
+  "mario": {
+   "n": 276,
+   "acc": 0.9927536231884058,
+   "nll": 0.013057849369943142,
+   "brier": 0.006284656934440136,
+   "ece": 0.010539730821830639,
+   "aurc": 3.9430219014339996e-05,
+   "acc_at_80": 1.0,
+   "acc_at_50": 1.0,
+   "chance": 0.3333333333333333,
+   "mean_conf": 0.9905721545219421,
+   "sec": 6.4,
+   "heldout": false,
+   "per_q": [
+    0.9855072463768116,
+    1.0
+   ]
   }
  },
  "agg": {
   "in_task": {
-   "acc": 0.8069814634685621,
-   "nll": 0.4598948841303354,
-   "brier": 0.25660377455642447,
-   "ece": 0.02860716135081228,
-   "aurc": 0.09736104400332174,
-   "acc_at_80": 0.8593853585400241,
-   "chance": 0.32912234745754104
+   "acc": 0.8126446732755706,
+   "nll": 0.4534154008986239,
+   "brier": 0.25158945492306806,
+   "ece": 0.03454096135309228,
+   "aurc": 0.09352685127729836,
+   "acc_at_80": 0.8639655870629418,
+   "chance": 0.32498304691713953
   },
   "heldout": {
-   "acc": 0.7450346431863512,
-   "nll": 0.6336416278196417,
-   "brier": 0.34327830520013103,
-   "ece": 0.0751272948477249,
-   "aurc": 0.1378520558704769,
-   "acc_at_80": 0.8024372456758276,
+   "acc": 0.7479565063694622,
+   "nll": 0.656638503074646,
+   "brier": 0.35096842001961626,
+   "ece": 0.08591399023846887,
+   "aurc": 0.13724056983642266,
+   "acc_at_80": 0.8022276412618095,
    "chance": 0.32053884112032693
   },
-  "n_in": 64,
+  "n_in": 69,
   "n_heldout": 23
  },
- "model": "runs/r3_v2/model"
+ "model": "runs/r7_v4/model",
+ "engine": "compile"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86ace0629908bff7583e67505958334f1ffd584cd9ed407e8c0fb23ef0249e0d
+oid sha256:005b182713b64d6a37f37b30740cd36e46225217c0183798fbc3f278bc7d7da3
 size 3763692048

tokenizer_config.json

@@ -9,7 +9,7 @@
   "eos_token": "<|endoftext|>",
   "errors": "replace",
   "image_token": "<|image_pad|>",
-  "is_local": false,
+  "is_local": true,
   "local_files_only": false,
   "model_max_length": 262144,
   "model_specific_special_tokens": {