typical-medium (tm1b): weights, evals, inference package, manifest

MANIFEST.json

@@ -0,0 +1,12 @@
+{
+ "release": "typical-medium",
+ "checkpoint": "tm1b",
+ "backbone": "Qwen/Qwen3-4B-Base",
+ "tap_layer": 26,
+ "sha256_best_pt": "0500c3faeff1b24a3c7e56a846346d9e33d88c8906c48c976f1ad19e95a7e14e",
+ "source_commit": "2fad326c4bcffec4dbc4e8341f9c8248a13f65ae",
+ "date": "2026-09-21",
+ "parent": "typical-small",
+ "private_source": "guychuk/pcdm-runs/typical-medium",
+ "public_repo": "OzLabs/typical-medium"
+}

bench.json

@@ -0,0 +1,425 @@
+{
+  "model": "runs/tm1b",
+  "energy_model": null,
+  "backbone": "Qwen/Qwen3-4B-Base",
+  "quick": false,
+  "b_chunk": 4,
+  "native": {
+    "256": {
+      "2": {
+        "state_tokens": 256,
+        "suffix_tokens": 22,
+        "nc_chunk": 8,
+        "native": [
+          {
+            "m": 1,
+            "t_state_ms": 17.69845560193062,
+            "t_queries_ms": 38.907185196876526,
+            "total_ms": 56.605640798807144,
+            "ratio": 1.0,
+            "marginal_ms": 38.907185196876526
+          },
+          {
+            "m": 32,
+            "t_state_ms": 17.29227602481842,
+            "t_queries_ms": 107.14780166745186,
+            "total_ms": 124.44007769227028,
+            "ratio": 2.1983688539904773,
+            "marginal_ms": 3.3483688021078706
+          },
+          {
+            "m": 256,
+            "t_state_ms": 17.395589500665665,
+            "t_queries_ms": 720.7270376384258,
+            "total_ms": 738.1226271390915,
+            "ratio": 13.039736265200023,
+            "marginal_ms": 2.815339990775101
+          }
+        ],
+        "native_peak_mem_mb": 14565.473792,
+        "b_batched": [
+          {
+            "m": 1,
+            "t_state_ms": 19.00634914636612,
+            "t_queries_ms": 65.67803397774696,
+            "total_ms": 84.68438312411308,
+            "ratio": 1.0,
+            "marginal_ms": 65.67803397774696
+          },
+          {
+            "m": 32,
+            "t_state_ms": 19.073322415351868,
+            "t_queries_ms": 408.54376927018166,
+            "total_ms": 427.6170916855335,
+            "ratio": 5.0495389576000065,
+            "marginal_ms": 12.766992789693177
+          },
+          {
+            "m": 256,
+            "t_state_ms": 19.45146918296814,
+            "t_queries_ms": 3124.6551759541035,
+            "total_ms": 3144.1066451370716,
+            "ratio": 37.127348976836515,
+            "marginal_ms": 12.205684281070717
+          }
+        ],
+        "b_batched_peak_mem_mb": 15050.985472
+      },
+      "4": {
+        "state_tokens": 256,
+        "suffix_tokens": 34,
+        "nc_chunk": 8,
+        "native": [
+          {
+            "m": 1,
+            "t_state_ms": 17.520517110824585,
+            "t_queries_ms": 39.11682963371277,
+            "total_ms": 56.63734674453735,
+            "ratio": 1.0,
+            "marginal_ms": 39.11682963371277
+          },
+          {
+            "m": 32,
+            "t_state_ms": 17.56681129336357,
+            "t_queries_ms": 109.09038409590721,
+            "total_ms": 126.65719538927078,
+            "ratio": 2.2362840540634403,
+            "marginal_ms": 3.4090745029971004
+          },
+          {
+            "m": 256,
+            "t_state_ms": 17.606686800718307,
+            "t_queries_ms": 820.20478323102,
+            "total_ms": 837.8114700317383,
+            "ratio": 14.792562120021005,
+            "marginal_ms": 3.2039249344961718
+          }
+        ],
+        "native_peak_mem_mb": 14579.467776,
+        "b_batched": [
+          {
+            "m": 1,
+            "t_state_ms": 19.45377141237259,
+            "t_queries_ms": 66.85380637645721,
+            "total_ms": 86.3075777888298,
+            "ratio": 1.0,
+            "marginal_ms": 66.85380637645721
+          },
+          {
+            "m": 32,
+            "t_state_ms": 19.49465647339821,
+            "t_queries_ms": 415.58846831321716,
+            "total_ms": 435.0831247866154,
+            "ratio": 5.041076762125575,
+            "marginal_ms": 12.987139634788036
+          },
+          {
+            "m": 256,
+            "t_state_ms": 19.42378655076027,
+            "t_queries_ms": 3201.4211788773537,
+            "total_ms": 3220.844965428114,
+            "ratio": 37.31821756495831,
+            "marginal_ms": 12.505551479989663
+          }
+        ],
+        "b_batched_peak_mem_mb": 15418.156544
+      },
+      "10": {
+        "state_tokens": 256,
+        "suffix_tokens": 70,
+        "nc_chunk": 8,
+        "native": [
+          {
+            "m": 1,
+            "t_state_ms": 17.71542802453041,
+            "t_queries_ms": 39.49485346674919,
+            "total_ms": 57.2102814912796,
+            "ratio": 1.0,
+            "marginal_ms": 39.49485346674919
+          },
+          {
+            "m": 32,
+            "t_state_ms": 17.667438834905624,
+            "t_queries_ms": 113.24474588036537,
+            "total_ms": 130.912184715271,
+            "ratio": 2.288263250989694,
+            "marginal_ms": 3.538898308761418
+          },
+          {
+            "m": 256,
+            "t_state_ms": 17.80020445585251,
+            "t_queries_ms": 772.7814465761185,
+            "total_ms": 790.581651031971,
+            "ratio": 13.818873643411054,
+            "marginal_ms": 3.0186775256879628
+          }
+        ],
+        "native_peak_mem_mb": 14627.14112,
+        "b_batched": [
+          {
+            "m": 1,
+            "t_state_ms": 19.491098821163177,
+            "t_queries_ms": 67.33075901865959,
+            "total_ms": 86.82185783982277,
+            "ratio": 1.0,
+            "marginal_ms": 67.33075901865959
+          },
+          {
+            "m": 32,
+            "t_state_ms": 19.476331770420074,
+            "t_queries_ms": 518.6272598803043,
+            "total_ms": 538.1035916507244,
+            "ratio": 6.19778941661752,
+            "marginal_ms": 16.20710187125951
+          },
+          {
+            "m": 256,
+            "t_state_ms": 19.5954367518425,
+            "t_queries_ms": 4071.4379996061325,
+            "total_ms": 4091.033436357975,
+            "ratio": 47.11985596882185,
+            "marginal_ms": 15.904054685961455
+          }
+        ],
+        "b_batched_peak_mem_mb": 16549.823488
+      },
+      "32": {
+        "state_tokens": 256,
+        "suffix_tokens": 224,
+        "nc_chunk": 8,
+        "native": [
+          {
+            "m": 1,
+            "t_state_ms": 17.933987081050873,
+            "t_queries_ms": 40.07749259471893,
+            "total_ms": 58.011479675769806,
+            "ratio": 1.0,
+            "marginal_ms": 40.07749259471893
+          },
+          {
+            "m": 32,
+            "t_state_ms": 17.875414341688156,
+            "t_queries_ms": 199.15443286299706,
+            "total_ms": 217.0298472046852,
+            "ratio": 3.74115344786377,
+            "marginal_ms": 6.223576026968658
+          },
+          {
+            "m": 256,
+            "t_state_ms": 17.83185824751854,
+            "t_queries_ms": 1439.1961731016636,
+            "total_ms": 1457.0280313491821,
+            "ratio": 25.116201818891934,
+            "marginal_ms": 5.621860051178373
+          }
+        ],
+        "native_peak_mem_mb": 14892.906496,
+        "b_batched": [
+          {
+            "m": 1,
+            "t_state_ms": 19.53354850411415,
+            "t_queries_ms": 75.62482729554176,
+            "total_ms": 95.15837579965591,
+            "ratio": 1.0,
+            "marginal_ms": 75.62482729554176
+          },
+          {
+            "m": 32,
+            "t_state_ms": 19.454356282949448,
+            "t_queries_ms": 903.2446146011353,
+            "total_ms": 922.6989708840847,
+            "ratio": 9.696455652277129,
+            "marginal_ms": 28.226394206285477
+          },
+          {
+            "m": 256,
+            "t_state_ms": 21.368220448493958,
+            "t_queries_ms": 7146.388553082943,
+            "total_ms": 7167.756773531437,
+            "ratio": 75.32449680123014,
+            "marginal_ms": 27.915580285480246
+          }
+        ],
+        "b_batched_peak_mem_mb": 20708.114944
+      },
+      "64": {
+        "state_tokens": 256,
+        "suffix_tokens": 460,
+        "nc_chunk": 8,
+        "native": [
+          {
+            "m": 1,
+            "t_state_ms": 17.79353618621826,
+            "t_queries_ms": 41.496291756629944,
+            "total_ms": 59.289827942848206,
+            "ratio": 1.0,
+            "marginal_ms": 41.496291756629944
+          },
+          {
+            "m": 32,
+            "t_state_ms": 17.75621995329857,
+            "t_queries_ms": 338.5215550661087,
+            "total_ms": 356.2777750194073,
+            "ratio": 6.009087686387578,
+            "marginal_ms": 10.578798595815897
+          },
+          {
+            "m": 256,
+            "t_state_ms": 17.848066985607147,
+            "t_queries_ms": 2598.344925791025,
+            "total_ms": 2616.1929927766323,
+            "ratio": 44.1254947694988,
+            "marginal_ms": 10.149784866371192
+          }
+        ],
+        "native_peak_mem_mb": 15333.215232,
+        "b_batched": [
+          {
+            "m": 1,
+            "t_state_ms": 19.58613097667694,
+            "t_queries_ms": 93.24939921498299,
+            "total_ms": 112.83553019165993,
+            "ratio": 1.0,
+            "marginal_ms": 93.24939921498299
+          },
+          {
+            "m": 32,
+            "t_state_ms": 19.635286182165146,
+            "t_queries_ms": 1480.6604906916618,
+            "total_ms": 1500.295776873827,
+            "ratio": 13.296306352488953,
+            "marginal_ms": 46.27064033411443
+          },
+          {
+            "m": 256,
+            "t_state_ms": 19.47050541639328,
+            "t_queries_ms": 11805.44163286686,
+            "total_ms": 11824.912138283253,
+            "ratio": 104.79777174971146,
+            "marginal_ms": 46.11500637838617
+          }
+        ],
+        "b_batched_peak_mem_mb": 26712.366592
+      },
+      "128": {
+        "state_tokens": 256,
+        "suffix_tokens": 1029,
+        "nc_chunk": 8,
+        "native": [
+          {
+            "m": 1,
+            "t_state_ms": 17.73315668106079,
+            "t_queries_ms": 48.95908385515213,
+            "total_ms": 66.69224053621292,
+            "ratio": 1.0,
+            "marginal_ms": 48.95908385515213
+          },
+          {
+            "m": 32,
+            "t_state_ms": 17.543818801641464,
+            "t_queries_ms": 716.6986837983131,
+            "total_ms": 734.2425025999546,
+            "ratio": 11.00941423914633,
+            "marginal_ms": 22.396833868697286
+          },
+          {
+            "m": 256,
+            "t_state_ms": 17.649024724960327,
+            "t_queries_ms": 5805.374853312969,
+            "total_ms": 5823.0238780379295,
+            "ratio": 87.31186463702792,
+            "marginal_ms": 22.677245520753786
+          }
+        ],
+        "native_peak_mem_mb": 16306.219008,
+        "b_batched": [
+          {
+            "m": 1,
+            "t_state_ms": 25.08896216750145,
+            "t_queries_ms": 140.0025337934494,
+            "total_ms": 165.09149596095085,
+            "ratio": 1.0,
+            "marginal_ms": 140.0025337934494
+          },
+          {
+            "m": 32,
+            "t_state_ms": 19.624758511781693,
+            "t_queries_ms": 2846.7205613851547,
+            "total_ms": 2866.3453198969364,
+            "ratio": 17.362162134474293,
+            "marginal_ms": 88.96001754328609
+          },
+          {
+            "m": 256,
+            "t_state_ms": 19.967511296272278,
+            "t_queries_ms": 22693.809997290373,
+            "total_ms": 22713.777508586645,
+            "ratio": 137.58296498784614,
+            "marginal_ms": 88.64769530191552
+          }
+        ],
+        "b_batched_peak_mem_mb": 33338.615296
+      },
+      "256": {
+        "state_tokens": 256,
+        "suffix_tokens": 2309,
+        "nc_chunk": 8,
+        "native": [
+          {
+            "m": 1,
+            "t_state_ms": 17.903797328472137,
+            "t_queries_ms": 78.47601175308228,
+            "total_ms": 96.37980908155441,
+            "ratio": 1.0,
+            "marginal_ms": 78.47601175308228
+          },
+          {
+            "m": 32,
+            "t_state_ms": 17.928332090377808,
+            "t_queries_ms": 1699.1598010063171,
+            "total_ms": 1717.088133096695,
+            "ratio": 17.81584908145786,
+            "marginal_ms": 53.09874378144741
+          },
+          {
+            "m": 256,
+            "t_state_ms": 17.86353439092636,
+            "t_queries_ms": 12893.904343247414,
+            "total_ms": 12911.76787763834,
+            "ratio": 133.9675602253237,
+            "marginal_ms": 50.36681384081021
+          }
+        ],
+        "native_peak_mem_mb": 18598.0416,
+        "b_batched": [
+          {
+            "m": 1,
+            "t_state_ms": 20.501967519521713,
+            "t_queries_ms": 222.51605615019798,
+            "total_ms": 243.0180236697197,
+            "ratio": 1.0,
+            "marginal_ms": 222.51605615019798
+          },
+          {
+            "m": 32,
+            "t_state_ms": 19.544314593076706,
+            "t_queries_ms": 5423.821311444044,
+            "total_ms": 5443.365626037121,
+            "ratio": 22.399020220142503,
+            "marginal_ms": 169.49441598262638
+          },
+          {
+            "m": 256,
+            "t_state_ms": 19.62793990969658,
+            "t_queries_ms": 43251.52888149023,
+            "total_ms": 43271.15682139993,
+            "ratio": 178.0573974225417,
+            "marginal_ms": 168.9512846933212
+          }
+        ],
+        "b_batched_peak_mem_mb": 27778.753536
+      }
+    }
+  },
+  "crossover": {}
+}

best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0500c3faeff1b24a3c7e56a846346d9e33d88c8906c48c976f1ad19e95a7e14e
+size 46546609

eval_wf.json

@@ -0,0 +1,3292 @@
+{
+ "run": "runs/tm1b",
+ "mode": "native",
+ "max_state": 4096,
+ "limit": 0,
+ "eval": {
+  "u_chaosnli": {
+   "n": 1594,
+   "n_total": 1594,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 1.014103315474928,
+    "brier": 0.17091631474795344,
+    "jsd": 0.06646093306121999,
+    "acc": 0.5131744040150564,
+    "acc_k": 0.5131744040150564,
+    "ece": 0.10623594570713328,
+    "sel_acc80": 0.5313479623824452,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "": {
+     "nll": 1.014103315474928,
+     "brier": 0.17091631474795344,
+     "jsd": 0.06646093306121999,
+     "acc": 0.5131744040150564,
+     "acc_k": 0.5131744040150564,
+     "ece": 0.10623594570713328,
+     "sel_acc80": 0.5313479623824452,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1594
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "answer": {
+     "nll": 1.014103315474928,
+     "brier": 0.17091631474795344,
+     "jsd": 0.06646093306121999,
+     "acc": 0.5131744040150564,
+     "acc_k": 0.5131744040150564,
+     "ece": 0.10623594570713328,
+     "sel_acc80": 0.5313479623824452,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1594
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.46361355081555833
+   }
+  },
+  "u_real_heldout": {
+   "n": 562,
+   "n_total": 562,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.6395687143934158,
+    "brier": 0.11684180639496176,
+    "jsd": 0.05530236546295147,
+    "acc": 0.6583629893238434,
+    "acc_k": 0.6583629893238434,
+    "ece": 0.04802694863696116,
+    "sel_acc80": 0.7,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "": {
+     "nll": 0.6395687143934158,
+     "brier": 0.11684180639496176,
+     "jsd": 0.05530236546295147,
+     "acc": 0.6583629893238434,
+     "acc_k": 0.6583629893238434,
+     "ece": 0.04802694863696116,
+     "sel_acc80": 0.7,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 562
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "answer": {
+     "nll": 0.6395687143934158,
+     "brier": 0.11684180639496176,
+     "jsd": 0.05530236546295147,
+     "acc": 0.6583629893238434,
+     "acc_k": 0.6583629893238434,
+     "ece": 0.04802694863696116,
+     "sel_acc80": 0.7,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 562
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.5071174377224199
+   }
+  },
+  "u_synthetic_heldout": {
+   "n": 475,
+   "n_total": 475,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.6619388394682302,
+    "brier": 0.029335292543417674,
+    "jsd": 0.01347952047234738,
+    "acc": 0.9073684210526316,
+    "acc_k": 0.9073684210526316,
+    "ece": 0.17639653883482279,
+    "sel_acc80": 0.9815789473684211,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "": {
+     "nll": 0.6619388394682302,
+     "brier": 0.029335292543417674,
+     "jsd": 0.01347952047234738,
+     "acc": 0.9073684210526316,
+     "acc_k": 0.9073684210526316,
+     "ece": 0.17639653883482279,
+     "sel_acc80": 0.9815789473684211,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 475
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "answer": {
+     "nll": 0.6619388394682302,
+     "brier": 0.029335292543417674,
+     "jsd": 0.01347952047234738,
+     "acc": 0.9073684210526316,
+     "acc_k": 0.9073684210526316,
+     "ece": 0.17639653883482279,
+     "sel_acc80": 0.9815789473684211,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 475
+    }
+   },
+   "baselines": {
+    "majority_class": 1,
+    "majority_acc": 0.5747368421052632
+   }
+  },
+  "wf_heldout_choice": {
+   "n": 1500,
+   "n_total": 1500,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.08894784373441748,
+    "brier": 0.03804468692396801,
+    "jsd": 0.017131709690636845,
+    "acc": 0.978,
+    "acc_k": 0.9907997169143666,
+    "ece": 0.011830694854259511,
+    "sel_acc80": 0.9958333333333333,
+    "conf_wrong": 0.0,
+    "coverage": 0.9273333333333333,
+    "false_abstain": 0.01556970983722576,
+    "null_recall": 1.0,
+    "auroc_null": 0.9994834500654839,
+    "auroc_conf": 0.9995119213217171
+   },
+   "by_family": {
+    "tool_select": {
+     "nll": 0.08894784373441748,
+     "brier": 0.03804468692396801,
+     "jsd": 0.017131709690636845,
+     "acc": 0.978,
+     "acc_k": 0.9907997169143666,
+     "ece": 0.011830694854259511,
+     "sel_acc80": 0.9958333333333333,
+     "conf_wrong": 0.0,
+     "coverage": 0.9273333333333333,
+     "false_abstain": 0.01556970983722576,
+     "null_recall": 1.0,
+     "auroc_null": 0.9994834500654839,
+     "auroc_conf": 0.9995119213217171,
+     "n": 1500
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 0.08894784373441748,
+     "brier": 0.03804468692396801,
+     "jsd": 0.017131709690636845,
+     "acc": 0.978,
+     "acc_k": 0.9907997169143666,
+     "ece": 0.011830694854259511,
+     "sel_acc80": 0.9958333333333333,
+     "conf_wrong": 0.0,
+     "coverage": 0.9273333333333333,
+     "false_abstain": 0.01556970983722576,
+     "null_recall": 1.0,
+     "auroc_null": 0.9994834500654839,
+     "auroc_conf": 0.9995119213217171,
+     "n": 1500
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.09437487539006816,
+     "brier": 0.04038581321521866,
+     "jsd": 0.018169384833798215,
+     "acc": 0.9766454352441614,
+     "acc_k": 0.9907997169143666,
+     "ece": 0.012608277211354707,
+     "sel_acc80": 0.9946949602122016,
+     "coverage": 0.9844302901627743,
+     "false_abstain": 0.01556970983722576,
+     "null_recall": "n/a",
+     "n": 1413
+    },
+    "null": {
+     "nll": 0.0008053640857458489,
+     "brier": 2.1566814345354523e-05,
+     "jsd": 0.0002784340896366467,
+     "acc": 1.0,
+     "ece": 0.0007983151523546228,
+     "sel_acc80": 1.0,
+     "conf_wrong": 0.0,
+     "coverage": 0.0,
+     "false_abstain": "n/a",
+     "null_recall": 1.0,
+     "n": 87
+    }
+   },
+   "baselines": {
+    "majority_class": 2,
+    "majority_acc": 0.21333333333333335
+   }
+  },
+  "wf_heldout_noul": {
+   "n": 1500,
+   "n_total": 1500,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.5731820141841979,
+    "brier": 0.30167717441890574,
+    "jsd": 0.10894462183976376,
+    "acc": 0.8106666666666666,
+    "acc_k": 0.8106666666666666,
+    "ece": 0.13052070760726933,
+    "sel_acc80": 0.88,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "eligibility": {
+     "nll": 0.5731820141841979,
+     "brier": 0.30167717441890574,
+     "jsd": 0.10894462183976376,
+     "acc": 0.8106666666666666,
+     "acc_k": 0.8106666666666666,
+     "ece": 0.13052070760726933,
+     "sel_acc80": 0.88,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1500
+    }
+   },
+   "by_qtype": {
+    "noul": {
+     "nll": 0.5731820141841979,
+     "brier": 0.30167717441890574,
+     "jsd": 0.10894462183976376,
+     "acc": 0.8106666666666666,
+     "acc_k": 0.8106666666666666,
+     "ece": 0.13052070760726933,
+     "sel_acc80": 0.88,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1500
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.5731820141841979,
+     "brier": 0.30167717441890574,
+     "jsd": 0.10894462183976376,
+     "acc": 0.8106666666666666,
+     "acc_k": 0.8106666666666666,
+     "ece": 0.13052070760726933,
+     "sel_acc80": 0.88,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1500
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.5833333333333334,
+    "always_positive_acc": 0.4166666666666667
+   }
+  },
+  "wf_heldout_score": {
+   "n": 1502,
+   "n_total": 1502,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 1.0185536074805717,
+    "brier": 0.5644523897741041,
+    "jsd": 0.2801056596145076,
+    "acc": 0.5279627163781625,
+    "acc_k": 0.5279627163781625,
+    "ece": 0.05101245270667476,
+    "sel_acc80": 0.5806988352745425,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "urgency": {
+     "nll": 1.0185536074805717,
+     "brier": 0.5644523897741041,
+     "jsd": 0.2801056596145076,
+     "acc": 0.5279627163781625,
+     "acc_k": 0.5279627163781625,
+     "ece": 0.05101245270667476,
+     "sel_acc80": 0.5806988352745425,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1502
+    }
+   },
+   "by_qtype": {
+    "score": {
+     "nll": 1.0185536074805717,
+     "brier": 0.5644523897741041,
+     "jsd": 0.2801056596145076,
+     "acc": 0.5279627163781625,
+     "acc_k": 0.5279627163781625,
+     "ece": 0.05101245270667476,
+     "sel_acc80": 0.5806988352745425,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1502
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 1.0185536074805717,
+     "brier": 0.5644523897741041,
+     "jsd": 0.2801056596145076,
+     "acc": 0.5279627163781625,
+     "acc_k": 0.5279627163781625,
+     "ece": 0.05101245270667476,
+     "sel_acc80": 0.5806988352745425,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1502
+    }
+   },
+   "baselines": {
+    "majority_class": 2,
+    "majority_acc": 0.3548601864181092
+   }
+  },
+  "wf_heldout_style": {
+   "n": 1627,
+   "n_total": 1627,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.35412793927257424,
+    "brier": 0.2060959847410378,
+    "jsd": 0.10287023165278557,
+    "acc": 0.8586355255070682,
+    "acc_k": 0.8564294631710362,
+    "ece": 0.05414908969592812,
+    "sel_acc80": 0.9308755760368663,
+    "conf_wrong": 0.0,
+    "coverage": 0.9846342962507683,
+    "false_abstain": 0.0,
+    "null_recall": 1.0,
+    "auroc_null": 1.0,
+    "auroc_conf": 1.0
+   },
+   "by_family": {
+    "relevance": {
+     "nll": 0.6980276300431268,
+     "brier": 0.42102710856000347,
+     "jsd": 0.21389350686536507,
+     "acc": 0.72,
+     "acc_k": 0.72,
+     "ece": 0.17414700448513035,
+     "sel_acc80": 0.775,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 150
+    },
+    "support": {
+     "nll": 0.20406603972307857,
+     "brier": 0.1248183124169987,
+     "jsd": 0.055781303712672675,
+     "acc": 0.912751677852349,
+     "acc_k": 0.912751677852349,
+     "ece": 0.043371342572589816,
+     "sel_acc80": 0.975,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 149
+    },
+    "adequacy": {
+     "nll": 0.17072481903574357,
+     "brier": 0.10599008454380157,
+     "jsd": 0.047014351937127816,
+     "acc": 0.9266666666666666,
+     "acc_k": 0.9266666666666666,
+     "ece": 0.04129962166150417,
+     "sel_acc80": 0.9916666666666667,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 150
+    },
+    "action_select": {
+     "nll": 0.231281707526007,
+     "brier": 0.13311344023592248,
+     "jsd": 0.058601355529894074,
+     "acc": 0.9133333333333333,
+     "acc_k": 0.9064748201438849,
+     "ece": 0.05428580959637955,
+     "sel_acc80": 0.9833333333333333,
+     "conf_wrong": 0.0,
+     "coverage": 0.9266666666666666,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 150
+    },
+    "completeness": {
+     "nll": 0.7433181718309183,
+     "brier": 0.4258628433595575,
+     "jsd": 0.22361098694674933,
+     "acc": 0.7248322147651006,
+     "acc_k": 0.7248322147651006,
+     "ece": 0.19188565155803758,
+     "sel_acc80": 0.75,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 149
+    },
+    "quality": {
+     "nll": 0.8425104976168987,
+     "brier": 0.4753130219919474,
+     "jsd": 0.24646774262839125,
+     "acc": 0.6259541984732825,
+     "acc_k": 0.6259541984732825,
+     "ece": 0.1351061500893294,
+     "sel_acc80": 0.6857142857142857,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 131
+    },
+    "enum_extract": {
+     "nll": 0.0033332703444646995,
+     "brier": 0.00024124939750805901,
+     "jsd": 0.0011463143230046637,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.003245462179183957,
+     "sel_acc80": 1.0,
+     "conf_wrong": 0.0,
+     "coverage": 0.9666666666666667,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 150
+    },
+    "categorical": {
+     "nll": 0.24690092856485552,
+     "brier": 0.14982400101731513,
+     "jsd": 0.07302109473186505,
+     "acc": 0.889763779527559,
+     "acc_k": 0.8870967741935484,
+     "ece": 0.053563406148294794,
+     "sel_acc80": 0.9607843137254902,
+     "conf_wrong": 0.0,
+     "coverage": 0.9763779527559056,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 127
+    },
+    "routing": {
+     "nll": 0.21774816429206942,
+     "brier": 0.13021437367644959,
+     "jsd": 0.055718942276041364,
+     "acc": 0.9090909090909091,
+     "acc_k": 0.9047619047619048,
+     "ece": 0.05593117032990313,
+     "sel_acc80": 0.9905660377358491,
+     "conf_wrong": 0.0,
+     "coverage": 0.9545454545454546,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 132
+    },
+    "severity": {
+     "nll": 0.6434271258602455,
+     "brier": 0.347468069379254,
+     "jsd": 0.19560876673722952,
+     "acc": 0.7974683544303798,
+     "acc_k": 0.7974683544303798,
+     "ece": 0.22888638286650936,
+     "sel_acc80": 0.875,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 79
+    },
+    "policy_permit": {
+     "nll": 0.08175127275497936,
+     "brier": 0.03988201939839468,
+     "jsd": 0.02053868180510852,
+     "acc": 0.9798657718120806,
+     "acc_k": 0.9798657718120806,
+     "ece": 0.031195846179987764,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 149
+    },
+    "fact": {
+     "nll": 0.3245140742970864,
+     "brier": 0.201230115726297,
+     "jsd": 0.09200789581633895,
+     "acc": 0.8558558558558559,
+     "acc_k": 0.8558558558558559,
+     "ece": 0.0778693476238766,
+     "sel_acc80": 0.9213483146067416,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 111
+    }
+   },
+   "by_qtype": {
+    "score": {
+     "nll": 0.7399964051867366,
+     "brier": 0.4249972756512392,
+     "jsd": 0.2222837523407285,
+     "acc": 0.7092337917485265,
+     "acc_k": 0.7092337917485265,
+     "ece": 0.1720800453354193,
+     "sel_acc80": 0.7549019607843137,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 509
+    },
+    "noul": {
+     "nll": 0.186433926049318,
+     "brier": 0.11229947221410162,
+     "jsd": 0.0512284562939753,
+     "acc": 0.9230769230769231,
+     "acc_k": 0.9230769230769231,
+     "ece": 0.024164358063972435,
+     "sel_acc80": 0.9754464285714286,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 559
+    },
+    "choice": {
+     "nll": 0.1704676606348132,
+     "brier": 0.10057092826387289,
+     "jsd": 0.04577948101845999,
+     "acc": 0.9302325581395349,
+     "acc_k": 0.9269662921348315,
+     "ece": 0.0223422101657164,
+     "sel_acc80": 0.9933035714285714,
+     "conf_wrong": 0.0,
+     "coverage": 0.9552772808586762,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 559
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.35959165383560376,
+     "brier": 0.2093046663699308,
+     "jsd": 0.10445406727566603,
+     "acc": 0.8564294631710362,
+     "acc_k": 0.8564294631710362,
+     "ece": 0.05493348601568356,
+     "sel_acc80": 0.9297971918876755,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1602
+    },
+    "null": {
+     "nll": 0.004013110073643866,
+     "brier": 0.0004836659615737134,
+     "jsd": 0.0013780449386066767,
+     "acc": 1.0,
+     "ece": 0.003884973526001012,
+     "sel_acc80": 1.0,
+     "conf_wrong": 0.0,
+     "coverage": 0.0,
+     "false_abstain": "n/a",
+     "null_recall": 1.0,
+     "n": 25
+    }
+   },
+   "baselines": {
+    "majority_class": 1,
+    "majority_acc": 0.36447449293177625
+   }
+  },
+  "wf_rubric_flip": {
+   "n": 2062,
+   "n_total": 2062,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.5140816127009797,
+    "brier": 0.27501173451171385,
+    "jsd": 0.1274834860045779,
+    "acc": 0.7856450048496605,
+    "acc_k": 0.7914645974781765,
+    "ece": 0.04033603424533379,
+    "sel_acc80": 0.8878787878787879,
+    "coverage": 0.9932104752667313,
+    "false_abstain": 0.006789524733268671,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "urgency": {
+     "nll": 1.0503140999721947,
+     "brier": 0.5826411232348119,
+     "jsd": 0.2870499222013987,
+     "acc": 0.5142857142857142,
+     "acc_k": 0.5142857142857142,
+     "ece": 0.061800167475427895,
+     "sel_acc80": 0.5464285714285714,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 700
+    },
+    "eligibility": {
+     "nll": 0.3789669930673108,
+     "brier": 0.19723455708945753,
+     "jsd": 0.07565788618043696,
+     "acc": 0.8704268292682927,
+     "acc_k": 0.8704268292682927,
+     "ece": 0.0704190259481349,
+     "sel_acc80": 0.9314285714285714,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 656
+    },
+    "tool_select": {
+     "nll": 0.10795193761576238,
+     "brier": 0.042265638595044515,
+     "jsd": 0.017428370065288763,
+     "acc": 0.9759206798866855,
+     "acc_k": 0.9929178470254958,
+     "ece": 0.015375537465381884,
+     "sel_acc80": 0.9946902654867257,
+     "coverage": 0.9801699716713881,
+     "false_abstain": 0.019830028328611898,
+     "null_recall": "n/a",
+     "n": 706
+    }
+   },
+   "by_qtype": {
+    "score": {
+     "nll": 1.0503140999721947,
+     "brier": 0.5826411232348119,
+     "jsd": 0.2870499222013987,
+     "acc": 0.5142857142857142,
+     "acc_k": 0.5142857142857142,
+     "ece": 0.061800167475427895,
+     "sel_acc80": 0.5464285714285714,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 700
+    },
+    "noul": {
+     "nll": 0.3789669930673108,
+     "brier": 0.19723455708945753,
+     "jsd": 0.07565788618043696,
+     "acc": 0.8704268292682927,
+     "acc_k": 0.8704268292682927,
+     "ece": 0.0704190259481349,
+     "sel_acc80": 0.9314285714285714,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 656
+    },
+    "choice": {
+     "nll": 0.10795193761576238,
+     "brier": 0.042265638595044515,
+     "jsd": 0.017428370065288763,
+     "acc": 0.9759206798866855,
+     "acc_k": 0.9929178470254958,
+     "ece": 0.015375537465381884,
+     "sel_acc80": 0.9946902654867257,
+     "coverage": 0.9801699716713881,
+     "false_abstain": 0.019830028328611898,
+     "null_recall": "n/a",
+     "n": 706
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.5140816127009797,
+     "brier": 0.27501173451171385,
+     "jsd": 0.1274834860045779,
+     "acc": 0.7856450048496605,
+     "acc_k": 0.7914645974781765,
+     "ece": 0.04033603424533379,
+     "sel_acc80": 0.8878787878787879,
+     "coverage": 0.9932104752667313,
+     "false_abstain": 0.006789524733268671,
+     "null_recall": "n/a",
+     "n": 2062
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.32104752667313285
+   },
+   "flip": {
+    "n_pairs": 1031,
+    "flip_rate": 0.8205625606207565,
+    "both_correct": 0.6702230843840931,
+    "acc": 0.7914645974781765
+   },
+   "flip_by_family": {
+    "eligibility": {
+     "n_pairs": 328,
+     "flip_rate": 0.7591463414634146,
+     "both_correct": 0.75,
+     "acc": 0.8704268292682927
+    },
+    "tool_select": {
+     "n_pairs": 353,
+     "flip_rate": 0.9915014164305949,
+     "both_correct": 0.9858356940509915,
+     "acc": 0.9929178470254958
+    },
+    "urgency": {
+     "n_pairs": 350,
+     "flip_rate": 0.7057142857142857,
+     "both_correct": 0.27714285714285714,
+     "acc": 0.5142857142857142
+    }
+   }
+  },
+  "wf_rubric_shuffled": {
+   "n": 3775,
+   "n_total": 3775,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 3.3019875953051896,
+    "brier": 0.9847791419883737,
+    "jsd": 0.38205552797027176,
+    "acc": 0.3801324503311258,
+    "acc_k": 0.3811920529801324,
+    "ece": 0.39123494437198764,
+    "sel_acc80": 0.39470198675496687,
+    "coverage": 0.9925827814569537,
+    "false_abstain": 0.007417218543046358,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "tool_select": {
+     "nll": 8.01855352489357,
+     "brier": 1.585383500778014,
+     "jsd": 0.5686605163566438,
+     "acc": 0.15359116022099448,
+     "acc_k": 0.1580110497237569,
+     "ece": 0.7718642068172686,
+     "sel_acc80": 0.13535911602209943,
+     "coverage": 0.969060773480663,
+     "false_abstain": 0.030939226519337018,
+     "null_recall": "n/a",
+     "n": 905
+    },
+    "eligibility": {
+     "nll": 2.0992631849983714,
+     "brier": 0.8251410346340735,
+     "jsd": 0.28859514090273525,
+     "acc": 0.5234758234057463,
+     "acc_k": 0.5234758234057463,
+     "ece": 0.39013672221684337,
+     "sel_acc80": 0.5332749562171629,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1427
+    },
+    "urgency": {
+     "nll": 1.5333081547163776,
+     "brier": 0.7659687708795461,
+     "jsd": 0.35744739065613984,
+     "acc": 0.3804573804573805,
+     "acc_k": 0.3804573804573805,
+     "ece": 0.15649522523952702,
+     "sel_acc80": 0.38961038961038963,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1443
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 8.01855352489357,
+     "brier": 1.585383500778014,
+     "jsd": 0.5686605163566438,
+     "acc": 0.15359116022099448,
+     "acc_k": 0.1580110497237569,
+     "ece": 0.7718642068172686,
+     "sel_acc80": 0.13535911602209943,
+     "coverage": 0.969060773480663,
+     "false_abstain": 0.030939226519337018,
+     "null_recall": "n/a",
+     "n": 905
+    },
+    "noul": {
+     "nll": 2.0992631849983714,
+     "brier": 0.8251410346340735,
+     "jsd": 0.28859514090273525,
+     "acc": 0.5234758234057463,
+     "acc_k": 0.5234758234057463,
+     "ece": 0.39013672221684337,
+     "sel_acc80": 0.5332749562171629,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1427
+    },
+    "score": {
+     "nll": 1.5333081547163776,
+     "brier": 0.7659687708795461,
+     "jsd": 0.35744739065613984,
+     "acc": 0.3804573804573805,
+     "acc_k": 0.3804573804573805,
+     "ece": 0.15649522523952702,
+     "sel_acc80": 0.38961038961038963,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1443
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 3.3019875953051896,
+     "brier": 0.9847791419883737,
+     "jsd": 0.38205552797027176,
+     "acc": 0.3801324503311258,
+     "acc_k": 0.3811920529801324,
+     "ece": 0.39123494437198764,
+     "sel_acc80": 0.39470198675496687,
+     "coverage": 0.9925827814569537,
+     "false_abstain": 0.007417218543046358,
+     "null_recall": "n/a",
+     "n": 3775
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.36
+   }
+  },
+  "cua_s1_forms_demo": {
+   "n": 196,
+   "n_total": 196,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.0006697119946596961,
+    "brier": 2.5804204528327477e-06,
+    "jsd": 0.00023202367765799874,
+    "acc": 1.0,
+    "acc_k": 1.0,
+    "ece": 0.0006688650773496452,
+    "sel_acc80": 1.0,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "cua_s1_forms": {
+     "nll": 0.0006697119946596961,
+     "brier": 2.5804204528327477e-06,
+     "jsd": 0.00023202367765799874,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.0006688650773496452,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 196
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 0.0006697119946596961,
+     "brier": 2.5804204528327477e-06,
+     "jsd": 0.00023202367765799874,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.0006688650773496452,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 196
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.0006697119946596961,
+     "brier": 2.5804204528327477e-06,
+     "jsd": 0.00023202367765799874,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.0006688650773496452,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 196
+    }
+   },
+   "baselines": {
+    "majority_class": 18,
+    "majority_acc": 0.27040816326530615
+   }
+  },
+  "cua_s1_forms_test": {
+   "n": 24370,
+   "n_total": 24370,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.032467243628837364,
+    "brier": 0.01572971627781683,
+    "jsd": 0.006402877388207127,
+    "acc": 0.9902749281903981,
+    "acc_k": 0.9902749281903981,
+    "ece": 0.004090282016471624,
+    "sel_acc80": 0.9998974148543291,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "cua_s1_forms": {
+     "nll": 0.032467243628837364,
+     "brier": 0.01572971627781683,
+     "jsd": 0.006402877388207127,
+     "acc": 0.9902749281903981,
+     "acc_k": 0.9902749281903981,
+     "ece": 0.004090282016471624,
+     "sel_acc80": 0.9998974148543291,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 24370
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 0.032467243628837364,
+     "brier": 0.01572971627781683,
+     "jsd": 0.006402877388207127,
+     "acc": 0.9902749281903981,
+     "acc_k": 0.9902749281903981,
+     "ece": 0.004090282016471624,
+     "sel_acc80": 0.9998974148543291,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 24370
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.032467243628837364,
+     "brier": 0.01572971627781683,
+     "jsd": 0.006402877388207127,
+     "acc": 0.9902749281903981,
+     "acc_k": 0.9902749281903981,
+     "ece": 0.004090282016471624,
+     "sel_acc80": 0.9998974148543291,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 24370
+    }
+   },
+   "baselines": {
+    "majority_class": 16,
+    "majority_acc": 0.07033237587197373
+   }
+  },
+  "jev4b_adversarial": {
+   "n": 600,
+   "n_total": 600,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.21596318335730846,
+    "brier": 0.11449747234656535,
+    "jsd": 0.0566693510735706,
+    "acc": 0.9266666666666666,
+    "acc_k": 0.9266666666666666,
+    "ece": 0.033218660900990224,
+    "sel_acc80": 0.98125,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "jev4b_distill": {
+     "nll": 0.21596318335730846,
+     "brier": 0.11449747234656535,
+     "jsd": 0.0566693510735706,
+     "acc": 0.9266666666666666,
+     "acc_k": 0.9266666666666666,
+     "ece": 0.033218660900990224,
+     "sel_acc80": 0.98125,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 600
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 0.15764157777699858,
+     "brier": 0.07730203604858563,
+     "jsd": 0.03353047215992909,
+     "acc": 0.9527777777777777,
+     "acc_k": 0.9527777777777777,
+     "ece": 0.027492753830220998,
+     "sel_acc80": 0.9895833333333334,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 360
+    },
+    "score": {
+     "nll": 0.588537222505538,
+     "brier": 0.3360254493815728,
+     "jsd": 0.17654749465911643,
+     "acc": 0.775,
+     "acc_k": 0.775,
+     "ece": 0.11445025528470672,
+     "sel_acc80": 0.8125,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 120
+    },
+    "noul": {
+     "nll": 0.018353960950008512,
+     "brier": 0.004555804205497079,
+     "jsd": 0.006207844228949319,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.01696336468060807,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 120
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.21596318335730846,
+     "brier": 0.11449747234656535,
+     "jsd": 0.0566693510735706,
+     "acc": 0.9266666666666666,
+     "acc_k": 0.9266666666666666,
+     "ece": 0.033218660900990224,
+     "sel_acc80": 0.98125,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 600
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.42833333333333334
+   }
+  },
+  "jevlogs_triage": {
+   "n": 5080,
+   "n_total": 5080,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.8200663549047869,
+    "brier": 0.5245085358820047,
+    "jsd": 0.19936215850961117,
+    "acc": 0.6734251968503937,
+    "acc_k": 0.6734251968503937,
+    "ece": 0.19179721678805164,
+    "sel_acc80": 0.6801181102362205,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "log_hdfs": {
+     "nll": 0.8322974851760632,
+     "brier": 0.5211627531136993,
+     "jsd": 0.1981452827621227,
+     "acc": 0.6664,
+     "acc_k": 0.6664,
+     "ece": 0.17615253834724426,
+     "sel_acc80": 0.6785,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 2500
+    },
+    "log_bgl": {
+     "nll": 0.7998629474253376,
+     "brier": 0.521462450412092,
+     "jsd": 0.1985896785858705,
+     "acc": 0.6856,
+     "acc_k": 0.6856,
+     "ece": 0.24997946472167967,
+     "sel_acc80": 0.659,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 2500
+    },
+    "log_adversarial": {
+     "nll": 1.0692000176602048,
+     "brier": 0.7242544183263233,
+     "jsd": 0.26152952323552203,
+     "acc": 0.5125,
+     "acc_k": 0.5125,
+     "ece": 0.304364737868309,
+     "sel_acc80": 0.515625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 80
+    }
+   },
+   "by_qtype": {
+    "noul": {
+     "nll": 0.8200663549047869,
+     "brier": 0.5245085358820047,
+     "jsd": 0.19936215850961117,
+     "acc": 0.6734251968503937,
+     "acc_k": 0.6734251968503937,
+     "ece": 0.19179721678805164,
+     "sel_acc80": 0.6801181102362205,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 5080
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.8200663549047869,
+     "brier": 0.5245085358820047,
+     "jsd": 0.19936215850961117,
+     "acc": 0.6734251968503937,
+     "acc_k": 0.6734251968503937,
+     "ece": 0.19179721678805164,
+     "sel_acc80": 0.6801181102362205,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 5080
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.6968503937007874,
+    "always_positive_acc": 0.3031496062992126
+   }
+  },
+  "mind2web_choice": {
+   "n": 1600,
+   "n_total": 1600,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 1.2406711424343253,
+    "brier": 0.5732404237664536,
+    "jsd": 0.2788736075063562,
+    "acc": 0.544375,
+    "acc_k": 0.544375,
+    "ece": 0.08040638350881635,
+    "sel_acc80": 0.615625,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "mind2web": {
+     "nll": 1.2406711424343253,
+     "brier": 0.5732404237664536,
+     "jsd": 0.2788736075063562,
+     "acc": 0.544375,
+     "acc_k": 0.544375,
+     "ece": 0.08040638350881635,
+     "sel_acc80": 0.615625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1600
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 1.8521192870863956,
+     "brier": 0.7087342212760166,
+     "jsd": 0.3756742847617898,
+     "acc": 0.44,
+     "acc_k": 0.44,
+     "ece": 0.08787944102659824,
+     "sel_acc80": 0.49375,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 800
+    },
+    "noul": {
+     "nll": 0.6292229977822548,
+     "brier": 0.4377466262568903,
+     "jsd": 0.18207293025092264,
+     "acc": 0.64875,
+     "acc_k": 0.64875,
+     "ece": 0.07293332599103454,
+     "sel_acc80": 0.6875,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 800
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 1.2406711424343253,
+     "brier": 0.5732404237664536,
+     "jsd": 0.2788736075063562,
+     "acc": 0.544375,
+     "acc_k": 0.544375,
+     "ece": 0.08040638350881635,
+     "sel_acc80": 0.615625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1600
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.426875
+   }
+  },
+  "pagerduty_trigger": {
+   "n": 6000,
+   "n_total": 6000,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.396707458891549,
+    "brier": 0.23002239391086174,
+    "jsd": 0.11322283340814882,
+    "acc": 0.838,
+    "acc_k": 0.838,
+    "ece": 0.07345882829030351,
+    "sel_acc80": 0.8979166666666667,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "replica_lag_data_risk": {
+     "nll": 0.29740572278535377,
+     "brier": 0.14798980251485705,
+     "jsd": 0.09449495177807844,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.2269586962566041,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 114
+    },
+    "tls_hours_left_serving": {
+     "nll": 2.462503098332096,
+     "brier": 1.4718128845820069,
+     "jsd": 0.5430561946798185,
+     "acc": 0.0,
+     "acc_k": 0.0,
+     "ece": 0.7213651537895203,
+     "sel_acc80": 0.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 98
+    },
+    "disk_full_writes_failing": {
+     "nll": 0.6112621743776719,
+     "brier": 0.43961606766122513,
+     "jsd": 0.1828291039570283,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.40561820756305345,
+     "sel_acc80": 0.625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 110
+    },
+    "revoked_key_used": {
+     "nll": 2.25962306280277,
+     "brier": 1.3607013127715883,
+     "jsd": 0.49743537591946874,
+     "acc": 0.0,
+     "acc_k": 0.0,
+     "ece": 0.7095300555229187,
+     "sel_acc80": 0.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 102
+    },
+    "checksum_corruption": {
+     "nll": 0.16655108974517663,
+     "brier": 0.05001836994744468,
+     "jsd": 0.05564429813812224,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.14739903873867463,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 90
+    },
+    "checkout_slo_burn": {
+     "nll": 0.8234049439981885,
+     "brier": 0.5886945952552634,
+     "jsd": 0.24262392305224867,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.5280300080776215,
+     "sel_acc80": 0.375,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 100
+    },
+    "payments_processor_down": {
+     "nll": 0.7458985031769221,
+     "brier": 0.5397173510738811,
+     "jsd": 0.21208411419957937,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.3847963486580139,
+     "sel_acc80": 0.618421052631579,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 94
+    },
+    "crash_loop_user_impact": {
+     "nll": 0.45723675404949216,
+     "brier": 0.28838964871857825,
+     "jsd": 0.14411361556383404,
+     "acc": 0.7916666666666666,
+     "acc_k": 0.7916666666666666,
+     "ece": 0.3439861076573531,
+     "sel_acc80": 0.7402597402597403,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 96
+    },
+    "credential_stuffing": {
+     "nll": 0.6855976842756064,
+     "brier": 0.5141043463970237,
+     "jsd": 0.19799073218670762,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.4205113059530656,
+     "sel_acc80": 0.6233766233766234,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 96
+    },
+    "queue_backup_checkout": {
+     "nll": 0.4943918081315674,
+     "brier": 0.3165144497680829,
+     "jsd": 0.1554112283610238,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.3690876066684723,
+     "sel_acc80": 0.625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 100
+    },
+    "queue_slow_growth": {
+     "nll": 0.8056600958296468,
+     "brier": 0.46479321367422693,
+     "jsd": 0.2233220382938683,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.2907683104276657,
+     "sel_acc80": 0.6222222222222222,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 56
+    },
+    "cert_30d": {
+     "nll": 0.5765844556293318,
+     "brier": 0.459360806409668,
+     "jsd": 0.1663273119345,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.34605565667152405,
+     "sel_acc80": 0.6170212765957447,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 58
+    },
+    "disk_80pct_48h": {
+     "nll": 0.6364577616833813,
+     "brier": 0.3603717509503686,
+     "jsd": 0.1950357110171284,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.44600894767791033,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 64
+    },
+    "cpu_high_no_impact": {
+     "nll": 0.7085339514792242,
+     "brier": 0.5101478528535615,
+     "jsd": 0.19822264049400234,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.37314837455749506,
+     "sel_acc80": 0.625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 50
+    },
+    "single_pod_oom": {
+     "nll": 0.597222631375,
+     "brier": 0.3511328330652838,
+     "jsd": 0.1808235790015383,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.32174784059707934,
+     "sel_acc80": 0.6190476190476191,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 52
+    },
+    "deprecation": {
+     "nll": 1.2131927809820504,
+     "brier": 0.8289228409108279,
+     "jsd": 0.2701922670787307,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.45578983426094055,
+     "sel_acc80": 0.625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 60
+    },
+    "canary_within_budget": {
+     "nll": 0.68097187510421,
+     "brier": 0.46432714017336996,
+     "jsd": 0.19212515668071925,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.33959993720054626,
+     "sel_acc80": 0.6153846153846154,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 48
+    },
+    "replica_lag_seconds": {
+     "nll": 0.48507738285690005,
+     "brier": 0.2914945314683177,
+     "jsd": 0.14510412619084756,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.21687236428260803,
+     "sel_acc80": 0.625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 50
+    },
+    "tls_expires_7d": {
+     "nll": 0.8200619861584703,
+     "brier": 0.6263737845037869,
+     "jsd": 0.21875945370810285,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.41936272382736206,
+     "sel_acc80": 0.6190476190476191,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 26
+    },
+    "slow_slo_burn": {
+     "nll": 0.4646857777411605,
+     "brier": 0.2633606501114195,
+     "jsd": 0.15017855189982032,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.3715018928050995,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 36
+    },
+    "injection_noise": {
+     "nll": 0.3191649156767602,
+     "brier": 0.15667065379041875,
+     "jsd": 0.10271966665831857,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.2534384096822431,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 310
+    },
+    "health_ok": {
+     "nll": 0.11813930759934398,
+     "brier": 0.039244944056013954,
+     "jsd": 0.03965546739519377,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.10602798960567297,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 306
+    },
+    "kafka_caught_up": {
+     "nll": 0.1239690909443528,
+     "brier": 0.03284989912051308,
+     "jsd": 0.04179599987809453,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.11298148923640626,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 278
+    },
+    "invalid_coupon_error": {
+     "nll": 0.4095562707396914,
+     "brier": 0.2070045846973819,
+     "jsd": 0.12855307030768603,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.3042865756505765,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 322
+    },
+    "stack_trace_handled": {
+     "nll": 0.1369772675048073,
+     "brier": 0.04242081045781862,
+     "jsd": 0.04599428397435028,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.12313097069062383,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 332
+    },
+    "retry_then_ok": {
+     "nll": 0.23577570062147413,
+     "brier": 0.09354840300021036,
+     "jsd": 0.07807766271472766,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.20314176128783692,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 284
+    },
+    "feature_flag": {
+     "nll": 0.02957982981648557,
+     "brier": 0.0025294010832958157,
+     "jsd": 0.010185593857561997,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.028911458685042968,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 282
+    },
+    "cache_hit": {
+     "nll": 0.05577471799801635,
+     "brier": 0.010092673936511587,
+     "jsd": 0.019045345808299963,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.05296187488349169,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 286
+    },
+    "metrics_scrape": {
+     "nll": 0.19512496383591874,
+     "brier": 0.06495891768903242,
+     "jsd": 0.06532458959189952,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.17378984264484268,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 328
+    },
+    "cron_ok": {
+     "nll": 0.8260068202421647,
+     "brier": 0.5908376052070468,
+     "jsd": 0.24215122771741,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.3292480845605173,
+     "sel_acc80": 0.375,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 310
+    },
+    "rate_limit_bot": {
+     "nll": 0.2224008145703419,
+     "brier": 0.10465719261617362,
+     "jsd": 0.0723341471248896,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.18123274389654398,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 256
+    },
+    "http_404": {
+     "nll": 0.0272498202190122,
+     "brier": 0.0025453067235740637,
+     "jsd": 0.009377000999055841,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.026570967720739525,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 318
+    },
+    "deploy_success": {
+     "nll": 0.11437534991524133,
+     "brier": 0.022894189185427016,
+     "jsd": 0.03898171724942419,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.10790608627874346,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 244
+    },
+    "user_not_found": {
+     "nll": 0.7467332928805962,
+     "brier": 0.4869266134970554,
+     "jsd": 0.20417162215696066,
+     "acc": 0.5,
+     "acc_k": 0.5,
+     "ece": 0.3224844210812835,
+     "sel_acc80": 0.6228813559322034,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 294
+    },
+    "request_ok": {
+     "nll": 0.15962773605536434,
+     "brier": 0.045111188981216126,
+     "jsd": 0.05396733304793474,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 0.14663683669907707,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 350
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 0.48022365288831015,
+     "brier": 0.25716945445373307,
+     "jsd": 0.14152253563996733,
+     "acc": 0.851,
+     "acc_k": 0.851,
+     "ece": 0.13448732795317966,
+     "sel_acc80": 0.905,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 3000
+    },
+    "noul": {
+     "nll": 0.31319126489478805,
+     "brier": 0.20287533336799038,
+     "jsd": 0.0849231311763303,
+     "acc": 0.825,
+     "acc_k": 0.825,
+     "ece": 0.10489587610960006,
+     "sel_acc80": 0.9566666666666667,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 3000
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.396707458891549,
+     "brier": 0.23002239391086174,
+     "jsd": 0.11322283340814882,
+     "acc": 0.838,
+     "acc_k": 0.838,
+     "ece": 0.07345882829030351,
+     "sel_acc80": 0.8979166666666667,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 6000
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.7916666666666666
+   }
+  },
+  "systemone_lite_hard": {
+   "n": 5400,
+   "n_total": 5400,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.43450476654815856,
+    "brier": 0.23583658134544866,
+    "jsd": 0.11106552302444929,
+    "acc": 0.850925925925926,
+    "acc_k": 0.850925925925926,
+    "ece": 0.05852483691440687,
+    "sel_acc80": 0.8976851851851851,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "systemone_lite": {
+     "nll": 0.43450476654815856,
+     "brier": 0.23583658134544866,
+     "jsd": 0.11106552302444929,
+     "acc": 0.850925925925926,
+     "acc_k": 0.850925925925926,
+     "ece": 0.05852483691440687,
+     "sel_acc80": 0.8976851851851851,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 5400
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 0.47634740362930744,
+     "brier": 0.22078883830669857,
+     "jsd": 0.0890289134343277,
+     "acc": 0.8522222222222222,
+     "acc_k": 0.8522222222222222,
+     "ece": 0.06805240516861279,
+     "sel_acc80": 0.9319444444444445,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1800
+    },
+    "noul": {
+     "nll": 0.23199631304256532,
+     "brier": 0.14232818861742158,
+     "jsd": 0.06226400940582674,
+     "acc": 0.8994444444444445,
+     "acc_k": 0.8994444444444445,
+     "ece": 0.01822677158647116,
+     "sel_acc80": 0.9597222222222223,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1800
+    },
+    "score": {
+     "nll": 0.595170582972603,
+     "brier": 0.3443927171122257,
+     "jsd": 0.18190364623319336,
+     "acc": 0.8011111111111111,
+     "acc_k": 0.8011111111111111,
+     "ece": 0.17747255620029237,
+     "sel_acc80": 0.8513888888888889,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1800
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 0.43450476654815856,
+     "brier": 0.23583658134544866,
+     "jsd": 0.11106552302444929,
+     "acc": 0.850925925925926,
+     "acc_k": 0.850925925925926,
+     "ece": 0.05852483691440687,
+     "sel_acc80": 0.8976851851851851,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 5400
+    }
+   },
+   "baselines": {
+    "majority_class": 1,
+    "majority_acc": 0.44166666666666665
+   }
+  },
+  "tree_choice_cap": {
+   "n": 720,
+   "n_total": 720,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 1.0937767414927673,
+    "brier": 0.4118990134350543,
+    "jsd": 0.2282905942697906,
+    "acc": 0.6902777777777778,
+    "acc_k": 0.7319444444444444,
+    "ece": 0.07905487779838341,
+    "sel_acc80": 0.7934027777777778,
+    "coverage": 0.8611111111111112,
+    "false_abstain": 0.1388888888888889,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "incident_catalog": {
+     "nll": 1.0937767414927673,
+     "brier": 0.4118990134350543,
+     "jsd": 0.2282905942697906,
+     "acc": 0.6902777777777778,
+     "acc_k": 0.7319444444444444,
+     "ece": 0.07905487779838341,
+     "sel_acc80": 0.7934027777777778,
+     "coverage": 0.8611111111111112,
+     "false_abstain": 0.1388888888888889,
+     "null_recall": "n/a",
+     "n": 720
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 1.0937767414927673,
+     "brier": 0.4118990134350543,
+     "jsd": 0.2282905942697906,
+     "acc": 0.6902777777777778,
+     "acc_k": 0.7319444444444444,
+     "ece": 0.07905487779838341,
+     "sel_acc80": 0.7934027777777778,
+     "coverage": 0.8611111111111112,
+     "false_abstain": 0.1388888888888889,
+     "null_recall": "n/a",
+     "n": 720
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 1.0937767414927673,
+     "brier": 0.4118990134350543,
+     "jsd": 0.2282905942697906,
+     "acc": 0.6902777777777778,
+     "acc_k": 0.7319444444444444,
+     "ece": 0.07905487779838341,
+     "sel_acc80": 0.7934027777777778,
+     "coverage": 0.8611111111111112,
+     "false_abstain": 0.1388888888888889,
+     "null_recall": "n/a",
+     "n": 720
+    }
+   },
+   "baselines": {
+    "majority_class": 3,
+    "majority_acc": 0.11388888888888889
+   }
+  },
+  "typed_decisions_test": {
+   "n": 2000,
+   "n_total": 2000,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 1.183971535602263,
+    "brier": 0.21124234378656148,
+    "jsd": 0.08978436233492346,
+    "acc": 0.5325,
+    "acc_k": 0.5325,
+    "ece": 0.0960818936675787,
+    "sel_acc80": 0.55125,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "agent_trace_observability": {
+     "nll": 1.4513733308753711,
+     "brier": 0.27746615896441984,
+     "jsd": 0.11437914671823042,
+     "acc": 0.4,
+     "acc_k": 0.4,
+     "ece": 0.21149230748414996,
+     "sel_acc80": 0.4125,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 500
+    },
+    "customer_service": {
+     "nll": 1.1394922824048543,
+     "brier": 0.1685314240724029,
+     "jsd": 0.07635420781877246,
+     "acc": 0.638,
+     "acc_k": 0.638,
+     "ece": 0.09256491619348527,
+     "sel_acc80": 0.6825,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 500
+    },
+    "invoice_processing": {
+     "nll": 1.0241192434455133,
+     "brier": 0.25477993975703184,
+     "jsd": 0.11173192033283712,
+     "acc": 0.582,
+     "acc_k": 0.582,
+     "ece": 0.08850071275234224,
+     "sel_acc80": 0.64,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 500
+    },
+    "security_incidents": {
+     "nll": 1.1209012856833143,
+     "brier": 0.14419185235239126,
+     "jsd": 0.056672174469853814,
+     "acc": 0.51,
+     "acc_k": 0.51,
+     "ece": 0.1298270362019539,
+     "sel_acc80": 0.505,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 500
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 1.4624093197198917,
+     "brier": 0.22882638324834792,
+     "jsd": 0.10929266556165232,
+     "acc": 0.49166666666666664,
+     "acc_k": 0.49166666666666664,
+     "ece": 0.12719455897808077,
+     "sel_acc80": 0.5333333333333333,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 600
+    },
+    "noul": {
+     "nll": 0.8045478477317457,
+     "brier": 0.25816601068728884,
+     "jsd": 0.08310478332934974,
+     "acc": 0.5783333333333334,
+     "acc_k": 0.5783333333333334,
+     "ece": 0.15133020063241323,
+     "sel_acc80": 0.5895833333333333,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 600
+    },
+    "score": {
+     "nll": 1.2597109634169303,
+     "brier": 0.16286156401467608,
+     "jsd": 0.08016281916905708,
+     "acc": 0.52875,
+     "acc_k": 0.52875,
+     "ece": 0.07651440937072038,
+     "sel_acc80": 0.55,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 800
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 1.183971535602263,
+     "brier": 0.21124234378656148,
+     "jsd": 0.08978436233492346,
+     "acc": 0.5325,
+     "acc_k": 0.5325,
+     "ece": 0.0960818936675787,
+     "sel_acc80": 0.55125,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 2000
+    }
+   },
+   "baselines": {
+    "majority_class": 1,
+    "majority_acc": 0.2905
+   }
+  },
+  "typed_decisions_train": {
+   "n": 6000,
+   "n_total": 6000,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 1.1985372981046378,
+    "brier": 0.2329991959274657,
+    "jsd": 0.09837575798408288,
+    "acc": 0.5223333333333333,
+    "acc_k": 0.5223333333333333,
+    "ece": 0.10562264278034368,
+    "sel_acc80": 0.5375,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "agent_trace_observability": {
+     "nll": 1.416481477970894,
+     "brier": 0.25422688167917284,
+     "jsd": 0.10777658676024122,
+     "acc": 0.4266666666666667,
+     "acc_k": 0.4266666666666667,
+     "ece": 0.18357536943753558,
+     "sel_acc80": 0.42916666666666664,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1500
+    },
+    "customer_service": {
+     "nll": 1.1214988914219903,
+     "brier": 0.19128662847914574,
+     "jsd": 0.08370220742725006,
+     "acc": 0.6106666666666667,
+     "acc_k": 0.6106666666666667,
+     "ece": 0.08198627370595929,
+     "sel_acc80": 0.6433333333333333,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1500
+    },
+    "invoice_processing": {
+     "nll": 1.1031419902595583,
+     "brier": 0.3050021381875509,
+     "jsd": 0.13027031117221108,
+     "acc": 0.5333333333333333,
+     "acc_k": 0.5333333333333333,
+     "ece": 0.10525429819027585,
+     "sel_acc80": 0.575,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1500
+    },
+    "security_incidents": {
+     "nll": 1.1530268327661088,
+     "brier": 0.1814811353639933,
+     "jsd": 0.07175392657662916,
+     "acc": 0.5186666666666667,
+     "acc_k": 0.5186666666666667,
+     "ece": 0.12120020204782486,
+     "sel_acc80": 0.5258333333333334,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1500
+    }
+   },
+   "by_qtype": {
+    "choice": {
+     "nll": 1.4635702511121844,
+     "brier": 0.2532055828202569,
+     "jsd": 0.11942735571638688,
+     "acc": 0.4961111111111111,
+     "acc_k": 0.4961111111111111,
+     "ece": 0.12604168651832473,
+     "sel_acc80": 0.5368055555555555,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1800
+    },
+    "noul": {
+     "nll": 0.8394133296564307,
+     "brier": 0.2816899204625882,
+     "jsd": 0.09059050796156125,
+     "acc": 0.5633333333333334,
+     "acc_k": 0.5633333333333334,
+     "ece": 0.1775682842400339,
+     "sel_acc80": 0.5701388888888889,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 1800
+    },
+    "score": {
+     "nll": 1.2691055596851333,
+     "brier": 0.18132636235653043,
+     "jsd": 0.08842599720174607,
+     "acc": 0.51125,
+     "acc_k": 0.51125,
+     "ece": 0.08015046964089076,
+     "sel_acc80": 0.5317708333333333,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 2400
+    }
+   },
+   "by_label_class": {
+    "answer": {
+     "nll": 1.1985372981046378,
+     "brier": 0.2329991959274657,
+     "jsd": 0.09837575798408288,
+     "acc": 0.5223333333333333,
+     "acc_k": 0.5223333333333333,
+     "ece": 0.10562264278034368,
+     "sel_acc80": 0.5375,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 6000
+    }
+   },
+   "baselines": {
+    "majority_class": 1,
+    "majority_acc": 0.3105
+   }
+  },
+  "wh_heldout_family": {
+   "n": 422,
+   "n_total": 422,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.3240292072876919,
+    "brier": 0.17772530751240193,
+    "jsd": 0.07779958816769972,
+    "acc": 0.8744075829383886,
+    "acc_k": 0.9045092838196287,
+    "ece": 0.04873869755256795,
+    "sel_acc80": 0.9437869822485208,
+    "conf_wrong": 0.0,
+    "coverage": 0.8625592417061612,
+    "false_abstain": 0.04774535809018567,
+    "null_recall": 0.8888888888888888,
+    "auroc_null": 0.9826112584733274,
+    "auroc_conf": 0.9737105806071323
+   },
+   "by_family": {
+    "procurement_request": {
+     "nll": 0.31726594171575667,
+     "brier": 0.15641665156788495,
+     "jsd": 0.06340224526164076,
+     "acc": 0.9014084507042254,
+     "acc_k": 0.9375,
+     "ece": 0.07553887535148941,
+     "sel_acc80": 0.956140350877193,
+     "conf_wrong": 0.0,
+     "coverage": 0.8802816901408451,
+     "false_abstain": 0.0390625,
+     "null_recall": 0.8571428571428571,
+     "auroc_null": 0.9827008928571429,
+     "auroc_conf": 0.9782366071428572,
+     "n": 142
+    },
+    "research_ethics_review": {
+     "nll": 0.41234907161332957,
+     "brier": 0.23254687981548977,
+     "jsd": 0.10529413759841766,
+     "acc": 0.8396946564885496,
+     "acc_k": 0.8376068376068376,
+     "ece": 0.07992691961863568,
+     "sel_acc80": 0.9238095238095239,
+     "conf_wrong": 0.0,
+     "coverage": 0.8549618320610687,
+     "false_abstain": 0.042735042735042736,
+     "null_recall": 1.0,
+     "auroc_null": 0.9865689865689865,
+     "auroc_conf": 0.9847374847374847,
+     "n": 131
+    },
+    "subscription_cancellation": {
+     "nll": 0.25282438503639165,
+     "brier": 0.14983405363600533,
+     "jsd": 0.06734748559881597,
+     "acc": 0.8791946308724832,
+     "acc_k": 0.9318181818181818,
+     "ece": 0.04965120893996835,
+     "sel_acc80": 0.9666666666666667,
+     "conf_wrong": 0.0,
+     "coverage": 0.8523489932885906,
+     "false_abstain": 0.06060606060606061,
+     "null_recall": 0.8235294117647058,
+     "auroc_null": 0.9795008912655971,
+     "auroc_conf": 0.9714795008912657,
+     "n": 149
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "answer": {
+     "nll": 0.3270902509793986,
+     "brier": 0.1794731921212136,
+     "jsd": 0.07638232337116445,
+     "acc": 0.8726790450928382,
+     "acc_k": 0.9045092838196287,
+     "ece": 0.04732005681536247,
+     "sel_acc80": 0.9403973509933775,
+     "coverage": 0.9522546419098143,
+     "false_abstain": 0.04774535809018567,
+     "null_recall": "n/a",
+     "n": 377
+    },
+    "null": {
+     "nll": 0.2983844634705046,
+     "brier": 0.16308191867857966,
+     "jsd": 0.08967311768533967,
+     "acc": 0.8888888888888888,
+     "ece": 0.13434556325276692,
+     "sel_acc80": 0.9722222222222222,
+     "conf_wrong": 0.0,
+     "coverage": 0.11111111111111116,
+     "false_abstain": "n/a",
+     "null_recall": 0.8888888888888888,
+     "n": 45
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.38388625592417064
+   }
+  },
+  "wh_heldout_grammar": {
+   "n": 816,
+   "n_total": 816,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.2472013947656661,
+    "brier": 0.14714674629576405,
+    "jsd": 0.0655881649228148,
+    "acc": 0.8933823529411765,
+    "acc_k": 0.925414364640884,
+    "ece": 0.02050217598968861,
+    "sel_acc80": 0.9647779479326187,
+    "conf_wrong": 0.0,
+    "coverage": 0.8455882352941176,
+    "false_abstain": 0.05386740331491713,
+    "null_recall": 0.9456521739130435,
+    "auroc_null": 0.9844012731203459,
+    "auroc_conf": 0.9806629834254144
+   },
+   "by_family": {
+    "visa_extension": {
+     "nll": 0.3876584563899138,
+     "brier": 0.23389678729229724,
+     "jsd": 0.10288573687884103,
+     "acc": 0.851063829787234,
+     "acc_k": 0.8782608695652174,
+     "ece": 0.0661268077843578,
+     "sel_acc80": 0.9203539823008849,
+     "conf_wrong": 0.0,
+     "coverage": 0.7588652482269503,
+     "false_abstain": 0.0782608695652174,
+     "null_recall": 0.9615384615384616,
+     "auroc_null": 0.9709030100334448,
+     "auroc_conf": 0.9692307692307692,
+     "n": 141
+    },
+    "loan_approval": {
+     "nll": 7.454925248028152e-05,
+     "brier": 1.315695971451629e-07,
+     "jsd": 2.5833865575709257e-05,
+     "acc": 1.0,
+     "acc_k": 1.0,
+     "ece": 7.45158125884382e-05,
+     "sel_acc80": 1.0,
+     "conf_wrong": 0.0,
+     "coverage": 0.9708029197080292,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 137
+    },
+    "content_moderation": {
+     "nll": 0.5935040969149519,
+     "brier": 0.3596948371934755,
+     "jsd": 0.16132759921807785,
+     "acc": 0.703125,
+     "acc_k": 0.8247422680412371,
+     "ece": 0.10629645804874599,
+     "sel_acc80": 0.7864077669902912,
+     "conf_wrong": 0.0,
+     "coverage": 0.5703125,
+     "false_abstain": 0.26804123711340205,
+     "null_recall": 0.9354838709677419,
+     "auroc_null": 0.9002327901563019,
+     "auroc_conf": 0.9068839374792153,
+     "n": 128
+    },
+    "warranty_claim": {
+     "nll": 0.04521754519140885,
+     "brier": 0.030164132379325935,
+     "jsd": 0.012534975904067302,
+     "acc": 0.971830985915493,
+     "acc_k": 0.9710144927536232,
+     "ece": 0.025068008563887925,
+     "sel_acc80": 1.0,
+     "conf_wrong": 0.0,
+     "coverage": 0.971830985915493,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 142
+    },
+    "shift_swap": {
+     "nll": 0.31995094781288197,
+     "brier": 0.1809786294088722,
+     "jsd": 0.07359853995759635,
+     "acc": 0.8615384615384616,
+     "acc_k": 0.8907563025210085,
+     "ece": 0.054814853805762016,
+     "sel_acc80": 0.9423076923076923,
+     "conf_wrong": 0.0,
+     "coverage": 0.9,
+     "false_abstain": 0.03361344537815126,
+     "null_recall": 0.8181818181818182,
+     "auroc_null": 0.985485103132162,
+     "auroc_conf": 0.9572192513368983,
+     "n": 130
+    },
+    "vendor_onboarding": {
+     "nll": 0.16712495284237217,
+     "brier": 0.09594791421965362,
+     "jsd": 0.05081018090389403,
+     "acc": 0.9565217391304348,
+     "acc_k": 0.9508196721311475,
+     "ece": 0.07468755521636082,
+     "sel_acc80": 0.990990990990991,
+     "conf_wrong": 0.0,
+     "coverage": 0.8840579710144928,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 138
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "null": {
+     "nll": 0.277453074058921,
+     "brier": 0.14561172018866156,
+     "jsd": 0.08687484324735249,
+     "acc": 0.9456521739130435,
+     "ece": 0.1358135566115379,
+     "sel_acc80": 0.972972972972973,
+     "conf_wrong": 0.0,
+     "coverage": 0.05434782608695654,
+     "false_abstain": "n/a",
+     "null_recall": 0.9456521739130435,
+     "n": 92
+    },
+    "answer": {
+     "nll": 0.24335725872287678,
+     "brier": 0.14734180486185994,
+     "jsd": 0.06288322789814979,
+     "acc": 0.8867403314917127,
+     "acc_k": 0.925414364640884,
+     "ece": 0.03288542010013568,
+     "sel_acc80": 0.9706896551724138,
+     "coverage": 0.9461325966850829,
+     "false_abstain": 0.05386740331491713,
+     "null_recall": "n/a",
+     "n": 724
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.38480392156862747
+   }
+  },
+  "wh_heldout_style": {
+   "n": 403,
+   "n_total": 403,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.26691198757004164,
+    "brier": 0.1464047042686628,
+    "jsd": 0.0662673268053048,
+    "acc": 0.890818858560794,
+    "acc_k": 0.890818858560794,
+    "ece": 0.031059944674039948,
+    "sel_acc80": 0.9845201238390093,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "feature_flag_rollout": {
+     "nll": 0.19010573836138,
+     "brier": 0.11432481201229953,
+     "jsd": 0.05277888022237791,
+     "acc": 0.9090909090909091,
+     "acc_k": 0.9090909090909091,
+     "ece": 0.039632322571494384,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 55
+    },
+    "shift_swap": {
+     "nll": 0.3043634096913596,
+     "brier": 0.1503282508374881,
+     "jsd": 0.06654764978820293,
+     "acc": 0.8939393939393939,
+     "acc_k": 0.8939393939393939,
+     "ece": 0.062232637044155215,
+     "sel_acc80": 0.9811320754716981,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 66
+    },
+    "content_moderation": {
+     "nll": 0.4789173250257905,
+     "brier": 0.26149167112429633,
+     "jsd": 0.1279300537119714,
+     "acc": 0.813953488372093,
+     "acc_k": 0.813953488372093,
+     "ece": 0.06183511365291682,
+     "sel_acc80": 0.8571428571428571,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 43
+    },
+    "insurance_claim": {
+     "nll": 0.29568089754215104,
+     "brier": 0.16360065632472767,
+     "jsd": 0.06646606171635495,
+     "acc": 0.8529411764705882,
+     "acc_k": 0.8529411764705882,
+     "ece": 0.1020367794177111,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 34
+    },
+    "visa_extension": {
+     "nll": 0.09558577342528488,
+     "brier": 0.04933176843684583,
+     "jsd": 0.02922906947317498,
+     "acc": 0.972972972972973,
+     "acc_k": 0.972972972972973,
+     "ece": 0.03434973310779885,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 37
+    },
+    "warranty_claim": {
+     "nll": 0.31847504141119937,
+     "brier": 0.1722281656536195,
+     "jsd": 0.07974897957552128,
+     "acc": 0.8813559322033898,
+     "acc_k": 0.8813559322033898,
+     "ece": 0.03718661453764316,
+     "sel_acc80": 0.9791666666666666,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 59
+    },
+    "loan_approval": {
+     "nll": 0.12007633533318733,
+     "brier": 0.06839648073371568,
+     "jsd": 0.027442892286818,
+     "acc": 0.9583333333333334,
+     "acc_k": 0.9583333333333334,
+     "ece": 0.04480526472131414,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 48
+    },
+    "vendor_onboarding": {
+     "nll": 0.299750322830979,
+     "brier": 0.17565957622934011,
+     "jsd": 0.07452437378826761,
+     "acc": 0.8524590163934426,
+     "acc_k": 0.8524590163934426,
+     "ece": 0.08769017262536971,
+     "sel_acc80": 0.9591836734693877,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 61
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "answer": {
+     "nll": 0.26691198757004164,
+     "brier": 0.1464047042686628,
+     "jsd": 0.0662673268053048,
+     "acc": 0.890818858560794,
+     "acc_k": 0.890818858560794,
+     "ece": 0.031059944674039948,
+     "sel_acc80": 0.9845201238390093,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 403
+    }
+   },
+   "baselines": {
+    "majority_class": 1,
+    "majority_acc": 0.4267990074441687
+   }
+  },
+  "wh_level7": {
+   "n": 880,
+   "n_total": 880,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.8425315491747045,
+    "brier": 0.566066637960518,
+    "jsd": 0.22709053783966102,
+    "acc": 0.5443181818181818,
+    "acc_k": 0.5443181818181818,
+    "ece": 0.17712931768460705,
+    "sel_acc80": 0.5738636363636364,
+    "coverage": 1.0,
+    "false_abstain": 0.0,
+    "null_recall": "n/a"
+   },
+   "by_family": {
+    "temporal": {
+     "nll": 0.7525894214290174,
+     "brier": 0.5283106465099253,
+     "jsd": 0.20930622376758312,
+     "acc": 0.5727272727272728,
+     "acc_k": 0.5727272727272728,
+     "ece": 0.18158261044458907,
+     "sel_acc80": 0.6193181818181818,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "probability": {
+     "nll": 0.804850408692942,
+     "brier": 0.5061907772166189,
+     "jsd": 0.22027018987749325,
+     "acc": 0.5727272727272728,
+     "acc_k": 0.5727272727272728,
+     "ece": 0.12663529461080378,
+     "sel_acc80": 0.6136363636363636,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "numeric": {
+     "nll": 0.854408628287633,
+     "brier": 0.6255809344308126,
+     "jsd": 0.24148170952539819,
+     "acc": 0.4909090909090909,
+     "acc_k": 0.4909090909090909,
+     "ece": 0.24755196327512913,
+     "sel_acc80": 0.4772727272727273,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "tradeoff": {
+     "nll": 0.9582777382892255,
+     "brier": 0.6041841936847155,
+     "jsd": 0.23730402818816942,
+     "acc": 0.5409090909090909,
+     "acc_k": 0.5409090909090909,
+     "ece": 0.21417698643424293,
+     "sel_acc80": 0.5965909090909091,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "answer": {
+     "nll": 0.8425315491747045,
+     "brier": 0.566066637960518,
+     "jsd": 0.22709053783966102,
+     "acc": 0.5443181818181818,
+     "acc_k": 0.5443181818181818,
+     "ece": 0.17712931768460705,
+     "sel_acc80": 0.5738636363636364,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 880
+    }
+   },
+   "baselines": {
+    "majority_class": 1,
+    "majority_acc": 0.4602272727272727
+   }
+  },
+  "wh_rubric_flip": {
+   "n": 2206,
+   "n_total": 2206,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 0.5215484658596726,
+    "brier": 0.33112145645101976,
+    "jsd": 0.13730397267005034,
+    "acc": 0.742973708068903,
+    "acc_k": 0.7501196744853997,
+    "ece": 0.08118663341860714,
+    "sel_acc80": 0.8164305949008499,
+    "conf_wrong": 0.0,
+    "coverage": 0.9301903898458749,
+    "false_abstain": 0.022498803255146003,
+    "null_recall": 0.9145299145299145,
+    "auroc_null": 0.9924267530777824,
+    "auroc_conf": 0.9735570530209114
+   },
+   "by_family": {
+    "loan_approval": {
+     "nll": 0.03986105987120578,
+     "brier": 0.022708606037172275,
+     "jsd": 0.00911794257520729,
+     "acc": 0.9861111111111112,
+     "acc_k": 0.9858156028368794,
+     "ece": 0.014913849118683096,
+     "sel_acc80": 1.0,
+     "conf_wrong": 0.0,
+     "coverage": 0.9791666666666666,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 144
+    },
+    "shift_swap": {
+     "nll": 0.3556272740222601,
+     "brier": 0.19030986550930082,
+     "jsd": 0.07994201505668534,
+     "acc": 0.8554216867469879,
+     "acc_k": 0.8782051282051282,
+     "ece": 0.05131704243550819,
+     "sel_acc80": 0.9398496240601504,
+     "conf_wrong": 0.0,
+     "coverage": 0.927710843373494,
+     "false_abstain": 0.02564102564102564,
+     "null_recall": 0.8,
+     "auroc_null": 0.9878205128205129,
+     "auroc_conf": 0.948076923076923,
+     "n": 166
+    },
+    "vendor_onboarding": {
+     "nll": 0.2460892508917783,
+     "brier": 0.14163160753660067,
+     "jsd": 0.06893476279267717,
+     "acc": 0.9102564102564102,
+     "acc_k": 0.9014084507042254,
+     "ece": 0.054101008826341355,
+     "sel_acc80": 0.976,
+     "conf_wrong": 0.0,
+     "coverage": 0.9102564102564102,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 156
+    },
+    "content_moderation": {
+     "nll": 0.5707868104784541,
+     "brier": 0.3399881864130089,
+     "jsd": 0.1565919344337606,
+     "acc": 0.7297297297297297,
+     "acc_k": 0.8166666666666667,
+     "ece": 0.07247401008734834,
+     "sel_acc80": 0.7983193277310925,
+     "conf_wrong": 0.0,
+     "coverage": 0.6891891891891893,
+     "false_abstain": 0.16666666666666666,
+     "null_recall": 0.9285714285714286,
+     "auroc_null": 0.9428571428571428,
+     "auroc_conf": 0.9383928571428571,
+     "n": 148
+    },
+    "warranty_claim": {
+     "nll": 0.1533494565143874,
+     "brier": 0.08736313763136712,
+     "jsd": 0.038794858019505045,
+     "acc": 0.935064935064935,
+     "acc_k": 0.9342105263157895,
+     "ece": 0.03250992956099568,
+     "sel_acc80": 1.0,
+     "conf_wrong": 0.0,
+     "coverage": 0.987012987012987,
+     "false_abstain": 0.0,
+     "null_recall": 1.0,
+     "auroc_null": 1.0,
+     "auroc_conf": 1.0,
+     "n": 154
+    },
+    "visa_extension": {
+     "nll": 0.3792518490833633,
+     "brier": 0.22995354363423323,
+     "jsd": 0.10051831683881524,
+     "acc": 0.8493150684931506,
+     "acc_k": 0.8790322580645161,
+     "ece": 0.06351610569104751,
+     "sel_acc80": 0.9230769230769231,
+     "conf_wrong": 0.0,
+     "coverage": 0.7945205479452055,
+     "false_abstain": 0.07258064516129033,
+     "null_recall": 0.9545454545454546,
+     "auroc_null": 0.9736070381231671,
+     "auroc_conf": 0.9721407624633431,
+     "n": 146
+    },
+    "procurement_request": {
+     "nll": 0.3881801381792412,
+     "brier": 0.18757470893100156,
+     "jsd": 0.07515705849986501,
+     "acc": 0.8909090909090909,
+     "acc_k": 0.9292929292929293,
+     "ece": 0.08982028690251438,
+     "sel_acc80": 0.9204545454545454,
+     "conf_wrong": 0.0,
+     "coverage": 0.8818181818181818,
+     "false_abstain": 0.04040404040404041,
+     "null_recall": 0.8181818181818182,
+     "auroc_null": 0.977961432506887,
+     "auroc_conf": 0.9687786960514233,
+     "n": 110
+    },
+    "research_ethics_review": {
+     "nll": 0.4421237581343384,
+     "brier": 0.24773885698642706,
+     "jsd": 0.11269835014231615,
+     "acc": 0.8333333333333334,
+     "acc_k": 0.8315789473684211,
+     "ece": 0.103033944412514,
+     "sel_acc80": 0.9080459770114943,
+     "conf_wrong": 0.0,
+     "coverage": 0.8425925925925926,
+     "false_abstain": 0.042105263157894736,
+     "null_recall": 1.0,
+     "auroc_null": 0.9854251012145749,
+     "auroc_conf": 0.9838056680161944,
+     "n": 108
+    },
+    "subscription_cancellation": {
+     "nll": 0.2644555372286351,
+     "brier": 0.15680793083414146,
+     "jsd": 0.06982178903755307,
+     "acc": 0.8728813559322034,
+     "acc_k": 0.9326923076923077,
+     "ece": 0.061388740095041576,
+     "sel_acc80": 0.9578947368421052,
+     "conf_wrong": 0.0,
+     "coverage": 0.8559322033898304,
+     "false_abstain": 0.057692307692307696,
+     "null_recall": 0.7857142857142857,
+     "auroc_null": 0.9745879120879122,
+     "auroc_conf": 0.9684065934065934,
+     "n": 118
+    },
+    "feature_flag_rollout": {
+     "nll": 0.2267314556395052,
+     "brier": 0.1367964508738713,
+     "jsd": 0.06290056031954905,
+     "acc": 0.8913043478260869,
+     "acc_k": 0.8913043478260869,
+     "ece": 0.04663619204707767,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 46
+    },
+    "insurance_claim": {
+     "nll": 0.33329503028518,
+     "brier": 0.185439438184308,
+     "jsd": 0.07467994362635967,
+     "acc": 0.8333333333333334,
+     "acc_k": 0.8333333333333334,
+     "ece": 0.11369339625040695,
+     "sel_acc80": 1.0,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 30
+    },
+    "temporal": {
+     "nll": 0.7525550379081508,
+     "brier": 0.5281081163658832,
+     "jsd": 0.20925588567068154,
+     "acc": 0.5727272727272728,
+     "acc_k": 0.5727272727272728,
+     "ece": 0.1753265808929097,
+     "sel_acc80": 0.625,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "numeric": {
+     "nll": 0.8554757856422319,
+     "brier": 0.6267431100109008,
+     "jsd": 0.2417873556438029,
+     "acc": 0.4909090909090909,
+     "acc_k": 0.4909090909090909,
+     "ece": 0.2482705327597531,
+     "sel_acc80": 0.4659090909090909,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "probability": {
+     "nll": 0.8047299060130065,
+     "brier": 0.5061678304732362,
+     "jsd": 0.220257933543653,
+     "acc": 0.5727272727272728,
+     "acc_k": 0.5727272727272728,
+     "ece": 0.11572440903295172,
+     "sel_acc80": 0.6136363636363636,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "tradeoff": {
+     "nll": 0.9591729335220512,
+     "brier": 0.6044553194922022,
+     "jsd": 0.2374179176613618,
+     "acc": 0.5454545454545454,
+     "acc_k": 0.5454545454545454,
+     "ece": 0.20948043235323643,
+     "sel_acc80": 0.5909090909090909,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "answer": {
+     "nll": 0.5344578349380356,
+     "brier": 0.340925948758752,
+     "jsd": 0.13998262548140297,
+     "acc": 0.7333652465294399,
+     "acc_k": 0.7501196744853997,
+     "ece": 0.09119710230610484,
+     "sel_acc80": 0.8068181818181818,
+     "coverage": 0.977501196744854,
+     "false_abstain": 0.022498803255146003,
+     "null_recall": "n/a",
+     "n": 2089
+    },
+    "null": {
+     "nll": 0.2910555427425767,
+     "brier": 0.15606517926424532,
+     "jsd": 0.08947742802974543,
+     "acc": 0.9145299145299145,
+     "ece": 0.1208790940606696,
+     "sel_acc80": 0.9680851063829787,
+     "conf_wrong": 0.0,
+     "coverage": 0.0854700854700855,
+     "false_abstain": "n/a",
+     "null_recall": 0.9145299145299145,
+     "n": 117
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.39981867633726204
+   }
+  },
+  "wh_rubric_shuffled": {
+   "n": 2437,
+   "n_total": 2437,
+   "suffix_overflow": 0,
+   "raw": {
+    "nll": 3.3731109702781983,
+    "brier": 1.0166531838701214,
+    "jsd": 0.3668337750954112,
+    "acc": 0.37792367665162085,
+    "acc_k": 0.4059104737070839,
+    "ece": 0.46684057980616855,
+    "sel_acc80": 0.3553846153846154,
+    "conf_wrong": 0.17647058823529413,
+    "coverage": 0.926138695116947,
+    "false_abstain": 0.049109083007388095,
+    "null_recall": 0.49264705882352944,
+    "auroc_null": 0.9573203466523506,
+    "auroc_conf": 0.8061951964618963
+   },
+   "by_family": {
+    "temporal": {
+     "nll": 1.0978670096057097,
+     "brier": 0.7736457181852393,
+     "jsd": 0.2832470099926861,
+     "acc": 0.4090909090909091,
+     "acc_k": 0.4090909090909091,
+     "ece": 0.34149774312973025,
+     "sel_acc80": 0.3693181818181818,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "shift_swap": {
+     "nll": 4.813958214372273,
+     "brier": 1.1627913595629051,
+     "jsd": 0.41357994903252204,
+     "acc": 0.35602094240837695,
+     "acc_k": 0.37222222222222223,
+     "ece": 0.5618253638919112,
+     "sel_acc80": 0.35294117647058826,
+     "conf_wrong": 0.2727272727272727,
+     "coverage": 0.9319371727748691,
+     "false_abstain": 0.044444444444444446,
+     "null_recall": 0.45454545454545453,
+     "auroc_null": 0.961111111111111,
+     "auroc_conf": 0.8565656565656565,
+     "n": 191
+    },
+    "vendor_onboarding": {
+     "nll": 4.468025309710248,
+     "brier": 1.1531663283624074,
+     "jsd": 0.4069967513654248,
+     "acc": 0.31958762886597936,
+     "acc_k": 0.39325842696629215,
+     "ece": 0.5820424177290239,
+     "sel_acc80": 0.34615384615384615,
+     "conf_wrong": 0.1875,
+     "coverage": 0.9175257731958762,
+     "false_abstain": 0.07303370786516854,
+     "null_recall": 0.1875,
+     "auroc_null": 0.9371488764044944,
+     "auroc_conf": 0.8433988764044944,
+     "n": 194
+    },
+    "loan_approval": {
+     "nll": 7.430180710009384,
+     "brier": 1.3288401062960586,
+     "jsd": 0.4609968152472018,
+     "acc": 0.33146067415730335,
+     "acc_k": 0.3505747126436782,
+     "ece": 0.6675498291347803,
+     "sel_acc80": 0.3356643356643357,
+     "conf_wrong": 0.5,
+     "coverage": 0.9662921348314607,
+     "false_abstain": 0.022988505747126436,
+     "null_recall": 0.5,
+     "auroc_null": 0.978448275862069,
+     "auroc_conf": 0.9396551724137931,
+     "n": 178
+    },
+    "probability": {
+     "nll": 1.5095582926439517,
+     "brier": 0.9146099048716316,
+     "jsd": 0.35344105098573364,
+     "acc": 0.35,
+     "acc_k": 0.35,
+     "ece": 0.3669138718735088,
+     "sel_acc80": 0.3181818181818182,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "feature_flag_rollout": {
+     "nll": 4.963032018178103,
+     "brier": 1.250659962667729,
+     "jsd": 0.4386572342122217,
+     "acc": 0.34782608695652173,
+     "acc_k": 0.34782608695652173,
+     "ece": 0.6342515031928602,
+     "sel_acc80": 0.3783783783783784,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 46
+    },
+    "research_ethics_review": {
+     "nll": 3.379301129427671,
+     "brier": 1.1293730274385836,
+     "jsd": 0.4101792599802621,
+     "acc": 0.31746031746031744,
+     "acc_k": 0.38392857142857145,
+     "ece": 0.5505591124769241,
+     "sel_acc80": 0.3069306930693069,
+     "conf_wrong": 0.07142857142857142,
+     "coverage": 0.8492063492063492,
+     "false_abstain": 0.125,
+     "null_recall": 0.35714285714285715,
+     "auroc_null": 0.8852040816326531,
+     "auroc_conf": 0.7633928571428572,
+     "n": 126
+    },
+    "warranty_claim": {
+     "nll": 5.425085813770316,
+     "brier": 1.222989340354703,
+     "jsd": 0.42426978503284835,
+     "acc": 0.35978835978835977,
+     "acc_k": 0.3675675675675676,
+     "ece": 0.6112382035091439,
+     "sel_acc80": 0.3684210526315789,
+     "conf_wrong": 0.5,
+     "coverage": 0.9788359788359788,
+     "false_abstain": 0.010810810810810811,
+     "null_recall": 0.5,
+     "auroc_null": 0.9905405405405405,
+     "auroc_conf": 0.8783783783783783,
+     "n": 189
+    },
+    "numeric": {
+     "nll": 0.7009633054040385,
+     "brier": 0.5083317633419103,
+     "jsd": 0.19623566233246967,
+     "acc": 0.5636363636363636,
+     "acc_k": 0.5636363636363636,
+     "ece": 0.20822090967134996,
+     "sel_acc80": 0.5795454545454546,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "content_moderation": {
+     "nll": 2.532750401458349,
+     "brier": 0.9723702132508161,
+     "jsd": 0.3676846987407566,
+     "acc": 0.3670886075949367,
+     "acc_k": 0.4094488188976378,
+     "ece": 0.4052060521856139,
+     "sel_acc80": 0.3464566929133858,
+     "conf_wrong": 0.06451612903225806,
+     "coverage": 0.6645569620253164,
+     "false_abstain": 0.23622047244094488,
+     "null_recall": 0.7419354838709677,
+     "auroc_null": 0.8270256540513081,
+     "auroc_conf": 0.7680975361950725,
+     "n": 158
+    },
+    "visa_extension": {
+     "nll": 3.4319388280589163,
+     "brier": 1.185777007774496,
+     "jsd": 0.4211514399505007,
+     "acc": 0.3023255813953488,
+     "acc_k": 0.3698630136986301,
+     "ece": 0.5790469986061717,
+     "sel_acc80": 0.2971014492753623,
+     "conf_wrong": 0.11538461538461539,
+     "coverage": 0.8023255813953488,
+     "false_abstain": 0.14383561643835616,
+     "null_recall": 0.5,
+     "auroc_null": 0.8730242360379347,
+     "auroc_conf": 0.8132244467860906,
+     "n": 172
+    },
+    "subscription_cancellation": {
+     "nll": 4.124055456988184,
+     "brier": 1.142628609921072,
+     "jsd": 0.40527967431266604,
+     "acc": 0.3546099290780142,
+     "acc_k": 0.424,
+     "ece": 0.5671253876483187,
+     "sel_acc80": 0.35398230088495575,
+     "conf_wrong": 0.3125,
+     "coverage": 0.8652482269503546,
+     "false_abstain": 0.096,
+     "null_recall": 0.4375,
+     "auroc_null": 0.9035,
+     "auroc_conf": 0.801,
+     "n": 141
+    },
+    "procurement_request": {
+     "nll": 5.064798743828172,
+     "brier": 1.1244811567056938,
+     "jsd": 0.3945059149254654,
+     "acc": 0.37593984962406013,
+     "acc_k": 0.4369747899159664,
+     "ece": 0.5752717774165303,
+     "sel_acc80": 0.3925233644859813,
+     "conf_wrong": 0.21428571428571427,
+     "coverage": 0.8796992481203008,
+     "false_abstain": 0.07563025210084033,
+     "null_recall": 0.5,
+     "auroc_null": 0.9357743097238895,
+     "auroc_conf": 0.8715486194477791,
+     "n": 133
+    },
+    "tradeoff": {
+     "nll": 1.295216810789425,
+     "brier": 0.8154554789002251,
+     "jsd": 0.30319613299194453,
+     "acc": 0.4318181818181818,
+     "acc_k": 0.4318181818181818,
+     "ece": 0.35886814662001354,
+     "sel_acc80": 0.3806818181818182,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 220
+    },
+    "insurance_claim": {
+     "nll": 5.987324790766054,
+     "brier": 1.150424499572334,
+     "jsd": 0.407765815124327,
+     "acc": 0.41379310344827586,
+     "acc_k": 0.41379310344827586,
+     "ece": 0.5872780434016523,
+     "sel_acc80": 0.4166666666666667,
+     "coverage": 1.0,
+     "false_abstain": 0.0,
+     "null_recall": "n/a",
+     "n": 29
+    }
+   },
+   "by_qtype": {},
+   "by_label_class": {
+    "answer": {
+     "nll": 3.4790294568471882,
+     "brier": 1.0318484361922742,
+     "jsd": 0.3710045446095289,
+     "acc": 0.37114298131247286,
+     "acc_k": 0.4059104737070839,
+     "ece": 0.47994829282352375,
+     "sel_acc80": 0.3508962520369365,
+     "coverage": 0.9508909169926119,
+     "false_abstain": 0.049109083007388095,
+     "null_recall": "n/a",
+     "n": 2301
+    },
+    "null": {
+     "nll": 1.5810636350190332,
+     "brier": 0.7595629221548795,
+     "jsd": 0.2962680350072858,
+     "acc": 0.49264705882352944,
+     "ece": 0.3134835373391124,
+     "sel_acc80": 0.45871559633027525,
+     "conf_wrong": 0.17647058823529413,
+     "coverage": 0.5073529411764706,
+     "false_abstain": "n/a",
+     "null_recall": 0.49264705882352944,
+     "n": 136
+    }
+   },
+   "baselines": {
+    "majority_class": 0,
+    "majority_acc": 0.4095199015182602
+   }
+  }
+ }
+}

inference/example.py

@@ -0,0 +1,27 @@
+"""uv run --no-sync python inference/example.py"""
+from typical import Typical
+
+STATE = (
+    "Ticket #4821: Customer says their package arrived damaged. They want a replacement "
+    "shipped overnight, not a refund. Order was placed 3 days ago, still under warranty."
+)
+
+m = Typical.from_pretrained("OzLabs/typical-small", device="auto")
+
+# K-way choice over a fixed label set.
+print(m.choice(STATE, "What does the customer want?", ["refund", "replacement", "repair"]))
+
+# Yes/no.
+print(m.noul(STATE, "Is the order still under warranty?"))
+
+# Ordinal levels + expected value.
+print(m.score(STATE, "How urgent is this ticket?", ["0", "1", "2", "3"]))
+
+# Full JevBench-style decide(), with the runtime block (latency, p_null, truncation flags).
+probs, runtime = m.decide(
+    STATE,
+    {"type": "choice", "instructions": "What does the customer want?",
+     "criteria": {"refund": "money back", "replacement": "a new item shipped"}},
+    ["refund", "replacement"],
+)
+print(probs, runtime)

inference/requirements.txt

@@ -0,0 +1,5 @@
+torch>=2.14.0
+transformers>=5.17.0
+safetensors>=0.4.0
+huggingface_hub>=0.25.0
+numpy>=2.0.0

inference/test_parity.py

@@ -0,0 +1,121 @@
+"""Parity check: inference/typical/ vs the training repo's pcdm_jev.decider.PCDMDecider
+(mode="native"), same checkpoint (guychuk/pcdm-runs/typical-small/best.pt), same device,
+same 6 items. Downloads a 1.7B checkpoint -> gated like the repo's other real-checkpoint
+tests (tests/test_pipeline.py, tests/test_jevbench.py): RUN_SLOW=1 required.
+
+uv run --no-sync pytest inference/test_parity.py -v   # (with RUN_SLOW=1 set)
+"""
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+INFERENCE_DIR = Path(__file__).resolve().parent
+for p in (str(REPO_ROOT), str(INFERENCE_DIR)):
+    if p not in sys.path:
+        sys.path.insert(0, p)
+
+CKPT_REPO = "guychuk/pcdm-runs"
+CKPT_FILE = "typical-small/best.pt"
+
+STATE = (
+    "Ticket #4821: Customer says their package arrived damaged. They want a replacement "
+    "shipped overnight, not a refund. Order was placed 3 days ago, still under warranty."
+)
+
+ITEMS = [
+    ("What does the customer want?", ["refund", "replacement", "repair"]),
+    ("Is the order still under warranty?", ["no", "yes"]),
+    ("How urgent is this ticket?", ["0", "1", "2", "3"]),
+    ("What is the customer's sentiment?", ["positive", "neutral", "negative"]),
+    ("Should this be escalated to a manager?", ["no", "yes"]),
+    ("Which department should own this ticket?", ["billing", "shipping", "support"]),
+]
+
+NOUL_ITEMS = [
+    "Is the order still under warranty?",
+    "Should this be escalated to a manager?",
+    "Was the package damaged?",
+]
+
+
+@pytest.mark.skipif(not os.environ.get("RUN_SLOW"), reason="downloads the real typical-small "
+                     "checkpoint (Qwen3-1.7B-Base + LoRA); set RUN_SLOW=1 to run")
+def test_typical_matches_pcdm_decider():
+    from huggingface_hub import hf_hub_download
+
+    from pcdm_jev.decider import PCDMDecider
+    from typical import Typical, to_labels
+
+    local_best_pt = hf_hub_download(CKPT_REPO, CKPT_FILE)
+    run_dir = os.path.dirname(local_best_pt)
+
+    orig = PCDMDecider(run_dir=run_dir, mode="native", device="auto")
+    mine = Typical.from_pretrained(CKPT_REPO, filename=CKPT_FILE, device=orig.device,
+                                   max_state=orig.max_state)
+    assert orig.device == mine.device
+
+    max_diff = 0.0
+    for query, labels in ITEMS:
+        p_orig = orig._probs("native", STATE, query, labels)
+        p_mine = mine._raw(STATE, query, labels)
+        diff = (p_orig.float().cpu() - p_mine.float().cpu()).abs().max().item()
+        max_diff = max(max_diff, diff)
+        assert diff < 1e-4, f"{query!r} {labels}: max abs diff {diff}"
+    print(f"[parity] {len(ITEMS)} items, max abs prob diff = {max_diff:.2e}")
+
+    # noul reversed-label control: the checkpoint's noul_head is bern (query-only, position-
+    # invariant by construction -- the rendered suffix never contains the candidate text, so
+    # P(yes) under ["no","yes"] and ["yes","no"] is the SAME forward pass mathematically).
+    # ponytail: exact float equality is verified on CPU (checked separately, bit-identical);
+    # on MPS the two calls can differ by ~1e-7 from non-deterministic reduction order, so this
+    # check uses a tight epsilon rather than `==` to stay device-agnostic.
+    eps = 1e-6
+    for q in NOUL_ITEMS:
+        p_yes_forward = mine.noul(STATE, q)
+        probs_rev, _ = to_labels(mine._raw(STATE, q, ["yes", "no"]), ["yes", "no"])
+        assert abs(probs_rev["yes"] - p_yes_forward) < eps, (q, probs_rev["yes"], p_yes_forward)
+
+        orig_fwd, _ = to_labels(orig._probs("native", STATE, q, ["no", "yes"]), ["no", "yes"])
+        orig_rev, _ = to_labels(orig._probs("native", STATE, q, ["yes", "no"]), ["yes", "no"])
+        assert abs(orig_fwd["yes"] - orig_rev["yes"]) < eps, (q, orig_fwd["yes"], orig_rev["yes"])
+        assert abs(orig_fwd["yes"] - p_yes_forward) < 1e-4, (q, orig_fwd["yes"], p_yes_forward)
+    print(f"[parity] noul reversed-label check OK over {len(NOUL_ITEMS)} items (eps={eps})")
+
+
+@pytest.mark.skipif(not os.environ.get("RUN_SLOW"), reason="downloads the real "
+                     "typical-small-preview checkpoint; set RUN_SLOW=1 to run")
+def test_typical_noul_via_choice_path_on_preview():
+    """typical-small-preview trains noul_head="choice" (no Bernoulli head): m.noul() must
+    still work by scoring the plain K-way ["no","yes"] choice and reading off P(yes) -- and
+    must still match the original decider exactly (order-invariance is a bern-only guarantee,
+    not required here)."""
+    from huggingface_hub import hf_hub_download
+
+    from pcdm_jev.decider import PCDMDecider
+    from typical import Typical, to_labels
+
+    local_best_pt = hf_hub_download(CKPT_REPO, "typical-small-preview/best.pt")
+    run_dir = os.path.dirname(local_best_pt)
+
+    orig = PCDMDecider(run_dir=run_dir, mode="native", device="auto")
+    assert orig.native["m"].noul_head == "choice"
+    mine = Typical.from_pretrained(CKPT_REPO, filename="typical-small-preview/best.pt",
+                                   device=orig.device, max_state=orig.max_state)
+    assert mine.model.noul_head == "choice"
+
+    for q in NOUL_ITEMS:
+        p_yes = mine.noul(STATE, q)
+        assert 0.0 <= p_yes <= 1.0
+        orig_probs, _ = to_labels(orig._probs("native", STATE, q, ["no", "yes"]), ["no", "yes"])
+        assert abs(orig_probs["yes"] - p_yes) < 1e-4, (q, orig_probs["yes"], p_yes)
+    print(f"[parity] preview noul() via K-way choice path OK over {len(NOUL_ITEMS)} items")
+
+
+if __name__ == "__main__":
+    if not os.environ.get("RUN_SLOW"):
+        os.environ["RUN_SLOW"] = "1"
+    test_typical_matches_pcdm_decider()
+    test_typical_noul_via_choice_path_on_preview()

inference/typical/__init__.py

@@ -0,0 +1,3 @@
+from .core import Typical, query_text, to_labels
+
+__all__ = ["Typical", "query_text", "to_labels"]

inference/typical/backbone.py

@@ -0,0 +1,90 @@
+"""Frozen backbone + LoRA adapter -- trimmed port of the training repo's encode.py, for
+inference only. Only what native_kv_decide touches: tokenizer, .model (the raw HF causal
+trunk, tap-truncated), LoRA weight loading. No FeatureCache/EmbedEncoder/forward() --
+the native serving path reads hidden states straight off Backbone.model, never through a
+Backbone.forward() (that method, and the frozen-feature/h_top machinery it supports, only
+exists in the training repo's energy-head path). extra_tap is also dropped: both shipped
+checkpoints (typical-small, typical-small-preview) train with extra_tap=0.
+"""
+import math
+
+import torch
+import torch.nn as nn
+from transformers import AutoModel, AutoTokenizer
+
+_LORA_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj")
+_printed_device = False
+
+
+def pick_device(device: str = "auto") -> str:
+    global _printed_device
+    if device != "auto":
+        resolved = device
+    elif torch.cuda.is_available():
+        resolved = "cuda"
+    elif torch.backends.mps.is_available():
+        resolved = "mps"
+    else:
+        resolved = "cpu"
+    if not _printed_device:
+        print(f"typical: device={resolved}")
+        _printed_device = True
+    return resolved
+
+
+class LoRALinear(nn.Module):
+    """Frozen base nn.Linear (bf16) + a low-rank fp32 adapter on top. Numerics verbatim
+    from encode.py's LoRALinear (scale = alpha / r); dropout is a no-op once .eval()'d."""
+
+    def __init__(self, base: nn.Linear, r: int, alpha: float, dropout: float):
+        super().__init__()
+        self.base = base
+        self.base.requires_grad_(False)
+        self.A = nn.Parameter(torch.empty(r, base.in_features, dtype=torch.float32))
+        nn.init.kaiming_uniform_(self.A, a=math.sqrt(5))
+        self.B = nn.Parameter(torch.zeros(base.out_features, r, dtype=torch.float32))
+        self.drop = nn.Dropout(dropout)
+        self.scale = alpha / r
+
+    def forward(self, x):
+        out = self.base(x)
+        lora = self.drop(x) @ self.A.to(x.dtype).T @ self.B.to(x.dtype).T * self.scale
+        return out + lora
+
+
+class Backbone(nn.Module):
+    """Frozen causal-LM trunk (Qwen3-family), truncated to tap_layer, with LoRA on its top
+    lora_layers blocks. Base weights come fresh from `name`; the fine-tuned delta is only
+    ever the LoRA A/B tensors, loaded via load_lora_state_dict from the checkpoint."""
+
+    def __init__(self, name: str, lora_layers: int = 8, lora_r: int = 16, lora_alpha: float = 32,
+                 lora_dropout: float = 0.05, device: str = "auto", tap_layer: int = 0):
+        super().__init__()
+        self.device = pick_device(device)
+        self.tokenizer = AutoTokenizer.from_pretrained(name, padding_side="right")
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        self.model = AutoModel.from_pretrained(name, dtype=torch.bfloat16, attn_implementation="sdpa")
+        self.model.requires_grad_(False)
+        self.d = self.model.config.hidden_size
+        n_layers = self.model.config.num_hidden_layers
+        # tap_layer: drop every layer above T (semantics live mid-depth, not at the final,
+        # next-token-shaped layer of a base LM) -- see encode.py's Backbone for the long form.
+        if 0 < tap_layer < n_layers:
+            self.model.layers = self.model.layers[:tap_layer]
+            self.model.config.num_hidden_layers = tap_layer
+            n_layers = tap_layer
+        if lora_r > 0:
+            for layer in self.model.layers[-lora_layers:]:
+                for parent in (layer.self_attn, layer.mlp):
+                    for mod_name in _LORA_MODULES:
+                        if hasattr(parent, mod_name):
+                            setattr(parent, mod_name,
+                                    LoRALinear(getattr(parent, mod_name), lora_r, lora_alpha, lora_dropout))
+        self.to(self.device)
+        self.eval()
+
+    def load_lora_state_dict(self, sd: dict):
+        own = dict(self.named_parameters())
+        for k, v in sd.items():
+            own[k].data.copy_(v.to(own[k].device, own[k].dtype))

inference/typical/core.py

@@ -0,0 +1,131 @@
+"""Typical: minimal from_pretrained + decide API around a native-readout PCDM checkpoint.
+Mirrors pcdm_jev.decider.PCDMDecider(mode="native") exactly (same query_text/to_labels,
+same native_kv_decide call) -- see that module's docstring for the mapping this
+reproduces. Everything the checkpoint needs to rebuild backbone+head lives in
+ckpt["args"]: backbone id, tap_layer, lora_r/lora_layers, nc_head, nc_render, null,
+noul_head, score_head. z-score calibration (--zscore) needs no special handling here --
+its buffers (mu_h/sd_h/mu_c/sd_c) are just part of ckpt["tower"]'s state_dict.
+"""
+import json
+import time
+
+import torch
+from huggingface_hub import hf_hub_download
+
+from .backbone import Backbone, pick_device
+from .native import NativeHead, native_kv_decide
+
+MAX_QUERY = 64   # trained query length (see pcdm_jev.decider) -- informational only below
+MAX_STATE = 256  # trained state length -- native_kv_decide's own max_state default is separate
+
+
+class _Head:
+    """Duck-types the training repo's mcq.MCQHead just enough for native_kv_decide: a
+    .backbone (with .tokenizer/.model) and .device. The real MCQHead also builds a
+    restricted LM head over single-token letters for the mcq_zero_shot/energy readout
+    paths -- native_kv_decide never touches it, so it's not ported here."""
+
+    def __init__(self, backbone: Backbone):
+        self.backbone = backbone
+        self.device = backbone.device
+
+
+def query_text(question: dict) -> str:
+    """instructions + criteria: `yes: .. no: ..` (noul), `0: .. 1: ..` (score), `label: desc`
+    (choice). Verbatim port of pcdm_jev.decider.query_text."""
+    q, crit = question["instructions"], question.get("criteria")
+    if not crit:
+        return q
+    if question["type"] == "noul":
+        rubric = f"yes: {crit.get('true', 'Yes')}  no: {crit.get('false', 'No')}"
+    elif isinstance(crit, list):
+        rubric = "  ".join(f"{i}: {c}" for i, c in enumerate(crit))
+    else:
+        rubric = "  ".join(f"{k}: {v or k}" for k, v in crit.items())
+    return f"{q}\n{rubric}"
+
+
+def to_labels(p: torch.Tensor, labels: list[str]) -> tuple[dict, float]:
+    """[K+1] (null last) -> (probs renormalised over labels, p_null). Verbatim port of
+    pcdm_jev.decider.to_labels."""
+    p = p.detach().float().cpu()
+    if not torch.isfinite(p).all():
+        raise RuntimeError("non-finite model output")
+    s = p[:-1].sum()
+    lab = p[:-1] / s if s > 0 else torch.zeros_like(p[:-1])
+    return dict(zip(labels, lab.tolist())), float(p[-1])
+
+
+class Typical:
+    """from_pretrained("OzLabs/typical-small") -> a ready-to-decide native-readout model."""
+
+    def __init__(self, head: _Head, model: NativeHead, device: str, max_state: int = 4096):
+        self.head, self.model, self.device, self.max_state = head, model, device, max_state
+        self.tok = head.backbone.tokenizer
+
+    @classmethod
+    def from_pretrained(cls, repo_id: str, device: str = "auto", filename: str = "best.pt",
+                        revision: str | None = None, cache_dir: str | None = None,
+                        max_state: int = 4096) -> "Typical":
+        path = hf_hub_download(repo_id, filename, revision=revision, cache_dir=cache_dir)
+        ckpt = torch.load(path, map_location="cpu", weights_only=True)
+        args = ckpt.get("args", {})
+        if args.get("readout") != "native":
+            raise ValueError(f"{repo_id}/{filename} is not a native-readout checkpoint (readout={args.get('readout')!r})")
+        dev = pick_device(device)
+        backbone = Backbone(args["backbone"], lora_layers=args.get("lora_layers", 8),
+                            lora_r=args.get("lora_r", 16), device=dev, tap_layer=args.get("tap_layer", 0))
+        backbone.load_lora_state_dict(ckpt["lora"])
+        model = NativeHead(backbone.d, nc_head=args.get("nc_head", "n2n3"), null=args.get("null", "factored"),
+                           render=args.get("nc_render", "letters"), score_head=args.get("score_head", "choice"),
+                           noul_head=args.get("noul_head", "choice")).to(dev)
+        model.load_state_dict(ckpt["tower"])
+        model.eval()
+        return cls(_Head(backbone), model, dev, max_state=max_state)
+
+    def _raw(self, state, query: str, labels: list[str]) -> torch.Tensor:
+        if not isinstance(state, str):
+            state = json.dumps(state, ensure_ascii=False)
+        with torch.inference_mode():
+            return native_kv_decide(self.head, self.model, state, [(query, labels)], max_state=self.max_state)[0]
+
+    def choice(self, state, question: str, labels: list[str]) -> dict:
+        """-> {label: p, ...} + "p_null"."""
+        probs, p_null = to_labels(self._raw(state, question, labels), labels)
+        return {**probs, "p_null": p_null}
+
+    def noul(self, state, question: str) -> float:
+        """P(yes), via the Bernoulli path if the checkpoint has one (noul_head="bern"), else
+        the plain K-way choice over ["no", "yes"] -- both paths are label-order invariant for
+        a genuine yes/no row (see native._is_bern_row / _yes_idx)."""
+        probs, _ = to_labels(self._raw(state, question, ["no", "yes"]), ["no", "yes"])
+        return probs["yes"]
+
+    def score(self, state, question: str, levels: list[str]) -> dict:
+        """-> {level: p, ...} + "p_null" + "expected" (E[index] under the candidate-
+        conditional distribution, matching the training repo's metrics.ordinal_metrics --
+        candidate index IS the ordinal level, by construction of how levels are rendered)."""
+        probs, p_null = to_labels(self._raw(state, question, levels), levels)
+        expected = sum(i * probs[lv] for i, lv in enumerate(levels))
+        return {**probs, "p_null": p_null, "expected": expected}
+
+    def decide(self, state, question: dict, labels: list[str]) -> tuple[dict, dict]:
+        """-> (probs over exactly `labels`, runtime block). Mirrors
+        pcdm_jev.decider.PCDMDecider.decide for a JevBench-style question dict
+        ({"type", "instructions", "criteria"})."""
+        if not isinstance(state, str):
+            state = json.dumps(state, ensure_ascii=False)
+        query = query_text(question)
+        n_state = len(self.tok(state, add_special_tokens=False)["input_ids"])
+        n_query = len(self.tok(query, add_special_tokens=False)["input_ids"])
+        t0 = time.perf_counter()
+        p = self._raw(state, query, labels)
+        probs, p_null = to_labels(p, labels)
+        latency = time.perf_counter() - t0
+        return probs, {"mode": "native", "device": self.device, "latency_s": latency, "p_null": p_null,
+                       "state_tokens": n_state, "max_state_tokens": self.max_state,
+                       "state_truncated": n_state > self.max_state,
+                       "trained_max_state": MAX_STATE, "state_beyond_train_len": n_state > MAX_STATE,
+                       "query_tokens": n_query, "query_truncated": False,
+                       "trained_max_query": MAX_QUERY, "query_beyond_train_len": False,
+                       "probability_origin": "native-softmax"}

inference/typical/native.py

@@ -0,0 +1,331 @@
+"""native_choice_v1/v2/v3 decision head + the KV-cached serving path -- trimmed port of the
+training repo's native.py + the two mcq.py helpers it needs (_label/_ids/_pack), for
+inference only. Kept: NativeHead (factored null, per-row Bernoulli "noul" routing via
+_is_bern_row, the letters/tags/letters_nonull/query_only renderers) and native_kv_decide,
+because pcdm_jev.decider.PCDMDecider(mode="native") calls exactly those two things and
+nothing else. Dropped: run_batch_native/native_features/_score_chunked/_fit_chunks (the
+training/eval batch path -- decider.py never calls them) and the n2 (Qwen3-Embedding
+candidate-vector) branch, since neither shipped checkpoint (typical-small,
+typical-small-preview) trains with nc_head n2/n2n3.
+"""
+import bisect
+import copy
+import string
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+EMBED_DIM = 1024  # Qwen3-Embedding-0.6B hidden size; only referenced by the (unsupported) n2 branch
+MAX_SUFFIX = 1024
+CHOICE_OPEN, CHOICE_CLOSE = "<choice>\n", "\n</choice>\n"
+LETTERS = string.ascii_uppercase + string.ascii_lowercase
+
+
+def _label(i: int) -> str:
+    return LETTERS[i] if i < len(LETTERS) else str(i + 1)
+
+
+def _render(query: str, cand_texts: list[str]):
+    """native_v1 (--nc_render letters): query, lettered options, a rendered "none of the
+    above" line, "Answer:". spans[k] = (start, end) of option k's text; spans[-1] is the
+    null line's text."""
+    text, spans = query + "\n", []
+    for i, c in enumerate(cand_texts + ["none of the above"]):
+        text += f"{_label(i)}. "
+        spans.append((len(text), len(text) + len(c)))
+        text += c + "\n"
+    return text + "Answer:", spans
+
+
+def _render_tags(query: str, cand_texts: list[str]):
+    """native_v2 (--nc_render tags): letter-free, one <choice>...</choice> block per option,
+    no null line, no "Answer:". len(spans) == K (the null head reads a learned constant,
+    not a rendered line)."""
+    text, spans = query + "\n", []
+    for c in cand_texts:
+        text += CHOICE_OPEN
+        spans.append((len(text), len(text) + len(c)))
+        text += c + CHOICE_CLOSE
+    return text, spans
+
+
+def _render_letters_nonull(query: str, cand_texts: list[str]):
+    """native_v3 (--nc_render letters_nonull): lettered options (keeps slot identity) but no
+    rendered null line and no "Answer:" -- both shipped checkpoints train with this render."""
+    text, spans = query + "\n", []
+    for i, c in enumerate(cand_texts):
+        text += f"{_label(i)}. "
+        spans.append((len(text), len(text) + len(c)))
+        text += c + "\n"
+    return text, spans
+
+
+def _render_query_only(query: str, cand_texts: list[str]):
+    """--noul_head bern: suffix is the query alone (the Bernoulli head reads h_D only, never
+    the candidate text -- that's what makes it label-order invariant). Empty spans."""
+    return query + "\n", []
+
+
+RENDERS = {"letters": _render, "tags": _render_tags, "letters_nonull": _render_letters_nonull,
+           "query_only": _render_query_only}
+
+
+def _is_bern_row(cand_texts, meta=None):
+    """A row is bern-eligible iff its candidate set is exactly {"yes","no"} (case-insensitive),
+    or meta.qtype == "noul" and K == 2."""
+    if {c.strip().lower() for c in cand_texts} == {"yes", "no"}:
+        return True
+    return bool(meta) and meta.get("qtype") == "noul" and len(cand_texts) == 2
+
+
+def _yes_idx(cand_lists, dev):
+    """[B] index of the literal "yes" candidate per row (case-insensitive) -- places P(yes)
+    there regardless of rendered order, so ["yes","no"] and ["no","yes"] give identical
+    P(yes). No literal "yes" -> defaults to index 0 (out-of-domain K-way row)."""
+    idx = []
+    for cl in cand_lists:
+        matches = [i for i, c in enumerate(cl) if c.strip().lower() == "yes"]
+        idx.append(matches[0] if matches else 0)
+    return torch.tensor(idx, dtype=torch.long, device=dev)
+
+
+def _ids(tok, texts: list[str], max_len: int, offsets: bool = False):
+    """Token ids (no specials, truncated), optionally with char offsets; empty text -> "."."""
+    texts = [t if t.strip() else "." for t in texts]
+    enc = tok(texts, truncation=True, max_length=max_len, add_special_tokens=False, return_offsets_mapping=offsets)
+    return (enc["input_ids"], enc["offset_mapping"]) if offsets else enc["input_ids"]
+
+
+def _pack(tok, s_ids, x_ids, tail=(), sink=True):
+    """rows = [eos] + state + suffix + tail, right-padded -> (input_ids, attention_mask,
+    lengths); sink=False drops the leading eos (native_kv_decide: the sink lives in the
+    cached prefix)."""
+    eos, pad = tok.eos_token_id, tok.pad_token_id
+    rows = [([eos] if sink else []) + s + x + list(tail) for s, x in zip(s_ids, x_ids)]
+    lengths = torch.tensor([len(r) for r in rows], dtype=torch.long)
+    T = max(len(r) for r in rows)
+    input_ids = torch.full((len(rows), T), pad, dtype=torch.long)
+    attention_mask = torch.zeros(len(rows), T, dtype=torch.long)
+    for i, r in enumerate(rows):
+        input_ids[i, :len(r)] = torch.tensor(r, dtype=torch.long)
+        attention_mask[i, :len(r)] = 1
+    return input_ids, attention_mask, lengths
+
+
+def factored_null_logits(gate, s, c, h, cmask, temperature: float = 1.0):
+    """r = sigmoid(gate(z)): z is O(K) permutation-invariant set statistics over the valid
+    candidate scores s + rep h + masked candidate-set mean/var of c. Composed as logits so
+    softmax(logits) gives P(a_j) = (1-r)*p_j and P(null) = r exactly. Verbatim port of
+    model.factored_null_logits (self-contained: only torch/F, no other model.py state)."""
+    K = cmask.sum(-1)
+    Kf = K.float().clamp_min(1)
+    top2 = torch.topk(s, k=min(2, s.shape[1]), dim=-1).values
+    margin = top2[:, 0] - top2[:, 1] if top2.shape[1] == 2 else torch.zeros_like(top2[:, 0])
+    margin = torch.where(K > 1, margin, torch.zeros_like(margin))
+    mean_s = s.masked_fill(~cmask, 0.0).sum(-1) / Kf
+    lse = torch.logsumexp(s, dim=-1) - torch.log(Kf)
+    c0 = c.masked_fill(~cmask.unsqueeze(-1), 0.0)
+    mean_c = c0.sum(1) / Kf.unsqueeze(-1)
+    var_c = (c0 - mean_c.unsqueeze(1)).pow(2).masked_fill(~cmask.unsqueeze(-1), 0.0).sum(1) / Kf.unsqueeze(-1)
+    z = torch.cat([top2[:, :1], margin.unsqueeze(-1), mean_s.unsqueeze(-1), lse.unsqueeze(-1),
+                   h, mean_c, var_c], dim=-1)
+    r_logit = gate(z).squeeze(-1)
+    log_1mr, log_r = -F.softplus(r_logit), -F.softplus(-r_logit)
+    s_t = (s.masked_fill(~cmask, 0.0) / temperature).masked_fill(~cmask, torch.finfo(s.dtype).min)
+    logp = F.log_softmax(s_t, dim=-1)
+    cand_logits = (logp + log_1mr.unsqueeze(-1)).masked_fill(~cmask, torch.finfo(s.dtype).min)
+    return torch.cat([cand_logits, log_r.unsqueeze(-1)], dim=-1)
+
+
+class NativeHead(nn.Module):
+    """Verbatim port of native.NativeHead. n2 (nc_head in {"n2","n2n3"}) is accepted for
+    state_dict shape compatibility but native_kv_decide below raises if a loaded checkpoint
+    actually needs it -- see module docstring."""
+
+    def __init__(self, d: int, nc_head: str = "n2n3", null: str = "factored", d_proj: int = 256,
+                 d_emb: int = EMBED_DIM, render: str = "letters", score_head: str = "choice",
+                 noul_head: str = "choice"):
+        super().__init__()
+        assert nc_head in ("n2", "n3", "n2n3"), nc_head
+        assert null in ("softmax", "factored"), null
+        assert render in RENDERS, render
+        assert score_head in ("choice", "cumlink"), score_head
+        assert noul_head in ("choice", "bern"), noul_head
+        self.nc_head, self.null, self.render = nc_head, null, render
+        self.score_head, self.noul_head = score_head, noul_head
+        self.use2, self.use3 = "n2" in nc_head, "n3" in nc_head
+        dv = d_proj * (self.use2 + self.use3)
+        self.dv = dv
+        self.register_buffer("mu_h", torch.zeros(d)); self.register_buffer("sd_h", torch.ones(d))
+        self.register_buffer("mu_c", torch.zeros(d)); self.register_buffer("sd_c", torch.ones(d))
+        self.proj_h = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, dv))
+        if self.use2:
+            self.proj_2 = nn.Sequential(nn.LayerNorm(d_emb), nn.Linear(d_emb, d_proj))
+            self.null_emb = nn.Parameter(torch.randn(d_emb) * 0.02)
+        if self.use3:
+            self.proj_3 = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, d_proj))
+        self.w = nn.Linear(dv, 1)
+        self.res = nn.Sequential(nn.Linear(dv, dv), nn.GELU(), nn.Linear(dv, 1))
+        if null == "factored":
+            self.null_gate = nn.Sequential(nn.Linear(4 + 4 * dv, 256), nn.GELU(), nn.Linear(256, 1))
+        else:
+            self.score_null = nn.Sequential(nn.Linear(2 * dv, dv // 2), nn.GELU(), nn.Linear(dv // 2, 1))
+        if score_head == "cumlink" or noul_head == "bern":
+            self.null_gate_typed = nn.Sequential(nn.Linear(4 + 4 * d, 256), nn.GELU(), nn.Linear(256, 1))
+        if score_head == "cumlink":
+            self.w_o = nn.Linear(d, 1)
+            self.w_t = nn.Linear(d, 1)
+            self.theta0 = nn.Parameter(torch.zeros(1))
+        if noul_head == "bern":
+            self.w_n = nn.Linear(d, 1)
+        self.temperature = 1.0
+
+    def forward(self, h, cmask, c3=None, c2=None, yes_idx=None, bern_mask=None):
+        """h [B, d]; cmask [B, Kmax]; c3 [B, Kmax+1, d] (null line last); c2 [B, Kmax, d_emb].
+        -> logits [B, Kmax+1], null last, pads finfo.min. Verbatim port of native.NativeHead.forward."""
+        hz = (h - self.mu_h) / self.sd_h
+        if self.score_head == "cumlink":
+            return self._typed_logits(self._cumlink_probs(hz, c3, cmask), hz, c3, cmask)
+        bm = None
+        if self.noul_head == "bern":
+            bm = bern_mask if bern_mask is not None else torch.ones(h.shape[0], dtype=torch.bool, device=h.device)
+            if bm.all():
+                return self._typed_logits(self._bern_probs(hz, cmask, yes_idx), hz, c3, cmask)
+
+        B = h.shape[0]
+        u = self.proj_h(hz)
+        vs = []
+        if self.use2:
+            vs.append(self.proj_2(torch.cat([c2, self.null_emb.expand(B, 1, -1)], 1)))
+        if self.use3:
+            vs.append(self.proj_3((c3 - self.mu_c) / self.sd_c))
+        v = torch.cat(vs, -1)
+        vc, vn = v[:, :-1], v[:, -1]
+        uv = u.unsqueeze(1) * vc
+        s = uv.sum(-1) * self.dv ** -0.5 + self.w(uv).squeeze(-1) + self.res(uv).squeeze(-1)
+        s = s.masked_fill(~cmask, torch.finfo(s.dtype).min)
+        hn = torch.cat([u, vn], -1)
+        if self.null == "factored":
+            choice_logits = factored_null_logits(self.null_gate, s, vc, hn, cmask, self.temperature)
+        else:
+            choice_logits = torch.cat([s, self.score_null(hn)], dim=-1)
+
+        if bm is not None and bm.any():
+            bern_logits = self._typed_logits(self._bern_probs(hz, cmask, yes_idx), hz, c3, cmask)
+            return torch.where(bm.unsqueeze(-1), bern_logits, choice_logits)
+        return choice_logits
+
+    def _bern_probs(self, hz, cmask, yes_idx):
+        B, Kmax = cmask.shape
+        p_yes = torch.sigmoid(self.w_n(hz).squeeze(-1))
+        idx = yes_idx if yes_idx is not None else torch.ones(B, dtype=torch.long, device=hz.device)
+        K = cmask.sum(-1)
+        other = (1 - p_yes) / (K - 1).clamp_min(1).float()
+        p = other.unsqueeze(-1).expand(B, Kmax) * cmask.float()
+        ar = torch.arange(B, device=hz.device)
+        p[ar, idx] = p_yes
+        return p.masked_fill(~cmask, 0.0)
+
+    def _cumlink_probs(self, hz, c3, cmask):
+        B, Kmax = cmask.shape
+        c3z = (c3 - self.mu_c) / self.sd_c
+        u = self.w_o(hz).squeeze(-1)
+        width = F.softplus(self.w_t(c3z[:, :-1]).squeeze(-1)).masked_fill(~cmask, 0.0)
+        theta = self.theta0 + torch.cumsum(width, dim=-1)
+        cdf = torch.sigmoid(theta - u.unsqueeze(-1))
+        p_prev = F.pad(cdf, (1, 0))[:, :-1]
+        p = (cdf - p_prev).clamp_min(0.0)
+        K = cmask.sum(-1)
+        last = (K - 1).clamp_min(0)
+        remainder = torch.where(K > 1, 1.0 - cdf.gather(1, (last - 1).clamp_min(0).unsqueeze(1)).squeeze(1),
+                                 torch.ones_like(u))
+        p = p.scatter(1, last.unsqueeze(1), remainder.unsqueeze(1)).masked_fill(~cmask, 0.0)
+        return p
+
+    def _typed_logits(self, p, hz, c3, cmask):
+        c3z = (c3 - self.mu_c) / self.sd_c
+        s = torch.log(p.clamp_min(1e-12)).masked_fill(~cmask, torch.finfo(p.dtype).min)
+        hn = torch.cat([hz, c3z[:, -1]], dim=-1)
+        return factored_null_logits(self.null_gate_typed, s, c3z[:, :-1], hn, cmask, self.temperature)
+
+
+def _pool_matrix(spans, offsets, base, Kmax, T, K=None):
+    """[Kmax+1, T] mean-pooling rows over each option's rendered char span (bisected against
+    the tokenizer's char offsets)."""
+    K = len(spans) - 1 if K is None else K
+    starts, ends = [o[0] for o in offsets], [o[1] for o in offsets]
+    pool = torch.zeros(Kmax + 1, T)
+    for k, (cs, ce) in enumerate(spans):
+        j0, j1 = bisect.bisect_right(ends, cs), bisect.bisect_left(starts, ce)
+        pool[Kmax if k == K else k, base + j0:base + j1] = 1.0
+    return pool / pool.sum(-1, keepdim=True).clamp_min(1.0)
+
+
+def _causal_pad_mask(attn2d, q_len, dtype):
+    """Explicit additive (batch,1,q_len,kv_len) mask -- transformers' automatic 2D-mask +
+    past_key_values path mis-handles a right-padded batch sharing one KV cache; this
+    reproduces the un-batched per-row computation exactly."""
+    batch, kv_len = attn2d.shape
+    past_len = kv_len - q_len
+    q_idx = torch.arange(q_len, device=attn2d.device).unsqueeze(1)
+    kv_idx = torch.arange(kv_len, device=attn2d.device).unsqueeze(0)
+    causal = kv_idx <= (past_len + q_idx)
+    allowed = causal.unsqueeze(0) & attn2d.bool().unsqueeze(1)
+    mask = torch.zeros(batch, 1, q_len, kv_len, dtype=dtype, device=attn2d.device)
+    return mask.masked_fill(~allowed.unsqueeze(1), torch.finfo(dtype).min)
+
+
+@torch.inference_mode()
+def native_kv_decide(head, model, state, queries, chunk: int = 32, max_state: int = 256,
+                     max_suffix: int = MAX_SUFFIX, vec_cache=None):
+    """Serving path: [eos] + state encoded ONCE into a KV cache on head.backbone.model; every
+    query's suffix runs against that cached prefix, `chunk` queries per forward. queries =
+    [(query, candidates)] -> list of [K_i + 1] probability vectors (null last). Verbatim port
+    of native.native_kv_decide (head/model contract unchanged -- head needs only .backbone
+    (tokenizer + model) and .device; model is a NativeHead). No hierarchical fallback: a
+    suffix longer than max_suffix raises (raise max_suffix instead)."""
+    if model.use2:
+        raise NotImplementedError(
+            "nc_head n2/n2n3 (Qwen3-Embedding candidate vectors) isn't ported to the "
+            "inference package -- neither typical-small nor typical-small-preview needs it. "
+            "Port encode.EmbedEncoder + native._n2_vectors from the training repo if a future "
+            "checkpoint requires it."
+        )
+    tok, dev, lm = head.backbone.tokenizer, head.device, head.backbone.model
+    prefix = torch.tensor([[tok.eos_token_id] + _ids(tok, [state], max_state)[0]], device=dev)
+    state_cache = lm(input_ids=prefix, use_cache=True).past_key_values
+    Ls = prefix.shape[1]
+    out = []
+    for start in range(0, len(queries), chunk):
+        qs = queries[start:start + chunk]
+        m = len(qs)
+        is_bern = [model.noul_head == "bern" and _is_bern_row(c) for _, c in qs]
+        rendered = [RENDERS["query_only" if b else getattr(model, "render", "letters")](q, c)
+                    for b, (q, c) in zip(is_bern, qs)]
+        enc = tok([r[0] for r in rendered], add_special_tokens=False, return_offsets_mapping=True)
+        x_ids, offs = enc["input_ids"], enc["offset_mapping"]
+        assert max(len(x) for x in x_ids) <= max_suffix, f"suffix > max_suffix={max_suffix}; raise it"
+        input_ids, am, lengths = _pack(tok, [[]] * m, x_ids, tail=[tok.eos_token_id], sink=False)
+        T = input_ids.shape[1]
+        Kmax = max(len(c) for _, c in qs)
+        pool = torch.stack([_pool_matrix(r[1], o, 0, Kmax, T, K=len(c)) for r, o, (_, c) in zip(rendered, offs, qs)])
+        cmask = torch.zeros(m, Kmax, dtype=torch.bool)
+        for i, (_, c) in enumerate(qs):
+            cmask[i, :len(c)] = True
+
+        cache = copy.deepcopy(state_cache)
+        cache.batch_repeat_interleave(m)
+        attn = torch.cat([torch.ones(m, Ls, dtype=torch.long), am], dim=1).to(dev)
+        position_ids = (torch.arange(T) + Ls).expand(m, -1).to(dev)
+        H = lm(input_ids=input_ids.to(dev), attention_mask=_causal_pad_mask(attn, T, lm.dtype),
+               position_ids=position_ids, past_key_values=cache, use_cache=False).last_hidden_state.float()
+        h = H[torch.arange(m, device=dev), (lengths - 1).to(dev)]
+        C3 = torch.bmm(pool.to(dev), H)
+        C2 = None  # n2 unsupported -- guarded above
+        yi = _yes_idx([c for _, c in qs], dev) if model.noul_head == "bern" else None
+        bm = torch.tensor(is_bern, dtype=torch.bool, device=dev) if model.noul_head == "bern" else None
+        probs = torch.softmax(model(h, cmask.to(dev), C3, C2, yes_idx=yi, bern_mask=bm), dim=-1)
+        for i, (_, c) in enumerate(qs):
+            out.append(torch.cat([probs[i, :len(c)], probs[i, -1:]]))
+    return out

jevbench_summary.json

@@ -0,0 +1,2824 @@
+{
+  "easy": {
+    "accuracy": 1.0,
+    "brier_mean": 0.011964783954900882,
+    "calibration_n": 48,
+    "complete": true,
+    "cost_basis": [
+      "local_gpu_no_provider_tariff"
+    ],
+    "coverage": 1.0,
+    "dataset_hash": "3cbd0e951fa8059741955ef2bd3273b0bb60aecae8f4a68b89872f55f7376b3d",
+    "ece": {
+      "bins": [
+        {
+          "accuracy": null,
+          "hi": 0.1,
+          "lo": 0.0,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": null,
+          "hi": 0.2,
+          "lo": 0.1,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": null,
+          "hi": 0.3,
+          "lo": 0.2,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": null,
+          "hi": 0.4,
+          "lo": 0.3,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": null,
+          "hi": 0.5,
+          "lo": 0.4,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": 1.0,
+          "hi": 0.6,
+          "lo": 0.5,
+          "mean_confidence": 0.5003303289413452,
+          "n": 1
+        },
+        {
+          "accuracy": null,
+          "hi": 0.7,
+          "lo": 0.6,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": null,
+          "hi": 0.8,
+          "lo": 0.7,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": 1.0,
+          "hi": 0.9,
+          "lo": 0.8,
+          "mean_confidence": 0.8724372982978821,
+          "n": 2
+        },
+        {
+          "accuracy": 1.0,
+          "hi": 1.0,
+          "lo": 0.9,
+          "mean_confidence": 0.9944215284453498,
+          "n": 45
+        }
+      ],
+      "ece": 0.020954714467128106,
+      "n": 48
+    },
+    "latency": {
+      "n": 48,
+      "p50_s": 0.06915471702814102,
+      "p95_s": 0.10234953109174966
+    },
+    "latency_failures": {
+      "n": 0,
+      "p50_s": null,
+      "p95_s": null
+    },
+    "ledger_charged_usd": null,
+    "macro_accuracy": 1.0,
+    "model_identities": [
+      "pcdm:native:runs/tm1b"
+    ],
+    "n_attempted": 48,
+    "n_correct": 48,
+    "n_planned": 48,
+    "n_renormalized": 0,
+    "n_scorable": 48,
+    "n_valid": 48,
+    "operational_success": 1.0,
+    "ordinal_mae": null,
+    "paraphrase_consistency": {
+      "agree": 0,
+      "agreement": null,
+      "both_correct_rate_all_pairs": null,
+      "both_valid": 0,
+      "pairs": 0
+    },
+    "per_family": {
+      "extraction": {
+        "accuracy": 1.0,
+        "brier_mean": 0.003787310148891915,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8558686375617981,
+              "n": 1
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9874209653247487,
+              "n": 11
+            }
+          ],
+          "ece": 0.023541728655497224,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.06931246258318424,
+          "p95_s": 0.08583159018307922
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 12,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "fact": {
+        "accuracy": 1.0,
+        "brier_mean": 0.044071809342586504,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5003303289413452,
+              "n": 1
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8890059590339661,
+              "n": 1
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.988783085346222,
+              "n": 10
+            }
+          ],
+          "ece": 0.06023607154687243,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.07128027640283108,
+          "p95_s": 0.1037625202909112
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 12,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "intent": {
+        "accuracy": 1.0,
+        "brier_mean": 1.4921552066029873e-08,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9999667853116989,
+              "n": 12
+            }
+          ],
+          "ece": 3.3214688301086426e-05,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.0686907097697258,
+          "p95_s": 0.1342240288853645
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 12,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "tool_selection": {
+        "accuracy": 1.0,
+        "brier_mean": 1.4065730446045023e-09,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9999921570221583,
+              "n": 12
+            }
+          ],
+          "ece": 7.842977841732157e-06,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.06903873011469841,
+          "p95_s": 0.07324958574026823
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 12,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      }
+    },
+    "price_per_1000_decisions_usd": null,
+    "probability_sources": [
+      "native"
+    ],
+    "schema_validity": 1.0,
+    "schema_validity_strict": 1.0,
+    "splits": {
+      "public": {
+        "accuracy": 1.0,
+        "brier_mean": 0.011964783954900882,
+        "calibration_n": 48,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5003303289413452,
+              "n": 1
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8724372982978821,
+              "n": 2
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9944215284453498,
+              "n": 45
+            }
+          ],
+          "ece": 0.020954714467128106,
+          "n": 48
+        },
+        "latency": {
+          "n": 48,
+          "p50_s": 0.06915471702814102,
+          "p95_s": 0.10234953109174966
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 48,
+        "n_correct": 48,
+        "n_planned": 48,
+        "n_renormalized": 0,
+        "n_scorable": 48,
+        "n_valid": 48,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      }
+    },
+    "version": "jevbench-v1"
+  },
+  "hard": {
+    "accuracy": 0.42342342342342343,
+    "brier_mean": 0.7731682475813886,
+    "calibration_n": 111,
+    "complete": true,
+    "cost_basis": [
+      "local_gpu_no_provider_tariff"
+    ],
+    "coverage": 1.0,
+    "dataset_hash": "e816e106f562dec75a8da12e69b75e957a4bbd9f6300f1145fc71837327a3c61",
+    "ece": {
+      "bins": [
+        {
+          "accuracy": null,
+          "hi": 0.1,
+          "lo": 0.0,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": null,
+          "hi": 0.2,
+          "lo": 0.1,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": 0.0,
+          "hi": 0.3,
+          "lo": 0.2,
+          "mean_confidence": 0.27849190682172775,
+          "n": 4
+        },
+        {
+          "accuracy": 0.125,
+          "hi": 0.4,
+          "lo": 0.3,
+          "mean_confidence": 0.37296687439084053,
+          "n": 8
+        },
+        {
+          "accuracy": 0.4166666666666667,
+          "hi": 0.5,
+          "lo": 0.4,
+          "mean_confidence": 0.4450460250178973,
+          "n": 12
+        },
+        {
+          "accuracy": 0.35,
+          "hi": 0.6,
+          "lo": 0.5,
+          "mean_confidence": 0.5447314709424973,
+          "n": 20
+        },
+        {
+          "accuracy": 0.5,
+          "hi": 0.7,
+          "lo": 0.6,
+          "mean_confidence": 0.6454204533781324,
+          "n": 14
+        },
+        {
+          "accuracy": 0.3333333333333333,
+          "hi": 0.8,
+          "lo": 0.7,
+          "mean_confidence": 0.7522401312987009,
+          "n": 6
+        },
+        {
+          "accuracy": 0.25,
+          "hi": 0.9,
+          "lo": 0.8,
+          "mean_confidence": 0.8441648408770561,
+          "n": 8
+        },
+        {
+          "accuracy": 0.5897435897435898,
+          "hi": 1.0,
+          "lo": 0.9,
+          "mean_confidence": 0.965268883949671,
+          "n": 39
+        }
+      ],
+      "ece": 0.2818109610596219,
+      "n": 111
+    },
+    "latency": {
+      "n": 111,
+      "p50_s": 0.11237071454524994,
+      "p95_s": 0.14899961836636066
+    },
+    "latency_failures": {
+      "n": 0,
+      "p50_s": null,
+      "p95_s": null
+    },
+    "ledger_charged_usd": null,
+    "macro_accuracy": 0.4814144429701705,
+    "model_identities": [
+      "pcdm:native:runs/tm1b"
+    ],
+    "n_attempted": 111,
+    "n_correct": 47,
+    "n_planned": 111,
+    "n_renormalized": 0,
+    "n_scorable": 111,
+    "n_valid": 111,
+    "operational_success": 1.0,
+    "ordinal_mae": 0.7110752028723558,
+    "paraphrase_consistency": {
+      "agree": 0,
+      "agreement": null,
+      "both_correct_rate_all_pairs": null,
+      "both_valid": 0,
+      "pairs": 0
+    },
+    "per_family": {
+      "adversarial": {
+        "accuracy": 0.6666666666666666,
+        "brier_mean": 0.5843859976304459,
+        "calibration_n": 6,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6800556182861328,
+              "n": 1
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8503317534923553,
+              "n": 2
+            },
+            {
+              "accuracy": 0.6666666666666666,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9747096498807272,
+              "n": 3
+            }
+          ],
+          "ece": 0.3241228063901266,
+          "n": 6
+        },
+        "latency": {
+          "n": 6,
+          "p50_s": 0.10217797197401524,
+          "p95_s": 0.10633919388055801
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 6,
+        "n_correct": 4,
+        "n_planned": 6,
+        "n_renormalized": 0,
+        "n_scorable": 6,
+        "n_valid": 6,
+        "operational_success": 1.0,
+        "ordinal_mae": 0.0494021475315094,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "ambiguous": {
+        "accuracy": 0.5714285714285714,
+        "brier_mean": 0.6891428318952786,
+        "calibration_n": 7,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5438312590122223,
+              "n": 2
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7460008561611176,
+              "n": 2
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8554508686065674,
+              "n": 1
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9503115713596344,
+              "n": 2
+            }
+          ],
+          "ece": 0.23211950063705444,
+          "n": 7
+        },
+        "latency": {
+          "n": 7,
+          "p50_s": 0.108931515365839,
+          "p95_s": 0.11300073452293874
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 7,
+        "n_correct": 4,
+        "n_planned": 7,
+        "n_renormalized": 0,
+        "n_scorable": 7,
+        "n_valid": 7,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "judge_hard": {
+        "accuracy": 0.5294117647058824,
+        "brier_mean": 0.8882525997132734,
+        "calibration_n": 17,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7447604537010193,
+              "n": 1
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8594815731048584,
+              "n": 1
+            },
+            {
+              "accuracy": 0.5333333333333333,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9754487832387289,
+              "n": 15
+            }
+          ],
+          "ece": 0.4556736981167513,
+          "n": 17
+        },
+        "latency": {
+          "n": 17,
+          "p50_s": 0.1022290475666523,
+          "p95_s": 0.11644351556897163
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 17,
+        "n_correct": 9,
+        "n_planned": 17,
+        "n_renormalized": 0,
+        "n_scorable": 17,
+        "n_valid": 17,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "long_policy": {
+        "accuracy": 0.21052631578947367,
+        "brier_mean": 0.956868090190627,
+        "calibration_n": 19,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": 0.2781612128019333,
+              "n": 2
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": 0.3873806794484456,
+              "n": 3
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": 0.43897129595279694,
+              "n": 4
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5172954201698303,
+              "n": 1
+            },
+            {
+              "accuracy": 0.6666666666666666,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6477566162745158,
+              "n": 3
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7533348798751831,
+              "n": 2
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8167205850283304,
+              "n": 3
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9532382488250732,
+              "n": 1
+            }
+          ],
+          "ece": 0.3919302429023541,
+          "n": 19
+        },
+        "latency": {
+          "n": 19,
+          "p50_s": 0.14529041200876236,
+          "p95_s": 0.16292529068887227
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 19,
+        "n_correct": 4,
+        "n_planned": 19,
+        "n_renormalized": 0,
+        "n_scorable": 19,
+        "n_valid": 19,
+        "operational_success": 1.0,
+        "ordinal_mae": 0.5590988993644714,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "multi_hop": {
+        "accuracy": 0.4444444444444444,
+        "brier_mean": 0.6872533730651412,
+        "calibration_n": 18,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": 0.33783966302871704,
+              "n": 1
+            },
+            {
+              "accuracy": 0.4,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": 0.44226173162460325,
+              "n": 5
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5598814710974693,
+              "n": 8
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6466758847236633,
+              "n": 2
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7700088620185852,
+              "n": 1
+            },
+            {
+              "accuracy": null,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9851491451263428,
+              "n": 1
+            }
+          ],
+          "ece": 0.19059623612297905,
+          "n": 18
+        },
+        "latency": {
+          "n": 18,
+          "p50_s": 0.128933347761631,
+          "p95_s": 0.13938846364617347
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 18,
+        "n_correct": 8,
+        "n_planned": 18,
+        "n_renormalized": 0,
+        "n_scorable": 18,
+        "n_valid": 18,
+        "operational_success": 1.0,
+        "ordinal_mae": 0.4032743573188782,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "probability": {
+        "accuracy": 0.4,
+        "brier_mean": 0.7593166581141717,
+        "calibration_n": 10,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": 0.4478049576282501,
+              "n": 1
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5372944176197052,
+              "n": 4
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6290208697319031,
+              "n": 2
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.3333333333333333,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9370233019193014,
+              "n": 3
+            }
+          ],
+          "ece": 0.2666094273328781,
+          "n": 10
+        },
+        "latency": {
+          "n": 10,
+          "p50_s": 0.11380846984684467,
+          "p95_s": 0.12198328394442795
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 10,
+        "n_correct": 4,
+        "n_planned": 10,
+        "n_renormalized": 0,
+        "n_scorable": 10,
+        "n_valid": 10,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "routing_hard": {
+        "accuracy": 1.0,
+        "brier_mean": 0.0021777005726523737,
+        "calibration_n": 5,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9799626469612122,
+              "n": 5
+            }
+          ],
+          "ece": 0.020037353038787842,
+          "n": 5
+        },
+        "latency": {
+          "n": 5,
+          "p50_s": 0.10272485390305519,
+          "p95_s": 0.10452966764569283
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 5,
+        "n_correct": 5,
+        "n_planned": 5,
+        "n_renormalized": 0,
+        "n_scorable": 5,
+        "n_valid": 5,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "temporal_numeric": {
+        "accuracy": 0.2,
+        "brier_mean": 0.9778733222516895,
+        "calibration_n": 15,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": 0.2788226008415222,
+              "n": 2
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": 0.3616597553094228,
+              "n": 3
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": 0.4627767503261566,
+              "n": 2
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5329880118370056,
+              "n": 3
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6510051488876343,
+              "n": 2
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.887561023235321,
+              "n": 1
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.963701069355011,
+              "n": 2
+            }
+          ],
+          "ece": 0.45526585777600603,
+          "n": 15
+        },
+        "latency": {
+          "n": 15,
+          "p50_s": 0.11358731612563133,
+          "p95_s": 0.1201222252100706
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 15,
+        "n_correct": 3,
+        "n_planned": 15,
+        "n_renormalized": 0,
+        "n_scorable": 15,
+        "n_valid": 15,
+        "operational_success": 1.0,
+        "ordinal_mae": 1.3456641659140587,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "tradeoff": {
+        "accuracy": 0.16666666666666666,
+        "brier_mean": 1.1503974298690245,
+        "calibration_n": 6,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5204590559005737,
+              "n": 1
+            },
+            {
+              "accuracy": 0.3333333333333333,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6232326428095499,
+              "n": 3
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9465669393539429,
+              "n": 2
+            }
+          ],
+          "ece": 0.5472151438395182,
+          "n": 6
+        },
+        "latency": {
+          "n": 6,
+          "p50_s": 0.10983631946146488,
+          "p95_s": 0.11511344648897648
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 6,
+        "n_correct": 1,
+        "n_planned": 6,
+        "n_renormalized": 0,
+        "n_scorable": 6,
+        "n_valid": 6,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "trap": {
+        "accuracy": 0.625,
+        "brier_mean": 0.3331839439462419,
+        "calibration_n": 8,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": 0.39877402782440186,
+              "n": 1
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5420189499855042,
+              "n": 1
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6894591450691223,
+              "n": 1
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9438392162322998,
+              "n": 5
+            }
+          ],
+          "ece": 0.23888200521469116,
+          "n": 8
+        },
+        "latency": {
+          "n": 8,
+          "p50_s": 0.10395996645092964,
+          "p95_s": 0.10869606416672467
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 8,
+        "n_correct": 5,
+        "n_planned": 8,
+        "n_renormalized": 0,
+        "n_scorable": 8,
+        "n_valid": 8,
+        "operational_success": 1.0,
+        "ordinal_mae": 1.3499127477407455,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      }
+    },
+    "price_per_1000_decisions_usd": null,
+    "probability_sources": [
+      "native"
+    ],
+    "schema_validity": 1.0,
+    "schema_validity_strict": 1.0,
+    "splits": {
+      "public": {
+        "accuracy": 0.42342342342342343,
+        "brier_mean": 0.7731682475813886,
+        "calibration_n": 111,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": 0.27849190682172775,
+              "n": 4
+            },
+            {
+              "accuracy": 0.125,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": 0.37296687439084053,
+              "n": 8
+            },
+            {
+              "accuracy": 0.4166666666666667,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": 0.4450460250178973,
+              "n": 12
+            },
+            {
+              "accuracy": 0.35,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5447314709424973,
+              "n": 20
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6454204533781324,
+              "n": 14
+            },
+            {
+              "accuracy": 0.3333333333333333,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7522401312987009,
+              "n": 6
+            },
+            {
+              "accuracy": 0.25,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8441648408770561,
+              "n": 8
+            },
+            {
+              "accuracy": 0.5897435897435898,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.965268883949671,
+              "n": 39
+            }
+          ],
+          "ece": 0.2818109610596219,
+          "n": 111
+        },
+        "latency": {
+          "n": 111,
+          "p50_s": 0.11237071454524994,
+          "p95_s": 0.14899961836636066
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 111,
+        "n_correct": 47,
+        "n_planned": 111,
+        "n_renormalized": 0,
+        "n_scorable": 111,
+        "n_valid": 111,
+        "operational_success": 1.0,
+        "ordinal_mae": 0.7110752028723558,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      }
+    },
+    "version": "jevbench-v1"
+  },
+  "original": {
+    "accuracy": 0.8055555555555556,
+    "brier_mean": 0.29652247410236326,
+    "calibration_n": 72,
+    "complete": true,
+    "cost_basis": [
+      "local_gpu_no_provider_tariff"
+    ],
+    "coverage": 1.0,
+    "dataset_hash": "914276022fa5f0ea70e4231f2af98e68c67b274b61bea8c440b33890401cd2f9",
+    "ece": {
+      "bins": [
+        {
+          "accuracy": null,
+          "hi": 0.1,
+          "lo": 0.0,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": null,
+          "hi": 0.2,
+          "lo": 0.1,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": null,
+          "hi": 0.3,
+          "lo": 0.2,
+          "mean_confidence": null,
+          "n": 0
+        },
+        {
+          "accuracy": 0.5,
+          "hi": 0.4,
+          "lo": 0.3,
+          "mean_confidence": 0.3444499373435974,
+          "n": 2
+        },
+        {
+          "accuracy": 1.0,
+          "hi": 0.5,
+          "lo": 0.4,
+          "mean_confidence": 0.4325794279575348,
+          "n": 1
+        },
+        {
+          "accuracy": 0.42857142857142855,
+          "hi": 0.6,
+          "lo": 0.5,
+          "mean_confidence": 0.5371253064700535,
+          "n": 7
+        },
+        {
+          "accuracy": 0.6666666666666666,
+          "hi": 0.7,
+          "lo": 0.6,
+          "mean_confidence": 0.6374844577577379,
+          "n": 9
+        },
+        {
+          "accuracy": 0.8571428571428571,
+          "hi": 0.8,
+          "lo": 0.7,
+          "mean_confidence": 0.7471651860645839,
+          "n": 7
+        },
+        {
+          "accuracy": 0.9,
+          "hi": 0.9,
+          "lo": 0.8,
+          "mean_confidence": 0.8518721401691437,
+          "n": 10
+        },
+        {
+          "accuracy": 0.8888888888888888,
+          "hi": 1.0,
+          "lo": 0.9,
+          "mean_confidence": 0.9730205883582433,
+          "n": 36
+        }
+      ],
+      "ece": 0.08584584999415612,
+      "n": 72
+    },
+    "latency": {
+      "n": 72,
+      "p50_s": 0.07467062026262283,
+      "p95_s": 0.1292133398354054
+    },
+    "latency_failures": {
+      "n": 0,
+      "p50_s": null,
+      "p95_s": null
+    },
+    "ledger_charged_usd": null,
+    "macro_accuracy": 0.8055555555555555,
+    "model_identities": [
+      "pcdm:native:runs/tm1b"
+    ],
+    "n_attempted": 72,
+    "n_correct": 58,
+    "n_planned": 72,
+    "n_renormalized": 0,
+    "n_scorable": 72,
+    "n_valid": 72,
+    "operational_success": 1.0,
+    "ordinal_mae": 0.39357847470091656,
+    "paraphrase_consistency": {
+      "agree": 30,
+      "agreement": 0.8333333333333334,
+      "both_correct_rate_all_pairs": 0.7222222222222222,
+      "both_valid": 36,
+      "pairs": 36
+    },
+    "per_family": {
+      "adequacy": {
+        "accuracy": 0.6666666666666666,
+        "brier_mean": 0.40836962254148307,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5817238092422485,
+              "n": 1
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6328441649675369,
+              "n": 4
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8540955781936646,
+              "n": 1
+            },
+            {
+              "accuracy": 0.8333333333333334,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.972666323184967,
+              "n": 6
+            }
+          ],
+          "ece": 0.1745835691690445,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.07050643116235733,
+          "p95_s": 0.08810084797441958
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 8,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "extraction": {
+        "accuracy": 1.0,
+        "brier_mean": 0.02007323946682944,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7408376932144165,
+              "n": 1
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8441984951496124,
+              "n": 2
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.968041201432546,
+              "n": 9
+            }
+          ],
+          "ece": 0.07153287529945371,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.05768968164920807,
+          "p95_s": 0.08274002466350794
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 12,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "intent": {
+        "accuracy": 0.9166666666666666,
+        "brier_mean": 0.22698352684655446,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5113146007061005,
+              "n": 2
+            },
+            {
+              "accuracy": null,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7275521159172058,
+              "n": 1
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8921765089035034,
+              "n": 1
+            },
+            {
+              "accuracy": 0.875,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9776975587010384,
+              "n": 8
+            }
+          ],
+          "ece": 0.1816018869479497,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.11113478429615498,
+          "p95_s": 0.19621948227286334
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 11,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "ordinal": {
+        "accuracy": 0.6666666666666666,
+        "brier_mean": 0.387267867931643,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": 0.3444499373435974,
+              "n": 2
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": 0.4325794279575348,
+              "n": 1
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5386167963345846,
+              "n": 3
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6215285360813141,
+              "n": 2
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.839689165353775,
+              "n": 4
+            },
+            {
+              "accuracy": null,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": null,
+              "n": 0
+            }
+          ],
+          "ece": 0.32437977939844137,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.07719611376523972,
+          "p95_s": 0.11410039383918047
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 8,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": 0.39357847470091656,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "policy": {
+        "accuracy": 0.75,
+        "brier_mean": 0.3966635065443795,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6735906898975372,
+              "n": 2
+            },
+            {
+              "accuracy": null,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.859863817691803,
+              "n": 1
+            },
+            {
+              "accuracy": 0.7777777777777778,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9743029342757331,
+              "n": 9
+            }
+          ],
+          "ece": 0.18800366421540576,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.10365468636155128,
+          "p95_s": 0.15637512318789956
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 9,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      },
+      "routing": {
+        "accuracy": 0.8333333333333334,
+        "brier_mean": 0.3397770812832901,
+        "calibration_n": 12,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5396737456321716,
+              "n": 1
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6157450079917908,
+              "n": 1
+            },
+            {
+              "accuracy": 0.8,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7523532986640931,
+              "n": 5
+            },
+            {
+              "accuracy": 0.0,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8654318451881409,
+              "n": 1
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9725163877010345,
+              "n": 4
+            }
+          ],
+          "ece": 0.17151508728663126,
+          "n": 12
+        },
+        "latency": {
+          "n": 12,
+          "p50_s": 0.053897660225629807,
+          "p95_s": 0.08854644279927014
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 12,
+        "n_correct": 10,
+        "n_planned": 12,
+        "n_renormalized": 0,
+        "n_scorable": 12,
+        "n_valid": 12,
+        "operational_success": 1.0,
+        "ordinal_mae": null,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      }
+    },
+    "price_per_1000_decisions_usd": null,
+    "probability_sources": [
+      "native"
+    ],
+    "schema_validity": 1.0,
+    "schema_validity_strict": 1.0,
+    "splits": {
+      "public": {
+        "accuracy": 0.8055555555555556,
+        "brier_mean": 0.29652247410236326,
+        "calibration_n": 72,
+        "cost_basis": [
+          "local_gpu_no_provider_tariff"
+        ],
+        "coverage": 1.0,
+        "ece": {
+          "bins": [
+            {
+              "accuracy": null,
+              "hi": 0.1,
+              "lo": 0.0,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.2,
+              "lo": 0.1,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": null,
+              "hi": 0.3,
+              "lo": 0.2,
+              "mean_confidence": null,
+              "n": 0
+            },
+            {
+              "accuracy": 0.5,
+              "hi": 0.4,
+              "lo": 0.3,
+              "mean_confidence": 0.3444499373435974,
+              "n": 2
+            },
+            {
+              "accuracy": 1.0,
+              "hi": 0.5,
+              "lo": 0.4,
+              "mean_confidence": 0.4325794279575348,
+              "n": 1
+            },
+            {
+              "accuracy": 0.42857142857142855,
+              "hi": 0.6,
+              "lo": 0.5,
+              "mean_confidence": 0.5371253064700535,
+              "n": 7
+            },
+            {
+              "accuracy": 0.6666666666666666,
+              "hi": 0.7,
+              "lo": 0.6,
+              "mean_confidence": 0.6374844577577379,
+              "n": 9
+            },
+            {
+              "accuracy": 0.8571428571428571,
+              "hi": 0.8,
+              "lo": 0.7,
+              "mean_confidence": 0.7471651860645839,
+              "n": 7
+            },
+            {
+              "accuracy": 0.9,
+              "hi": 0.9,
+              "lo": 0.8,
+              "mean_confidence": 0.8518721401691437,
+              "n": 10
+            },
+            {
+              "accuracy": 0.8888888888888888,
+              "hi": 1.0,
+              "lo": 0.9,
+              "mean_confidence": 0.9730205883582433,
+              "n": 36
+            }
+          ],
+          "ece": 0.08584584999415612,
+          "n": 72
+        },
+        "latency": {
+          "n": 72,
+          "p50_s": 0.07467062026262283,
+          "p95_s": 0.1292133398354054
+        },
+        "latency_failures": {
+          "n": 0,
+          "p50_s": null,
+          "p95_s": null
+        },
+        "n_attempted": 72,
+        "n_correct": 58,
+        "n_planned": 72,
+        "n_renormalized": 0,
+        "n_scorable": 72,
+        "n_valid": 72,
+        "operational_success": 1.0,
+        "ordinal_mae": 0.39357847470091656,
+        "price_per_1000_decisions_usd": null,
+        "schema_validity": 1.0,
+        "schema_validity_strict": 1.0
+      }
+    },
+    "version": "jevbench-v1"
+  }
+}

probe_results.json

@@ -0,0 +1,155 @@
+{
+  "T": 1.1242100350620867,
+  "null_offset": 0.0,
+  "calib_sets": [
+    "val"
+  ],
+  "val_nll": 0.33989632085942884,
+  "args": {
+    "name": "probe_tm1b",
+    "backbone": "Qwen/Qwen3-4B-Base",
+    "lora_layers": 8,
+    "lora_r": 16,
+    "steps": 12000,
+    "bs": 64,
+    "grad_accum": 1,
+    "lr": 0.0003,
+    "lora_lr": 0.0001,
+    "seed": 0,
+    "no_hybrid": false,
+    "no_cand_null": false,
+    "no_null": false,
+    "null": "factored",
+    "hard_only": false,
+    "mix": "full",
+    "eval_every": 4000,
+    "val_every": 1000,
+    "ckpt_every": 2000,
+    "eval_bs": 16,
+    "device": "auto",
+    "wandb": false,
+    "hf_repo": null,
+    "ckpt_upload": false,
+    "data": "data_probe",
+    "extra_data": null,
+    "eval_cap": null,
+    "calib_sets": null,
+    "family_weights": null,
+    "bucket_map": null,
+    "null_aug": null,
+    "smoke": false,
+    "eval_limit": 0,
+    "tap_layer": 26,
+    "zscore": false,
+    "eval_only": true,
+    "dump_logits": null,
+    "tower_d": 512,
+    "tower_layers": 2,
+    "tower_heads": 8,
+    "joint": false,
+    "listwise": false,
+    "extra_tap": 0,
+    "init_from": null,
+    "readout": "native",
+    "nc_head": "n3",
+    "nc_render": "letters_nonull",
+    "score_head": "choice",
+    "noul_head": "bern",
+    "qtype_filter": null,
+    "ordinal_smooth": 0.0,
+    "max_state": 1024,
+    "perm_lambda": 0.0,
+    "no_shuffle": false,
+    "zero_shot": false,
+    "shots": 0,
+    "cand_encoder": "backbone",
+    "head": "mlp",
+    "z_dim": 128,
+    "z_probes": 8,
+    "tiny_layers": 2,
+    "distill_alpha": 1.0,
+    "distill_beta": 0.0,
+    "distill_T": 2.0,
+    "delta_gamma": 0.0
+  },
+  "eval": {
+    "mmlu_choicesonly": {
+      "raw": {
+        "nll": 2.039828592927867,
+        "brier": 0.8168081995898367,
+        "jsd": 0.4534946954898565,
+        "acc": 0.29,
+        "acc_k": 0.29083333333333333,
+        "ece": 0.04304502696419755,
+        "sel_acc80": 0.3229166666666667,
+        "coverage": 0.9966666666666667,
+        "false_abstain": 0.0033333333333333335,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 2.0257116311704886,
+        "brier": 0.8150993429201715,
+        "jsd": 0.4567105766310495,
+        "acc": 0.29,
+        "acc_k": 0.29083333333333333,
+        "ece": 0.03703421314557393,
+        "sel_acc80": 0.32395833333333335,
+        "coverage": 0.9966666666666667,
+        "false_abstain": 0.0033333333333333335,
+        "null_recall": "n/a"
+      }
+    },
+    "mmlu_pro": {
+      "raw": {
+        "nll": 1.6070274236514384,
+        "brier": 0.6767720553604334,
+        "jsd": 0.3678978835677458,
+        "acc": 0.455,
+        "acc_k": 0.4583333333333333,
+        "ece": 0.04268285360187291,
+        "sel_acc80": 0.5166666666666667,
+        "coverage": 0.9958333333333333,
+        "false_abstain": 0.004166666666666667,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.6083517592633751,
+        "brier": 0.6793089765777306,
+        "jsd": 0.3746115181488698,
+        "acc": 0.455,
+        "acc_k": 0.4583333333333333,
+        "ece": 0.05830767397458354,
+        "sel_acc80": 0.5166666666666667,
+        "coverage": 0.9958333333333333,
+        "false_abstain": 0.004166666666666667,
+        "null_recall": "n/a"
+      }
+    },
+    "mmlu_shuffledq": {
+      "raw": {
+        "nll": 2.0994824466839295,
+        "brier": 0.839300334664014,
+        "jsd": 0.4620919566125671,
+        "acc": 0.2633333333333333,
+        "acc_k": 0.265,
+        "ece": 0.06165247778718671,
+        "sel_acc80": 0.29270833333333335,
+        "coverage": 0.9933333333333333,
+        "false_abstain": 0.006666666666666667,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 2.0802900159544735,
+        "brier": 0.8357595643299349,
+        "jsd": 0.46472209676556786,
+        "acc": 0.2633333333333333,
+        "acc_k": 0.265,
+        "ece": 0.03918854349603254,
+        "sel_acc80": 0.29270833333333335,
+        "coverage": 0.9933333333333333,
+        "false_abstain": 0.006666666666666667,
+        "null_recall": "n/a"
+      }
+    }
+  }
+}

results.json

@@ -0,0 +1,2248 @@
+{
+  "T": 1.1242100350620867,
+  "null_offset": 0.0,
+  "calib_sets": [
+    "val"
+  ],
+  "val_nll": 0.33989632085942884,
+  "args": {
+    "name": "tm1b",
+    "backbone": "Qwen/Qwen3-4B-Base",
+    "lora_layers": 8,
+    "lora_r": 16,
+    "steps": 12000,
+    "bs": 64,
+    "grad_accum": 4,
+    "lr": 0.0003,
+    "lora_lr": 0.0001,
+    "seed": 0,
+    "no_hybrid": false,
+    "no_cand_null": false,
+    "no_null": false,
+    "null": "factored",
+    "hard_only": false,
+    "mix": "full",
+    "eval_every": 4000,
+    "val_every": 1000,
+    "ckpt_every": 2000,
+    "eval_bs": 16,
+    "device": "auto",
+    "wandb": true,
+    "hf_repo": "guychuk/pcdm-runs",
+    "ckpt_upload": true,
+    "data": "data_v5",
+    "extra_data": "data_kb,data_wf,data_wf_hf,data_wf_long,data_wh,data_u",
+    "eval_cap": 1500,
+    "calib_sets": null,
+    "family_weights": "E:0.40,K:0.15,W:0.35,U:0.10",
+    "bucket_map": "data_wf_long=W,data_wh=W,data_u=U",
+    "null_aug": "W:0.20",
+    "smoke": false,
+    "eval_limit": 0,
+    "tap_layer": 26,
+    "zscore": true,
+    "eval_only": false,
+    "dump_logits": null,
+    "tower_d": 512,
+    "tower_layers": 2,
+    "tower_heads": 8,
+    "joint": false,
+    "listwise": false,
+    "extra_tap": 0,
+    "init_from": null,
+    "readout": "native",
+    "nc_head": "n3",
+    "nc_render": "letters_nonull",
+    "score_head": "choice",
+    "noul_head": "bern",
+    "qtype_filter": null,
+    "ordinal_smooth": 0.7,
+    "max_state": 1024,
+    "perm_lambda": 0.0,
+    "no_shuffle": false,
+    "zero_shot": false,
+    "shots": 0,
+    "cand_encoder": "backbone",
+    "head": "mlp",
+    "z_dim": 128,
+    "z_probes": 8,
+    "tiny_layers": 2,
+    "distill_alpha": 1.0,
+    "distill_beta": 0.0,
+    "distill_T": 2.0,
+    "delta_gamma": 0.0
+  },
+  "eval": {
+    "anli_test": {
+      "raw": {
+        "nll": 1.0757932427921113,
+        "brier": 0.6307436394770407,
+        "jsd": 0.2572920094851673,
+        "acc": 0.54375,
+        "acc_k": 0.54375,
+        "ece": 0.18585077961906793,
+        "sel_acc80": 0.572265625,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.029092141141034,
+        "brier": 0.6156411190974276,
+        "jsd": 0.2564680245018555,
+        "acc": 0.54375,
+        "acc_k": 0.54375,
+        "ece": 0.1616302808281034,
+        "sel_acc80": 0.572265625,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "banking77_k": {
+      "raw": {
+        "nll": 0.8387672812179306,
+        "brier": 0.40366817680288347,
+        "jsd": 0.16477596631036817,
+        "acc": 0.7278933680104032,
+        "acc_k": 0.8718626155878467,
+        "ece": 0.1269044502406964,
+        "sel_acc80": 0.7996749288906948,
+        "conf_wrong": 0.11203585147247119,
+        "coverage": 0.6089076723016905,
+        "false_abstain": 0.11690885072655217,
+        "null_recall": 0.6568501920614597,
+        "auroc_null": 0.8789133262406189,
+        "auroc_conf": 0.8724135977145449
+      },
+      "scaled": {
+        "nll": 0.8239931862146926,
+        "brier": 0.4001605248196045,
+        "jsd": 0.16518185949943257,
+        "acc": 0.7298439531859557,
+        "acc_k": 0.8718626155878467,
+        "ece": 0.12038085420261124,
+        "sel_acc80": 0.8017066233238521,
+        "conf_wrong": 0.10627400768245839,
+        "coverage": 0.6059817945383614,
+        "false_abstain": 0.11889035667107001,
+        "null_recall": 0.6606914212548015,
+        "auroc_null": 0.878913326240619,
+        "auroc_conf": 0.8719218155093647
+      }
+    },
+    "banking77_test": {
+      "raw": {
+        "nll": 2.052295154509988,
+        "brier": 0.6301511272273523,
+        "jsd": 0.2807325439322352,
+        "acc": 0.5689206762028609,
+        "acc_k": 0.6404421326397919,
+        "ece": 0.1865983266862998,
+        "sel_acc80": 0.6468915075172694,
+        "coverage": 0.8299739921976593,
+        "false_abstain": 0.1700260078023407,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.9304311410624564,
+        "brier": 0.6258988479081237,
+        "jsd": 0.2832575644475134,
+        "acc": 0.5679453836150845,
+        "acc_k": 0.6404421326397919,
+        "ece": 0.17139127035231674,
+        "sel_acc80": 0.6452661519707436,
+        "coverage": 0.8267230169050716,
+        "false_abstain": 0.17327698309492848,
+        "null_recall": "n/a"
+      }
+    },
+    "boolq_val": {
+      "raw": {
+        "nll": 0.39950305672739084,
+        "brier": 0.23047799103736755,
+        "jsd": 0.09612269180931764,
+        "acc": 0.8425076452599388,
+        "acc_k": 0.8440366972477065,
+        "ece": 0.049182466018819634,
+        "sel_acc80": 0.9063455657492355,
+        "coverage": 0.9969418960244648,
+        "false_abstain": 0.0030581039755351682,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.3860076226788216,
+        "brier": 0.22776378177404963,
+        "jsd": 0.09772515374173538,
+        "acc": 0.8425076452599388,
+        "acc_k": 0.8440366972477065,
+        "ece": 0.036158523428330716,
+        "sel_acc80": 0.9063455657492355,
+        "coverage": 0.9969418960244648,
+        "false_abstain": 0.0030581039755351682,
+        "null_recall": "n/a"
+      }
+    },
+    "chaos_mnli": {
+      "raw": {
+        "nll": 1.2692985455080275,
+        "brier": 0.23347494851122355,
+        "jsd": 0.09021653349830507
+      },
+      "scaled": {
+        "nll": 1.188978150676804,
+        "brier": 0.21544714736486784,
+        "jsd": 0.0820067606796143
+      }
+    },
+    "clinc_heldout": {
+      "raw": {
+        "nll": 0.20331486754583272,
+        "brier": 0.09009287230064558,
+        "jsd": 0.04199571607272658,
+        "acc": 0.9365256124721604,
+        "acc_k": 0.9810690423162584,
+        "ece": 0.016201265172862793,
+        "sel_acc80": 0.9916550764951322,
+        "coverage": 0.9454342984409799,
+        "false_abstain": 0.05456570155902005,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.19976629489598294,
+        "brier": 0.09080975891959425,
+        "jsd": 0.04278228699236031,
+        "acc": 0.9365256124721604,
+        "acc_k": 0.9810690423162584,
+        "ece": 0.017676310344103448,
+        "sel_acc80": 0.9902642559109874,
+        "coverage": 0.9443207126948775,
+        "false_abstain": 0.0556792873051225,
+        "null_recall": "n/a"
+      }
+    },
+    "clinc_k": {
+      "raw": {
+        "nll": 0.262570050582597,
+        "brier": 0.13814550219910562,
+        "jsd": 0.06344528720435406,
+        "acc": 0.9089478417266187,
+        "acc_k": 0.9789897183728208,
+        "ece": 0.011057701008729387,
+        "sel_acc80": 0.9674065748805845,
+        "conf_wrong": 0.03211216644052465,
+        "coverage": 0.5454136690647482,
+        "false_abstain": 0.04202056325435852,
+        "null_recall": 0.872003618272275,
+        "auroc_null": 0.9765736279790952,
+        "auroc_conf": 0.9762074740290501
+      },
+      "scaled": {
+        "nll": 0.2619289437378122,
+        "brier": 0.13735334589241208,
+        "jsd": 0.06378631528836894,
+        "acc": 0.9091726618705036,
+        "acc_k": 0.9789897183728208,
+        "ece": 0.00921771111864619,
+        "sel_acc80": 0.9676875526833381,
+        "conf_wrong": 0.03075531433740389,
+        "coverage": 0.5447392086330936,
+        "false_abstain": 0.0424675905230219,
+        "null_recall": 0.8729081863410222,
+        "auroc_null": 0.9765736279790951,
+        "auroc_conf": 0.9761183111952734
+      }
+    },
+    "clinc_oos": {
+      "raw": {
+        "nll": 0.8829221451581013,
+        "brier": 0.4253317641029075,
+        "jsd": 0.22263034211911978,
+        "acc": 0.7427427427427428,
+        "ece": 0.17314278620022072,
+        "sel_acc80": 0.775,
+        "conf_wrong": 0.044044044044044044,
+        "coverage": 0.25725725725725723,
+        "false_abstain": "n/a",
+        "null_recall": 0.7427427427427428
+      },
+      "scaled": {
+        "nll": 0.8829221442206856,
+        "brier": 0.4089999529102035,
+        "jsd": 0.2226303399234561,
+        "acc": 0.7607607607607607,
+        "ece": 0.185149793420826,
+        "sel_acc80": 0.8,
+        "conf_wrong": 0.042042042042042045,
+        "coverage": 0.2392392392392393,
+        "false_abstain": "n/a",
+        "null_recall": 0.7607607607607607
+      }
+    },
+    "clinc_test": {
+      "raw": {
+        "nll": 0.5477434777450255,
+        "brier": 0.21618169032775858,
+        "jsd": 0.10833343624727476,
+        "acc": 0.8502697841726619,
+        "acc_k": 0.8711780575539568,
+        "ece": 0.014199602169741796,
+        "sel_acc80": 0.9364990165776904,
+        "coverage": 0.9651528776978417,
+        "false_abstain": 0.03484712230215827,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.5432309676259418,
+        "brier": 0.21901325867275181,
+        "jsd": 0.11382309839047851,
+        "acc": 0.8471223021582733,
+        "acc_k": 0.8711780575539568,
+        "ece": 0.01667415301764399,
+        "sel_acc80": 0.9342511941556617,
+        "coverage": 0.960431654676259,
+        "false_abstain": 0.039568345323741004,
+        "null_recall": "n/a"
+      }
+    },
+    "cse_banking77": {
+      "raw": {
+        "nll": 1.013373283262765,
+        "brier": 0.3896295995875706,
+        "jsd": 0.1607590626376104,
+        "acc": 0.7016666666666667,
+        "acc_k": 0.7608,
+        "ece": 0.14997901448110743,
+        "sel_acc80": 0.7683333333333333,
+        "conf_wrong": 0.118,
+        "coverage": 0.7903333333333333,
+        "false_abstain": 0.1108,
+        "null_recall": 0.704,
+        "auroc_null": 0.89648,
+        "auroc_conf": 0.8598568
+      },
+      "scaled": {
+        "nll": 0.9641588631983036,
+        "brier": 0.3855041258460942,
+        "jsd": 0.16056262866375498,
+        "acc": 0.702,
+        "acc_k": 0.7608,
+        "ece": 0.14450320369005204,
+        "sel_acc80": 0.7679166666666667,
+        "conf_wrong": 0.112,
+        "coverage": 0.789,
+        "false_abstain": 0.1116,
+        "null_recall": 0.708,
+        "auroc_null": 0.89648,
+        "auroc_conf": 0.8583504
+      },
+      "cse": {
+        "add_irr/p_irr": 0.000970285543377067,
+        "add_irr/dlo_top2": 0.22229034927480423,
+        "remove_gold/dp_null": 0.5173476701720647,
+        "dup/mass_err": 0.04696189815764079,
+        "dup/slot_gap": 0.20254385519145984,
+        "dup/acc_dedup": 0.852,
+        "reorder/max_dp": 0.09346072051447049,
+        "near_dup/cluster_mass_err": 0.0383210542500016
+      }
+    },
+    "cse_clinc": {
+      "raw": {
+        "nll": 0.27065101309237005,
+        "brier": 0.11199271731259507,
+        "jsd": 0.05256673519412974,
+        "acc": 0.8859800206682742,
+        "acc_k": 0.9152471956792688,
+        "ece": 0.013434116671178152,
+        "sel_acc80": 0.965992251399053,
+        "conf_wrong": 0.020161290322580645,
+        "coverage": 0.8157078883913194,
+        "false_abstain": 0.03946821769837973,
+        "null_recall": 0.8870967741935484,
+        "auroc_null": 0.9826153892544593,
+        "auroc_conf": 0.9730247463714703
+      },
+      "scaled": {
+        "nll": 0.2682153721308544,
+        "brier": 0.11078170689271184,
+        "jsd": 0.052865233459530446,
+        "acc": 0.8859800206682742,
+        "acc_k": 0.9152471956792688,
+        "ece": 0.01334414801801273,
+        "sel_acc80": 0.9655617735686612,
+        "conf_wrong": 0.018145161290322582,
+        "coverage": 0.8153634171546675,
+        "false_abstain": 0.039883672621520566,
+        "null_recall": 0.8870967741935484,
+        "auroc_null": 0.9826153892544593,
+        "auroc_conf": 0.9724878378921694
+      },
+      "cse": {
+        "add_irr/p_irr": 0.0004140982716602347,
+        "add_irr/dlo_top2": 0.21209305091759995,
+        "remove_gold/dp_null": 0.7651275384538431,
+        "dup/mass_err": 0.01625872549363192,
+        "dup/slot_gap": 0.16577797743548894,
+        "dup/acc_dedup": 0.9879032258064516,
+        "reorder/max_dp": 0.03674042043095158,
+        "near_dup/cluster_mass_err": 0.021475064313451826
+      }
+    },
+    "cse_hwu64": {
+      "raw": {
+        "nll": 0.47411119093999,
+        "brier": 0.21987119444109693,
+        "jsd": 0.10141276861597559,
+        "acc": 0.8153333333333334,
+        "acc_k": 0.8708,
+        "ece": 0.03724427296717967,
+        "sel_acc80": 0.8858333333333334,
+        "conf_wrong": 0.05,
+        "coverage": 0.8326666666666667,
+        "false_abstain": 0.0524,
+        "null_recall": 0.742,
+        "auroc_null": 0.9480720000000001,
+        "auroc_conf": 0.9206480000000001
+      },
+      "scaled": {
+        "nll": 0.4676719236027076,
+        "brier": 0.2173795947712966,
+        "jsd": 0.10222786247607862,
+        "acc": 0.814,
+        "acc_k": 0.8708,
+        "ece": 0.031223723649978607,
+        "sel_acc80": 0.8866666666666667,
+        "conf_wrong": 0.044,
+        "coverage": 0.8303333333333334,
+        "false_abstain": 0.0548,
+        "null_recall": 0.744,
+        "auroc_null": 0.9480720000000001,
+        "auroc_conf": 0.9197439999999999
+      },
+      "cse": {
+        "add_irr/p_irr": 0.006002017148867547,
+        "add_irr/dlo_top2": 0.1683097961744787,
+        "remove_gold/dp_null": 0.5516747428024682,
+        "dup/mass_err": 0.025967722096361542,
+        "dup/slot_gap": 0.17362533946861175,
+        "dup/acc_dedup": 0.96,
+        "reorder/max_dp": 0.05326351690868955,
+        "near_dup/cluster_mass_err": 0.03816221129417807
+      }
+    },
+    "cse_snli": {
+      "raw": {
+        "nll": 0.3785246814347904,
+        "brier": 0.15583854229184074,
+        "jsd": 0.07841771629000768,
+        "acc": 0.8942451420029895,
+        "acc_k": 0.9076286764705882,
+        "ece": 0.060462515327873466,
+        "sel_acc80": 0.932274638019617,
+        "conf_wrong": 0.032,
+        "coverage": 0.843796711509716,
+        "false_abstain": 0.0,
+        "null_recall": 0.836,
+        "auroc_null": 0.9999972426470588,
+        "auroc_conf": 0.9509972426470589
+      },
+      "scaled": {
+        "nll": 0.3844776583138124,
+        "brier": 0.1567990789175919,
+        "jsd": 0.08211913713577323,
+        "acc": 0.8942451420029895,
+        "acc_k": 0.9076286764705882,
+        "ece": 0.07534116962075416,
+        "sel_acc80": 0.932274638019617,
+        "conf_wrong": 0.032,
+        "coverage": 0.843796711509716,
+        "false_abstain": 0.0,
+        "null_recall": 0.836,
+        "auroc_null": 0.9999972426470588,
+        "auroc_conf": 0.9473409926470588
+      },
+      "cse": {
+        "add_irr/p_irr": 0.012405680973958624,
+        "add_irr/dlo_top2": 0.08886174408259696,
+        "remove_gold/dp_null": 0.7629259843477127,
+        "dup/mass_err": 0.04644630262634746,
+        "dup/slot_gap": 0.14531334546010474,
+        "dup/acc_dedup": 0.916,
+        "reorder/max_dp": 0.013759512538003036,
+        "near_dup/cluster_mass_err": 0.0401906658817924
+      }
+    },
+    "hwu64_test": {
+      "raw": {
+        "nll": 0.8544879862574016,
+        "brier": 0.3244043618126192,
+        "jsd": 0.16828853097185387,
+        "acc": 0.7732342007434945,
+        "acc_k": 0.8113382899628253,
+        "ece": 0.03379586631203675,
+        "sel_acc80": 0.8745644599303136,
+        "coverage": 0.9107806691449815,
+        "false_abstain": 0.08921933085501858,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.8432026245083506,
+        "brier": 0.3286730332135295,
+        "jsd": 0.17449841753703743,
+        "acc": 0.7685873605947955,
+        "acc_k": 0.8113382899628253,
+        "ece": 0.03884527408500588,
+        "sel_acc80": 0.8722415795586528,
+        "coverage": 0.9014869888475836,
+        "false_abstain": 0.09851301115241635,
+        "null_recall": "n/a"
+      }
+    },
+    "ksweep_banking77": {
+      "raw": {
+        "nll": 0.9123393477006552,
+        "brier": 0.4099352651060742,
+        "jsd": 0.17143707584095402,
+        "acc": 0.7276785714285714,
+        "acc_k": 0.8607142857142858,
+        "ece": 0.12577233606151167,
+        "sel_acc80": 0.8020089285714286,
+        "conf_wrong": 0.09857142857142857,
+        "coverage": 0.6269642857142856,
+        "false_abstain": 0.095,
+        "null_recall": 0.6510714285714285,
+        "auroc_null": 0.8887267857142856,
+        "auroc_conf": 0.8845818877551019
+      },
+      "scaled": {
+        "nll": 0.8946802412802192,
+        "brier": 0.4049772048479024,
+        "jsd": 0.17224385846685789,
+        "acc": 0.7283928571428572,
+        "acc_k": 0.8607142857142858,
+        "ece": 0.11672890212652934,
+        "sel_acc80": 0.8040178571428571,
+        "conf_wrong": 0.09071428571428572,
+        "coverage": 0.6242857142857143,
+        "false_abstain": 0.09714285714285714,
+        "null_recall": 0.6542857142857142,
+        "auroc_null": 0.8887267857142856,
+        "auroc_conf": 0.8840718112244897
+      },
+      "ksweep": {
+        "K2/p_null_absent": 0.8811372525494516,
+        "K2/p_null_present": 0.15638932955197787,
+        "K2/acc": 0.975,
+        "K2/auroc": 0.9538375,
+        "K3/p_null_absent": 0.8291295790093136,
+        "K3/p_null_present": 0.11529847987123049,
+        "K3/acc": 0.9625,
+        "K3/auroc": 0.95703125,
+        "K5/p_null_absent": 0.7253072958214034,
+        "K5/p_null_present": 0.0856258794779842,
+        "K5/acc": 0.93,
+        "K5/auroc": 0.94483125,
+        "K10/p_null_absent": 0.6407146220555296,
+        "K10/p_null_present": 0.09918227566511632,
+        "K10/acc": 0.8925,
+        "K10/auroc": 0.9150375000000001,
+        "K20/p_null_absent": 0.4894634351064451,
+        "K20/p_null_present": 0.0933621416045935,
+        "K20/acc": 0.84,
+        "K20/auroc": 0.8745937500000001,
+        "K50/p_null_absent": 0.27896184352750425,
+        "K50/p_null_present": 0.05611725780938286,
+        "K50/acc": 0.735,
+        "K50/auroc": 0.809825,
+        "K77/p_null_absent": 0.39302291929721833,
+        "K77/p_null_present": 0.14252538830245612,
+        "K77/acc": 0.69,
+        "K77/auroc": 0.7562,
+        "p_null_absent_range": 0.6021754090219473
+      }
+    },
+    "ksweep_clinc": {
+      "raw": {
+        "nll": 0.33326689492364,
+        "brier": 0.1664739459458728,
+        "jsd": 0.0781240443304308,
+        "acc": 0.8875313283208021,
+        "acc_k": 0.956140350877193,
+        "ece": 0.014531420793543075,
+        "sel_acc80": 0.9541895066562255,
+        "conf_wrong": 0.03853383458646616,
+        "coverage": 0.5571741854636592,
+        "false_abstain": 0.039160401002506263,
+        "null_recall": 0.8464912280701754,
+        "auroc_null": 0.9752394378804153,
+        "auroc_conf": 0.9706104625599087
+      },
+      "scaled": {
+        "nll": 0.3334823865761253,
+        "brier": 0.16524676939838495,
+        "jsd": 0.07912781817534134,
+        "acc": 0.8894110275689223,
+        "acc_k": 0.956140350877193,
+        "ece": 0.014194780991022886,
+        "sel_acc80": 0.9549725920125294,
+        "conf_wrong": 0.03696741854636591,
+        "coverage": 0.5552944862155389,
+        "false_abstain": 0.039160401002506263,
+        "null_recall": 0.850250626566416,
+        "auroc_null": 0.9752394378804151,
+        "auroc_conf": 0.9697932958963825
+      },
+      "ksweep": {
+        "K2/p_null_absent": 0.9124401700321146,
+        "K2/p_null_present": 0.03731175375713476,
+        "K2/acc": 1.0,
+        "K2/auroc": 0.9960301756898512,
+        "K3/p_null_absent": 0.8935738604922515,
+        "K3/p_null_present": 0.03285068903447748,
+        "K3/acc": 0.9974937343358395,
+        "K3/auroc": 0.9916143742815686,
+        "K5/p_null_absent": 0.8634682853127313,
+        "K5/p_null_present": 0.038481267121283975,
+        "K5/acc": 0.9949874686716792,
+        "K5/auroc": 0.989679713067129,
+        "K10/p_null_absent": 0.8389577590959627,
+        "K10/p_null_present": 0.0566416007986982,
+        "K10/acc": 0.9874686716791979,
+        "K10/auroc": 0.9814197147002844,
+        "K20/p_null_absent": 0.7870685033885911,
+        "K20/p_null_present": 0.0636121057681325,
+        "K20/acc": 0.9573934837092731,
+        "K20/auroc": 0.971645906746817,
+        "K50/p_null_absent": 0.7133180369491127,
+        "K50/p_null_present": 0.08933361823714915,
+        "K50/acc": 0.9448621553884712,
+        "K50/auroc": 0.9461686798449758,
+        "K100/p_null_absent": 0.4153825638495983,
+        "K100/p_null_present": 0.05320865131507892,
+        "K100/acc": 0.9147869674185464,
+        "K100/auroc": 0.9146739028021182,
+        "K150/p_null_absent": 0.8475411443417906,
+        "K150/p_null_present": 0.054351499962986896,
+        "K150/acc": 0.8521303258145363,
+        "K150/auroc": 0.9914259332541882,
+        "p_null_absent_range": 0.4970576061825163
+      }
+    },
+    "mmlu_cf": {
+      "raw": {
+        "nll": 1.7757699338936928,
+        "brier": 0.6361527077025765,
+        "jsd": 0.3573300858114095,
+        "acc": 0.43933333333333335,
+        "acc_k": 0.4513333333333333,
+        "ece": 0.059149413963158926,
+        "sel_acc80": 0.495,
+        "coverage": 0.9806666666666667,
+        "false_abstain": 0.019333333333333334,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.7661442234094802,
+        "brier": 0.6354676987043012,
+        "jsd": 0.363000626599059,
+        "acc": 0.43933333333333335,
+        "acc_k": 0.4513333333333333,
+        "ece": 0.06224065233767032,
+        "sel_acc80": 0.49416666666666664,
+        "coverage": 0.98,
+        "false_abstain": 0.02,
+        "null_recall": "n/a"
+      },
+      "cf": {
+        "orig/acc": 0.472,
+        "orig/among_k": 0.472,
+        "remove3/acc": 0.476,
+        "remove3/among_k": 0.536,
+        "add3_unrel/acc": 0.444,
+        "add3_unrel/among_k": 0.452,
+        "replace_hard/acc": 0.452,
+        "replace_hard/among_k": 0.456,
+        "add_neardup/acc": 0.292,
+        "add_neardup/among_k": 0.292,
+        "reorder/acc": 0.5,
+        "reorder/among_k": 0.5,
+        "add3_unrel/dlo_top2": 0.19286915780863595,
+        "reorder/max_dp": 0.1489373840354383,
+        "add_neardup/slot_gap": 0.13364317205362022,
+        "add_neardup/cluster_mass_err": 0.1036915674302727
+      }
+    },
+    "mmlu_cf_teacher": {
+      "raw": {
+        "nll": 1.7413541804238846,
+        "brier": 0.6266079920746009,
+        "jsd": 0.3541206344382742,
+        "acc": 0.4297222222222222,
+        "acc_k": 0.44305555555555554,
+        "ece": 0.03678819169600805,
+        "sel_acc80": 0.48420138888888886,
+        "coverage": 0.9794444444444445,
+        "false_abstain": 0.020555555555555556,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.7379008911888407,
+        "brier": 0.6271273947648627,
+        "jsd": 0.36024415790391084,
+        "acc": 0.4297222222222222,
+        "acc_k": 0.44305555555555554,
+        "ece": 0.045424607180886804,
+        "sel_acc80": 0.4852430555555556,
+        "coverage": 0.9791666666666666,
+        "false_abstain": 0.020833333333333332,
+        "null_recall": "n/a"
+      },
+      "cf": {
+        "orig/acc": 0.4558333333333333,
+        "orig/among_k": 0.45916666666666667,
+        "orig/kl_teacher": 0.44995319883388635,
+        "remove3/acc": 0.4766666666666667,
+        "remove3/among_k": 0.5366666666666666,
+        "remove3/kl_teacher": 1.2494058071732164,
+        "add3_unrel/acc": 0.4575,
+        "add3_unrel/among_k": 0.4633333333333333,
+        "add3_unrel/kl_teacher": 0.5880642448495372,
+        "replace_hard/acc": 0.45166666666666666,
+        "replace_hard/among_k": 0.4583333333333333,
+        "replace_hard/kl_teacher": 0.48544187695503743,
+        "add_neardup/acc": 0.28,
+        "add_neardup/among_k": 0.28,
+        "add_neardup/kl_teacher": 0.42830956656334196,
+        "reorder/acc": 0.45666666666666667,
+        "reorder/among_k": 0.4608333333333333,
+        "reorder/kl_teacher": 0.44567269445063523,
+        "remove3/delta_mae": 1.2099147179202685,
+        "remove3/delta_teacher_abs": 1.135840148062828,
+        "add3_unrel/delta_mae": 0.7671681344547404,
+        "add3_unrel/delta_teacher_abs": 0.7390631803199154,
+        "replace_hard/delta_mae": 0.7565496808691786,
+        "replace_hard/delta_teacher_abs": 0.6681882120617108,
+        "add_neardup/delta_mae": 0.7833307280802294,
+        "add_neardup/delta_teacher_abs": 0.7305883395327554,
+        "reorder/delta_mae": 1.059772158733795,
+        "reorder/delta_teacher_abs": 0.7763704285195197,
+        "add3_unrel/dlo_top2": 0.17924617675632248,
+        "reorder/max_dp": 0.1370161229314774,
+        "add_neardup/slot_gap": 0.13539707640166548,
+        "add_neardup/cluster_mass_err": 0.10971532651535627
+      }
+    },
+    "mmlu_choicesonly": {
+      "raw": {
+        "nll": 2.039828592927867,
+        "brier": 0.8168081995898367,
+        "jsd": 0.4534946954898565,
+        "acc": 0.29,
+        "acc_k": 0.29083333333333333,
+        "ece": 0.04304502696419755,
+        "sel_acc80": 0.3229166666666667,
+        "coverage": 0.9966666666666667,
+        "false_abstain": 0.0033333333333333335,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 2.0257116311704886,
+        "brier": 0.8150993429201715,
+        "jsd": 0.4567105766310495,
+        "acc": 0.29,
+        "acc_k": 0.29083333333333333,
+        "ece": 0.03703421314557393,
+        "sel_acc80": 0.32395833333333335,
+        "coverage": 0.9966666666666667,
+        "false_abstain": 0.0033333333333333335,
+        "null_recall": "n/a"
+      }
+    },
+    "mmlu_pro": {
+      "raw": {
+        "nll": 1.6070274236514384,
+        "brier": 0.6767720553604334,
+        "jsd": 0.3678978835677458,
+        "acc": 0.455,
+        "acc_k": 0.4583333333333333,
+        "ece": 0.04268285360187291,
+        "sel_acc80": 0.5166666666666667,
+        "coverage": 0.9958333333333333,
+        "false_abstain": 0.004166666666666667,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.6083517592633751,
+        "brier": 0.6793089765777306,
+        "jsd": 0.3746115181488698,
+        "acc": 0.455,
+        "acc_k": 0.4583333333333333,
+        "ece": 0.05830767397458354,
+        "sel_acc80": 0.5166666666666667,
+        "coverage": 0.9958333333333333,
+        "false_abstain": 0.004166666666666667,
+        "null_recall": "n/a"
+      }
+    },
+    "mmlu_shuffledq": {
+      "raw": {
+        "nll": 2.0994824466839295,
+        "brier": 0.839300334664014,
+        "jsd": 0.4620919566125671,
+        "acc": 0.2633333333333333,
+        "acc_k": 0.265,
+        "ece": 0.06165247778718671,
+        "sel_acc80": 0.29270833333333335,
+        "coverage": 0.9933333333333333,
+        "false_abstain": 0.006666666666666667,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 2.0802900159544735,
+        "brier": 0.8357595643299349,
+        "jsd": 0.46472209676556786,
+        "acc": 0.2633333333333333,
+        "acc_k": 0.265,
+        "ece": 0.03918854349603254,
+        "sel_acc80": 0.29270833333333335,
+        "coverage": 0.9933333333333333,
+        "false_abstain": 0.006666666666666667,
+        "null_recall": "n/a"
+      }
+    },
+    "mnli_val": {
+      "raw": {
+        "nll": 0.3773646737665917,
+        "brier": 0.20883140960217866,
+        "jsd": 0.09443824642830204,
+        "acc": 0.8605442176870748,
+        "acc_k": 0.8605442176870748,
+        "ece": 0.020842784633036352,
+        "sel_acc80": 0.9200680272108843,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.3750955821527883,
+        "brier": 0.20850910256182303,
+        "jsd": 0.09836140994889432,
+        "acc": 0.8605442176870748,
+        "acc_k": 0.8605442176870748,
+        "ece": 0.009489905398313684,
+        "sel_acc80": 0.9200680272108843,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "ng20_test": {
+      "raw": {
+        "nll": 1.487761037323019,
+        "brier": 0.5656505449790027,
+        "jsd": 0.26489029125897295,
+        "acc": 0.5913875598086125,
+        "acc_k": 0.6397812713602188,
+        "ece": 0.13154189228726346,
+        "sel_acc80": 0.6712235133287765,
+        "coverage": 0.8489405331510594,
+        "false_abstain": 0.15105946684894053,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.4293305916943673,
+        "brier": 0.5619083493741223,
+        "jsd": 0.2683506922940156,
+        "acc": 0.5882433356117567,
+        "acc_k": 0.6397812713602188,
+        "ece": 0.11515617097841906,
+        "sel_acc80": 0.6669514695830485,
+        "coverage": 0.8414217361585783,
+        "false_abstain": 0.15857826384142173,
+        "null_recall": "n/a"
+      }
+    },
+    "null_irrq_intent": {
+      "raw": {
+        "nll": 0.2228418101184203,
+        "brier": 0.1143710591296055,
+        "jsd": 0.06114704381982695,
+        "acc": 0.9266936299292214,
+        "acc_k": 0.9816326530612245,
+        "ece": 0.031551026387933015,
+        "sel_acc80": 0.981680353758686,
+        "conf_wrong": 0.006012024048096192,
+        "coverage": 0.5197168857431749,
+        "false_abstain": 0.04387755102040816,
+        "null_recall": 0.908817635270541,
+        "auroc_null": 0.9819393889820457,
+        "auroc_conf": 0.9848083922947937
+      },
+      "scaled": {
+        "nll": 0.2227184423843362,
+        "brier": 0.11267167675609457,
+        "jsd": 0.06145260385603397,
+        "acc": 0.9277047522750252,
+        "acc_k": 0.9816326530612245,
+        "ece": 0.036607042256332135,
+        "sel_acc80": 0.9823120656980417,
+        "conf_wrong": 0.004008016032064128,
+        "coverage": 0.5182002022244692,
+        "false_abstain": 0.044897959183673466,
+        "null_recall": 0.9108216432865731,
+        "auroc_null": 0.9819393889820457,
+        "auroc_conf": 0.9853574495930636
+      }
+    },
+    "null_irrq_squad": {
+      "raw": {
+        "nll": 0.11958802708388418,
+        "brier": 0.06691821229442936,
+        "jsd": 0.031051279929708648,
+        "acc": 0.9555,
+        "acc_k": 0.995,
+        "ece": 0.006837947338819474,
+        "sel_acc80": 0.993125,
+        "conf_wrong": 0.015,
+        "coverage": 0.5355,
+        "false_abstain": 0.007,
+        "null_recall": 0.922,
+        "auroc_null": 0.9963829999999999,
+        "auroc_conf": 0.996046
+      },
+      "scaled": {
+        "nll": 0.11969051312469098,
+        "brier": 0.06648639497090965,
+        "jsd": 0.031141734225993013,
+        "acc": 0.9555,
+        "acc_k": 0.995,
+        "ece": 0.009631279647350293,
+        "sel_acc80": 0.994375,
+        "conf_wrong": 0.014,
+        "coverage": 0.5355,
+        "false_abstain": 0.007,
+        "null_recall": 0.922,
+        "auroc_null": 0.996383,
+        "auroc_conf": 0.996008
+      }
+    },
+    "null_nearmiss_banking77": {
+      "raw": {
+        "nll": 0.9175728007055943,
+        "brier": 0.4502252277332544,
+        "jsd": 0.1821161578591737,
+        "acc": 0.704,
+        "acc_k": 0.882,
+        "ece": 0.14530414193868638,
+        "sel_acc80": 0.75875,
+        "conf_wrong": 0.148,
+        "coverage": 0.6639999999999999,
+        "false_abstain": 0.092,
+        "null_recall": 0.58,
+        "auroc_null": 0.8760680000000001,
+        "auroc_conf": 0.8604099999999999
+      },
+      "scaled": {
+        "nll": 0.9055373980443426,
+        "brier": 0.44583399808413937,
+        "jsd": 0.18265736516534373,
+        "acc": 0.706,
+        "acc_k": 0.882,
+        "ece": 0.13480573588609693,
+        "sel_acc80": 0.7625,
+        "conf_wrong": 0.136,
+        "coverage": 0.6619999999999999,
+        "false_abstain": 0.092,
+        "null_recall": 0.584,
+        "auroc_null": 0.8760680000000001,
+        "auroc_conf": 0.859052
+      }
+    },
+    "null_nearmiss_clinc": {
+      "raw": {
+        "nll": 0.5506594233951506,
+        "brier": 0.2877703155362219,
+        "jsd": 0.11687679987648988,
+        "acc": 0.8135764944275583,
+        "acc_k": 0.9494949494949495,
+        "ece": 0.08668260612990229,
+        "sel_acc80": 0.8708860759493671,
+        "conf_wrong": 0.13414634146341464,
+        "coverage": 0.6322188449848024,
+        "false_abstain": 0.03636363636363636,
+        "null_recall": 0.7012195121951219,
+        "auroc_null": 0.9421162847992116,
+        "auroc_conf": 0.9255440584708877
+      },
+      "scaled": {
+        "nll": 0.5473392438642083,
+        "brier": 0.28632701776783964,
+        "jsd": 0.1173640732811478,
+        "acc": 0.8135764944275583,
+        "acc_k": 0.9494949494949495,
+        "ece": 0.08807264780201081,
+        "sel_acc80": 0.8734177215189873,
+        "conf_wrong": 0.13008130081300814,
+        "coverage": 0.6322188449848024,
+        "false_abstain": 0.03636363636363636,
+        "null_recall": 0.7012195121951219,
+        "auroc_null": 0.9421162847992116,
+        "auroc_conf": 0.9237004188223701
+      }
+    },
+    "null_nearmiss_hwu64": {
+      "raw": {
+        "nll": 0.6930345077628086,
+        "brier": 0.37550991179408877,
+        "jsd": 0.1627737776269167,
+        "acc": 0.734,
+        "acc_k": 0.924,
+        "ece": 0.08103191289305686,
+        "sel_acc80": 0.80125,
+        "conf_wrong": 0.092,
+        "coverage": 0.6799999999999999,
+        "false_abstain": 0.058,
+        "null_recall": 0.582,
+        "auroc_null": 0.914212,
+        "auroc_conf": 0.8997919999999999
+      },
+      "scaled": {
+        "nll": 0.691222061632767,
+        "brier": 0.3714586770346436,
+        "jsd": 0.16380177240954913,
+        "acc": 0.736,
+        "acc_k": 0.924,
+        "ece": 0.07761898154020311,
+        "sel_acc80": 0.80625,
+        "conf_wrong": 0.092,
+        "coverage": 0.6779999999999999,
+        "false_abstain": 0.058,
+        "null_recall": 0.586,
+        "auroc_null": 0.914212,
+        "auroc_conf": 0.898608
+      }
+    },
+    "snli_null": {
+      "raw": {
+        "nll": 0.3419294368754166,
+        "brier": 0.1853403799127337,
+        "jsd": 0.08550839024504887,
+        "acc": 0.8741230707556624,
+        "acc_k": 0.9477732793522268,
+        "ece": 0.019044966377746518,
+        "sel_acc80": 0.9418837675350702,
+        "conf_wrong": 0.037316395394998016,
+        "coverage": 0.5742633794347565,
+        "false_abstain": 0.023076923076923078,
+        "null_recall": 0.8205637157602224,
+        "auroc_null": 0.9805860882395012,
+        "auroc_conf": 0.9665643940063613
+      },
+      "scaled": {
+        "nll": 0.343350635842443,
+        "brier": 0.18461307863262783,
+        "jsd": 0.08667314826430747,
+        "acc": 0.8751252756063339,
+        "acc_k": 0.9477732793522268,
+        "ece": 0.019226861922385895,
+        "sel_acc80": 0.9418837675350702,
+        "conf_wrong": 0.03652242953552997,
+        "coverage": 0.573261174584085,
+        "false_abstain": 0.023076923076923078,
+        "null_recall": 0.8225486304088924,
+        "auroc_null": 0.9805860882395012,
+        "auroc_conf": 0.9652861732613514
+      }
+    },
+    "snli_test": {
+      "raw": {
+        "nll": 0.25593494718671267,
+        "brier": 0.14036088145874998,
+        "jsd": 0.0665975840595549,
+        "acc": 0.9089997995590299,
+        "acc_k": 0.9089997995590299,
+        "ece": 0.008833283340077942,
+        "sel_acc80": 0.9611723446893787,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.25976074002227,
+        "brier": 0.14143341682032448,
+        "jsd": 0.07073793640612969,
+        "acc": 0.9089997995590299,
+        "acc_k": 0.9089997995590299,
+        "ece": 0.023527867845648823,
+        "sel_acc80": 0.9614228456913828,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "snli_test_hyponly": {
+      "raw": {
+        "nll": 1.7751328278047886,
+        "brier": 0.9288499904530326,
+        "jsd": 0.350175634798463,
+        "acc": 0.3742232912407296,
+        "acc_k": 0.3742232912407296,
+        "ece": 0.42920332531170124,
+        "sel_acc80": 0.3905310621242485,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.6366306739667331,
+        "brier": 0.894582538842687,
+        "jsd": 0.34275924283904896,
+        "acc": 0.3742232912407296,
+        "acc_k": 0.3742232912407296,
+        "ece": 0.4047492533521964,
+        "sel_acc80": 0.39203406813627256,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "snli_test_paraphrase": {
+      "raw": {
+        "nll": 0.32587851940146534,
+        "brier": 0.17727392068849784,
+        "jsd": 0.0913833239073259,
+        "acc": 0.8877530567247945,
+        "acc_k": 0.8877530567247945,
+        "ece": 0.04598980332023037,
+        "sel_acc80": 0.9438877755511023,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.33854533963891326,
+        "brier": 0.18208532264372873,
+        "jsd": 0.09776032407399843,
+        "acc": 0.8877530567247945,
+        "acc_k": 0.8877530567247945,
+        "ece": 0.06781542564016661,
+        "sel_acc80": 0.9443887775551102,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "snli_test_qpara": {
+      "raw": {
+        "nll": 0.24387772870151897,
+        "brier": 0.13415882157921932,
+        "jsd": 0.061499692705830125,
+        "acc": 0.9112337011033099,
+        "acc_k": 0.9112337011033099,
+        "ece": 0.01218783982479228,
+        "sel_acc80": 0.9655388471177945,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.24449059889889446,
+        "brier": 0.13416361658852846,
+        "jsd": 0.06481072262916734,
+        "acc": 0.9112337011033099,
+        "acc_k": 0.9112337011033099,
+        "ece": 0.01282951259684777,
+        "sel_acc80": 0.9655388471177945,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "snli_test_soft": {
+      "raw": {
+        "nll": 0.5597517215152261,
+        "brier": 0.09879588537311783,
+        "jsd": 0.05306827216224841,
+        "acc": 0.9089997995590299,
+        "acc_k": 0.9089997995590299,
+        "ece": 0.008833283340077942,
+        "sel_acc80": 0.9611723446893787,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.5300098613125277,
+        "brier": 0.09536717726609922,
+        "jsd": 0.053384305599445174,
+        "acc": 0.9089997995590299,
+        "acc_k": 0.9089997995590299,
+        "ece": 0.023527867845648823,
+        "sel_acc80": 0.9614228456913828,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "squad_null": {
+      "raw": {
+        "nll": 0.0653928564491927,
+        "brier": 0.03340959325302636,
+        "jsd": 0.014394054113564767,
+        "acc": 0.9775,
+        "acc_k": 0.997,
+        "ece": 0.007179727219045159,
+        "sel_acc80": 0.9975,
+        "conf_wrong": 0.0115,
+        "coverage": 0.507,
+        "false_abstain": 0.0145,
+        "null_recall": 0.9715,
+        "auroc_null": 0.997093,
+        "auroc_conf": 0.9969047500000001
+      },
+      "scaled": {
+        "nll": 0.06562872611683704,
+        "brier": 0.03343094242134052,
+        "jsd": 0.014530513671621156,
+        "acc": 0.97775,
+        "acc_k": 0.997,
+        "ece": 0.007410270601511033,
+        "sel_acc80": 0.9975,
+        "conf_wrong": 0.0105,
+        "coverage": 0.50675,
+        "false_abstain": 0.0145,
+        "null_recall": 0.972,
+        "auroc_null": 0.997093,
+        "auroc_conf": 0.99687025
+      }
+    },
+    "trec_coarse": {
+      "raw": {
+        "nll": 1.001176772570978,
+        "brier": 0.45645891235995145,
+        "jsd": 0.23270860839413132,
+        "acc": 0.708,
+        "acc_k": 0.744,
+        "ece": 0.07937431913614273,
+        "sel_acc80": 0.7725,
+        "coverage": 0.91,
+        "false_abstain": 0.09,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.9972639353463436,
+        "brier": 0.46413852182874643,
+        "jsd": 0.24018308427283502,
+        "acc": 0.706,
+        "acc_k": 0.744,
+        "ece": 0.10051598793268204,
+        "sel_acc80": 0.765,
+        "coverage": 0.908,
+        "false_abstain": 0.092,
+        "null_recall": "n/a"
+      }
+    },
+    "trec_fine": {
+      "raw": {
+        "nll": 1.9270376611515554,
+        "brier": 0.7866914685130448,
+        "jsd": 0.3710943789415943,
+        "acc": 0.42,
+        "acc_k": 0.51,
+        "ece": 0.20852845552563673,
+        "sel_acc80": 0.44,
+        "coverage": 0.752,
+        "false_abstain": 0.248,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.890897607132656,
+        "brier": 0.7828980197231503,
+        "jsd": 0.37848747499812385,
+        "acc": 0.414,
+        "acc_k": 0.51,
+        "ece": 0.18710149055719377,
+        "sel_acc80": 0.4275,
+        "coverage": 0.734,
+        "false_abstain": 0.266,
+        "null_recall": "n/a"
+      }
+    },
+    "unli_test": {
+      "raw": {
+        "nll": 0.5144665216131059,
+        "brier": 0.11676567490046662,
+        "jsd": 0.04409186280928513,
+        "auroc_null": 0.932519288829252,
+        "auroc_conf": 0.932519288829252
+      },
+      "scaled": {
+        "nll": 0.5144665214847596,
+        "brier": 0.11676567514171127,
+        "jsd": 0.044091862724640454,
+        "auroc_null": 0.932519288829252,
+        "auroc_conf": 0.932519288829252
+      }
+    },
+    "data_kb_val": {
+      "raw": {
+        "nll": 0.7956040995278478,
+        "brier": 0.4090308878761947,
+        "jsd": 0.2035840258631193,
+        "acc": 0.6826666666666666,
+        "acc_k": 0.6659634317862166,
+        "ece": 0.021778101732333507,
+        "sel_acc80": 0.7691666666666667,
+        "conf_wrong": 0.0,
+        "coverage": 0.948,
+        "false_abstain": 0.0007032348804500703,
+        "null_recall": 0.9871794871794872,
+        "auroc_null": 0.9999819683363987,
+        "auroc_conf": 0.9962313823073317
+      },
+      "scaled": {
+        "nll": 0.7956797234491352,
+        "brier": 0.409640835609177,
+        "jsd": 0.2081642635211449,
+        "acc": 0.6826666666666666,
+        "acc_k": 0.6659634317862166,
+        "ece": 0.023031593660513546,
+        "sel_acc80": 0.7683333333333333,
+        "conf_wrong": 0.0,
+        "coverage": 0.948,
+        "false_abstain": 0.0007032348804500703,
+        "null_recall": 0.9871794871794872,
+        "auroc_null": 0.9999819683363987,
+        "auroc_conf": 0.9961502398211259
+      }
+    },
+    "truthfulqa_mc1": {
+      "raw": {
+        "nll": 2.1805148426121623,
+        "brier": 0.8215843259734116,
+        "jsd": 0.380775613960407,
+        "acc": 0.408812729498164,
+        "acc_k": 0.46511627906976744,
+        "ece": 0.1856523987676406,
+        "sel_acc80": 0.44954128440366975,
+        "coverage": 0.8629130966952264,
+        "false_abstain": 0.13708690330477355,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 2.1274534999375962,
+        "brier": 0.8176668121988437,
+        "jsd": 0.383300010835186,
+        "acc": 0.40758873929008566,
+        "acc_k": 0.46511627906976744,
+        "ece": 0.1614636940245529,
+        "sel_acc80": 0.44801223241590216,
+        "coverage": 0.8604651162790697,
+        "false_abstain": 0.13953488372093023,
+        "null_recall": "n/a"
+      }
+    },
+    "data_wf_val": {
+      "raw": {
+        "nll": 0.338919784759892,
+        "brier": 0.18760861684389032,
+        "jsd": 0.09645376573672314,
+        "acc": 0.878,
+        "acc_k": 0.8759372869802318,
+        "ece": 0.05888278289635974,
+        "sel_acc80": 0.9466666666666667,
+        "conf_wrong": 0.0,
+        "coverage": 0.9786666666666667,
+        "false_abstain": 0.0,
+        "null_recall": 0.9696969696969697,
+        "auroc_null": 0.9999793435376257,
+        "auroc_conf": 1.0
+      },
+      "scaled": {
+        "nll": 0.349651697481338,
+        "brier": 0.19284379766016657,
+        "jsd": 0.10093981758344396,
+        "acc": 0.878,
+        "acc_k": 0.8759372869802318,
+        "ece": 0.07291284207503003,
+        "sel_acc80": 0.9433333333333334,
+        "conf_wrong": 0.0,
+        "coverage": 0.9786666666666667,
+        "false_abstain": 0.0,
+        "null_recall": 0.9696969696969697,
+        "auroc_null": 0.9999793435376257,
+        "auroc_conf": 1.0
+      },
+      "ordinal": {
+        "ordinal_mae": 0.26497695852534564,
+        "exp_score_err": 0.43905842999686256,
+        "n": 434
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.010068416595458984,
+        "mean_abs_diff": 0.00043900286989631126
+      }
+    },
+    "wf_heldout_choice": {
+      "raw": {
+        "nll": 0.08894784373441748,
+        "brier": 0.03804468692396801,
+        "jsd": 0.017131709690636845,
+        "acc": 0.978,
+        "acc_k": 0.9907997169143666,
+        "ece": 0.011830694854259511,
+        "sel_acc80": 0.9958333333333333,
+        "conf_wrong": 0.0,
+        "coverage": 0.9273333333333333,
+        "false_abstain": 0.01556970983722576,
+        "null_recall": 1.0,
+        "auroc_null": 0.9994834500654839,
+        "auroc_conf": 0.9995119213217171
+      },
+      "scaled": {
+        "nll": 0.09155318678100496,
+        "brier": 0.03891874334203766,
+        "jsd": 0.018270416526501614,
+        "acc": 0.978,
+        "acc_k": 0.9907997169143666,
+        "ece": 0.012840404589970836,
+        "sel_acc80": 0.995,
+        "conf_wrong": 0.0,
+        "coverage": 0.9273333333333333,
+        "false_abstain": 0.01556970983722576,
+        "null_recall": 1.0,
+        "auroc_null": 0.9994834500654839,
+        "auroc_conf": 0.9995119213217171
+      }
+    },
+    "wf_heldout_noul": {
+      "raw": {
+        "nll": 0.5731820141841979,
+        "brier": 0.30167717441890574,
+        "jsd": 0.10894462183976376,
+        "acc": 0.8106666666666666,
+        "acc_k": 0.8106666666666666,
+        "ece": 0.13052070760726933,
+        "sel_acc80": 0.88,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.5272018810924806,
+        "brier": 0.2944517623199594,
+        "jsd": 0.10771026191000946,
+        "acc": 0.8106666666666666,
+        "acc_k": 0.8106666666666666,
+        "ece": 0.12216018517812091,
+        "sel_acc80": 0.88,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.0,
+        "mean_abs_diff": 0.0
+      }
+    },
+    "wf_heldout_score": {
+      "raw": {
+        "nll": 1.0186137959697332,
+        "brier": 0.5645217285335032,
+        "jsd": 0.28011783812053404,
+        "acc": 0.528,
+        "acc_k": 0.528,
+        "ece": 0.04999171092112859,
+        "sel_acc80": 0.5808333333333333,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.0259130994214425,
+        "brier": 0.5673302940420648,
+        "jsd": 0.2850322863655089,
+        "acc": 0.528,
+        "acc_k": 0.528,
+        "ece": 0.06743905701239906,
+        "sel_acc80": 0.5808333333333333,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "ordinal": {
+        "ordinal_mae": 0.5706666666666667,
+        "exp_score_err": 0.6252469550896435,
+        "n": 1500
+      }
+    },
+    "wf_heldout_style": {
+      "raw": {
+        "nll": 0.3463679467600616,
+        "brier": 0.2021043802230954,
+        "jsd": 0.10170733756745631,
+        "acc": 0.8606666666666667,
+        "acc_k": 0.8584969532836831,
+        "ece": 0.05806009686986599,
+        "sel_acc80": 0.9341666666666667,
+        "conf_wrong": 0.0,
+        "coverage": 0.9846666666666667,
+        "false_abstain": 0.0,
+        "null_recall": 1.0,
+        "auroc_null": 1.0,
+        "auroc_conf": 1.0
+      },
+      "scaled": {
+        "nll": 0.3576204161242343,
+        "brier": 0.20659511693859356,
+        "jsd": 0.10588178815700212,
+        "acc": 0.8606666666666667,
+        "acc_k": 0.8584969532836831,
+        "ece": 0.07112260150909422,
+        "sel_acc80": 0.9325,
+        "conf_wrong": 0.0,
+        "coverage": 0.9846666666666667,
+        "false_abstain": 0.0,
+        "null_recall": 1.0,
+        "auroc_null": 1.0,
+        "auroc_conf": 1.0
+      },
+      "ordinal": {
+        "ordinal_mae": 0.3010752688172043,
+        "exp_score_err": 0.46150196176081854,
+        "n": 465
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.012003302574157715,
+        "mean_abs_diff": 0.0005715727954548829
+      }
+    },
+    "wf_rubric_flip": {
+      "raw": {
+        "nll": 0.5047635419647872,
+        "brier": 0.2691039167691245,
+        "jsd": 0.12483415140150374,
+        "acc": 0.794,
+        "acc_k": 0.7986666666666666,
+        "ece": 0.03788818931579589,
+        "sel_acc80": 0.895,
+        "coverage": 0.994,
+        "false_abstain": 0.006,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.49939328249927545,
+        "brier": 0.26871069651279345,
+        "jsd": 0.12677251415967972,
+        "acc": 0.794,
+        "acc_k": 0.7986666666666666,
+        "ece": 0.03212277259429295,
+        "sel_acc80": 0.8958333333333334,
+        "coverage": 0.994,
+        "false_abstain": 0.006,
+        "null_recall": "n/a"
+      },
+      "flip": {
+        "n_pairs": 750,
+        "flip_rate": 0.8173333333333334,
+        "both_correct": 0.6733333333333333,
+        "acc": 0.7986666666666666
+      },
+      "ordinal": {
+        "ordinal_mae": 0.5816733067729084,
+        "exp_score_err": 0.6296289763572166,
+        "n": 502
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.015544265508651733,
+        "mean_abs_diff": 0.0003942134614645715
+      }
+    },
+    "wf_rubric_shuffled": {
+      "raw": {
+        "nll": 3.2318217026493268,
+        "brier": 0.9866861870364446,
+        "jsd": 0.38150478086393286,
+        "acc": 0.38066666666666665,
+        "acc_k": 0.382,
+        "ece": 0.40178403099377946,
+        "sel_acc80": 0.395,
+        "coverage": 0.9906666666666667,
+        "false_abstain": 0.009333333333333334,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 2.936559013634907,
+        "brier": 0.9733415497626388,
+        "jsd": 0.3787540582851265,
+        "acc": 0.38066666666666665,
+        "acc_k": 0.382,
+        "ece": 0.3852959664066633,
+        "sel_acc80": 0.39666666666666667,
+        "coverage": 0.9906666666666667,
+        "false_abstain": 0.009333333333333334,
+        "null_recall": "n/a"
+      },
+      "ordinal": {
+        "ordinal_mae": 0.890652557319224,
+        "exp_score_err": 0.8865040883733496,
+        "n": 567
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.008019983768463135,
+        "mean_abs_diff": 0.00052099735086022
+      }
+    },
+    "data_wf_hf_val": {
+      "raw": {
+        "nll": 0.014253304002249504,
+        "brier": 0.008345681951172091,
+        "jsd": 0.003829924295894302,
+        "acc": 0.9946666666666667,
+        "acc_k": 0.9946666666666667,
+        "ece": 0.0024553716182709445,
+        "sel_acc80": 1.0,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.014647498088970042,
+        "brier": 0.008344898141976866,
+        "jsd": 0.004117263061674672,
+        "acc": 0.9946666666666667,
+        "acc_k": 0.9946666666666667,
+        "ece": 0.0016058325767517242,
+        "sel_acc80": 1.0,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "cua_s1_forms_demo": {
+      "raw": {
+        "nll": 0.0006697119946596961,
+        "brier": 2.5804204528327477e-06,
+        "jsd": 0.00023202367765799874,
+        "acc": 1.0,
+        "acc_k": 1.0,
+        "ece": 0.0006688650773496452,
+        "sel_acc80": 1.0,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.0016030520060776182,
+        "brier": 1.0894076450595428e-05,
+        "jsd": 0.0005552100790503665,
+        "acc": 1.0,
+        "acc_k": 1.0,
+        "ece": 0.0015992209011194136,
+        "sel_acc80": 1.0,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "cua_s1_forms_test": {
+      "raw": {
+        "nll": 0.03166430327218538,
+        "brier": 0.015101025211502896,
+        "jsd": 0.0058346216304238216,
+        "acc": 0.9913333333333333,
+        "acc_k": 0.9913333333333333,
+        "ece": 0.0068630737860996775,
+        "sel_acc80": 1.0,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.029713019758039725,
+        "brier": 0.014819644050189798,
+        "jsd": 0.005967063943986709,
+        "acc": 0.9913333333333333,
+        "acc_k": 0.9913333333333333,
+        "ece": 0.005603543957074477,
+        "sel_acc80": 1.0,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "jev4b_adversarial": {
+      "raw": {
+        "nll": 0.38843911469630704,
+        "brier": 0.21720070872016706,
+        "jsd": 0.09923600700781257,
+        "acc": 0.835,
+        "acc_k": 0.835,
+        "ece": 0.04567231977979346,
+        "sel_acc80": 0.9354166666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.3825427230468783,
+        "brier": 0.21588778092143868,
+        "jsd": 0.10090386299009478,
+        "acc": 0.835,
+        "acc_k": 0.835,
+        "ece": 0.03748601749539371,
+        "sel_acc80": 0.9333333333333333,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "ordinal": {
+        "ordinal_mae": 0.43333333333333335,
+        "exp_score_err": 0.4572230171763431,
+        "n": 120
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.0018151402473449707,
+        "mean_abs_diff": 0.0003225292331383874
+      }
+    },
+    "jevlogs_triage": {
+      "raw": {
+        "nll": 1.2262092962906508,
+        "brier": 0.7776982801860635,
+        "jsd": 0.2786176452889147,
+        "acc": 0.48,
+        "acc_k": 0.48,
+        "ece": 0.3425089488426844,
+        "sel_acc80": 0.4725,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.1367764885369938,
+        "brier": 0.7531789068144168,
+        "jsd": 0.2718081592157289,
+        "acc": 0.48,
+        "acc_k": 0.48,
+        "ece": 0.3260512084364891,
+        "sel_acc80": 0.4725,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.0,
+        "mean_abs_diff": 0.0
+      }
+    },
+    "mind2web_choice": {
+      "raw": {
+        "nll": 1.2412872188409299,
+        "brier": 0.572483286457417,
+        "jsd": 0.27838292216102917,
+        "acc": 0.5466666666666666,
+        "acc_k": 0.5466666666666666,
+        "ece": 0.0791386130352815,
+        "sel_acc80": 0.6166666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.2202121294813058,
+        "brier": 0.5664862341494601,
+        "jsd": 0.28233206877326084,
+        "acc": 0.5466666666666666,
+        "acc_k": 0.5466666666666666,
+        "ece": 0.060432610829671184,
+        "sel_acc80": 0.6166666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.016718044877052307,
+        "mean_abs_diff": 0.0021705068728576104
+      }
+    },
+    "pagerduty_trigger": {
+      "raw": {
+        "nll": 0.8354194549929469,
+        "brier": 0.5352882009354301,
+        "jsd": 0.22011285498403638,
+        "acc": 0.5533333333333333,
+        "acc_k": 0.5533333333333333,
+        "ece": 0.2813940734863281,
+        "sel_acc80": 0.6325,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.8100947578470528,
+        "brier": 0.5245902994052029,
+        "jsd": 0.21967471371958058,
+        "acc": 0.5533333333333333,
+        "acc_k": 0.5533333333333333,
+        "ece": 0.24505597927172978,
+        "sel_acc80": 0.6316666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.0083027184009552,
+        "mean_abs_diff": 0.0007018453646451234
+      }
+    },
+    "systemone_lite_hard": {
+      "raw": {
+        "nll": 0.44872061751977815,
+        "brier": 0.2415014631857414,
+        "jsd": 0.1130754906430559,
+        "acc": 0.8526666666666667,
+        "acc_k": 0.8526666666666667,
+        "ece": 0.06893132446209589,
+        "sel_acc80": 0.8908333333333334,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.44582079406698827,
+        "brier": 0.24517414554091083,
+        "jsd": 0.11670442592270194,
+        "acc": 0.8526666666666667,
+        "acc_k": 0.8526666666666667,
+        "ece": 0.07723588303724928,
+        "sel_acc80": 0.8908333333333334,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "ordinal": {
+        "ordinal_mae": 0.182,
+        "exp_score_err": 0.35481958847247086,
+        "n": 500
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.014602690935134888,
+        "mean_abs_diff": 0.001006261961236305
+      }
+    },
+    "tree_choice_cap": {
+      "raw": {
+        "nll": 1.0944042301182224,
+        "brier": 0.41195583724182677,
+        "jsd": 0.22831650697305347,
+        "acc": 0.6902777777777778,
+        "acc_k": 0.7319444444444444,
+        "ece": 0.07779081524349749,
+        "sel_acc80": 0.7934027777777778,
+        "coverage": 0.8611111111111112,
+        "false_abstain": 0.1388888888888889,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.1347869650083346,
+        "brier": 0.4228106671667945,
+        "jsd": 0.23927792250464092,
+        "acc": 0.6833333333333333,
+        "acc_k": 0.7319444444444444,
+        "ece": 0.0688384915339864,
+        "sel_acc80": 0.7881944444444444,
+        "coverage": 0.85,
+        "false_abstain": 0.15,
+        "null_recall": "n/a"
+      }
+    },
+    "typed_decisions_test": {
+      "raw": {
+        "nll": 1.2050029006817273,
+        "brier": 0.23359803910932034,
+        "jsd": 0.10082391914655038,
+        "acc": 0.54,
+        "acc_k": 0.54,
+        "ece": 0.08852838679154712,
+        "sel_acc80": 0.5691666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.1675730711137542,
+        "brier": 0.22356043087364605,
+        "jsd": 0.09743992727868328,
+        "acc": 0.54,
+        "acc_k": 0.54,
+        "ece": 0.06417149796088538,
+        "sel_acc80": 0.5683333333333334,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "ordinal": {
+        "ordinal_mae": 0.5883333333333334,
+        "exp_score_err": 0.7085446309794221,
+        "n": 600
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.013534486293792725,
+        "mean_abs_diff": 0.0021557581052184106
+      }
+    },
+    "typed_decisions_train": {
+      "raw": {
+        "nll": 1.416492816208917,
+        "brier": 0.25422257216150745,
+        "jsd": 0.10777533042065943,
+        "acc": 0.4266666666666667,
+        "acc_k": 0.4266666666666667,
+        "ece": 0.18357078438997268,
+        "sel_acc80": 0.42916666666666664,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.3595566999012847,
+        "brier": 0.2359260337969907,
+        "jsd": 0.1005492616508014,
+        "acc": 0.4266666666666667,
+        "acc_k": 0.4266666666666667,
+        "ece": 0.1572492956717809,
+        "sel_acc80": 0.4266666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "ordinal": {
+        "ordinal_mae": 0.75,
+        "exp_score_err": 0.7864355162782587,
+        "n": 600
+      },
+      "noul_reversed": {
+        "max_abs_diff": 0.011488527059555054,
+        "mean_abs_diff": 0.003036180833975474
+      }
+    },
+    "data_wh_val": {
+      "raw": {
+        "nll": 0.2300384575464517,
+        "brier": 0.14204218823632153,
+        "jsd": 0.06341173013641205,
+        "acc": 0.894,
+        "acc_k": 0.9278737791134485,
+        "ece": 0.015060881972312945,
+        "sel_acc80": 0.9716666666666667,
+        "conf_wrong": 0.005917159763313609,
+        "coverage": 0.8546666666666667,
+        "false_abstain": 0.048835462058602556,
+        "null_recall": 0.9053254437869822,
+        "auroc_null": 0.9826530748336214,
+        "auroc_conf": 0.9783230120165911
+      },
+      "scaled": {
+        "nll": 0.23008159136699488,
+        "brier": 0.14167703723149266,
+        "jsd": 0.0641277525235366,
+        "acc": 0.8926666666666667,
+        "acc_k": 0.9278737791134485,
+        "ece": 0.016159646193186408,
+        "sel_acc80": 0.9733333333333334,
+        "conf_wrong": 0.005917159763313609,
+        "coverage": 0.8533333333333333,
+        "false_abstain": 0.05033809166040571,
+        "null_recall": 0.9053254437869822,
+        "auroc_null": 0.9826530748336215,
+        "auroc_conf": 0.9778339905485487
+      }
+    },
+    "wh_heldout_family": {
+      "raw": {
+        "nll": 0.3386832933282492,
+        "brier": 0.19001383971199456,
+        "jsd": 0.08311882716530435,
+        "acc": 0.8649289099526066,
+        "acc_k": 0.8938992042440318,
+        "ece": 0.04368148955971149,
+        "sel_acc80": 0.9408284023668639,
+        "conf_wrong": 0.0,
+        "coverage": 0.8625592417061612,
+        "false_abstain": 0.04774535809018567,
+        "null_recall": 0.8888888888888888,
+        "auroc_null": 0.9831417624521073,
+        "auroc_conf": 0.9725316828765105
+      },
+      "scaled": {
+        "nll": 0.3327042389265942,
+        "brier": 0.1900852589651073,
+        "jsd": 0.08425147199473242,
+        "acc": 0.8649289099526066,
+        "acc_k": 0.8938992042440318,
+        "ece": 0.04666075068062519,
+        "sel_acc80": 0.9408284023668639,
+        "conf_wrong": 0.0,
+        "coverage": 0.8625592417061612,
+        "false_abstain": 0.04774535809018567,
+        "null_recall": 0.8888888888888888,
+        "auroc_null": 0.9831417624521073,
+        "auroc_conf": 0.9718243442381373
+      }
+    },
+    "wh_heldout_grammar": {
+      "raw": {
+        "nll": 0.2637673505050913,
+        "brier": 0.15817524496433566,
+        "jsd": 0.07089041621921395,
+        "acc": 0.8799019607843137,
+        "acc_k": 0.914364640883978,
+        "ece": 0.024724171843891023,
+        "sel_acc80": 0.9601837672281777,
+        "conf_wrong": 0.0,
+        "coverage": 0.8480392156862745,
+        "false_abstain": 0.05386740331491713,
+        "null_recall": 0.9239130434782609,
+        "auroc_null": 0.9828849387460965,
+        "auroc_conf": 0.9772249579630075
+      },
+      "scaled": {
+        "nll": 0.2635058749685546,
+        "brier": 0.15850269393002997,
+        "jsd": 0.07194949020789403,
+        "acc": 0.8799019607843137,
+        "acc_k": 0.914364640883978,
+        "ece": 0.025690338117819185,
+        "sel_acc80": 0.9586523736600306,
+        "conf_wrong": 0.0,
+        "coverage": 0.8480392156862745,
+        "false_abstain": 0.05386740331491713,
+        "null_recall": 0.9239130434782609,
+        "auroc_null": 0.9828849387460965,
+        "auroc_conf": 0.9767145087677156
+      }
+    },
+    "wh_heldout_style": {
+      "raw": {
+        "nll": 0.26686252182988546,
+        "brier": 0.15311981455702697,
+        "jsd": 0.06948256408727153,
+        "acc": 0.8858560794044665,
+        "acc_k": 0.8858560794044665,
+        "ece": 0.03080959015389527,
+        "sel_acc80": 0.9752321981424149,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.2633042948240913,
+        "brier": 0.1516918189603084,
+        "jsd": 0.07049474346320594,
+        "acc": 0.8858560794044665,
+        "acc_k": 0.8858560794044665,
+        "ece": 0.029319018938996952,
+        "sel_acc80": 0.978328173374613,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "wh_level7": {
+      "raw": {
+        "nll": 0.8425315491747045,
+        "brier": 0.566066637960518,
+        "jsd": 0.22709053783966102,
+        "acc": 0.5443181818181818,
+        "acc_k": 0.5443181818181818,
+        "ece": 0.17712931768460705,
+        "sel_acc80": 0.5738636363636364,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.815751176572115,
+        "brier": 0.5531539212337735,
+        "jsd": 0.22560885355170332,
+        "acc": 0.5443181818181818,
+        "acc_k": 0.5443181818181818,
+        "ece": 0.15701854222200134,
+        "sel_acc80": 0.5724431818181818,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "wh_rubric_flip": {
+      "raw": {
+        "nll": 0.37024391536252643,
+        "brier": 0.2251475372121566,
+        "jsd": 0.09686676026339679,
+        "acc": 0.832,
+        "acc_k": 0.8517715112075199,
+        "ece": 0.03836428254842761,
+        "sel_acc80": 0.9116666666666666,
+        "conf_wrong": 0.0,
+        "coverage": 0.898,
+        "false_abstain": 0.03398409255242227,
+        "null_recall": 0.905982905982906,
+        "auroc_null": 0.9883258863736086,
+        "auroc_conf": 0.9758978067004097
+      },
+      "scaled": {
+        "nll": 0.36406225678752874,
+        "brier": 0.22335213586456443,
+        "jsd": 0.09762219605986137,
+        "acc": 0.8326666666666667,
+        "acc_k": 0.8517715112075199,
+        "ece": 0.03447558822234474,
+        "sel_acc80": 0.9125,
+        "conf_wrong": 0.0,
+        "coverage": 0.8973333333333333,
+        "false_abstain": 0.03398409255242227,
+        "null_recall": 0.9145299145299145,
+        "auroc_null": 0.9883258863736086,
+        "auroc_conf": 0.9749893394145022
+      }
+    },
+    "wh_rubric_shuffled": {
+      "raw": {
+        "nll": 3.3784176615021364,
+        "brier": 1.0176747777780963,
+        "jsd": 0.36765612382768675,
+        "acc": 0.374,
+        "acc_k": 0.4036697247706422,
+        "ece": 0.466156586488088,
+        "sel_acc80": 0.3441666666666667,
+        "conf_wrong": 0.18072289156626506,
+        "coverage": 0.9299999999999999,
+        "false_abstain": 0.0465772759350741,
+        "null_recall": 0.46987951807228917,
+        "auroc_null": 0.9583967486034469,
+        "auroc_conf": 0.7938543163479606
+      },
+      "scaled": {
+        "nll": 3.0621024358996203,
+        "brier": 1.0025028615908116,
+        "jsd": 0.36380861609352405,
+        "acc": 0.37333333333333335,
+        "acc_k": 0.4036697247706422,
+        "ece": 0.4562540921767553,
+        "sel_acc80": 0.3458333333333333,
+        "conf_wrong": 0.1686746987951807,
+        "coverage": 0.9293333333333333,
+        "false_abstain": 0.04728299223712068,
+        "null_recall": 0.46987951807228917,
+        "auroc_null": 0.9583967486034469,
+        "auroc_conf": 0.7857258249653519
+      }
+    },
+    "data_u_val": {
+      "raw": {
+        "nll": 0.4886842598845047,
+        "brier": 0.04234767853598601,
+        "jsd": 0.018090470816900562,
+        "acc": 0.8953333333333333,
+        "acc_k": 0.8953333333333333,
+        "ece": 0.12599515644709272,
+        "sel_acc80": 0.9616666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.49177310430162957,
+        "brier": 0.04392926004492564,
+        "jsd": 0.019119015669118427,
+        "acc": 0.8953333333333333,
+        "acc_k": 0.8953333333333333,
+        "ece": 0.14489340595404307,
+        "sel_acc80": 0.9616666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "u_chaosnli": {
+      "raw": {
+        "nll": 1.0139885174394003,
+        "brier": 0.17014809164519165,
+        "jsd": 0.06626210261670297,
+        "acc": 0.5166666666666667,
+        "acc_k": 0.5166666666666667,
+        "ece": 0.10358940768241887,
+        "sel_acc80": 0.5341666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 1.0020134621764547,
+        "brier": 0.16190185540335636,
+        "jsd": 0.06505670021636623,
+        "acc": 0.5166666666666667,
+        "acc_k": 0.5166666666666667,
+        "ece": 0.08189480453729626,
+        "sel_acc80": 0.5341666666666667,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "u_real_heldout": {
+      "raw": {
+        "nll": 0.6395687144671027,
+        "brier": 0.11684180579038546,
+        "jsd": 0.055302365340723074,
+        "acc": 0.6583629893238434,
+        "acc_k": 0.6583629893238434,
+        "ece": 0.04802694863696116,
+        "sel_acc80": 0.7,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.6408081760955402,
+        "brier": 0.11675911344973736,
+        "jsd": 0.05675639741980845,
+        "acc": 0.6583629893238434,
+        "acc_k": 0.6583629893238434,
+        "ece": 0.06303854366214252,
+        "sel_acc80": 0.7022222222222222,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    },
+    "u_synthetic_heldout": {
+      "raw": {
+        "nll": 0.6619388393182489,
+        "brier": 0.029335292468287227,
+        "jsd": 0.013479520479808405,
+        "acc": 0.9073684210526316,
+        "acc_k": 0.9073684210526316,
+        "ece": 0.17639653883482279,
+        "sel_acc80": 0.9815789473684211,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      },
+      "scaled": {
+        "nll": 0.6541064203422494,
+        "brier": 0.025880943825774404,
+        "jsd": 0.012156440703303576,
+        "acc": 0.9073684210526316,
+        "acc_k": 0.9073684210526316,
+        "ece": 0.19372765189722962,
+        "sel_acc80": 0.9789473684210527,
+        "coverage": 1.0,
+        "false_abstain": 0.0,
+        "null_recall": "n/a"
+      }
+    }
+  }
+}