Update weights and evaluation

MERGE_REPORT.json

@@ -1,15 +1,15 @@
 {
   "snapshot": "mistralai/Ministral-3-3B-Instruct-2512-BF16@b6d637bef2393152b3da2b2fde72eecdee30557e",
-  "adapter": "StandardOne-3B-LoRA (training run final2-3b-long)",
-  "adapter_sha256": "ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf",
+  "adapter": "StandardOne-3B-LoRA (training run hard25-3b-r1)",
+  "adapter_sha256": "7bc8bfff4be52e0c9c6f605faf1eebc5e8a966ce103793c7f6970e29056ef5d3",
   "changed_params": 182,
   "unchanged_params": 276,
   "changed_outside_lm_projections": [],
-  "max_abs_delta": 0.002506256103515625,
+  "max_abs_delta": 0.0022978782653808594,
   "changed_sample": [
     [
       "model.language_model.layers.0.self_attn.q_proj.weight",
-      0.001220703125
+      0.0011186599731445312
     ],
     [
       "model.language_model.layers.0.self_attn.k_proj.weight",
@@ -17,19 +17,19 @@
     ],
     [
       "model.language_model.layers.0.self_attn.v_proj.weight",
-      0.001373291015625
+      0.0014190673828125
     ],
     [
       "model.language_model.layers.0.self_attn.o_proj.weight",
-      0.000946044921875
+      0.001007080078125
     ],
     [
       "model.language_model.layers.0.mlp.gate_proj.weight",
-      0.00200653076171875
+      0.00202178955078125
     ],
     [
       "model.language_model.layers.0.mlp.up_proj.weight",
-      0.0017948150634765625
+      0.0017843246459960938
     ],
     [
       "model.language_model.layers.0.mlp.down_proj.weight",
@@ -37,7 +37,7 @@
     ],
     [
       "model.language_model.layers.1.self_attn.q_proj.weight",
-      0.00146484375
+      0.001434326171875
     ]
   ],
   "file_sha256": {
@@ -45,15 +45,15 @@
     "chat_template.jinja": "0701cfbdc2b7d44fdbad104dff604faee4b0543e8247624568777fe465746f9b",
     "config.json": "c4446cf970fefd8b65f2e3119c30d406ea7026dca362dfb319c9aa08643131c3",
     "generation_config.json": "e0923390059f84a9180b00e5501778acc45ea9856cd7f2fd68208b360927c677",
-    "model-00001-of-00002.safetensors": "6f675d7bb2615275072980b36a0d94356c4b866d829b1cfeb472294a0caabc57",
-    "model-00002-of-00002.safetensors": "edcb0fd56e0a5da4e6c057166ac812587636e8cda20fa23776d465d448f3dfdd",
+    "model-00001-of-00002.safetensors": "8a02fdf56e2c3fdf75b303a1c617ed8bebe06d7e934e7ef56af3689a7a2552a5",
+    "model-00002-of-00002.safetensors": "1d8099875a6e4466dace9ceab20c38ffee02903dbb2acee9d5b0565b6e64da2a",
     "model.safetensors.index.json": "55641440eb385d8bc83a319b0a92f60fe45e6aef4b6992245eee91142237786e",
     "params.json": "d15587c57574d2a92c66ff2552c6db5173d4c440ad47f7454dc51bad0ecaeb59",
-    "processor_config.json": "84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491",
+    "processor_config.json": "ece2373c2ae391bce18785a1810543bc6173a1f28b6767cffab2c35dbea5f002",
     "special_tokens_map.json": "0a5c981e8c5c6f8886ee007a6d4543a0be6b221cb9ca32a8709384a4c6fc8cbb",
     "tekken.json": "600bb27946565481ecf51ba8aee252e49b9a68507866080ac9c30185bb312843",
     "tokenizer.json": "d5f6046775b112f0e2d456ee9dba450684ab964fe5c4e231599bdc6773028135",
-    "tokenizer_config.json": "de368f6e0f14b41c380b29278159f8897f01ec6b3208a91096d2deaf2bcd19f9"
+    "tokenizer_config.json": "f59f7294e4f26383d0ea93840fe21cf197784be0842a8301a0343e8c34ed0d6d"
   },
-  "seconds": 64.6
+  "seconds": 26.7
 }

QUICKSTART.md

@@ -40,7 +40,7 @@ jev-adapter --engine-url http://127.0.0.1:30000 --model standard-one-8b --alias
   --host 0.0.0.0 --port 30120 --max-concurrency 1 \
   --tokenizer-model mistralai/Ministral-3-8B-Instruct-2512-BF16 \
   --tokenizer-revision f6fae9795746f63c9be8344932f01275f3c63734 \
-  --prompt-wording native --native-system-prompt none --default-temperature 1.35
+  --prompt-wording native --native-system-prompt none --default-temperature 1.65
 ```
 
 6. Check health and models.
@@ -105,7 +105,7 @@ jev-adapter --engine-url http://127.0.0.1:30000 --model standard-one-3b --alias
   --host 0.0.0.0 --port 30120 --max-concurrency 1 \
   --tokenizer-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \
   --tokenizer-revision b6d637bef2393152b3da2b2fde72eecdee30557e \
-  --prompt-wording native --native-system-prompt none --default-temperature 1.45
+  --prompt-wording native --native-system-prompt none --default-temperature 1.55
 ```
 
 4. Check health, then send a request or run JevBench exactly as in steps 6-9 above, with

README.md

@@ -45,16 +45,14 @@ for the measurement conditions and limitations.
 ![Standard One benchmark card: JevBench public tiers, held-out suites, stated-distribution probability, hard-tier calibration, latency and throughput for Standard One 8B, Standard One 3B and Jev 1.13.](docs/assets/00-benchmark-card.png)
 
 *The figure combines results from different measurement paths. See [Benchmarks](#benchmarks) for
-served versus offline conditions; measured 24 September 2026.*
+served versus offline conditions; measured 24–25 September 2026.*
 
 ## At a glance
 
 - Send a state and a bounded rubric to receive probabilities for the supplied labels:
   `choice` selects among labeled options, `noul` is yes/no, and `score` uses an ordinal scale.
   The endpoint scores the labels in one forward pass without decoding answer text.
-- In the same-run **offline** comparison with its untuned base, 3B improves on all six suites; the
-  public hard gain is just 1.80 percentage points. These are not served-endpoint results, and that
-  small hard-tier gain is not evidence of broad hard-task improvement.
+- In the same-run **offline** comparison with its untuned base, 3B improves on five of six suites and declines by 0.90 percentage points on public hard. These are not served-endpoint results.
 - Probabilities are temperature-scaled and calibration-checked (hard-tier ECE, distribution
   total-variation) — see Benchmarks below.
 - The shared training mixture covers English, Japanese, Chinese, Spanish, French, German,
@@ -91,7 +89,7 @@ Adapter:
   --host 0.0.0.0 --port 30120 --max-concurrency 1 \
   --tokenizer-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \
   --tokenizer-revision b6d637bef2393152b3da2b2fde72eecdee30557e \
-  --prompt-wording native --native-system-prompt none --default-temperature 1.45
+  --prompt-wording native --native-system-prompt none --default-temperature 1.55
 ```
 
 Try it:
@@ -110,7 +108,7 @@ curl -s http://127.0.0.1:30120/v1/systemone -X POST -H 'content-type: applicatio
 }'
 ```
 
-Response shape (illustrative values, default temperature applied):
+Response shape (example values, default temperature applied):
 
 ```json
 {
@@ -119,7 +117,7 @@ Response shape (illustrative values, default temperature applied):
   "usage": {"input_tokens": 96, "output_tokens": 0},
   "metadata": {
     "confidence_method": "1 - normalized_entropy",
-    "temperature": 1.45,
+    "temperature": 1.55,
     "evaluations": 1,
     "adapter_elapsed_ms": 22.4
   }
@@ -132,19 +130,19 @@ More (client command, 8B variant, request format): see
 ## Benchmarks
 
 **Served endpoint results (the release configuration).** Merged BF16 weights through SGLang 0.5.20
-and `jev-adapter`, native wording, no system prompt, one option order, T=1.45. Jev 1.13 was
+and `jev-adapter`, native wording, no system prompt, one option order, T=1.55. Jev 1.13 was
 measured on the same items through its hosted endpoint; its probabilities are raw, with no
 temperature applied. These are our measurements, not official sealed-set JevBench scores.
 
 | Suite | **Standard One 3B** | Jev 1.13 |
 |---|---|---|
 | JevBench public easy (48) | **100.00 %** | 100.00 % |
-| JevBench public standard (72) | **86.11 %** | 98.61 % |
-| JevBench public hard (111) | **48.65 %** | 72.07 % |
-| judge proxy (600: routing + answer adequacy) | **85.67 %** | 90.50 % |
-| realistic transfer set (600) | **88.17 %** | 86.67 % |
-| stated-distribution probability (1,036) | **80.12 %** | 72.97 % |
-| hard proxy (600) | **43.67 %** | 54.83 % |
+| JevBench public standard (72) | **88.89 %** | 98.61 % |
+| JevBench public hard (111) | **45.95 %** | 72.07 % |
+| judge proxy (600: routing + answer adequacy) | **89.00 %** | 90.50 % |
+| realistic transfer set (600) | **90.83 %** | 86.67 % |
+| stated-distribution probability (1,036) | **80.50 %** | 72.97 % |
+| hard proxy (600) | **42.17 %** | 54.83 % |
 
 **Offline comparison with the untuned base.** This separate `transformers` runner used native
 wording, no system prompt, one option order, T=1. The base and tuned checkpoint were scored by the
@@ -153,16 +151,15 @@ same offline path; these numbers are indicative of the base-model change, not th
 | Suite | Untuned base | **Standard One 3B** |
 |---|---|---|
 | JevBench public easy (48) | 97.92 % | **100.00 %** |
-| JevBench public standard (72) | 70.83 % | **91.67 %** |
-| JevBench public hard (111) | 47.75 % | **49.55 %** |
-| judge proxy (600: routing + answer adequacy) | 63.17 % | **85.67 %** |
-| realistic transfer set (600) | 62.50 % | **88.67 %** |
-| stated-distribution probability (1,036) | 31.56 % | **79.73 %** |
-
-Against the untuned base, all six suites improve in this offline run; the public hard gain is only
-1.80 percentage points. Served and offline probabilities differ even on identical prompts, so use
-the served table for expected endpoint behavior. Hard-tier ECE at served T is 0.158 (T=1.45) against
-Jev 1.13's 0.099 raw, and mean TV to the stated distributions is 0.135 at served T against Jev's
+| JevBench public standard (72) | 70.83 % | **93.06 %** |
+| JevBench public hard (111) | 47.75 % | **46.85 %** |
+| judge proxy (600: routing + answer adequacy) | 63.17 % | **88.83 %** |
+| realistic transfer set (600) | 62.50 % | **90.67 %** |
+| stated-distribution probability (1,036) | 31.56 % | **79.83 %** |
+
+Against the untuned base, five suites improve in this offline run; public hard falls by 0.90 percentage points. Served and offline probabilities differ even on identical prompts, so use
+the served table for expected endpoint behavior. Hard-tier ECE at served T is 0.198 (T=1.55) against
+Jev 1.13's 0.099 raw, and mean TV to the stated distributions is 0.139 at served T against Jev's
 0.192 raw; full calibration table: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#calibration).
 
 **Speed** — raw serial latency on one H200 with SGLang 0.5.20, using a 242-decision profile averaging
@@ -179,16 +176,16 @@ for speed only; no accuracy comparison is implied.
 
 Throughput has been measured for the 8B only (see `docs/BENCHMARKS.md`); no equivalent sweep has been run for the 3B.
 
-On the public classification and decision suites (400 cases/suite, seed 13, served endpoints): AG News 83.8 %, typed decisions 69.3 %, MASSIVE intent mean 84.0 %, email spam 91.8 %, phishing 89.5 %. Full table, per-language and per-workflow breakdown: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#public-classification-and-decision-suites) and [`docs/public-classification-suites.md`](docs/public-classification-suites.md).
+On the public classification and decision suites (400 cases/suite, seed 13, served endpoints): AG News 83.0 %, typed decisions 69.0 %, MASSIVE intent mean 83.3 %, email spam 93.0 %, phishing 89.8 %. Full table, per-language and per-workflow breakdown: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#public-classification-and-decision-suites) and [`docs/public-classification-suites.md`](docs/public-classification-suites.md).
 
 A JevBench v1.4.1 run has been requested; the sealed-set result is not yet available. Full report: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md).
 
 ## Model details
 
 - **Base model:** `mistralai/Ministral-3-3B-Instruct-2512-BF16`, revision `b6d637bef2393152b3da2b2fde72eecdee30557e` (Apache-2.0).
-- **Adapter:** LoRA r=16, α=32, dropout 0, on `q_proj k_proj v_proj o_proj gate_proj up_proj down_proj` of the language-model projections only (vision tower and multimodal projector excluded), **24,707,072** trainable parameters, PEFT 0.21.0. Adapter file `adapter_model.safetensors`, 135,113,048 bytes, sha256 `ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf`.
-- **Merged BF16 checkpoint:** merging the adapter into the base changed **182 tensors**, none of them outside the language-model projections, maximum absolute weight change **0.0025**.
-- **Serving details:** native chat-template wording, served without a system prompt, at a fixed temperature (T = 1.45, fitted on held-out calibration data); served model name `standard-one-3b` behind stock SGLang 0.5.20 via `jev-adapter` (`POST /v1/systemone`); single, caller-supplied option order, no rotation ensemble; 8,192-token context.
+- **Adapter:** LoRA r=16, α=32, dropout 0, on `q_proj k_proj v_proj o_proj gate_proj up_proj down_proj` of the language-model projections only (vision tower and multimodal projector excluded), **24,707,072** trainable parameters, PEFT 0.21.0. Adapter file `adapter_model.safetensors`, 135,113,048 bytes, sha256 `7bc8bfff4be52e0c9c6f605faf1eebc5e8a966ce103793c7f6970e29056ef5d3`.
+- **Merged BF16 checkpoint:** merging the adapter into the base changed **182 tensors**, none of them outside the language-model projections, maximum absolute weight change **0.0023**.
+- **Serving details:** native chat-template wording, served without a system prompt, at a fixed temperature (T = 1.55, fitted on held-out calibration data); served model name `standard-one-3b` behind stock SGLang 0.5.20 via `jev-adapter` (`POST /v1/systemone`); single, caller-supplied option order, no rotation ensemble; 8,192-token context.
 
 | Path | Contents |
 |---|---|
@@ -201,25 +198,24 @@ The server code and full quick-start guide live in `StandardThinking/StandardOne
 
 ## Training data
 
-Trains on the identical mixture as `StandardThinking/StandardOne-8B` — same cohorts, same row counts, same overlap-audit result, not a reduced subset. Training data spans document and field normalisation, judge/routing and answer-adequacy, stated-distribution probability, verification/scoring, adequacy-rubric style, and game- and Tetris board-state cohorts — every row is synthetic and solver-generated (deterministic generators with visible checkers; a subset was teacher-reviewed), licensed Apache-2.0 (ours); the JevBench public tiers used only for evaluation carry MIT. Full per-cohort breakdown: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#training-data-provenance).
+Trains on the identical mixture as `StandardThinking/StandardOne-8B` — same cohorts, same row counts, same overlap-audit result, not a reduced subset. Training data spans document and field normalisation, judge/routing and answer-adequacy, stated-distribution probability, verification/scoring, adequacy-rubric style, abstention and robustness (paraphrase, trap, hard natural-language), and game- and Tetris board-state cohorts — every row is synthetic and solver-generated (deterministic generators with visible checkers; a subset was teacher-reviewed), licensed Apache-2.0 (ours); the JevBench public tiers used only for evaluation carry MIT. Full per-cohort breakdown: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#training-data-provenance).
 
-An exact-text overlap audit against the public JevBench tiers found 0 exact scenario matches and 181 exact instruction matches — rows in two adequacy-rubric cohorts whose entire instruction field, a generic 58-character adequacy question, is byte-identical to one public hard-tier instruction (0.05 % of the 359,497-row training mixture). These rows are kept and disclosed here rather than regenerated, since the overlap is limited to one rubric question's wording and never touches a scenario or an answer.
+An exact-text overlap audit against the public JevBench tiers found 0 exact scenario matches and 181 exact instruction matches — rows in two adequacy-rubric cohorts whose entire instruction field, a generic 58-character adequacy question, is byte-identical to one public hard-tier instruction (0.05 % of the 382,576-row training mixture). These rows are kept and disclosed here rather than regenerated, since the overlap is limited to one rubric question's wording and never touches a scenario or an answer.
 
 ## Limitations
 
-- **Public hard tier:** the served 3B score is **48.65 %**, versus **72.07 %** for Jev 1.13. In the
-  separate offline base comparison, Standard One 3B scores **49.55 %**, only **1.80 percentage
-  points** above the untuned base's **47.75 %**.
+- **Public hard tier:** the served 3B score is **45.95 %**, versus **72.07 %** for Jev 1.13. In the
+  separate offline base comparison, Standard One 3B scores **46.85 %**, **0.90 percentage points** below the untuned base's **47.75 %**.
 - Served probabilities are temperature-scaled by a single global value; if you apply this model to a
   materially different question distribution, re-fitting that temperature is advisable rather than
-  assuming 1.45 transfers.
+  assuming 1.55 transfers.
 - At most 26 options per question (one uppercase letter per option, `A`–`Z`).
 - The sealed JevBench set has not been measured for this model.
-- Served and offline probabilities can differ on identical prompts (mean total-variation ≈0.07 on the
+- Served and offline probabilities can differ on identical prompts (mean total-variation ≈0.07–0.09 on the
   hard tier); served numbers are treated as authoritative.
 - Korean is a small share of multilingual training alongside English.
 - The card reports text benchmarks; it does not establish decision accuracy on image inputs.
-- Ten-way support triage (34–40 %) and RAG passage relevance (58–61 %) are weak zero-shot; fine-tune
+- Ten-way support triage (35–40 %) and RAG passage relevance (57–60 %) are weak zero-shot; fine-tune
   for those.
 
 ## Licence

SHA256SUMS

@@ -1,36 +1,36 @@
 1495e1e757ef4d0925a2350563cf5754bb23c51701a8ec4fb3c5cdcbedae6747  LICENSE
-db003518d8d78b50128bd0166322cd0c230c199b047a01ff646ea6529538f05a  MERGE_REPORT.json
+6756de0e7de5d85d7f5b4ff2e5e75b4b74e2d2ad7f0addf5719e652ba53e15dc  MERGE_REPORT.json
 f10343f951ecb3a72670e12720de6e04d59e9cb872e84b2ceddd7d54853bb7e7  NOTICE
-cccd4aeea4b5ca9c98b6ea5f638e113175847526bfd748c488e417040ecb5930  QUICKSTART.md
-b93c7d30f112209abb6f2188084661dffb73179e1718c4de692610f57ddc7b67  README.md
+372d9160e95493d1d8e5c94d82b12fb0693fd36e4a50f5206d287ee74f189046  QUICKSTART.md
+8599881209b2f75da16e6f45608bbab66f76f0375ce67ead01bd42bbabb8bf61  README.md
 331b249682cd52226c50e533f59825184997f3b31f9a62b2c3fea940db6999c5  SYSTEM_PROMPT.txt
 0701cfbdc2b7d44fdbad104dff604faee4b0543e8247624568777fe465746f9b  chat_template.jinja
 c4446cf970fefd8b65f2e3119c30d406ea7026dca362dfb319c9aa08643131c3  config.json
-41f6da260cb61e59780081fa5b5e97c63413721889faf34f7ae4dc65fca7291b  docs/BENCHMARKS.md
-2300b40c53e596eacf93f152375c68bd78a9d77c03fb15ec17e5a76ca9f632bd  docs/assets/00-benchmark-card.png
-4d36cb45e5445a672de2dcbe0cfb273226354af8c842ca8d0925698033efe1a3  docs/assets/00-benchmark-card.svg
+868cb61534ddc6f57b8718113c54391b2d7fae46d9b72b6ae59b3c96d4994d2c  docs/BENCHMARKS.md
+96127670ad1e5bdd336a33118d6246330c27ac9e4fcb3f4f22767033809cd889  docs/assets/00-benchmark-card.png
+28c67a8c861f421fae5889af259d0850e7fb6eadb11f247adef72b94bf00b187  docs/assets/00-benchmark-card.svg
 7313f2b0645ee401d082572986dcee0c08d3daed1ae79fa650cfa685570ad7e2  docs/assets/02-latency-vs-qwen.png
 0b48a0dd4b4c9ac7c1842836940348edc2e1ba697184943f0c84c3e3d3c7f2d0  docs/assets/02-latency-vs-qwen.svg
 5f430f8841e393548f080833524840adb93deef1c6f75dca93d0571262a2ac68  docs/assets/04-throughput.png
 dd888be78c0e32894cf04a8e3fbf8757e7218873668fa4443ba47c7c1c48b6dc  docs/assets/04-throughput.svg
-f3211f7d1cbc624b0f558e7f9e7edac3ea51a381287cb7409719a12d3f11edaa  docs/assets/05-gain-over-base.png
-855f5e1d0871d89f5e92dbf8087282d787f9bf7c7c6a2b25b936cffd33f6c219  docs/assets/05-gain-over-base.svg
-4d41f1b782502403109c4074e7705fc42f18dfb9fc865b5aca5771f4c4991264  docs/assets/06-vs-jev.png
-946703a63a228ae328dc713186a6b16b682b193fcfbc6303e17fa143cbd58ddb  docs/assets/06-vs-jev.svg
-c3eacfddd6755a60b741d8929f3a41e84d2a862629dce7458ca96d248b4378fd  docs/public-classification-suites.md
-5da7c598decc1df7d385f835c378e4a1e14aaf8d54c25b84834f0649cd11f713  evidence/calibration-leaderboard-ablate-nosys.json
-b8c5441a673229a58223214f65b5150b67e659947dfd6edc6216df793c79975f  evidence/calibration-leaderboard-merged-native.json
-c887ca6b3d73571207c4f39a619c409685b217a4207b8d0486219650aafb5a71  evidence/served-nosys-easy-report.json
-08ef8e3a16bc49c865d44961d026d282df5647b4e3d3d023baa900ac76272636  evidence/served-nosys-hard-report.json
+2d056ba120ea239ee7a2f6840187589f779d5e6266185d835c36a4693984e4b8  docs/assets/05-gain-over-base.png
+0009567d9bd46390f95b6e8e8f1defd02c86b49c62bb40fec28431bf25725f4d  docs/assets/05-gain-over-base.svg
+837f07a1934509e3dffd4bc1b824d7c983b8dccd47457e5fd881a17f6b2e047c  docs/assets/06-vs-jev.png
+085a13c149a8245f046beeca014fe90b85a8faa379639c2cdedcd443ad941a4f  docs/assets/06-vs-jev.svg
+773cdf5156019a276bb7b7153a1962f38a9439a323644cd7a7291fb08a60788f  docs/public-classification-suites.md
+95188c27727bf11bde728f1e4dde96e5023501efb67fd2a790e41011cff16984  evidence/calibration-leaderboard-ablate-nosys.json
+2d70606bee5c231976b8d3efacbb6bde460373a5285ec23290140a12a963016e  evidence/calibration-leaderboard-merged-native.json
+0f433655d8e129dbbc11c31e3ed19cba8047719bffb2882b9c4e0b8eb4bc7085  evidence/served-nosys-easy-report.json
+8a7dc4ee7b607a650d787ce513b22064b0227c723514d56702c1529f505982b2  evidence/served-nosys-hard-report.json
 7b9b1ee2f93166a817ead61bbc0f4c62984afe7d6c8affefd051007582d0a485  evidence/served-nosys-latency-idle-gpu.json
-22b8e9b0ae331fb195271ae434083059041131ab3b317c6b56f6d7e93593001c  evidence/served-nosys-original-report.json
+5b217fe780b4c8c7cb292d57630b0f0a0bb5dcbe275f731272d6f0804cc14a68  evidence/served-nosys-original-report.json
 e0923390059f84a9180b00e5501778acc45ea9856cd7f2fd68208b360927c677  generation_config.json
-6f675d7bb2615275072980b36a0d94356c4b866d829b1cfeb472294a0caabc57  model-00001-of-00002.safetensors
-edcb0fd56e0a5da4e6c057166ac812587636e8cda20fa23776d465d448f3dfdd  model-00002-of-00002.safetensors
+8a02fdf56e2c3fdf75b303a1c617ed8bebe06d7e934e7ef56af3689a7a2552a5  model-00001-of-00002.safetensors
+1d8099875a6e4466dace9ceab20c38ffee02903dbb2acee9d5b0565b6e64da2a  model-00002-of-00002.safetensors
 55641440eb385d8bc83a319b0a92f60fe45e6aef4b6992245eee91142237786e  model.safetensors.index.json
 d15587c57574d2a92c66ff2552c6db5173d4c440ad47f7454dc51bad0ecaeb59  params.json
 ece2373c2ae391bce18785a1810543bc6173a1f28b6767cffab2c35dbea5f002  processor_config.json
-42352f35f7e9424e228f437943b78d67c007cf458613d9076371fb42d9d38a4f  release-manifest.json
+fa228b3ddc6db30e10a55b8b7039646572f53e66f96c04a416f34480e9415e54  release-manifest.json
 eef2a88a000d2ec294379755770c147327cb79d3d7278c711899bd36797d7a02  server/.gitignore
 1495e1e757ef4d0925a2350563cf5754bb23c51701a8ec4fb3c5cdcbedae6747  server/LICENSE
 0f1f023b916d38d7c12af3a71e09f5e82b819a7e156e4992118bb160ee63ede1  server/NOTICE

docs/BENCHMARKS.md

@@ -36,8 +36,8 @@ hard) are used as-is in both places.
 ## Scope
 
 - Both candidates: merged BF16 weights, LoRA r=16/α=32 on `q/k/v/o/gate/up/down` of the language model
-  only (vision tower + projector excluded), same three-stage training recipe on the **identical final
-  training mixture** (29 cohorts, 359,497 rows) — see `MODEL_CARD_8B.md` / `MODEL_CARD_3B.md` §
+  only (vision tower + projector excluded), same five-stage training recipe on the **identical final
+  training mixture** (34 cohorts, 382,576 rows) — see `MODEL_CARD_8B.md` / `MODEL_CARD_3B.md` §
   *Training* for the recipe and per-cohort breakdown, and `release-manifest.json` for the exact
   per-cohort row counts and hashes. The 3B is a smaller base/adapter pair, not a reduced-data variant.
   **Image input is supported** (the Pixtral vision tower and projector are retained; images are passed
@@ -45,7 +45,7 @@ hard) are used as-is in both places.
   detail image or content-moderation training or results further.
 - Release serving configuration for both: native wording, **no system prompt**
   (`--prompt-wording native --native-system-prompt none`), single option order, server-side default
-  temperature **T = 1.35 (8B)** / **T = 1.45 (3B)** — both refit directly on served probabilities
+  temperature **T = 1.65 (8B)** / **T = 1.55 (3B)** — both refit directly on served probabilities
   (see § Calibration). See `QUICKSTART.md` for the exact commands.
 - What is **not** covered here: any official JevBench sealed-set score (we have none yet), any additional peer-system comparison beyond Jev 1.13, and image/content-moderation
   training detail beyond the one sentence above.
@@ -60,14 +60,14 @@ Two different measurement paths appear throughout this report and are not interc
    external caller would use — see `QUICKSTART.md` for how to reproduce this endpoint.
 
 **These two paths do not agree exactly.** Served probabilities differ measurably from offline-runner
-probabilities for both candidates: mean total-variation distance ≈0.07 on the hard tier, with 8–12 of
+probabilities for both candidates: mean total-variation distance ≈0.07–0.09 on the hard tier, with 8–11 of
 111 argmax label changes, even though the rendered prompts were verified byte-identical between the two
 paths. **Treat served numbers as authoritative and offline numbers as indicative.**
 
-Temperature is **final and served-path-fit** for both candidates: T = 1.35 (8B) and T = 1.45 (3B) were
+Temperature is **final and served-path-fit** for both candidates: T = 1.65 (8B) and T = 1.55 (3B) were
 refit directly on served probabilities from the running endpoint, on
 `internal-eval-hardproxy-calib-v1` + `distprob-calib-v2`, no system prompt. This supersedes an earlier
-offline-harness fit (T = 1.5 / T = 1.3).
+offline-harness fit (T = 1.55 / T = 1.5).
 
 Most of the offline text-suite table below reflects the **with-system-prompt** configuration, while the
 release ships **without** the system prompt. Where we independently re-measured the no-system-prompt
@@ -81,14 +81,14 @@ columns are shown.
 | Suite | n | With system prompt | No system prompt (shipped) | Notes |
 |---|---|---|---|---|
 | JevBench public easy | 48 | 100.00 % | **100.00 %** | |
-| JevBench public original | 72 | 98.61 % | **97.22 %** | |
-| JevBench public hard | 111 | 56.76 % | **54.95 %** | never used for training/T-fitting/selection |
-| judge-proxy-v1 (routing + adequacy) | 600 | 89.33 % | **88.67 %** | leaderboard judge text not public; proxy only |
-| internal-eval-v2-realistic | 600 | 88.83 % | **88.83 %** | unchanged with/without system prompt |
-| distprob-val-v2 | 1,036 | 81.27 % | **81.08 %** | |
-| internal-eval-v1 | 600 | 78.50 % | not separately re-measured | |
-| internal-eval-hardproxy-v1 | 600 | 48.67 % | not separately re-measured | hard-tier proxy |
-| games-harness-eval-v1 | 360 | 97.78 % | not separately re-measured | othello/chess/tic-tac-toe/snake/graph, text |
+| JevBench public original | 72 | 95.83 % | **97.22 %** | |
+| JevBench public hard | 111 | 51.35 % | **52.25 %** | never used for training or T-fitting; informed the prompt-wording choice |
+| judge-proxy-v1 (routing + adequacy) | 600 | 89.00 % | **88.83 %** | leaderboard judge text not public; proxy only |
+| internal-eval-v2-realistic | 600 | 91.33 % | **90.00 %** | |
+| distprob-val-v2 | 1,036 | 80.89 % | **81.56 %** | |
+| internal-eval-v1 | 600 | 76.50 % | not separately re-measured | |
+| internal-eval-hardproxy-v1 | 600 | 53.00 % | not separately re-measured | hard-tier proxy |
+| games-harness-eval-v1 | 360 | 96.94 % | not separately re-measured | othello/chess/tic-tac-toe/snake/graph, text |
 | tetris-drought-eval-v1 (text) | 120 | 67.50 % | not separately re-measured | |
 
 ### Standard One 3B
@@ -96,58 +96,57 @@ columns are shown.
 | Suite | n | With system prompt | No system prompt (shipped) | Notes |
 |---|---|---|---|---|
 | JevBench public easy | 48 | 100.00 % | **100.00 %** | |
-| JevBench public original | 72 | 90.28 % | **91.67 %** | one of the few rows where dropping the prompt *helps* |
-| JevBench public hard | 111 | 52.25 % | **49.55 %** | never used for training/T-fitting/selection |
-| judge-proxy-v1 (routing + adequacy) | 600 | 86.83 % | **85.67 %** | |
-| internal-eval-v2-realistic | 600 | 88.83 % | **88.67 %** | |
-| distprob-val-v2 | 1,036 | 80.98 % | **79.73 %** | |
-| internal-eval-v1 | 600 | 77.17 % | not separately re-measured | |
-| internal-eval-hardproxy-v1 | 600 | 43.33 % | not separately re-measured | hard-tier proxy |
-| games-harness-eval-v1 | 360 | 95.83 % | not separately re-measured | |
-| tetris-drought-eval-v1 (text) | 120 | 64.17 % | not separately re-measured | |
+| JevBench public original | 72 | 93.06 % | **93.06 %** | unchanged with/without system prompt |
+| JevBench public hard | 111 | 48.65 % | **46.85 %** | never used for training or T-fitting; informed the prompt-wording choice |
+| judge-proxy-v1 (routing + adequacy) | 600 | 89.00 % | **88.83 %** | |
+| internal-eval-v2-realistic | 600 | 90.00 % | **90.67 %** | |
+| distprob-val-v2 | 1,036 | 81.18 % | **79.83 %** | |
+| internal-eval-v1 | 600 | 71.67 % | not separately re-measured | |
+| internal-eval-hardproxy-v1 | 600 | 43.50 % | not separately re-measured | hard-tier proxy |
+| games-harness-eval-v1 | 360 | 96.39 % | not separately re-measured | |
+| tetris-drought-eval-v1 (text) | 120 | 68.33 % | not separately re-measured | |
 
 ## Served parity
 
 Both offline and served measurement paths were run; served numbers below are authoritative, on the
 final served-path temperature fit:
 
-| | Standard One 8B (no system prompt, T=1.35) | Standard One 3B (no system prompt, T=1.45) |
+| | Standard One 8B (no system prompt, T=1.65) | Standard One 3B (no system prompt, T=1.55) |
 |---|---|---|
 | easy (48) | 100.00 % | 100.00 % |
-| original (72) | 95.83 % | 86.11 % |
-| hard (111) | 54.05 % | 48.65 % |
-| judge-proxy-v1 | 90.33 % | 85.67 % |
-| internal-eval-hardproxy-v1 | 48.33 % | 43.67 % |
-| internal-eval-v2-realistic | 89.50 % | 88.17 % |
-| distprob-val-v2 | 81.56 % | 80.12 % |
-| Calibration axis (leaderboard formula) | 75.9 | 77.5 |
+| original (72) | 94.44 % | 88.89 % |
+| hard (111) | 54.95 % | 45.95 % |
+| judge-proxy-v1 | 90.50 % | 89.00 % |
+| internal-eval-hardproxy-v1 | 52.50 % | 42.17 % |
+| internal-eval-v2-realistic | 90.33 % | 90.83 % |
+| distprob-val-v2 | 81.37 % | 80.50 % |
+| Calibration axis (leaderboard formula) | 78.1 | 73.3 |
 
 As noted above, served numbers differ somewhat from the corresponding offline numbers for both
-candidates (mean TV ≈0.07 hard tier, 8–12/111 argmax changes) — these are the figures used for the
+candidates (mean TV ≈0.07–0.09 hard tier, 8–11/111 argmax changes) — these are the figures used for the
 Accuracy does not depend on temperature, so the earlier T=1.0-forced served run
-(before the temperature refit) reported the same easy/original/hard accuracy: 8B 100.00/95.83/54.05, 3B
-100.00/87.50/48.65 no-system-prompt (100.00/88.89/46.85 with-system-prompt ablation).
+(before the temperature refit) reported the same easy/original/hard accuracy: 8B 100.00/94.44/54.95, 3B 100.00/88.89/45.95 no-system-prompt.
 
 ## Public classification and decision suites
 
 400 cases per suite, seed 13, sampled from public test splits, one request per case, served endpoints
 (merged BF16, SGLang 0.5.20, `jev-adapter`, native wording, no system prompt, single option order),
-measured 24 September 2026. Standard One only; no comparison column.
+measured 25 September 2026. Standard One only; no comparison column.
 
 | Suite | Standard One 8B | Standard One 3B |
 |---|---|---|
-| AG News (4 topics) | 84.8 % | 83.8 % |
-| DAIR Emotion (6 labels) | 57.0 % | 58.0 % |
-| Typed decisions (400 cases, 2,000 decisions) | 69.8 % | 69.3 % |
-| MASSIVE intent, 20 options, 9 languages, mean (en 88.0 / 85.8, ko 89.2 / 84.2) | 86.4 % | 84.0 % |
-| XNLI, 3 labels, 6 languages, mean | 61.4 % | 62.2 % |
-| Email spam | 87.2 % | 91.8 % |
-| Phishing | 84.8 % | 89.5 % |
-| Moderation (toxic vs benign) | 82.5 % | 78.8 % |
-| Model routing (domain) | 92.7 % | 87.2 % |
+| AG News (4 topics) | 84.2 % | 83.0 % |
+| DAIR Emotion (6 labels) | 56.5 % | 58.8 % |
+| Typed decisions (400 cases, 2,000 decisions) | 69.3 % | 69.0 % |
+| MASSIVE intent, 20 options, 9 languages, mean (en 88.0 / 85.5, ko 88.2 / 83.8) | 85.6 % | 83.3 % |
+| XNLI, 3 labels, 6 languages, mean | 60.1 % | 62.0 % |
+| Email spam | 92.2 % | 93.0 % |
+| Phishing | 86.0 % | 89.8 % |
+| Moderation (toxic vs benign) | 85.0 % | 81.5 % |
+| Model routing (domain) | 93.0 % | 83.7 % |
 
 Option-order flip rate (share of cases whose answer changes under a permuted option order) ranges from
-3.5 % (AG News) to 12 % (MASSIVE en) for the 8B. Full per-language and per-workflow numbers, macro-F1
+3.3 % (AG News) to 13 % (MASSIVE en) for the 8B. Full per-language and per-workflow numbers, macro-F1
 and ECE: [`public-classification-suites.md`](public-classification-suites.md).
 
 ## Latency and throughput
@@ -204,13 +203,13 @@ same host used for our own runs.
 
 | Suite (n) | Jev 1.13 | Standard One 8B | Standard One 3B |
 |---|---|---|---|
-| distprob-val-v2 (1,036) | 72.97 % | **81.56 %** | 80.12 % |
-| internal-eval-v2-realistic (600) | 86.67 % | **89.50 %** | 88.17 % |
-| judge-proxy-v1 (600) | **90.50 %** | 90.33 % | 85.67 % |
+| distprob-val-v2 (1,036) | 72.97 % | **81.37 %** | 80.50 % |
+| internal-eval-v2-realistic (600) | 86.67 % | 90.33 % | **90.83 %** |
+| judge-proxy-v1 (600) | **90.50 %** | **90.50 %** | 89.00 % |
 | JevBench easy (48) | 100.00 % | 100.00 % | 100.00 % |
-| JevBench standard (72) | **98.61 %** | 95.83 % | 86.11 % |
-| internal-eval-hardproxy-v1 (600) | **54.83 %** | 48.33 % | 43.67 % |
-| JevBench hard (111) | **72.07 %** | 54.05 % | 48.65 % |
+| JevBench standard (72) | **98.61 %** | 94.44 % | 88.89 % |
+| internal-eval-hardproxy-v1 (600) | **54.83 %** | 52.50 % | 42.17 % |
+| JevBench hard (111) | **72.07 %** | 54.95 % | 45.95 % |
 
 Jev leads on the public hard tier and the hard proxy; Standard One leads on stated-distribution probability and
 the realistic transfer set; judge and easy tie.
@@ -219,8 +218,8 @@ Calibration on the same items, Jev raw (no temperature applied):
 
 | | Jev 1.13 | Standard One 8B | Standard One 3B |
 |---|---|---|---|
-| Hard-tier ECE (10-bin, top label) | **0.099** | 0.232 raw (0.178 at T=1.35) | 0.227 raw (0.158 at T=1.45) |
-| Mean TV to stated distributions | **0.192** | 0.111 raw (0.127 at T) | 0.121 raw (0.135 at T) |
+| Hard-tier ECE (10-bin, top label) | **0.099** | 0.231 raw (0.147 at T=1.65) | 0.286 raw (0.198 at T=1.55) |
+| Mean TV to stated distributions | **0.192** | 0.109 raw (0.145 at T) | 0.112 raw (0.139 at T) |
 
 Latency, same 242-decision profile, same host:
 
@@ -234,17 +233,16 @@ Jev's latency includes network round-trip to a hosted API; Standard One's is loc
 
 ## Calibration
 
-- **T fit (release, no system prompt, served path, final)**: 8B T = 1.35, 3B T = 1.45, both refit
+- **T fit (release, no system prompt, served path, final)**: 8B T = 1.65, 3B T = 1.55, both refit
   directly on served probabilities from the running endpoint, on
   `internal-eval-hardproxy-calib-v1` + `distprob-calib-v2`. This is the final, shipped configuration.
-- **Historical reference (offline-harness fit, superseded)**: 8B T = 1.5 (no system prompt) / T = 1.4
-  (with system prompt, merged-checkpoint offline check, hard-tier ECE 0.211 at T=1, 0.148 at T=1.4);
-  3B T = 1.3 (no system prompt) / T = 1.35 (with system prompt, merged-checkpoint offline check,
-  hard-tier ECE 0.236 at T=1, 0.190 at T=1.35). These predate the served-path refit and are kept for
+- **Historical reference (offline-harness fit, superseded)**: 8B T = 1.55 (no system prompt) / T = 1.45
+  (with system prompt, merged-checkpoint offline check, hard-tier ECE 0.269 at T=1, 0.206 at T=1.45);
+  3B T = 1.5 (no system prompt) / T = 1.35 (with system prompt, merged-checkpoint offline check,
+  hard-tier ECE 0.269 at T=1, 0.212 at T=1.35). These predate the served-path refit and are kept for
   record only.
-- **Calibration axis (leaderboard formula)**: 8B **75.9**, 3B **77.5** (see § Served parity).
-- **Distribution (TV) fidelity**, `distprob-val-v2`, offline, with system prompt, at T=1: 8B mean TV
-  ≈0.107–0.128 across suite slices; 3B mean TV ≈0.118–0.128. See § *Measured against Jev 1.13* for the
+- **Calibration axis (leaderboard formula)**: 8B **78.1**, 3B **73.3** (see § Served parity).
+- **Distribution (TV) fidelity**, `distprob-val-v2`, offline, with system prompt, at T=1: 8B mean TV ≈0.103; 3B mean TV ≈0.110. See § *Measured against Jev 1.13* for the
   served-path TV comparison against Jev.
 
 ## Decision quality vs. the untuned base
@@ -254,19 +252,18 @@ Jev's latency includes network round-trip to a hosted API; Standard One's is loc
 *Accuracy change versus the untuned Ministral 3 base, same prompt, same offline runner, no system prompt, T=1. Public JevBench tiers are the 231 public items; the other suites are held-out proxies built from public data. The hard tier is placed last on purpose (see below).*
 
 The only base-model comparison in this report. On the public **hard** tier, the untuned Ministral 3 8B
-base is measured **ahead of Standard One 8B** (60.4 % vs. 55.0 %, a 6-of-111-item gap) — hard is 30 % of the
+base is measured **ahead of Standard One 8B** (60.4 % vs. 52.3 %, a 9-of-111-item gap) — hard is 30 % of the
 v1.3.0 Intelligence axis, so this is material, not a rounding artifact. All rows below come from one
-run of the same offline runner (native wording, no system prompt, T=1, single option order); the
-untuned 3B base is behind on every suite, the untuned 8B base is ahead only on the public hard tier.
+run of the same offline runner (native wording, no system prompt, T=1, single option order); the untuned 3B base is behind on every suite except the public hard tier (by 1 of 111 items), the untuned 8B base is ahead only on the public hard tier.
 
 | Suite | Base 8B | **Standard One 8B** | Base 3B | **Standard One 3B** |
 |---|---|---|---|---|
-| distprob-val-v2 (stated-distribution probability, 1,036) | 34.85 % | **81.08 %** (+46.2) | 31.56 % | **79.73 %** (+48.2) |
-| internal-eval-v2-realistic (realistic transfer set, 600) | 72.83 % | **88.83 %** (+16.0) | 62.50 % | **88.67 %** (+26.2) |
-| JevBench public standard tier (72) | 79.17 % | **97.22 %** (+18.0) | 70.83 % | **91.67 %** (+20.8) |
-| judge-proxy-v1 (routing + adequacy, 600) | 79.33 % | **88.67 %** (+9.3) | 63.17 % | **85.67 %** (+22.5) |
+| distprob-val-v2 (stated-distribution probability, 1,036) | 34.85 % | **81.56 %** (+46.7) | 31.56 % | **79.83 %** (+48.3) |
+| internal-eval-v2-realistic (realistic transfer set, 600) | 72.83 % | **90.00 %** (+17.2) | 62.50 % | **90.67 %** (+28.2) |
+| JevBench public standard tier (72) | 79.17 % | **97.22 %** (+18.0) | 70.83 % | **93.06 %** (+22.2) |
+| judge-proxy-v1 (routing + adequacy, 600) | 79.33 % | **88.83 %** (+9.5) | 63.17 % | **88.83 %** (+25.7) |
 | JevBench public easy tier (48) | 100.00 % | **100.00 %** (+0.0) | 97.92 % | **100.00 %** (+2.1) |
-| JevBench public hard tier (111) | 60.36 % | **54.95 %** (-5.4) | 47.75 % | **49.55 %** (+1.8) |
+| JevBench public hard tier (111) | 60.36 % | **52.25 %** (-8.1) | 47.75 % | **46.85 %** (-0.9) |
 
 
 ## Data provenance and licences
@@ -283,7 +280,7 @@ and in `release-manifest.json`, identical for both candidates.
 
 ## Training data provenance
 
-The following cohort families make up the identical 29-cohort, 359,497-row training mixture used by
+The following cohort families make up the identical 34-cohort, 382,576-row training mixture used by
 both Standard One 8B and Standard One 3B. Every row is synthetic and solver-generated (deterministic generators with
 visible checkers; a subset was teacher-reviewed). No JevBench evaluation scenario appears in any
 training cohort.
@@ -295,13 +292,14 @@ training cohort.
 | Stated-distribution probability (`distprob-scale30k`, `distprob-multilingual`, `distprob-pilot2k`, `narrative-prob-train`, `probability`) | Estimating a probability distribution over stated outcomes | Apache-2.0 (ours) |
 | Verification & scoring (`noul-score-10k`) | Yes/no verification and ordinal scoring | Apache-2.0 (ours) |
 | Adequacy-rubric style (`hardstyle-train`) | Hard-style adequacy questions (see § Public-overlap audit and disclosure) | Apache-2.0 (ours) |
+| Abstention & robustness (`abstain`, `paraphrase`, `trap`, `hard-natural`) | Abstaining when the state does not settle the question, paraphrased and adversarial wording, harder natural-language decisions | Apache-2.0 (ours) |
 | Game & board-state (`games-text-train`, `games-harness-train`, `games-harness-multilingual`) | Othello, chess, tic-tac-toe, snake and graph-problem decisions | Apache-2.0 (ours) |
 | Tetris board-state (`games-text-train-tetris`, `games-harness-tetris`, `tetris-drought-train`) | Tetris-specific board-state decisions | Apache-2.0 (ours) |
 | JevBench public tiers (evaluation only, not training) | Easy / standard / hard held-out accuracy measurement | MIT |
 
 ## Public-overlap audit and disclosure
 
-Run on the exact final 29-cohort, 359,497-row mixture that **both candidates train on**:
+Run on the exact final 34-cohort, 382,576-row mixture that **both candidates train on**:
 
 - **0 exact state matches** — no training row's state/scenario duplicates any public JevBench item.
 - **181 exact instruction matches**: `hardstyle-train-v1` (87 of 3,900 rows) and `hardstyle-train-v2`
@@ -313,7 +311,7 @@ Run on the exact final 29-cohort, 359,497-row mixture that **both candidates tra
   rubric sentence above (also in `judge-realistic-train-v3`, `judge-multilingual-train-v1`,
   `judge-proxy-v1`), an insurance "cancellation takes effect" boilerplate clause, and a "coverage
   reviewer" claims-adjudication phrase.
-- All other 27 cohorts: 0 exact state and 0 exact instruction matches.
+- All other 32 cohorts: 0 exact state and 0 exact instruction matches.
 
 **These 181 rows are kept and disclosed here, not regenerated.** They are 0.05 % of the mixture, and
 the exposure is limited to one hard-tier *question's* generic wording, never a *state* or an answer.
@@ -324,25 +322,25 @@ the exposure is limited to one hard-tier *question's* generic wording, never a *
 
 | File | sha256 |
 |---|---|
-| `adapter_model.safetensors` | `a2d9f02d5dd5d3ade3fab88145fc7321315609aa52d3847319dc4ec827f0c684` |
-| `adapter_config.json` | `a6a57c918f63358bc6c6b031319b124d843de612423906cdcdd777f38321fc95` |
-| `manifest.json` | `250891a115ee6c490b93fd9c9f0d63faf0a2d4cc446efd62481340c3a214b821` |
-| `MERGE_REPORT.json` | `5e98d117e0b06bf3d2272df65d49b8084469ed7718f579a9c68ee98483b04d70` |
-| `calibration-leaderboard.json` | `5384f476f04530a271a86a023fb73134633585f55d684cd30eaa58e8f57d9c2b` |
-| `latency-report-shared-gpu.json` | `2942f20a19632569828762558132706d7ac61b50660e7e6e36bed0c31d7846b5` |
-| `served-easy-report.json` | `62bd49df29c528c350bc779bf568f05a0f7715a9898ed538d5604053bc21ed29` |
-| `served-original-report.json` | `fe276e076192f037cf7e56a8495e666dbfd6ca42caad04b4548439ce70b8fa04` |
-| `served-hard-report.json` | `8a27f9cb5639eb63191dc27c08347f825556915cd6553a596c6a3ab597dbd67b` |
+| `adapter_model.safetensors` | `9551923c162970ea8bc2c3a5faf3ea3762b48591c9f460064e531d09427fa41a` |
+| `adapter_config.json` | `545101ebcafd9fb90723284518d744ca01de46534e04a5dfd4e8f86bc4de0ef2` |
+| `manifest.json` | `62bd99b6d4a49926840c258874a3b804a868626761fa57a39e8bfe10e3192904` |
+| `MERGE_REPORT.json` | `1f39839e969d874d397d3842eb702f39516ec784f546ce18220e9836bffb2793` |
+| `calibration-leaderboard.json` | `a28ada7681a3ae8edcce3f7885dec63ecfb9bc6a9c9794204fa040ae1dd1065f` |
+| `latency-report-shared-gpu.json` | `f6734e5cb4a210e9284632ef97c922876f4aaced3106461ec9cede475a640fa9` |
+| `served-easy-report.json` | `f68aa8f1d1bbc60b045592713188563643e07d0e8d8337df4df9387bf87254f3` |
+| `served-original-report.json` | `b87e0a9a48ebb9fa933fe0db470b506cbe1850824abf65dfebf6ad6c5f58cdda` |
+| `served-hard-report.json` | `c0eb0fde38c44b1f3f62f6a79cc45c3e89cad1f782d1c6cc40526f69e8595ff6` |
 
 `evidence/` (3B):
 
 | File | sha256 |
 |---|---|
-| `adapter_model.safetensors` | `ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf` |
-| `adapter_config.json` | `8326f9e4092ed74a3e68978df37bc9c0a5d8f47b90fbea41ea27d592dd9bda64` |
-| `manifest.json` | `67a81590d59cebe6da802682d1d5c017cff04e518e7fbb752609ed038d31772a` |
-| `calibration-leaderboard.json` | `555d8f7f34a30fcadf1d428e34f8084ee9e13c9f127128e66e808959c81c69ca` |
-| `offline-native-result.json` | `a8efd79e389fc125ce38ab6001186e190eb818982290568577f9147429809f46` |
+| `adapter_model.safetensors` | `7bc8bfff4be52e0c9c6f605faf1eebc5e8a966ce103793c7f6970e29056ef5d3` |
+| `adapter_config.json` | `43b99a646a535b15e0f8bceebe2cfff03dd8cc519f5e21f68856b885657d7179` |
+| `manifest.json` | `ecfd448e212e09df55c86c93233abe857282b8d9d6154dceddb681663b6ebea2` |
+| `calibration-leaderboard.json` | `de13ebd925123ade9f065d7ae3a3955e7f55ddc29ce1d598624039fcdc75567e` |
+| `offline-native-result.json` | `2a53c2dc7c82dcf971110e2f788c750c63a09c1c0ea844a371e6abdd087adf24` |
 
 `docs/assets/` (figures referenced by this report):
 
@@ -352,5 +350,5 @@ the exposure is limited to one hard-tier *question's* generic wording, never a *
 | `04-throughput.png` | `bdb8cb392f6f288fe7ce7fde78e39759cd688340be4fe605890dc79c58e83f73` |
 
 The 3B evidence set does not yet include a separate `MERGE_REPORT.json` or served-endpoint report
-files; the merge-tensor numbers (182 tensors changed, max |Δ| 0.0025) and served numbers (§ Served
+files; the merge-tensor numbers (182 tensors changed, max |Δ| 0.0023) and served numbers (§ Served
 parity) are recorded narratively in `MODEL_CARD_3B.md`.

docs/public-classification-suites.md

@@ -1,52 +1,52 @@
 # Public classification and decision suites
 
-Measured 24 September 2026 through the served endpoints (merged BF16, SGLang 0.5.20, jev-adapter, native wording, no system prompt, single option order). Each suite: 400 cases sampled with seed 13 from the public test split, one request per case; ECE (10-bin, top label) uses the served temperature (8B T=1.35, 3B T=1.45). Accuracy and macro-F1 are argmax metrics and do not depend on the temperature. banking77 (77 labels) is not runnable: the endpoint accepts at most 26 options per question.
+Measured 25 September 2026 through the served endpoints (merged BF16, SGLang 0.5.20, jev-adapter, native wording, no system prompt, single option order). Each suite: 400 cases sampled with seed 13 from the public test split, one request per case; ECE (10-bin, top label) uses the served temperature (8B T=1.65, 3B T=1.55). Accuracy and macro-F1 are argmax metrics and do not depend on the temperature. banking77 (77 labels) is not runnable: the endpoint accepts at most 26 options per question.
 
 ## Public datasets
 
 | Suite | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
 |---|---|---|---|---|---|---|---|
-| AG News (4 topics) | 400 | 84.8 | 83.8 | 83.3 | 82.3 | 0.095 | 0.106 |
-| DAIR Emotion (6 labels) | 400 | 57.0 | 58.0 | 47.2 | 48.3 | 0.217 | 0.052 |
+| AG News (4 topics) | 400 | 84.2 | 83.0 | 82.9 | 81.4 | 0.084 | 0.094 |
+| DAIR Emotion (6 labels) | 400 | 56.5 | 58.8 | 46.6 | 48.6 | 0.191 | 0.064 |
 | SST-5 (5-level sentiment, ordinal score) | 400 | — | — | — | — | — | — |
-| Typed decisions (400 cases, 2,000 decisions) | 400 | 69.8 | 69.3 | 61.4 | 60.5 | 0.060 | 0.070 |
+| Typed decisions (400 cases, 2,000 decisions) | 400 | 69.3 | 69.0 | 60.2 | 58.9 | 0.056 | 0.048 |
 
 ## Application workflows (400 cases each, public sources)
 
 | Suite | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
 |---|---|---|---|---|---|---|---|
-| Email spam | 400 | 87.2 | 91.8 | 87.2 | 91.7 | 0.038 | 0.054 |
-| Phishing | 400 | 84.8 | 89.5 | 82.5 | 88.4 | 0.064 | 0.042 |
-| LLM guardrails (jailbreak vs benign) | 400 | 58.0 | 63.0 | 56.9 | 61.0 | 0.198 | 0.139 |
-| Moderation (toxic vs benign) | 400 | 82.5 | 78.8 | 82.5 | 78.5 | 0.047 | 0.080 |
-| RAG passage relevance | 400 | 60.5 | 57.2 | 57.5 | 53.4 | 0.310 | 0.292 |
-| Support triage (10 queues) | 400 | 34.0 | 40.2 | 35.0 | 37.5 | 0.463 | 0.301 |
-| Model routing (domain) | 399 | 92.7 | 87.2 | 47.0 | 45.7 | 0.042 | 0.042 |
+| Email spam | 400 | 92.2 | 93.0 | 92.2 | 93.0 | 0.047 | 0.045 |
+| Phishing | 400 | 86.0 | 89.8 | 84.2 | 88.6 | 0.048 | 0.037 |
+| LLM guardrails (jailbreak vs benign) | 400 | 50.7 | 57.5 | 50.5 | 56.7 | 0.314 | 0.181 |
+| Moderation (toxic vs benign) | 400 | 85.0 | 81.5 | 85.0 | 81.4 | 0.079 | 0.046 |
+| RAG passage relevance | 400 | 60.2 | 57.2 | 56.9 | 53.7 | 0.307 | 0.289 |
+| Support triage (10 queues) | 400 | 35.2 | 39.5 | 36.1 | 37.5 | 0.435 | 0.359 |
+| Model routing (domain) | 399 | 93.0 | 83.7 | 56.5 | 52.8 | 0.048 | 0.046 |
 
 ## MASSIVE intent, 20 options (chance = 5 %)
 
 | Language | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
 |---|---|---|---|---|---|---|---|
-| en | 400 | 88.0 | 85.8 | 61.3 | 55.5 | 0.025 | 0.058 |
-| ja | 400 | 87.5 | 85.8 | 62.0 | 62.9 | 0.024 | 0.035 |
-| zh | 400 | 88.0 | 85.5 | 67.5 | 60.0 | 0.030 | 0.057 |
-| es | 400 | 82.5 | 82.0 | 59.0 | 52.8 | 0.020 | 0.043 |
-| fr | 400 | 85.8 | 84.5 | 59.9 | 56.7 | 0.034 | 0.061 |
-| de | 400 | 86.0 | 84.5 | 59.9 | 56.2 | 0.022 | 0.043 |
-| pt | 400 | 84.5 | 81.5 | 50.4 | 51.3 | 0.030 | 0.045 |
-| ru | 400 | 85.8 | 82.5 | 61.3 | 58.2 | 0.026 | 0.041 |
-| ko | 400 | 89.2 | 84.2 | 67.6 | 59.4 | 0.034 | 0.035 |
+| en | 400 | 88.0 | 85.5 | 61.7 | 54.3 | 0.024 | 0.055 |
+| ja | 400 | 87.0 | 86.0 | 60.1 | 65.2 | 0.032 | 0.037 |
+| zh | 400 | 86.0 | 84.0 | 63.2 | 60.7 | 0.042 | 0.054 |
+| es | 400 | 82.0 | 82.0 | 55.5 | 53.5 | 0.025 | 0.064 |
+| fr | 400 | 84.5 | 82.8 | 59.2 | 54.2 | 0.040 | 0.061 |
+| de | 400 | 85.8 | 83.8 | 62.6 | 52.8 | 0.039 | 0.043 |
+| pt | 400 | 83.5 | 80.5 | 53.4 | 48.5 | 0.044 | 0.056 |
+| ru | 400 | 85.0 | 81.8 | 62.0 | 55.5 | 0.038 | 0.062 |
+| ko | 400 | 88.2 | 83.8 | 67.2 | 58.0 | 0.034 | 0.029 |
 
 ## XNLI, 3 labels
 
 | Language | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
 |---|---|---|---|---|---|---|---|
-| en | 400 | 65.0 | 67.8 | 65.3 | 66.3 | 0.126 | 0.094 |
-| es | 400 | 58.2 | 63.5 | 58.0 | 62.4 | 0.170 | 0.077 |
-| fr | 400 | 63.5 | 64.8 | 63.1 | 63.9 | 0.137 | 0.092 |
-| de | 400 | 61.8 | 60.8 | 61.6 | 60.1 | 0.147 | 0.054 |
-| ru | 400 | 58.0 | 60.2 | 57.0 | 59.1 | 0.196 | 0.053 |
-| zh | 400 | 62.0 | 56.2 | 61.6 | 55.6 | 0.159 | 0.032 |
+| en | 400 | 63.5 | 69.2 | 64.2 | 68.7 | 0.136 | 0.080 |
+| es | 400 | 57.2 | 64.0 | 57.5 | 63.7 | 0.186 | 0.057 |
+| fr | 400 | 61.0 | 64.2 | 61.3 | 63.7 | 0.152 | 0.067 |
+| de | 400 | 60.5 | 60.5 | 60.3 | 60.2 | 0.160 | 0.024 |
+| ru | 400 | 57.2 | 60.8 | 56.3 | 60.2 | 0.212 | 0.047 |
+| zh | 400 | 61.0 | 53.2 | 61.0 | 52.6 | 0.169 | 0.061 |
 
 ## Option-order robustness
 
@@ -54,8 +54,8 @@ Share of cases whose answer changes when the options are presented in a permuted
 
 | Suite | 8B | 3B |
 |---|---|---|
-| AG News (4 topics) | 0.035 | 0.028 |
-| DAIR Emotion (6 labels) | 0.068 | 0.160 |
-| massive_intent.en | 0.122 | 0.125 |
+| AG News (4 topics) | 0.033 | 0.037 |
+| DAIR Emotion (6 labels) | 0.072 | 0.185 |
+| massive_intent.en | 0.133 | 0.135 |
 
-SST-5 score MAE on the 0–4 scale: 8B 0.563, 3B 0.655.
+SST-5 score MAE on the 0–4 scale: 8B 0.566, 3B 0.653.

evidence/calibration-leaderboard-ablate-nosys.json

@@ -1,165 +1,237 @@
 {
  "arm_fitted": "original",
  "inputs": [
-  "runs/ablate-final3b-nosys/predictions.jsonl"
+  "runs/ablate-hard25-3b-r1-nosys/predictions.jsonl"
  ],
  "fit_suites": [
   "distprob-calib-v2",
   "internal-eval-hardproxy-calib-v1"
  ],
- "temperature": 1.3,
+ "temperature": 1.5,
  "objective": "leaderboard",
- "axis_proxy_at_T1": 85.34441046550333,
- "axis_proxy_at_T": 88.60090642532538,
+ "axis_proxy_at_T1": 83.82616050141019,
+ "axis_proxy_at_T": 87.03644794016614,
  "at_T1": {
   "jevbench-easy": {
    "original": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.0018748964085212687,
-    "calibration_axis_ece_part": 99.62502071829574
+    "ece_10bin": 0.001149104215916778,
+    "calibration_axis_ece_part": 99.77017915681664
    },
    "rotation": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.0018748964085212687,
-    "calibration_axis_ece_part": 99.62502071829574
+    "ece_10bin": 0.001149104215916778,
+    "calibration_axis_ece_part": 99.77017915681664
    }
   },
   "jevbench-original": {
    "original": {
     "questions": 72,
-    "accuracy": 0.9166666666666666,
-    "ece_10bin": 0.07102508567572954,
-    "calibration_axis_ece_part": 85.79498286485409
+    "accuracy": 0.9305555555555556,
+    "ece_10bin": 0.016422420620663625,
+    "calibration_axis_ece_part": 96.71551587586727
    },
    "rotation": {
     "questions": 72,
-    "accuracy": 0.9166666666666666,
-    "ece_10bin": 0.07102508567572954,
-    "calibration_axis_ece_part": 85.79498286485409
+    "accuracy": 0.9305555555555556,
+    "ece_10bin": 0.016422420620663625,
+    "calibration_axis_ece_part": 96.71551587586727
    }
   },
   "jevbench-hard": {
    "original": {
     "questions": 111,
-    "accuracy": 0.4954954954954955,
-    "ece_10bin": 0.2444415956193727,
-    "calibration_axis_ece_part": 51.111680876125455
+    "accuracy": 0.46846846846846846,
+    "ece_10bin": 0.2863840041801175,
+    "calibration_axis_ece_part": 42.7231991639765
    },
    "rotation": {
     "questions": 111,
-    "accuracy": 0.4954954954954955,
-    "ece_10bin": 0.2444415956193727,
-    "calibration_axis_ece_part": 51.111680876125455
+    "accuracy": 0.46846846846846846,
+    "ece_10bin": 0.2863840041801175,
+    "calibration_axis_ece_part": 42.7231991639765
    }
   },
   "judge-proxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8566666666666667,
-    "ece_10bin": 0.06232230220956217,
-    "calibration_axis_ece_part": 87.53553955808758
+    "accuracy": 0.8883333333333333,
+    "ece_10bin": 0.033608477626319376,
+    "calibration_axis_ece_part": 93.27830447473613
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8566666666666667,
-    "ece_10bin": 0.06232230220956217,
-    "calibration_axis_ece_part": 87.53553955808758
+    "accuracy": 0.8883333333333333,
+    "ece_10bin": 0.033608477626319376,
+    "calibration_axis_ece_part": 93.27830447473613
    }
   },
   "internal-eval-hardproxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.44,
-    "ece_10bin": 0.0929088833737751,
-    "calibration_axis_ece_part": 81.41822332524498
+    "accuracy": 0.43666666666666665,
+    "ece_10bin": 0.11217360972848735,
+    "calibration_axis_ece_part": 77.56527805430254
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.44,
-    "ece_10bin": 0.0929088833737751,
-    "calibration_axis_ece_part": 81.41822332524498
+    "accuracy": 0.43666666666666665,
+    "ece_10bin": 0.11217360972848735,
+    "calibration_axis_ece_part": 77.56527805430254
+   }
+  },
+  "internal-eval-hardproxy-v2": {
+   "original": {
+    "questions": 900,
+    "accuracy": 0.8066666666666666,
+    "ece_10bin": 0.0440976735832413,
+    "calibration_axis_ece_part": 91.18046528335174,
+    "probability_items": 80,
+    "mean_tv_distance": 0.1277800123178266,
+    "fidelity_part": 87.22199876821735,
+    "calibration_axis_estimate": 89.20123202578455
+   },
+   "rotation": {
+    "questions": 900,
+    "accuracy": 0.8066666666666666,
+    "ece_10bin": 0.0440976735832413,
+    "calibration_axis_ece_part": 91.18046528335174,
+    "probability_items": 80,
+    "mean_tv_distance": 0.1277800123178266,
+    "fidelity_part": 87.22199876821735,
+    "calibration_axis_estimate": 89.20123202578455
+   }
+  },
+  "internal-eval-hardproxy-v3": {
+   "original": {
+    "questions": 960,
+    "accuracy": 0.6416666666666667,
+    "ece_10bin": 0.140114867135236,
+    "calibration_axis_ece_part": 71.9770265729528,
+    "probability_items": 120,
+    "mean_tv_distance": 0.3854807681079812,
+    "fidelity_part": 61.45192318920187,
+    "calibration_axis_estimate": 66.71447488107734
+   },
+   "rotation": {
+    "questions": 960,
+    "accuracy": 0.6416666666666667,
+    "ece_10bin": 0.140114867135236,
+    "calibration_axis_ece_part": 71.9770265729528,
+    "probability_items": 120,
+    "mean_tv_distance": 0.3854807681079812,
+    "fidelity_part": 61.45192318920187,
+    "calibration_axis_estimate": 66.71447488107734
    }
   },
   "internal-eval-hardproxy-calib-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.4683333333333333,
-    "ece_10bin": 0.0668145573567345,
-    "calibration_axis_ece_part": 86.63708852865311
+    "accuracy": 0.48,
+    "ece_10bin": 0.08306918196174966,
+    "calibration_axis_ece_part": 83.38616360765006
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.4683333333333333,
-    "ece_10bin": 0.0668145573567345,
-    "calibration_axis_ece_part": 86.63708852865311
+    "accuracy": 0.48,
+    "ece_10bin": 0.08306918196174966,
+    "calibration_axis_ece_part": 83.38616360765006
    }
   },
   "distprob-calib-v2": {
    "original": {
     "questions": 916,
-    "accuracy": 0.7925764192139738,
-    "ece_10bin": 0.18166524599524145,
-    "calibration_axis_ece_part": 63.6669508009517,
+    "accuracy": 0.7827510917030568,
+    "ece_10bin": 0.17041662070579264,
+    "calibration_axis_ece_part": 65.91667585884147,
     "probability_items": 916,
-    "mean_tv_distance": 0.1594826759764644,
-    "fidelity_part": 84.05173240235357,
-    "calibration_axis_estimate": 73.85934160165263
+    "mean_tv_distance": 0.15733842604829681,
+    "fidelity_part": 84.26615739517031,
+    "calibration_axis_estimate": 75.09141662700588
    },
    "rotation": {
     "questions": 916,
-    "accuracy": 0.7925764192139738,
-    "ece_10bin": 0.18166524599524145,
-    "calibration_axis_ece_part": 63.6669508009517,
+    "accuracy": 0.7827510917030568,
+    "ece_10bin": 0.17041662070579264,
+    "calibration_axis_ece_part": 65.91667585884147,
     "probability_items": 916,
-    "mean_tv_distance": 0.1594826759764644,
-    "fidelity_part": 84.05173240235357,
-    "calibration_axis_estimate": 73.85934160165263
+    "mean_tv_distance": 0.15733842604829681,
+    "fidelity_part": 84.26615739517031,
+    "calibration_axis_estimate": 75.09141662700588
    }
   },
   "distprob-val-v2": {
    "original": {
     "questions": 1036,
-    "accuracy": 0.7972972972972973,
-    "ece_10bin": 0.18815120252393266,
-    "calibration_axis_ece_part": 62.36975949521346,
+    "accuracy": 0.7982625482625483,
+    "ece_10bin": 0.1934569384835919,
+    "calibration_axis_ece_part": 61.30861230328162,
     "probability_items": 1036,
-    "mean_tv_distance": 0.12080046492926981,
-    "fidelity_part": 87.91995350707302,
-    "calibration_axis_estimate": 75.14485650114324
+    "mean_tv_distance": 0.11388774889852549,
+    "fidelity_part": 88.61122511014744,
+    "calibration_axis_estimate": 74.95991870671453
    },
    "rotation": {
     "questions": 1036,
-    "accuracy": 0.7972972972972973,
-    "ece_10bin": 0.18815120252393266,
-    "calibration_axis_ece_part": 62.36975949521346,
+    "accuracy": 0.7982625482625483,
+    "ece_10bin": 0.1934569384835919,
+    "calibration_axis_ece_part": 61.30861230328162,
     "probability_items": 1036,
-    "mean_tv_distance": 0.12080046492926981,
-    "fidelity_part": 87.91995350707302,
-    "calibration_axis_estimate": 75.14485650114324
+    "mean_tv_distance": 0.11388774889852549,
+    "fidelity_part": 88.61122511014744,
+    "calibration_axis_estimate": 74.95991870671453
    }
   },
   "internal-eval-v2-realistic": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8866666666666667,
-    "ece_10bin": 0.07163316371310577,
-    "calibration_axis_ece_part": 85.67336725737884,
+    "accuracy": 0.9066666666666666,
+    "ece_10bin": 0.0825985048108747,
+    "calibration_axis_ece_part": 83.48029903782506,
     "probability_items": 120,
-    "mean_tv_distance": 0.12894082495178552,
-    "fidelity_part": 87.10591750482145,
-    "calibration_axis_estimate": 86.38964238110015
+    "mean_tv_distance": 0.11120485898001545,
+    "fidelity_part": 88.87951410199845,
+    "calibration_axis_estimate": 86.17990656991176
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8866666666666667,
-    "ece_10bin": 0.07163316371310577,
-    "calibration_axis_ece_part": 85.67336725737884,
+    "accuracy": 0.9066666666666666,
+    "ece_10bin": 0.0825985048108747,
+    "calibration_axis_ece_part": 83.48029903782506,
     "probability_items": 120,
-    "mean_tv_distance": 0.12894082495178552,
-    "fidelity_part": 87.10591750482145,
-    "calibration_axis_estimate": 86.38964238110015
+    "mean_tv_distance": 0.11120485898001545,
+    "fidelity_part": 88.87951410199845,
+    "calibration_axis_estimate": 86.17990656991176
+   }
+  },
+  "games-harness-eval-v1": {
+   "original": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.009742608154769303,
+    "calibration_axis_ece_part": 98.05147836904614
+   },
+   "rotation": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.009742608154769303,
+    "calibration_axis_ece_part": 98.05147836904614
+   }
+  },
+  "tetris-drought-eval-v1": {
+   "original": {
+    "questions": 120,
+    "accuracy": 0.675,
+    "ece_10bin": 0.11965060424992888,
+    "calibration_axis_ece_part": 76.06987915001422
+   },
+   "rotation": {
+    "questions": 120,
+    "accuracy": 0.675,
+    "ece_10bin": 0.11965060424992888,
+    "calibration_axis_ece_part": 76.06987915001422
    }
   }
  },
@@ -168,151 +240,223 @@
    "original": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.006819216668603041,
-    "calibration_axis_ece_part": 98.63615666627939
+    "ece_10bin": 0.008738382420806143,
+    "calibration_axis_ece_part": 98.25232351583877
    },
    "rotation": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.006819216668603041,
-    "calibration_axis_ece_part": 98.63615666627939
+    "ece_10bin": 0.008738382420806143,
+    "calibration_axis_ece_part": 98.25232351583877
    }
   },
   "jevbench-original": {
    "original": {
     "questions": 72,
-    "accuracy": 0.9166666666666666,
-    "ece_10bin": 0.07749353079037855,
-    "calibration_axis_ece_part": 84.50129384192428
+    "accuracy": 0.9305555555555556,
+    "ece_10bin": 0.05473490425907009,
+    "calibration_axis_ece_part": 89.05301914818598
    },
    "rotation": {
     "questions": 72,
-    "accuracy": 0.9166666666666666,
-    "ece_10bin": 0.07749353079037855,
-    "calibration_axis_ece_part": 84.50129384192428
+    "accuracy": 0.9305555555555556,
+    "ece_10bin": 0.05473490425907009,
+    "calibration_axis_ece_part": 89.05301914818598
    }
   },
   "jevbench-hard": {
    "original": {
     "questions": 111,
-    "accuracy": 0.4954954954954955,
-    "ece_10bin": 0.18228644509614172,
-    "calibration_axis_ece_part": 63.54271098077165
+    "accuracy": 0.46846846846846846,
+    "ece_10bin": 0.209595391205915,
+    "calibration_axis_ece_part": 58.080921758817006
    },
    "rotation": {
     "questions": 111,
-    "accuracy": 0.4954954954954955,
-    "ece_10bin": 0.18228644509614172,
-    "calibration_axis_ece_part": 63.54271098077165
+    "accuracy": 0.46846846846846846,
+    "ece_10bin": 0.209595391205915,
+    "calibration_axis_ece_part": 58.080921758817006
    }
   },
   "judge-proxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8566666666666667,
-    "ece_10bin": 0.03179219593693721,
-    "calibration_axis_ece_part": 93.64156081261255
+    "accuracy": 0.8883333333333333,
+    "ece_10bin": 0.02419928317439568,
+    "calibration_axis_ece_part": 95.16014336512086
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8566666666666667,
-    "ece_10bin": 0.03179219593693721,
-    "calibration_axis_ece_part": 93.64156081261255
+    "accuracy": 0.8883333333333333,
+    "ece_10bin": 0.02419928317439568,
+    "calibration_axis_ece_part": 95.16014336512086
    }
   },
   "internal-eval-hardproxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.44,
-    "ece_10bin": 0.052255175773237995,
-    "calibration_axis_ece_part": 89.5489648453524
+    "accuracy": 0.43666666666666665,
+    "ece_10bin": 0.05620456615385875,
+    "calibration_axis_ece_part": 88.75908676922825
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.44,
-    "ece_10bin": 0.052255175773237995,
-    "calibration_axis_ece_part": 89.5489648453524
+    "accuracy": 0.43666666666666665,
+    "ece_10bin": 0.05620456615385875,
+    "calibration_axis_ece_part": 88.75908676922825
+   }
+  },
+  "internal-eval-hardproxy-v2": {
+   "original": {
+    "questions": 900,
+    "accuracy": 0.8066666666666666,
+    "ece_10bin": 0.07746394431116115,
+    "calibration_axis_ece_part": 84.50721113776777,
+    "probability_items": 80,
+    "mean_tv_distance": 0.14965105729945485,
+    "fidelity_part": 85.03489427005452,
+    "calibration_axis_estimate": 84.77105270391115
+   },
+   "rotation": {
+    "questions": 900,
+    "accuracy": 0.8066666666666666,
+    "ece_10bin": 0.07746394431116115,
+    "calibration_axis_ece_part": 84.50721113776777,
+    "probability_items": 80,
+    "mean_tv_distance": 0.14965105729945485,
+    "fidelity_part": 85.03489427005452,
+    "calibration_axis_estimate": 84.77105270391115
+   }
+  },
+  "internal-eval-hardproxy-v3": {
+   "original": {
+    "questions": 960,
+    "accuracy": 0.6416666666666667,
+    "ece_10bin": 0.10491496455454948,
+    "calibration_axis_ece_part": 79.01700708909011,
+    "probability_items": 120,
+    "mean_tv_distance": 0.31211445746013206,
+    "fidelity_part": 68.7885542539868,
+    "calibration_axis_estimate": 73.90278067153847
+   },
+   "rotation": {
+    "questions": 960,
+    "accuracy": 0.6416666666666667,
+    "ece_10bin": 0.10491496455454948,
+    "calibration_axis_ece_part": 79.01700708909011,
+    "probability_items": 120,
+    "mean_tv_distance": 0.31211445746013206,
+    "fidelity_part": 68.7885542539868,
+    "calibration_axis_estimate": 73.90278067153847
    }
   },
   "internal-eval-hardproxy-calib-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.4683333333333333,
-    "ece_10bin": 0.028497932213315664,
-    "calibration_axis_ece_part": 94.30041355733687
+    "accuracy": 0.48,
+    "ece_10bin": 0.03986368848484867,
+    "calibration_axis_ece_part": 92.02726230303027
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.4683333333333333,
-    "ece_10bin": 0.028497932213315664,
-    "calibration_axis_ece_part": 94.30041355733687
+    "accuracy": 0.48,
+    "ece_10bin": 0.03986368848484867,
+    "calibration_axis_ece_part": 92.02726230303027
    }
   },
   "distprob-calib-v2": {
    "original": {
     "questions": 916,
-    "accuracy": 0.7925764192139738,
-    "ece_10bin": 0.22110326195148225,
-    "calibration_axis_ece_part": 55.77934760970356,
+    "accuracy": 0.7827510917030568,
+    "ece_10bin": 0.23232298669097232,
+    "calibration_axis_ece_part": 53.53540266180554,
     "probability_items": 916,
-    "mean_tv_distance": 0.17098600706686123,
-    "fidelity_part": 82.90139929331389,
-    "calibration_axis_estimate": 69.34037345150873
+    "mean_tv_distance": 0.17954366422697995,
+    "fidelity_part": 82.045633577302,
+    "calibration_axis_estimate": 67.79051811955377
    },
    "rotation": {
     "questions": 916,
-    "accuracy": 0.7925764192139738,
-    "ece_10bin": 0.22110326195148225,
-    "calibration_axis_ece_part": 55.77934760970356,
+    "accuracy": 0.7827510917030568,
+    "ece_10bin": 0.23232298669097232,
+    "calibration_axis_ece_part": 53.53540266180554,
     "probability_items": 916,
-    "mean_tv_distance": 0.17098600706686123,
-    "fidelity_part": 82.90139929331389,
-    "calibration_axis_estimate": 69.34037345150873
+    "mean_tv_distance": 0.17954366422697995,
+    "fidelity_part": 82.045633577302,
+    "calibration_axis_estimate": 67.79051811955377
    }
   },
   "distprob-val-v2": {
    "original": {
     "questions": 1036,
-    "accuracy": 0.7972972972972973,
-    "ece_10bin": 0.23073672294686529,
-    "calibration_axis_ece_part": 53.85265541062694,
+    "accuracy": 0.7982625482625483,
+    "ece_10bin": 0.2562140570862304,
+    "calibration_axis_ece_part": 48.75718858275392,
     "probability_items": 1036,
-    "mean_tv_distance": 0.13039772223878196,
-    "fidelity_part": 86.9602277761218,
-    "calibration_axis_estimate": 70.40644159337437
+    "mean_tv_distance": 0.1372665400776386,
+    "fidelity_part": 86.27334599223614,
+    "calibration_axis_estimate": 67.51526728749504
    },
    "rotation": {
     "questions": 1036,
-    "accuracy": 0.7972972972972973,
-    "ece_10bin": 0.23073672294686529,
-    "calibration_axis_ece_part": 53.85265541062694,
+    "accuracy": 0.7982625482625483,
+    "ece_10bin": 0.2562140570862304,
+    "calibration_axis_ece_part": 48.75718858275392,
     "probability_items": 1036,
-    "mean_tv_distance": 0.13039772223878196,
-    "fidelity_part": 86.9602277761218,
-    "calibration_axis_estimate": 70.40644159337437
+    "mean_tv_distance": 0.1372665400776386,
+    "fidelity_part": 86.27334599223614,
+    "calibration_axis_estimate": 67.51526728749504
    }
   },
   "internal-eval-v2-realistic": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8866666666666667,
-    "ece_10bin": 0.10111166335309768,
-    "calibration_axis_ece_part": 79.77766732938046,
+    "accuracy": 0.9066666666666666,
+    "ece_10bin": 0.12869745290106477,
+    "calibration_axis_ece_part": 74.26050941978704,
     "probability_items": 120,
-    "mean_tv_distance": 0.1319216870666447,
-    "fidelity_part": 86.80783129333554,
-    "calibration_axis_estimate": 83.292749311358
+    "mean_tv_distance": 0.13473642194515204,
+    "fidelity_part": 86.52635780548479,
+    "calibration_axis_estimate": 80.39343361263592
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8866666666666667,
-    "ece_10bin": 0.10111166335309768,
-    "calibration_axis_ece_part": 79.77766732938046,
+    "accuracy": 0.9066666666666666,
+    "ece_10bin": 0.12869745290106477,
+    "calibration_axis_ece_part": 74.26050941978704,
     "probability_items": 120,
-    "mean_tv_distance": 0.1319216870666447,
-    "fidelity_part": 86.80783129333554,
-    "calibration_axis_estimate": 83.292749311358
+    "mean_tv_distance": 0.13473642194515204,
+    "fidelity_part": 86.52635780548479,
+    "calibration_axis_estimate": 80.39343361263592
+   }
+  },
+  "games-harness-eval-v1": {
+   "original": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.029814957164165524,
+    "calibration_axis_ece_part": 94.0370085671669
+   },
+   "rotation": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.029814957164165524,
+    "calibration_axis_ece_part": 94.0370085671669
+   }
+  },
+  "tetris-drought-eval-v1": {
+   "original": {
+    "questions": 120,
+    "accuracy": 0.675,
+    "ece_10bin": 0.05513521009547392,
+    "calibration_axis_ece_part": 88.97295798090522
+   },
+   "rotation": {
+    "questions": 120,
+    "accuracy": 0.675,
+    "ece_10bin": 0.05513521009547392,
+    "calibration_axis_ece_part": 88.97295798090522
    }
   }
  }
-}
+}

evidence/calibration-leaderboard-merged-native.json

@@ -1,107 +1,207 @@
 {
  "arm_fitted": "original",
  "inputs": [
-  "runs/merged-3b-native/predictions.jsonl"
+  "runs/merged-v11-3b-native/predictions.jsonl"
  ],
  "fit_suites": [
   "distprob-calib-v2",
   "internal-eval-hardproxy-calib-v1"
  ],
- "temperature": 1.5,
+ "temperature": 1.35,
  "objective": "leaderboard",
- "axis_proxy_at_T1": 85.42044135230344,
- "axis_proxy_at_T": 88.14183810147179,
+ "axis_proxy_at_T1": 84.21389291588972,
+ "axis_proxy_at_T": 88.67401841583384,
  "at_T1": {
   "jevbench-easy": {
    "original": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.0016624161117089937,
-    "calibration_axis_ece_part": 99.66751677765821
+    "ece_10bin": 0.0014843125501875498,
+    "calibration_axis_ece_part": 99.70313748996249
    },
    "rotation": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.0016624161117089937,
-    "calibration_axis_ece_part": 99.66751677765821
+    "ece_10bin": 0.0014843125501875498,
+    "calibration_axis_ece_part": 99.70313748996249
    }
   },
   "jevbench-original": {
    "original": {
     "questions": 72,
-    "accuracy": 0.9027777777777778,
-    "ece_10bin": 0.029791712502210094,
-    "calibration_axis_ece_part": 94.04165749955799
+    "accuracy": 0.9305555555555556,
+    "ece_10bin": 0.0228556343408844,
+    "calibration_axis_ece_part": 95.42887313182312
    },
    "rotation": {
     "questions": 72,
-    "accuracy": 0.9027777777777778,
-    "ece_10bin": 0.029791712502210094,
-    "calibration_axis_ece_part": 94.04165749955799
+    "accuracy": 0.9305555555555556,
+    "ece_10bin": 0.0228556343408844,
+    "calibration_axis_ece_part": 95.42887313182312
    }
   },
   "jevbench-hard": {
    "original": {
     "questions": 111,
-    "accuracy": 0.5135135135135135,
-    "ece_10bin": 0.23823396943984734,
-    "calibration_axis_ece_part": 52.353206112030534
+    "accuracy": 0.4864864864864865,
+    "ece_10bin": 0.26941446853676937,
+    "calibration_axis_ece_part": 46.11710629264613
    },
    "rotation": {
     "questions": 111,
-    "accuracy": 0.5135135135135135,
-    "ece_10bin": 0.23823396943984734,
-    "calibration_axis_ece_part": 52.353206112030534
+    "accuracy": 0.4864864864864865,
+    "ece_10bin": 0.26941446853676937,
+    "calibration_axis_ece_part": 46.11710629264613
    }
   },
   "judge-proxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.87,
-    "ece_10bin": 0.040704387200682556,
-    "calibration_axis_ece_part": 91.8591225598635
+    "accuracy": 0.89,
+    "ece_10bin": 0.021799163372850473,
+    "calibration_axis_ece_part": 95.6401673254299
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.87,
-    "ece_10bin": 0.040704387200682556,
-    "calibration_axis_ece_part": 91.8591225598635
+    "accuracy": 0.89,
+    "ece_10bin": 0.021799163372850473,
+    "calibration_axis_ece_part": 95.6401673254299
+   }
+  },
+  "internal-eval-v2-realistic": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.9,
+    "ece_10bin": 0.08024207896892373,
+    "calibration_axis_ece_part": 83.95158420621526,
+    "probability_items": 120,
+    "mean_tv_distance": 0.10918668729159332,
+    "fidelity_part": 89.08133127084066,
+    "calibration_axis_estimate": 86.51645773852796
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.9,
+    "ece_10bin": 0.08024207896892373,
+    "calibration_axis_ece_part": 83.95158420621526,
+    "probability_items": 120,
+    "mean_tv_distance": 0.10918668729159332,
+    "fidelity_part": 89.08133127084066,
+    "calibration_axis_estimate": 86.51645773852796
+   }
+  },
+  "distprob-val-v2": {
+   "original": {
+    "questions": 1036,
+    "accuracy": 0.8117760617760618,
+    "ece_10bin": 0.19675310407034366,
+    "calibration_axis_ece_part": 60.649379185931274,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.11023935512937952,
+    "fidelity_part": 88.97606448706205,
+    "calibration_axis_estimate": 74.81272183649666
+   },
+   "rotation": {
+    "questions": 1036,
+    "accuracy": 0.8117760617760618,
+    "ece_10bin": 0.19675310407034366,
+    "calibration_axis_ece_part": 60.649379185931274,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.11023935512937952,
+    "fidelity_part": 88.97606448706205,
+    "calibration_axis_estimate": 74.81272183649666
+   }
+  },
+  "internal-eval-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.7166666666666667,
+    "ece_10bin": 0.14650752255863214,
+    "calibration_axis_ece_part": 70.69849548827357
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.7166666666666667,
+    "ece_10bin": 0.14650752255863214,
+    "calibration_axis_ece_part": 70.69849548827357
+   }
+  },
+  "internal-eval-hardproxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.435,
+    "ece_10bin": 0.11452618198956604,
+    "calibration_axis_ece_part": 77.09476360208679
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.435,
+    "ece_10bin": 0.11452618198956604,
+    "calibration_axis_ece_part": 77.09476360208679
+   }
+  },
+  "games-harness-eval-v1": {
+   "original": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.016584023903433262,
+    "calibration_axis_ece_part": 96.68319521931335
+   },
+   "rotation": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.016584023903433262,
+    "calibration_axis_ece_part": 96.68319521931335
+   }
+  },
+  "tetris-drought-eval-v1": {
+   "original": {
+    "questions": 120,
+    "accuracy": 0.6833333333333333,
+    "ece_10bin": 0.09488361820723981,
+    "calibration_axis_ece_part": 81.02327635855204
+   },
+   "rotation": {
+    "questions": 120,
+    "accuracy": 0.6833333333333333,
+    "ece_10bin": 0.09488361820723981,
+    "calibration_axis_ece_part": 81.02327635855204
    }
   },
   "internal-eval-hardproxy-calib-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.475,
-    "ece_10bin": 0.06795799130970562,
-    "calibration_axis_ece_part": 86.40840173805888
+    "accuracy": 0.4816666666666667,
+    "ece_10bin": 0.08163742104516354,
+    "calibration_axis_ece_part": 83.6725157909673
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.475,
-    "ece_10bin": 0.06795799130970562,
-    "calibration_axis_ece_part": 86.40840173805888
+    "accuracy": 0.4816666666666667,
+    "ece_10bin": 0.08163742104516354,
+    "calibration_axis_ece_part": 83.6725157909673
    }
   },
   "distprob-calib-v2": {
    "original": {
     "questions": 916,
-    "accuracy": 0.7827510917030568,
-    "ece_10bin": 0.16288129692265946,
-    "calibration_axis_ece_part": 67.4237406154681,
+    "accuracy": 0.7838427947598253,
+    "ece_10bin": 0.164477610622995,
+    "calibration_axis_ece_part": 67.10447787540099,
     "probability_items": 916,
-    "mean_tv_distance": 0.1556751903345202,
-    "fidelity_part": 84.43248096654798,
-    "calibration_axis_estimate": 75.92811079100804
+    "mean_tv_distance": 0.15244729959187872,
+    "fidelity_part": 84.75527004081212,
+    "calibration_axis_estimate": 75.92987395810655
    },
    "rotation": {
     "questions": 916,
-    "accuracy": 0.7827510917030568,
-    "ece_10bin": 0.16288129692265946,
-    "calibration_axis_ece_part": 67.4237406154681,
+    "accuracy": 0.7838427947598253,
+    "ece_10bin": 0.164477610622995,
+    "calibration_axis_ece_part": 67.10447787540099,
     "probability_items": 916,
-    "mean_tv_distance": 0.1556751903345202,
-    "fidelity_part": 84.43248096654798,
-    "calibration_axis_estimate": 75.92811079100804
+    "mean_tv_distance": 0.15244729959187872,
+    "fidelity_part": 84.75527004081212,
+    "calibration_axis_estimate": 75.92987395810655
    }
   }
  },
@@ -110,92 +210,192 @@
    "original": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.01212479121865218,
-    "calibration_axis_ece_part": 97.57504175626957
+    "ece_10bin": 0.006004891425301917,
+    "calibration_axis_ece_part": 98.79902171493961
    },
    "rotation": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.01212479121865218,
-    "calibration_axis_ece_part": 97.57504175626957
+    "ece_10bin": 0.006004891425301917,
+    "calibration_axis_ece_part": 98.79902171493961
    }
   },
   "jevbench-original": {
    "original": {
     "questions": 72,
-    "accuracy": 0.9027777777777778,
-    "ece_10bin": 0.09724393334626145,
-    "calibration_axis_ece_part": 80.5512133307477
+    "accuracy": 0.9305555555555556,
+    "ece_10bin": 0.043759725982928606,
+    "calibration_axis_ece_part": 91.24805480341428
    },
    "rotation": {
     "questions": 72,
-    "accuracy": 0.9027777777777778,
-    "ece_10bin": 0.09724393334626145,
-    "calibration_axis_ece_part": 80.5512133307477
+    "accuracy": 0.9305555555555556,
+    "ece_10bin": 0.043759725982928606,
+    "calibration_axis_ece_part": 91.24805480341428
    }
   },
   "jevbench-hard": {
    "original": {
     "questions": 111,
-    "accuracy": 0.5135135135135135,
-    "ece_10bin": 0.17510273140034285,
-    "calibration_axis_ece_part": 64.97945371993143
+    "accuracy": 0.4864864864864865,
+    "ece_10bin": 0.21242280730976387,
+    "calibration_axis_ece_part": 57.51543853804723
    },
    "rotation": {
     "questions": 111,
-    "accuracy": 0.5135135135135135,
-    "ece_10bin": 0.17510273140034285,
-    "calibration_axis_ece_part": 64.97945371993143
+    "accuracy": 0.4864864864864865,
+    "ece_10bin": 0.21242280730976387,
+    "calibration_axis_ece_part": 57.51543853804723
    }
   },
   "judge-proxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.87,
-    "ece_10bin": 0.03615621129247121,
-    "calibration_axis_ece_part": 92.76875774150575
+    "accuracy": 0.89,
+    "ece_10bin": 0.029870314961169264,
+    "calibration_axis_ece_part": 94.02593700776615
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.87,
-    "ece_10bin": 0.03615621129247121,
-    "calibration_axis_ece_part": 92.76875774150575
+    "accuracy": 0.89,
+    "ece_10bin": 0.029870314961169264,
+    "calibration_axis_ece_part": 94.02593700776615
+   }
+  },
+  "internal-eval-v2-realistic": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.9,
+    "ece_10bin": 0.11506992601190205,
+    "calibration_axis_ece_part": 76.98601479761959,
+    "probability_items": 120,
+    "mean_tv_distance": 0.12110827936168912,
+    "fidelity_part": 87.88917206383108,
+    "calibration_axis_estimate": 82.43759343072534
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.9,
+    "ece_10bin": 0.11506992601190205,
+    "calibration_axis_ece_part": 76.98601479761959,
+    "probability_items": 120,
+    "mean_tv_distance": 0.12110827936168912,
+    "fidelity_part": 87.88917206383108,
+    "calibration_axis_estimate": 82.43759343072534
+   }
+  },
+  "distprob-val-v2": {
+   "original": {
+    "questions": 1036,
+    "accuracy": 0.8117760617760618,
+    "ece_10bin": 0.24602121170213875,
+    "calibration_axis_ece_part": 50.795757659572246,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.12238352239415004,
+    "fidelity_part": 87.76164776058499,
+    "calibration_axis_estimate": 69.27870271007862
+   },
+   "rotation": {
+    "questions": 1036,
+    "accuracy": 0.8117760617760618,
+    "ece_10bin": 0.24602121170213875,
+    "calibration_axis_ece_part": 50.795757659572246,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.12238352239415004,
+    "fidelity_part": 87.76164776058499,
+    "calibration_axis_estimate": 69.27870271007862
+   }
+  },
+  "internal-eval-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.7166666666666667,
+    "ece_10bin": 0.10387948891595138,
+    "calibration_axis_ece_part": 79.22410221680973
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.7166666666666667,
+    "ece_10bin": 0.10387948891595138,
+    "calibration_axis_ece_part": 79.22410221680973
+   }
+  },
+  "internal-eval-hardproxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.435,
+    "ece_10bin": 0.06736918921772936,
+    "calibration_axis_ece_part": 86.52616215645412
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.435,
+    "ece_10bin": 0.06736918921772936,
+    "calibration_axis_ece_part": 86.52616215645412
+   }
+  },
+  "games-harness-eval-v1": {
+   "original": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.028917472692378383,
+    "calibration_axis_ece_part": 94.21650546152432
+   },
+   "rotation": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.028917472692378383,
+    "calibration_axis_ece_part": 94.21650546152432
+   }
+  },
+  "tetris-drought-eval-v1": {
+   "original": {
+    "questions": 120,
+    "accuracy": 0.6833333333333333,
+    "ece_10bin": 0.05340595065703371,
+    "calibration_axis_ece_part": 89.31880986859326
+   },
+   "rotation": {
+    "questions": 120,
+    "accuracy": 0.6833333333333333,
+    "ece_10bin": 0.05340595065703371,
+    "calibration_axis_ece_part": 89.31880986859326
    }
   },
   "internal-eval-hardproxy-calib-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.475,
-    "ece_10bin": 0.030585414443062297,
-    "calibration_axis_ece_part": 93.88291711138754
+    "accuracy": 0.4816666666666667,
+    "ece_10bin": 0.03032421514666685,
+    "calibration_axis_ece_part": 93.93515697066664
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.475,
-    "ece_10bin": 0.030585414443062297,
-    "calibration_axis_ece_part": 93.88291711138754
+    "accuracy": 0.4816666666666667,
+    "ece_10bin": 0.03032421514666685,
+    "calibration_axis_ece_part": 93.93515697066664
    }
   },
   "distprob-calib-v2": {
    "original": {
     "questions": 916,
-    "accuracy": 0.7827510917030568,
-    "ece_10bin": 0.22864736346190892,
-    "calibration_axis_ece_part": 54.27052730761821,
+    "accuracy": 0.7838427947598253,
+    "ece_10bin": 0.21579301823334204,
+    "calibration_axis_ece_part": 56.84139635333159,
     "probability_items": 916,
-    "mean_tv_distance": 0.17599240908443944,
-    "fidelity_part": 82.40075909155605,
-    "calibration_axis_estimate": 68.33564319958714
+    "mean_tv_distance": 0.16587120138998968,
+    "fidelity_part": 83.41287986100103,
+    "calibration_axis_estimate": 70.1271381071663
    },
    "rotation": {
     "questions": 916,
-    "accuracy": 0.7827510917030568,
-    "ece_10bin": 0.22864736346190892,
-    "calibration_axis_ece_part": 54.27052730761821,
+    "accuracy": 0.7838427947598253,
+    "ece_10bin": 0.21579301823334204,
+    "calibration_axis_ece_part": 56.84139635333159,
     "probability_items": 916,
-    "mean_tv_distance": 0.17599240908443944,
-    "fidelity_part": 82.40075909155605,
-    "calibration_axis_estimate": 68.33564319958714
+    "mean_tv_distance": 0.16587120138998968,
+    "fidelity_part": 83.41287986100103,
+    "calibration_axis_estimate": 70.1271381071663
    }
   }
  }

evidence/served-nosys-easy-report.json

@@ -8,22 +8,22 @@
   "served_models": [
     "decision-model"
   ],
-  "elapsed_measurement_s": 1.120218633004697,
-  "requests_per_second": 42.84877843109287,
-  "questions_per_second": 42.84877843109287,
+  "elapsed_measurement_s": 1.8050968870520592,
+  "requests_per_second": 26.5913704379546,
+  "questions_per_second": 26.5913704379546,
   "latency_ms": {
     "n": 48,
-    "mean": 23.234710584802087,
-    "p50": 22.789281996665522,
-    "p95": 27.86436639871681,
-    "p99": 31.33449414919596
+    "mean": 37.49669147994913,
+    "p50": 37.14705549646169,
+    "p95": 37.637978242128156,
+    "p99": 49.57394877215848
   },
   "adapter_ms": {
     "n": 48,
-    "mean": 21.890798979560106,
-    "p50": 21.413121503428556,
-    "p95": 26.402303371287417,
-    "p99": 30.082574323459994
+    "mean": 36.123791850210786,
+    "p50": 35.78712046146393,
+    "p95": 36.14921441185288,
+    "p99": 48.11713631264866
   },
   "input_tokens_per_request": {
     "n": 48,
@@ -37,11 +37,11 @@
       "clean": {
         "n": 48,
         "acc": 1.0,
-        "nll": 0.0017789755673829387,
-        "brier": 4.807367058120001e-05,
-        "ece": 0.0017633189725172471,
-        "mean_conf": 0.9982366810274828,
-        "confidence_bias": -0.0017633189725172471,
+        "nll": 0.0015276622415421612,
+        "brier": 5.584960223915265e-05,
+        "ece": 0.0015118869298550441,
+        "mean_conf": 0.998488113070145,
+        "confidence_bias": -0.0015118869298550441,
         "confident_error_rate": 0.0,
         "coverage_at_0_9": 1.0,
         "accuracy_at_0_9": 1.0,
@@ -52,11 +52,11 @@
         "extraction": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.003626219592460447,
-          "brier": 0.00011029343263911612,
-          "ece": 0.0035859255722816563,
-          "mean_conf": 0.9964140744277183,
-          "confidence_bias": -0.0035859255722816563,
+          "nll": 0.002813571161406935,
+          "brier": 7.836254226801523e-05,
+          "ece": 0.0027875193213375837,
+          "mean_conf": 0.9972124806786624,
+          "confidence_bias": -0.0027875193213375837,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -66,11 +66,11 @@
         "fact": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.002241130474217068,
-          "brier": 7.005642102392954e-05,
-          "ece": 0.002223378978578716,
-          "mean_conf": 0.9977766210214213,
-          "confidence_bias": -0.002223378978578716,
+          "nll": 0.0029452787672731777,
+          "brier": 0.0001444467258469531,
+          "ece": 0.0029084513057315853,
+          "mean_conf": 0.9970915486942684,
+          "confidence_bias": -0.0029084513057315853,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -80,11 +80,11 @@
         "intent": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.0010573720877922199,
-          "brier": 1.175346410816985e-05,
-          "ece": 0.0010528560188417524,
-          "mean_conf": 0.9989471439811582,
-          "confidence_bias": -0.0010528560188417524,
+          "nll": 0.00026626090898396263,
+          "brier": 5.533060241933733e-07,
+          "ece": 0.0002660503696976235,
+          "mean_conf": 0.9997339496303024,
+          "confidence_bias": -0.0002660503696976235,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -94,11 +94,11 @@
         "tool_selection": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.00019118011506202054,
-          "brier": 1.9136455358453179e-07,
-          "ece": 0.0001911153203667526,
-          "mean_conf": 0.9998088846796332,
-          "confidence_bias": -0.0001911153203667526,
+          "nll": 8.553812850457014e-05,
+          "brier": 3.5834817448884555e-08,
+          "ece": 8.552672265349504e-05,
+          "mean_conf": 0.9999144732773465,
+          "confidence_bias": -8.552672265349504e-05,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -110,11 +110,11 @@
         "extraction": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.003626219592460447,
-          "brier": 0.00011029343263911612,
-          "ece": 0.0035859255722816563,
-          "mean_conf": 0.9964140744277183,
-          "confidence_bias": -0.0035859255722816563,
+          "nll": 0.002813571161406935,
+          "brier": 7.836254226801523e-05,
+          "ece": 0.0027875193213375837,
+          "mean_conf": 0.9972124806786624,
+          "confidence_bias": -0.0027875193213375837,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -124,11 +124,11 @@
         "fact": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.002241130474217068,
-          "brier": 7.005642102392954e-05,
-          "ece": 0.002223378978578716,
-          "mean_conf": 0.9977766210214213,
-          "confidence_bias": -0.002223378978578716,
+          "nll": 0.0029452787672731777,
+          "brier": 0.0001444467258469531,
+          "ece": 0.0029084513057315853,
+          "mean_conf": 0.9970915486942684,
+          "confidence_bias": -0.0029084513057315853,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -138,11 +138,11 @@
         "intent": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.0010573720877922199,
-          "brier": 1.175346410816985e-05,
-          "ece": 0.0010528560188417524,
-          "mean_conf": 0.9989471439811582,
-          "confidence_bias": -0.0010528560188417524,
+          "nll": 0.00026626090898396263,
+          "brier": 5.533060241933733e-07,
+          "ece": 0.0002660503696976235,
+          "mean_conf": 0.9997339496303024,
+          "confidence_bias": -0.0002660503696976235,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -152,11 +152,11 @@
         "tool_selection": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.00019118011506202054,
-          "brier": 1.9136455358453179e-07,
-          "ece": 0.0001911153203667526,
-          "mean_conf": 0.9998088846796332,
-          "confidence_bias": -0.0001911153203667526,
+          "nll": 8.553812850457014e-05,
+          "brier": 3.5834817448884555e-08,
+          "ece": 8.552672265349504e-05,
+          "mean_conf": 0.9999144732773465,
+          "confidence_bias": -8.552672265349504e-05,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -168,11 +168,11 @@
         "clean": {
           "n": 48,
           "acc": 1.0,
-          "nll": 0.0017789755673829387,
-          "brier": 4.807367058120001e-05,
-          "ece": 0.0017633189725172471,
-          "mean_conf": 0.9982366810274828,
-          "confidence_bias": -0.0017633189725172471,
+          "nll": 0.0015276622415421612,
+          "brier": 5.584960223915265e-05,
+          "ece": 0.0015118869298550441,
+          "mean_conf": 0.998488113070145,
+          "confidence_bias": -0.0015118869298550441,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -181,7 +181,7 @@
         }
       },
       "macro_task_acc": 1.0,
-      "objective": -0.001778975567382939,
+      "objective": -0.0015276622415421614,
       "paired_flip": {
         "pairs": 0,
         "incomplete_pairs": 0,
@@ -211,10 +211,10 @@
       },
       "latency_ms": {
         "n": 48,
-        "mean": 23.234710584802087,
-        "p50": 22.789281996665522,
-        "p95": 27.86436639871681,
-        "p99": 31.33449414919596
+        "mean": 37.49669147994913,
+        "p50": 37.14705549646169,
+        "p95": 37.637978242128156,
+        "p99": 49.57394877215848
       }
     }
   },

evidence/served-nosys-hard-report.json

@@ -8,22 +8,22 @@
   "served_models": [
     "decision-model"
   ],
-  "elapsed_measurement_s": 3.5634483199974056,
-  "requests_per_second": 31.14960286559756,
-  "questions_per_second": 31.14960286559756,
+  "elapsed_measurement_s": 6.400594180973712,
+  "requests_per_second": 17.342139942250448,
+  "questions_per_second": 17.342139942250448,
   "latency_ms": {
     "n": 111,
-    "mean": 31.97643356628481,
-    "p50": 26.17260901024565,
-    "p95": 50.40793698572088,
-    "p99": 60.8419331139885
+    "mean": 57.53704737808958,
+    "p50": 41.14527994534001,
+    "p95": 107.0126669947058,
+    "p99": 121.36280137347059
   },
   "adapter_ms": {
     "n": 111,
-    "mean": 30.40883497378017,
-    "p50": 24.645024997880682,
-    "p95": 48.79910999443382,
-    "p99": 59.05504081456457
+    "mean": 55.97996988348445,
+    "p50": 39.66773202409968,
+    "p95": 105.28535101911984,
+    "p99": 119.4705346308183
   },
   "input_tokens_per_request": {
     "n": 111,
@@ -36,105 +36,105 @@
     "jevbench-hard/public": {
       "clean": {
         "n": 111,
-        "acc": 0.4864864864864865,
-        "nll": 1.2210790654837587,
-        "brier": 0.6754949211362803,
-        "ece": 0.22721326697517824,
-        "mean_conf": 0.7136997534616648,
-        "confidence_bias": 0.2272132669751783,
-        "confident_error_rate": 0.0990990990990991,
-        "coverage_at_0_9": 0.27927927927927926,
-        "accuracy_at_0_9": 0.6451612903225806,
+        "acc": 0.4594594594594595,
+        "nll": 1.2962948856809682,
+        "brier": 0.6968902514322239,
+        "ece": 0.28535210981983,
+        "mean_conf": 0.7361970214660999,
+        "confidence_bias": 0.2767375620066404,
+        "confident_error_rate": 0.10810810810810811,
+        "coverage_at_0_9": 0.36036036036036034,
+        "accuracy_at_0_9": 0.7,
         "coverage_at_5pct_error": 0.0990990990990991,
         "coverage_at_1pct_error": 0.0990990990990991,
-        "score_mae": 0.6476665946070091,
-        "ranked_probability_score": 0.18348001503670752
+        "score_mae": 0.6876357781329778,
+        "ranked_probability_score": 0.19267980315847502
       },
       "tasks": {
         "adversarial": {
           "n": 6,
-          "acc": 1.0,
-          "nll": 0.35251123143079216,
-          "brier": 0.20041343565378164,
-          "ece": 0.26837694440376436,
-          "mean_conf": 0.7316230555962356,
-          "confidence_bias": -0.26837694440376436,
+          "acc": 0.6666666666666666,
+          "nll": 0.3804766699344641,
+          "brier": 0.24696365952999444,
+          "ece": 0.10635190756625662,
+          "mean_conf": 0.7502708058819635,
+          "confidence_bias": 0.08360413921529686,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.3333333333333333,
+          "coverage_at_0_9": 0.5,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 1.0,
-          "coverage_at_1pct_error": 1.0,
-          "score_mae": 0.3523800709707958,
-          "ranked_probability_score": 0.04304514044526898
+          "coverage_at_5pct_error": 0.6666666666666666,
+          "coverage_at_1pct_error": 0.6666666666666666,
+          "score_mae": 0.6004958589124929,
+          "ranked_probability_score": 0.10993958949738573
         },
         "ambiguous": {
           "n": 7,
-          "acc": 0.42857142857142855,
-          "nll": 1.3565237367001013,
-          "brier": 0.6452232869191212,
-          "ece": 0.32251571368107274,
-          "mean_conf": 0.6502180852536663,
-          "confidence_bias": 0.22164665668223776,
+          "acc": 0.2857142857142857,
+          "nll": 1.1965487237375452,
+          "brier": 0.6527176169156463,
+          "ece": 0.4011055273042717,
+          "mean_conf": 0.6739533055078394,
+          "confidence_bias": 0.3882390197935537,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.0,
-          "accuracy_at_0_9": null,
+          "coverage_at_0_9": 0.2857142857142857,
+          "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 0.2857142857142857,
           "coverage_at_1pct_error": 0.2857142857142857
         },
         "judge_hard": {
           "n": 17,
-          "acc": 0.6470588235294118,
-          "nll": 1.043153687758328,
-          "brier": 0.6092315129495184,
-          "ece": 0.28681625447610926,
-          "mean_conf": 0.8928580032603509,
-          "confidence_bias": 0.24579917973093912,
-          "confident_error_rate": 0.23529411764705882,
-          "coverage_at_0_9": 0.7058823529411765,
-          "accuracy_at_0_9": 0.6666666666666666,
+          "acc": 0.7058823529411765,
+          "nll": 0.9352993338445932,
+          "brier": 0.5776553044911235,
+          "ece": 0.34600650565663504,
+          "mean_conf": 0.8837164999753305,
+          "confidence_bias": 0.177834147034154,
+          "confident_error_rate": 0.29411764705882354,
+          "coverage_at_0_9": 0.7647058823529411,
+          "accuracy_at_0_9": 0.6153846153846154,
           "coverage_at_5pct_error": 0.23529411764705882,
           "coverage_at_1pct_error": 0.23529411764705882
         },
         "long_policy": {
           "n": 19,
-          "acc": 0.15789473684210525,
-          "nll": 1.6308614483111026,
-          "brier": 0.9792997943021993,
-          "ece": 0.4952746829682073,
-          "mean_conf": 0.6531694198103126,
-          "confidence_bias": 0.49527468296820737,
+          "acc": 0.21052631578947367,
+          "nll": 1.969063478310097,
+          "brier": 1.0575910654140444,
+          "ece": 0.5443446760383921,
+          "mean_conf": 0.6978151313376447,
+          "confidence_bias": 0.48728881554817105,
           "confident_error_rate": 0.10526315789473684,
           "coverage_at_0_9": 0.10526315789473684,
           "accuracy_at_0_9": 0.0,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0,
-          "score_mae": 0.477994310305481,
-          "ranked_probability_score": 0.15165035324462384
+          "score_mae": 0.5265190609151413,
+          "ranked_probability_score": 0.15218789918371528
         },
         "multi_hop": {
           "n": 18,
           "acc": 0.5,
-          "nll": 1.0688746124802804,
-          "brier": 0.6372311093159718,
-          "ece": 0.24119875598255616,
-          "mean_conf": 0.6399374680333065,
-          "confidence_bias": 0.13993746803330653,
+          "nll": 1.1199038798912422,
+          "brier": 0.6376540745233639,
+          "ece": 0.3014772683765995,
+          "mean_conf": 0.6628249641338441,
+          "confidence_bias": 0.16282496413384406,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.05555555555555555,
+          "coverage_at_0_9": 0.1111111111111111,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 0.1111111111111111,
           "coverage_at_1pct_error": 0.1111111111111111,
-          "score_mae": 0.3264137607819857,
-          "ranked_probability_score": 0.10477293291947704
+          "score_mae": 0.38716699283188594,
+          "ranked_probability_score": 0.1118292374551618
         },
         "probability": {
           "n": 10,
-          "acc": 0.7,
-          "nll": 0.7923180459636632,
-          "brier": 0.5234434268632783,
-          "ece": 0.37541941555644287,
-          "mean_conf": 0.6176246938708918,
-          "confidence_bias": -0.08237530612910815,
+          "acc": 0.5,
+          "nll": 0.7271252358087724,
+          "brier": 0.483854142289723,
+          "ece": 0.14057929069212244,
+          "mean_conf": 0.6261857999614627,
+          "confidence_bias": 0.12618579996146273,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 0.1,
           "accuracy_at_0_9": 1.0,
@@ -144,149 +144,149 @@
         "routing_hard": {
           "n": 5,
           "acc": 1.0,
-          "nll": 0.03599323458923339,
-          "brier": 0.005530167936106714,
-          "ece": 0.03444735897283405,
-          "mean_conf": 0.9655526410271659,
-          "confidence_bias": -0.03444735897283413,
+          "nll": 0.02045794307903345,
+          "brier": 0.0024295831118786066,
+          "ece": 0.019812811096257654,
+          "mean_conf": 0.9801871889037423,
+          "confidence_bias": -0.019812811096257654,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.8,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
         },
         "temporal_numeric": {
           "n": 15,
-          "acc": 0.06666666666666667,
-          "nll": 2.178260177219352,
-          "brier": 1.1166410815303502,
-          "ece": 0.5320347358601226,
-          "mean_conf": 0.5987014025267893,
-          "confidence_bias": 0.5320347358601226,
+          "acc": 0.13333333333333333,
+          "nll": 2.2634257023598283,
+          "brier": 1.119741919350661,
+          "ece": 0.487236140970707,
+          "mean_conf": 0.6205694743040404,
+          "confidence_bias": 0.4872361409707071,
           "confident_error_rate": 0.2,
           "coverage_at_0_9": 0.2,
           "accuracy_at_0_9": 0.0,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0,
-          "score_mae": 1.9515866130907424,
-          "ranked_probability_score": 0.6353113738050508
+          "score_mae": 1.9373593402178928,
+          "ranked_probability_score": 0.626412898273334
         },
         "tradeoff": {
           "n": 6,
-          "acc": 0.3333333333333333,
-          "nll": 2.182300081255963,
-          "brier": 0.9455793810780045,
-          "ece": 0.5467033524554479,
-          "mean_conf": 0.7305070046153205,
-          "confidence_bias": 0.39717367128198716,
+          "acc": 0.16666666666666666,
+          "nll": 2.697830255531367,
+          "brier": 1.1394884476835054,
+          "ece": 0.6244343812760749,
+          "mean_conf": 0.7911010479427416,
+          "confidence_bias": 0.624434381276075,
           "confident_error_rate": 0.3333333333333333,
-          "coverage_at_0_9": 0.3333333333333333,
-          "accuracy_at_0_9": 0.0,
+          "coverage_at_0_9": 0.5,
+          "accuracy_at_0_9": 0.3333333333333333,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0
         },
         "trap": {
           "n": 8,
           "acc": 0.875,
-          "nll": 0.2623087137944162,
-          "brier": 0.1427404033078605,
-          "ece": 0.1551691982278216,
-          "mean_conf": 0.8505193797539718,
-          "confidence_bias": -0.02448062024602815,
+          "nll": 0.28094468518213467,
+          "brier": 0.17851500724166375,
+          "ece": 0.1602692173284523,
+          "mean_conf": 0.8835139641418724,
+          "confidence_bias": 0.008513964141872443,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.5,
+          "coverage_at_0_9": 0.75,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 0.875,
-          "coverage_at_1pct_error": 0.875,
-          "score_mae": 0.29963050218756937,
-          "ranked_probability_score": 0.014449936561200668
+          "coverage_at_5pct_error": 0.75,
+          "coverage_at_1pct_error": 0.75,
+          "score_mae": 0.1477543550053129,
+          "ranked_probability_score": 0.0035212953575381283
         }
       },
       "sources": {
         "adversarial": {
           "n": 6,
-          "acc": 1.0,
-          "nll": 0.35251123143079216,
-          "brier": 0.20041343565378164,
-          "ece": 0.26837694440376436,
-          "mean_conf": 0.7316230555962356,
-          "confidence_bias": -0.26837694440376436,
+          "acc": 0.6666666666666666,
+          "nll": 0.3804766699344641,
+          "brier": 0.24696365952999444,
+          "ece": 0.10635190756625662,
+          "mean_conf": 0.7502708058819635,
+          "confidence_bias": 0.08360413921529686,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.3333333333333333,
+          "coverage_at_0_9": 0.5,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 1.0,
-          "coverage_at_1pct_error": 1.0,
-          "score_mae": 0.3523800709707958,
-          "ranked_probability_score": 0.04304514044526898
+          "coverage_at_5pct_error": 0.6666666666666666,
+          "coverage_at_1pct_error": 0.6666666666666666,
+          "score_mae": 0.6004958589124929,
+          "ranked_probability_score": 0.10993958949738573
         },
         "ambiguous": {
           "n": 7,
-          "acc": 0.42857142857142855,
-          "nll": 1.3565237367001013,
-          "brier": 0.6452232869191212,
-          "ece": 0.32251571368107274,
-          "mean_conf": 0.6502180852536663,
-          "confidence_bias": 0.22164665668223776,
+          "acc": 0.2857142857142857,
+          "nll": 1.1965487237375452,
+          "brier": 0.6527176169156463,
+          "ece": 0.4011055273042717,
+          "mean_conf": 0.6739533055078394,
+          "confidence_bias": 0.3882390197935537,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.0,
-          "accuracy_at_0_9": null,
+          "coverage_at_0_9": 0.2857142857142857,
+          "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 0.2857142857142857,
           "coverage_at_1pct_error": 0.2857142857142857
         },
         "judge_hard": {
           "n": 17,
-          "acc": 0.6470588235294118,
-          "nll": 1.043153687758328,
-          "brier": 0.6092315129495184,
-          "ece": 0.28681625447610926,
-          "mean_conf": 0.8928580032603509,
-          "confidence_bias": 0.24579917973093912,
-          "confident_error_rate": 0.23529411764705882,
-          "coverage_at_0_9": 0.7058823529411765,
-          "accuracy_at_0_9": 0.6666666666666666,
+          "acc": 0.7058823529411765,
+          "nll": 0.9352993338445932,
+          "brier": 0.5776553044911235,
+          "ece": 0.34600650565663504,
+          "mean_conf": 0.8837164999753305,
+          "confidence_bias": 0.177834147034154,
+          "confident_error_rate": 0.29411764705882354,
+          "coverage_at_0_9": 0.7647058823529411,
+          "accuracy_at_0_9": 0.6153846153846154,
           "coverage_at_5pct_error": 0.23529411764705882,
           "coverage_at_1pct_error": 0.23529411764705882
         },
         "long_policy": {
           "n": 19,
-          "acc": 0.15789473684210525,
-          "nll": 1.6308614483111026,
-          "brier": 0.9792997943021993,
-          "ece": 0.4952746829682073,
-          "mean_conf": 0.6531694198103126,
-          "confidence_bias": 0.49527468296820737,
+          "acc": 0.21052631578947367,
+          "nll": 1.969063478310097,
+          "brier": 1.0575910654140444,
+          "ece": 0.5443446760383921,
+          "mean_conf": 0.6978151313376447,
+          "confidence_bias": 0.48728881554817105,
           "confident_error_rate": 0.10526315789473684,
           "coverage_at_0_9": 0.10526315789473684,
           "accuracy_at_0_9": 0.0,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0,
-          "score_mae": 0.477994310305481,
-          "ranked_probability_score": 0.15165035324462384
+          "score_mae": 0.5265190609151413,
+          "ranked_probability_score": 0.15218789918371528
         },
         "multi_hop": {
           "n": 18,
           "acc": 0.5,
-          "nll": 1.0688746124802804,
-          "brier": 0.6372311093159718,
-          "ece": 0.24119875598255616,
-          "mean_conf": 0.6399374680333065,
-          "confidence_bias": 0.13993746803330653,
+          "nll": 1.1199038798912422,
+          "brier": 0.6376540745233639,
+          "ece": 0.3014772683765995,
+          "mean_conf": 0.6628249641338441,
+          "confidence_bias": 0.16282496413384406,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.05555555555555555,
+          "coverage_at_0_9": 0.1111111111111111,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 0.1111111111111111,
           "coverage_at_1pct_error": 0.1111111111111111,
-          "score_mae": 0.3264137607819857,
-          "ranked_probability_score": 0.10477293291947704
+          "score_mae": 0.38716699283188594,
+          "ranked_probability_score": 0.1118292374551618
         },
         "probability": {
           "n": 10,
-          "acc": 0.7,
-          "nll": 0.7923180459636632,
-          "brier": 0.5234434268632783,
-          "ece": 0.37541941555644287,
-          "mean_conf": 0.6176246938708918,
-          "confidence_bias": -0.08237530612910815,
+          "acc": 0.5,
+          "nll": 0.7271252358087724,
+          "brier": 0.483854142289723,
+          "ece": 0.14057929069212244,
+          "mean_conf": 0.6261857999614627,
+          "confidence_bias": 0.12618579996146273,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 0.1,
           "accuracy_at_0_9": 1.0,
@@ -296,84 +296,84 @@
         "routing_hard": {
           "n": 5,
           "acc": 1.0,
-          "nll": 0.03599323458923339,
-          "brier": 0.005530167936106714,
-          "ece": 0.03444735897283405,
-          "mean_conf": 0.9655526410271659,
-          "confidence_bias": -0.03444735897283413,
+          "nll": 0.02045794307903345,
+          "brier": 0.0024295831118786066,
+          "ece": 0.019812811096257654,
+          "mean_conf": 0.9801871889037423,
+          "confidence_bias": -0.019812811096257654,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.8,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
         },
         "temporal_numeric": {
           "n": 15,
-          "acc": 0.06666666666666667,
-          "nll": 2.178260177219352,
-          "brier": 1.1166410815303502,
-          "ece": 0.5320347358601226,
-          "mean_conf": 0.5987014025267893,
-          "confidence_bias": 0.5320347358601226,
+          "acc": 0.13333333333333333,
+          "nll": 2.2634257023598283,
+          "brier": 1.119741919350661,
+          "ece": 0.487236140970707,
+          "mean_conf": 0.6205694743040404,
+          "confidence_bias": 0.4872361409707071,
           "confident_error_rate": 0.2,
           "coverage_at_0_9": 0.2,
           "accuracy_at_0_9": 0.0,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0,
-          "score_mae": 1.9515866130907424,
-          "ranked_probability_score": 0.6353113738050508
+          "score_mae": 1.9373593402178928,
+          "ranked_probability_score": 0.626412898273334
         },
         "tradeoff": {
           "n": 6,
-          "acc": 0.3333333333333333,
-          "nll": 2.182300081255963,
-          "brier": 0.9455793810780045,
-          "ece": 0.5467033524554479,
-          "mean_conf": 0.7305070046153205,
-          "confidence_bias": 0.39717367128198716,
+          "acc": 0.16666666666666666,
+          "nll": 2.697830255531367,
+          "brier": 1.1394884476835054,
+          "ece": 0.6244343812760749,
+          "mean_conf": 0.7911010479427416,
+          "confidence_bias": 0.624434381276075,
           "confident_error_rate": 0.3333333333333333,
-          "coverage_at_0_9": 0.3333333333333333,
-          "accuracy_at_0_9": 0.0,
+          "coverage_at_0_9": 0.5,
+          "accuracy_at_0_9": 0.3333333333333333,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0
         },
         "trap": {
           "n": 8,
           "acc": 0.875,
-          "nll": 0.2623087137944162,
-          "brier": 0.1427404033078605,
-          "ece": 0.1551691982278216,
-          "mean_conf": 0.8505193797539718,
-          "confidence_bias": -0.02448062024602815,
+          "nll": 0.28094468518213467,
+          "brier": 0.17851500724166375,
+          "ece": 0.1602692173284523,
+          "mean_conf": 0.8835139641418724,
+          "confidence_bias": 0.008513964141872443,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.5,
+          "coverage_at_0_9": 0.75,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 0.875,
-          "coverage_at_1pct_error": 0.875,
-          "score_mae": 0.29963050218756937,
-          "ranked_probability_score": 0.014449936561200668
+          "coverage_at_5pct_error": 0.75,
+          "coverage_at_1pct_error": 0.75,
+          "score_mae": 0.1477543550053129,
+          "ranked_probability_score": 0.0035212953575381283
         }
       },
       "variants": {
         "clean": {
           "n": 111,
-          "acc": 0.4864864864864865,
-          "nll": 1.2210790654837587,
-          "brier": 0.6754949211362803,
-          "ece": 0.22721326697517824,
-          "mean_conf": 0.7136997534616648,
-          "confidence_bias": 0.2272132669751783,
-          "confident_error_rate": 0.0990990990990991,
-          "coverage_at_0_9": 0.27927927927927926,
-          "accuracy_at_0_9": 0.6451612903225806,
+          "acc": 0.4594594594594595,
+          "nll": 1.2962948856809682,
+          "brier": 0.6968902514322239,
+          "ece": 0.28535210981983,
+          "mean_conf": 0.7361970214660999,
+          "confidence_bias": 0.2767375620066404,
+          "confident_error_rate": 0.10810810810810811,
+          "coverage_at_0_9": 0.36036036036036034,
+          "accuracy_at_0_9": 0.7,
           "coverage_at_5pct_error": 0.0990990990990991,
           "coverage_at_1pct_error": 0.0990990990990991,
-          "score_mae": 0.6476665946070091,
-          "ranked_probability_score": 0.18348001503670752
+          "score_mae": 0.6876357781329778,
+          "ranked_probability_score": 0.19267980315847502
         }
       },
-      "macro_task_acc": 0.5708524988942946,
-      "objective": -1.0903104969503232,
+      "macro_task_acc": 0.5043789621111603,
+      "objective": -1.1591075907679076,
       "paired_flip": {
         "pairs": 0,
         "incomplete_pairs": 0,
@@ -385,9 +385,9 @@
       },
       "reference_distribution": {
         "n": 10,
-        "squared_l2": 0.10905696764037734,
-        "total_variation": 0.1997306177200928,
-        "kl_reference_to_model": 0.1374588310661992,
+        "squared_l2": 0.0923480672287547,
+        "total_variation": 0.18456016438122402,
+        "kl_reference_to_model": 0.11494476732281071,
         "note": "Exact reference-distribution fidelity, separate from hard-label Brier/ECE."
       },
       "unknowable": null,
@@ -409,10 +409,10 @@
       },
       "latency_ms": {
         "n": 111,
-        "mean": 31.97643356628481,
-        "p50": 26.17260901024565,
-        "p95": 50.40793698572088,
-        "p99": 60.8419331139885
+        "mean": 57.53704737808958,
+        "p50": 41.14527994534001,
+        "p95": 107.0126669947058,
+        "p99": 121.36280137347059
       }
     }
   },

evidence/served-nosys-original-report.json

@@ -8,22 +8,22 @@
   "served_models": [
     "decision-model"
   ],
-  "elapsed_measurement_s": 1.6658977939805482,
-  "requests_per_second": 43.219938378068775,
-  "questions_per_second": 43.219938378068775,
+  "elapsed_measurement_s": 2.666326795006171,
+  "requests_per_second": 27.00344163920588,
+  "questions_per_second": 27.00344163920588,
   "latency_ms": {
     "n": 72,
-    "mean": 23.027374291511176,
-    "p50": 22.36310299485922,
-    "p95": 30.51040534919594,
-    "p99": 34.34268011507812
+    "mean": 36.9241978882605,
+    "p50": 37.165592511883005,
+    "p95": 39.12460421561264,
+    "p99": 43.66849939222452
   },
   "adapter_ms": {
     "n": 72,
-    "mean": 21.671233003467528,
-    "p50": 21.02683151315432,
-    "p95": 29.079331248067323,
-    "p99": 31.649900618358537
+    "mean": 35.50634430495039,
+    "p50": 35.74040098465048,
+    "p95": 37.74651304702275,
+    "p99": 41.85468893789225
   },
   "input_tokens_per_request": {
     "n": 72,
@@ -36,45 +36,45 @@
     "jevbench-original/public": {
       "clean": {
         "n": 72,
-        "acc": 0.875,
-        "nll": 0.2811701595983037,
-        "brier": 0.16998368696190747,
-        "ece": 0.027585444231134813,
-        "mean_conf": 0.9003661785243816,
-        "confidence_bias": 0.02536617852438161,
+        "acc": 0.8888888888888888,
+        "nll": 0.2445291027435668,
+        "brier": 0.14146770822826904,
+        "ece": 0.07089677407755061,
+        "mean_conf": 0.9290339623031046,
+        "confidence_bias": 0.040145073414215804,
         "confident_error_rate": 0.013888888888888888,
-        "coverage_at_0_9": 0.6666666666666666,
-        "accuracy_at_0_9": 0.9791666666666666,
-        "coverage_at_5pct_error": 0.8333333333333334,
-        "coverage_at_1pct_error": 0.5277777777777778,
-        "score_mae": 0.14433068031614896,
-        "ranked_probability_score": 0.021378525778158384
+        "coverage_at_0_9": 0.7777777777777778,
+        "accuracy_at_0_9": 0.9821428571428571,
+        "coverage_at_5pct_error": 0.875,
+        "coverage_at_1pct_error": 0.4722222222222222,
+        "score_mae": 0.14699746098780417,
+        "ranked_probability_score": 0.027507417748960535
       },
       "tasks": {
         "adequacy": {
           "n": 12,
           "acc": 0.6666666666666666,
-          "nll": 0.7816227650376891,
-          "brier": 0.5076781498841229,
-          "ece": 0.19641208292198364,
-          "mean_conf": 0.8096085306199567,
-          "confidence_bias": 0.14294186395329012,
+          "nll": 0.7362793168402896,
+          "brier": 0.4277953207024657,
+          "ece": 0.2804673152635425,
+          "mean_conf": 0.8273459270172787,
+          "confidence_bias": 0.16067926035061209,
           "confident_error_rate": 0.08333333333333333,
           "coverage_at_0_9": 0.3333333333333333,
           "accuracy_at_0_9": 0.75,
-          "coverage_at_5pct_error": 0.25,
-          "coverage_at_1pct_error": 0.25
+          "coverage_at_5pct_error": 0.16666666666666666,
+          "coverage_at_1pct_error": 0.16666666666666666
         },
         "extraction": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.0542116119441933,
-          "brier": 0.01493325495130398,
-          "ece": 0.0495188270475546,
-          "mean_conf": 0.9504811729524453,
-          "confidence_bias": -0.049518827047554725,
+          "nll": 0.029664413922215655,
+          "brier": 0.00422488499266305,
+          "ece": 0.028473599180563025,
+          "mean_conf": 0.971526400819437,
+          "confidence_bias": -0.028473599180563025,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.8333333333333334,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
@@ -82,13 +82,13 @@
         "intent": {
           "n": 12,
           "acc": 0.8333333333333334,
-          "nll": 0.30572276294355083,
-          "brier": 0.20045464895299747,
-          "ece": 0.14304785668742614,
-          "mean_conf": 0.9322074612496651,
-          "confidence_bias": 0.09887412791633177,
+          "nll": 0.3530283282604048,
+          "brier": 0.24567434750699083,
+          "ece": 0.1528447829790891,
+          "mean_conf": 0.9619515814755606,
+          "confidence_bias": 0.12861824814222722,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.75,
+          "coverage_at_0_9": 0.8333333333333334,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 0.8333333333333334,
           "coverage_at_1pct_error": 0.8333333333333334
@@ -96,27 +96,27 @@
         "ordinal": {
           "n": 12,
           "acc": 0.9166666666666666,
-          "nll": 0.20756268320968063,
-          "brier": 0.10919592930395382,
-          "ece": 0.12099966786008741,
-          "mean_conf": 0.9077797081312969,
-          "confidence_bias": -0.008886958535369693,
+          "nll": 0.20100958799798233,
+          "brier": 0.10103710326805294,
+          "ece": 0.0759267994222482,
+          "mean_conf": 0.9162006485755824,
+          "confidence_bias": -0.00046601809108426373,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.6666666666666666,
+          "coverage_at_0_9": 0.75,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 0.9166666666666666,
           "coverage_at_1pct_error": 0.9166666666666666,
-          "score_mae": 0.14433068031614896,
-          "ranked_probability_score": 0.021378525778158384
+          "score_mae": 0.14699746098780417,
+          "ranked_probability_score": 0.027507417748960535
         },
         "policy": {
           "n": 12,
           "acc": 0.9166666666666666,
-          "nll": 0.10467945507562453,
-          "brier": 0.05772104532732034,
-          "ece": 0.08135385206374438,
-          "mean_conf": 0.9290088980838886,
-          "confidence_bias": 0.012342231417222016,
+          "nll": 0.10621874252814244,
+          "brier": 0.06285648604607869,
+          "ece": 0.079613773421764,
+          "mean_conf": 0.9358306599039143,
+          "confidence_bias": 0.019163993237247712,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 0.8333333333333334,
           "accuracy_at_0_9": 1.0,
@@ -125,44 +125,44 @@
         },
         "routing": {
           "n": 12,
-          "acc": 0.9166666666666666,
-          "nll": 0.23322167937908408,
-          "brier": 0.12991909335174637,
-          "ece": 0.1168843091364116,
-          "mean_conf": 0.8731113001090374,
-          "confidence_bias": -0.04355536655762926,
+          "acc": 1.0,
+          "nll": 0.04097422691236581,
+          "brier": 0.007218106853362921,
+          "ece": 0.03865144397314558,
+          "mean_conf": 0.9613485560268544,
+          "confidence_bias": -0.03865144397314557,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.5833333333333334,
+          "coverage_at_0_9": 0.9166666666666666,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 0.75,
-          "coverage_at_1pct_error": 0.75
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
         }
       },
       "sources": {
         "adequacy": {
           "n": 12,
           "acc": 0.6666666666666666,
-          "nll": 0.7816227650376891,
-          "brier": 0.5076781498841229,
-          "ece": 0.19641208292198364,
-          "mean_conf": 0.8096085306199567,
-          "confidence_bias": 0.14294186395329012,
+          "nll": 0.7362793168402896,
+          "brier": 0.4277953207024657,
+          "ece": 0.2804673152635425,
+          "mean_conf": 0.8273459270172787,
+          "confidence_bias": 0.16067926035061209,
           "confident_error_rate": 0.08333333333333333,
           "coverage_at_0_9": 0.3333333333333333,
           "accuracy_at_0_9": 0.75,
-          "coverage_at_5pct_error": 0.25,
-          "coverage_at_1pct_error": 0.25
+          "coverage_at_5pct_error": 0.16666666666666666,
+          "coverage_at_1pct_error": 0.16666666666666666
         },
         "extraction": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.0542116119441933,
-          "brier": 0.01493325495130398,
-          "ece": 0.0495188270475546,
-          "mean_conf": 0.9504811729524453,
-          "confidence_bias": -0.049518827047554725,
+          "nll": 0.029664413922215655,
+          "brier": 0.00422488499266305,
+          "ece": 0.028473599180563025,
+          "mean_conf": 0.971526400819437,
+          "confidence_bias": -0.028473599180563025,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.8333333333333334,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
@@ -170,13 +170,13 @@
         "intent": {
           "n": 12,
           "acc": 0.8333333333333334,
-          "nll": 0.30572276294355083,
-          "brier": 0.20045464895299747,
-          "ece": 0.14304785668742614,
-          "mean_conf": 0.9322074612496651,
-          "confidence_bias": 0.09887412791633177,
+          "nll": 0.3530283282604048,
+          "brier": 0.24567434750699083,
+          "ece": 0.1528447829790891,
+          "mean_conf": 0.9619515814755606,
+          "confidence_bias": 0.12861824814222722,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.75,
+          "coverage_at_0_9": 0.8333333333333334,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 0.8333333333333334,
           "coverage_at_1pct_error": 0.8333333333333334
@@ -184,27 +184,27 @@
         "ordinal": {
           "n": 12,
           "acc": 0.9166666666666666,
-          "nll": 0.20756268320968063,
-          "brier": 0.10919592930395382,
-          "ece": 0.12099966786008741,
-          "mean_conf": 0.9077797081312969,
-          "confidence_bias": -0.008886958535369693,
+          "nll": 0.20100958799798233,
+          "brier": 0.10103710326805294,
+          "ece": 0.0759267994222482,
+          "mean_conf": 0.9162006485755824,
+          "confidence_bias": -0.00046601809108426373,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.6666666666666666,
+          "coverage_at_0_9": 0.75,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 0.9166666666666666,
           "coverage_at_1pct_error": 0.9166666666666666,
-          "score_mae": 0.14433068031614896,
-          "ranked_probability_score": 0.021378525778158384
+          "score_mae": 0.14699746098780417,
+          "ranked_probability_score": 0.027507417748960535
         },
         "policy": {
           "n": 12,
           "acc": 0.9166666666666666,
-          "nll": 0.10467945507562453,
-          "brier": 0.05772104532732034,
-          "ece": 0.08135385206374438,
-          "mean_conf": 0.9290088980838886,
-          "confidence_bias": 0.012342231417222016,
+          "nll": 0.10621874252814244,
+          "brier": 0.06285648604607869,
+          "ece": 0.079613773421764,
+          "mean_conf": 0.9358306599039143,
+          "confidence_bias": 0.019163993237247712,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 0.8333333333333334,
           "accuracy_at_0_9": 1.0,
@@ -213,39 +213,39 @@
         },
         "routing": {
           "n": 12,
-          "acc": 0.9166666666666666,
-          "nll": 0.23322167937908408,
-          "brier": 0.12991909335174637,
-          "ece": 0.1168843091364116,
-          "mean_conf": 0.8731113001090374,
-          "confidence_bias": -0.04355536655762926,
+          "acc": 1.0,
+          "nll": 0.04097422691236581,
+          "brier": 0.007218106853362921,
+          "ece": 0.03865144397314558,
+          "mean_conf": 0.9613485560268544,
+          "confidence_bias": -0.03865144397314557,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.5833333333333334,
+          "coverage_at_0_9": 0.9166666666666666,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 0.75,
-          "coverage_at_1pct_error": 0.75
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
         }
       },
       "variants": {
         "clean": {
           "n": 72,
-          "acc": 0.875,
-          "nll": 0.2811701595983037,
-          "brier": 0.16998368696190747,
-          "ece": 0.027585444231134813,
-          "mean_conf": 0.9003661785243816,
-          "confidence_bias": 0.02536617852438161,
+          "acc": 0.8888888888888888,
+          "nll": 0.2445291027435668,
+          "brier": 0.14146770822826904,
+          "ece": 0.07089677407755061,
+          "mean_conf": 0.9290339623031046,
+          "confidence_bias": 0.040145073414215804,
           "confident_error_rate": 0.013888888888888888,
-          "coverage_at_0_9": 0.6666666666666666,
-          "accuracy_at_0_9": 0.9791666666666666,
-          "coverage_at_5pct_error": 0.8333333333333334,
-          "coverage_at_1pct_error": 0.5277777777777778,
-          "score_mae": 0.14433068031614896,
-          "ranked_probability_score": 0.021378525778158384
+          "coverage_at_0_9": 0.7777777777777778,
+          "accuracy_at_0_9": 0.9821428571428571,
+          "coverage_at_5pct_error": 0.875,
+          "coverage_at_1pct_error": 0.4722222222222222,
+          "score_mae": 0.14699746098780417,
+          "ranked_probability_score": 0.027507417748960535
         }
       },
-      "macro_task_acc": 0.875,
-      "objective": -0.28117015959830377,
+      "macro_task_acc": 0.8888888888888888,
+      "objective": -0.24452910274356676,
       "paired_flip": {
         "pairs": 0,
         "incomplete_pairs": 0,
@@ -275,10 +275,10 @@
       },
       "latency_ms": {
         "n": 72,
-        "mean": 23.027374291511176,
-        "p50": 22.36310299485922,
-        "p95": 30.51040534919594,
-        "p99": 34.34268011507812
+        "mean": 36.9241978882605,
+        "p50": 37.165592511883005,
+        "p95": 39.12460421561264,
+        "p99": 43.66849939222452
       }
     }
   },

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f675d7bb2615275072980b36a0d94356c4b866d829b1cfeb472294a0caabc57
+oid sha256:8a02fdf56e2c3fdf75b303a1c617ed8bebe06d7e934e7ef56af3689a7a2552a5
 size 4995664864

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:edcb0fd56e0a5da4e6c057166ac812587636e8cda20fa23776d465d448f3dfdd
+oid sha256:1d8099875a6e4466dace9ceab20c38ffee02903dbb2acee9d5b0565b6e64da2a
 size 2702576200

release-manifest.json

@@ -5,19 +5,19 @@
     "revision": "b6d637bef2393152b3da2b2fde72eecdee30557e"
   },
   "adapter": {
-    "source_run": "final2-3b-long",
-    "adapter_model_sha256": "ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf"
+    "source_run": "hard25-3b-r1",
+    "adapter_model_sha256": "7bc8bfff4be52e0c9c6f605faf1eebc5e8a966ce103793c7f6970e29056ef5d3"
   },
   "merge_report_summary": {
     "changed_params": 182,
     "unchanged_params": 276,
     "changed_outside_lm_projections": [],
-    "max_abs_delta": 0.002506256103515625
+    "max_abs_delta": 0.0022978782653808594
   },
   "serving": {
     "wording": "native",
     "system_prompt": null,
-    "temperature": 1.45,
+    "temperature": 1.55,
     "served_model_name": "standard-one-3b",
     "engine": "SGLang 0.5.20"
   }