Serving package (same as GitHub chenyuwang166-oss/MetaDecision-31B-Rain@6b8ed2d0)

service/.gitattributes

@@ -0,0 +1 @@
+* -text

service/.gitignore

@@ -0,0 +1,5 @@
+runs/
+__pycache__/
+*.pyc
+state/
+*.log

service/ADAPTER_MANIFEST.json

@@ -0,0 +1,134 @@
+{
+  "adapter_config_normalization": {
+    "changed_fields": [
+      "base_model_name_or_path",
+      "revision"
+    ],
+    "rule": "Only these keys differ from the selected checkpoint adapter_config.json.",
+    "serving_values": {
+      "base_model_name_or_path": "google/gemma-4-31B-it-qat-q4_0-unquantized",
+      "revision": "1e4d8beecacb8b7590c1d8bedd7335f687bf311f"
+    }
+  },
+  "files": {
+    "adapter_config.json": {
+      "sha256": "140bb00c832d117f57d29650dabe50e992b2e5064e908aaa29230b3ce19ca2c4",
+      "size": 22953
+    },
+    "adapter_model.safetensors": {
+      "sha256": "2b9568998ffed5d133223ffdff03056be297285c643122db4d45365d0bd86970",
+      "size": 494595208
+    }
+  },
+  "key_value_alignment": {
+    "added_count": 20,
+    "added_tensors": [
+      "base_model.model.model.language_model.layers.11.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.11.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.17.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.17.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.23.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.23.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.29.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.29.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.35.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.35.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.41.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.41.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.47.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.47.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.5.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.5.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.53.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.53.self_attn.v_proj.lora_B.weight",
+      "base_model.model.model.language_model.layers.59.self_attn.v_proj.lora_A.weight",
+      "base_model.model.model.language_model.layers.59.self_attn.v_proj.lora_B.weight"
+    ],
+    "applied": true,
+    "layers": [
+      5,
+      11,
+      17,
+      23,
+      29,
+      35,
+      41,
+      47,
+      53,
+      59
+    ],
+    "rule": "v_proj.lora_A/B = byte copy of k_proj.lora_A/B in every attention_k_eq_v layer; all source tensors and their data unchanged"
+  },
+  "kind": "unmerged_lora_serving_adapter",
+  "lora": {
+    "lora_alpha": 32,
+    "merged": false,
+    "peft_type": "LORA",
+    "r": 16,
+    "stored_tensors": {
+      "dtypes": {
+        "F32": 840
+      },
+      "k_eq_v_layers_without_v_proj_lora": [],
+      "layers": 60,
+      "module_count": 420,
+      "module_prefix": "model.language_model.",
+      "rank": 16,
+      "targets": [
+        "down_proj",
+        "gate_proj",
+        "k_proj",
+        "o_proj",
+        "q_proj",
+        "up_proj",
+        "v_proj"
+      ],
+      "tensor_count": 840,
+      "v_proj_copied_from_k_proj_layers": [
+        5,
+        11,
+        17,
+        23,
+        29,
+        35,
+        41,
+        47,
+        53,
+        59
+      ],
+      "variant": "key_value_aligned"
+    },
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "up_proj",
+      "down_proj"
+    ]
+  },
+  "protocol": "adapter_arm_31b_v1",
+  "schema_version": 1,
+  "served_base": {
+    "files_byte_identical": true,
+    "model_sha256": "ca750c75ea690276947dfc6d3c775668282e9180e4c56f180bfad912fed606a5",
+    "repo_id": "google/gemma-4-31B-it-qat-w4a16-ct",
+    "revision": "52f3f65bc7a02d555763bc923bd1d9094898219d"
+  },
+  "source": {
+    "adapter_model_sha256": "9dbf1acc152ef1a327b934b9674c232eec1129bee0fad51707021ad16285fc27",
+    "checkpoint_adapter_config_sha256": "52553a60f6155af2707d05d11d17a4e0a3d20c35233ea9261fdc56a7e7b4c4ca",
+    "checkpoint_complete_sha256": "056e81fa6b71062de77b6a640ffdb91c3680a16ffeb0154a6a10cf58b4c325fb",
+    "checkpoint_step": 150,
+    "run_identity_sha256": "7c94956f79b06b60a1066ebbf094e346ade3c70e9b900888563d878b2850b2b3",
+    "selection_sha256": "621fde9159b24e773298511418d3373dcd2da45bed535327c5be01facf218994",
+    "training_manifest_sha256": "ced7511dd24f895ad18a65616736ddbad4a4a906c3c015879e7d8148a49fa879"
+  },
+  "subdir": "adapter",
+  "training_base": {
+    "repo_id": "google/gemma-4-31B-it-qat-q4_0-unquantized",
+    "revision": "1e4d8beecacb8b7590c1d8bedd7335f687bf311f"
+  },
+  "variant": "key_value_aligned"
+}

service/DISCLOSURE.md

@@ -0,0 +1,81 @@
+# MetaDecision-31B-Rain: disclosure
+
+## Model
+
+- Base model: `google/gemma-4-31B-it`. The weights used are Google's official
+  quantization-aware-training (QAT) releases of this model:
+  - training base: `google/gemma-4-31B-it-qat-q4_0-unquantized` at revision
+    `1e4d8beecacb8b7590c1d8bedd7335f687bf311f` (BF16);
+  - served base: `google/gemma-4-31B-it-qat-w4a16-ct` at revision
+    `52f3f65bc7a02d555763bc923bd1d9094898219d`, Google's compressed-tensors W4A16 checkpoint
+    (4-bit weights, group size 32, symmetric; activations unquantized), used unchanged. We did not
+    quantize anything.
+- Own trained weights: the LoRA adapter in `adapter/` (PEFT format), rank 16, alpha 32, on the
+  language model's q/k/v/o/gate/up/down projections, trained by us in BF16 on the training base.
+- The adapter is not merged: vLLM 0.31.0 loads it with `enable_lora` and `max_lora_rank=16` and
+  applies it to every request on top of the W4A16 base.
+- In the 10 full-attention layers (5, 11, …, 59) Gemma 4 computes the attention values with the key
+  projection (`attention_k_eq_v`); these layers have no v_proj, and vLLM serves them with separate
+  key and value slots. `adapter/adapter_model.safetensors` therefore contains the trained checkpoint
+  tensors unchanged plus, for these 10 layers, a v_proj LoRA that is a byte copy of the trained
+  k_proj LoRA (20 tensors), so that the update applies to keys and values as in training.
+- The base files and the adapter are exactly those of `MeTask-Rain/metask-jev-rain-31B-trained`
+  (same SHA-256 for every file, `MODEL_MANIFEST.json`); no weight was trained or changed for this
+  entry. It differs from that entry only in the serving rules below.
+
+## Data
+
+- The adapter is the one of `MeTask-Rain/metask-jev-rain-31B-trained`, trained on our own generated
+  data as disclosed for that entry.
+- Inherited from the earlier service of these weights, unchanged: the per-type temperatures of the
+  first calibration step, fitted on the 231 public JevBench items, and the drafting prompt limit
+  (840 tokens, below), chosen using the 231 public items together with our own development data.
+  Fits and choices made on these public items are in-sample. The yes/no confidence constant 0.898
+  (first-step answers carry 0.898 or 0.102) is the yes/no accuracy of our own held-out development
+  data (1,419 yes/no items), rounded to 3 decimals; its value does not depend on any leaderboard
+  result.
+- The serving parameters new in this entry (drafting threshold and share, draft length, the re-read
+  rule and the post-calibration below) were fitted on our own independently built development set.
+  The 9 s deadline and the drafting chunk settings are fixed rules, not fitted.
+- The 231 public items were also used for the validation run of the workflow in README.md; no
+  parameter of this entry was chosen from that run.
+
+## Serving
+
+- Each decision is first read in one prompt pass: the answer distribution comes from the next-token
+  log-probabilities of the answer forms. If the gap between the two highest answer scores of this
+  pass is below a threshold (fitted so that 52.1% of the decisions of our development set
+  that can take the drafting pass fall below it), the decision may take a second, drafting pass: the
+  model writes at most 192 tokens of reasoning (the instruction asks for at most
+  50 words), then the answer distribution is read again. If the draft ends without an
+  answer line, an answer line is appended and the answer is read after the draft.
+- Deadline: a decision that takes the drafting pass has a deadline of 9 s from the
+  moment the service starts answering the request. Drafting runs in chunks sized so that the answer
+  can still be read before the deadline (an engine call in progress is always completed, never
+  abandoned); a draft that reaches its time budget is ended with an answer line, and a drafting pass
+  that cannot be read in time returns the first-pass answer (with a `warning` note, also listed in
+  `partial_errors`).
+- Decisions whose drafting prompt (the first-pass prompt plus the drafting instruction) is
+  longer than 840 tokens are answered in one pass. Inputs longer than 6,653 tokens never take the drafting
+  pass and are answered from a context shortened to at most 6,653 tokens by the service's existing
+  extraction rule; an input is not refused because of its length.
+- Drafting share (slow-path quota 49%): a counter kept below the service's state
+  directory admits a drafting pass only while drafting decisions stay
+  at most 49% of all decisions so far.
+- Re-read: a choice decision answered in one pass although its length allowed the drafting pass (for example because the drafting share or the cost fuse refused it), whose first-pass margin (the log-score difference between its two highest answers) is below 4, is read once more in one pass with the options in reverse order (each option keeps its text and label). The two answer distributions are averaged by label. The second read is included in the decision's token usage and in the cost fuse; it is not a drafting pass and does not count toward the drafting share.
+- Cost fuse: from the 101st decision on, when the running engine cost per decision reaches 90% of
+  USD 0.0646 per 1,000 decisions, drafting passes are limited to the share that brings the running
+  cost back to 85% of that limit (none while first-pass decisions alone cost more than 85% of it),
+  until the running cost falls to 85% of the limit. The fuse prices engine token counts at
+  USD 0.0598 per million tokens, input and output alike (the GPU-time basis of the
+  leaderboard's cost rule for this architecture). The fuse never changes the first pass.
+- Answers after a drafting pass use the same first-step calibration parameters (per-type
+  temperatures and yes/no confidence) as first-pass answers.
+- Post-calibration: after the first-step calibration, the reported probabilities are transformed per type, without any model call or token (a temperature t replaces the probabilities p by p^(1/t), renormalised; parameters fitted on our own independently built development set; values in `config.json`, block `post_calibration`). yes/no: temperature 0.5 (the first-step 0.898/0.102 become 0.987/0.013); score: all probability mass is put on the most probable level. The answer label is recomputed from the transformed probabilities; types not listed are unchanged.
+
+## Run conditions and usage accounting
+
+- vLLM 0.31.0 with `VLLM_BATCH_INVARIANT=1` (also set by the launcher for serving) and
+  `VLLM_USE_FLASHINFER_SAMPLER=0`, one request at a time; model loading and warm-up are excluded.
+- Cost basis: measured engine token counts (input tokens minus cached prefix tokens; generated
+  tokens).

service/FREEZE.json

@@ -0,0 +1,54 @@
+{
+  "config": {
+    "change": "package.finalized: false -> true (single textual replacement)",
+    "frozen_sha256": "94d5765076019628a0dc736f79e470eefeea72661895599ea2670ef391b0fed0",
+    "prepared_sha256": "0c84b933ac60e43394dbd748da21cb27bf91fe640cac7674acc3d02b80e8de11"
+  },
+  "frozen_utc": "2026-10-30T20:10:25Z",
+  "model_identity_sha256": "0513d121db967ec1a6aaccf627d3c42149d092dd027298da5b53684b030821ea",
+  "prepared_files": {
+    ".gitattributes": "705fd4d6451a31d36b3df7de96f83f30ac976c9b4a6d1e51671d8e2f33e2d0da",
+    ".gitignore": "4254e460a0ff807e7fd68e104efa8935a769377f09876b8b4fec0fb0b544a0e6",
+    "ADAPTER_MANIFEST.json": "6c40a7534f5d2b87cc8591748f85c7958f0259821c98659a60e25308904aa430",
+    "DISCLOSURE.md": "f50f14b50c9399ebf1cbf8867a8f807374448e0fbc9218dd3f3cd4a350a36135",
+    "LICENSE": "199f78d6c6e3b98c1601e4407aa092af57e42842d195409569c199b0393afac9",
+    "MODEL_CARD.md": "311bf55c3eda8fb9e464c0870d662c87fc99ae8f9d1a38fa64e12b9cfed8b59f",
+    "MODEL_MANIFEST.json": "d82f71ba20a912a3d5fe1ef8a5a2f8e004be16e7c67df2ee5308864b6832eb5f",
+    "PIPELINE_PROVENANCE.json": "b069a1fdbe6b56867a84ecaad10ce7521cfe135cead029a002c83383f8bf6f44",
+    "README.md": "91995c8d1a63816e14ec73a3d2c9d25993072592c98ddbbd073ac211d1f445c5",
+    "SERVICE_LOCK.json": "f777656d2cd1d40d1df1bb8a9d55661175663564f5ad5c1de1db57d9be72af3b",
+    "attention.py": "9c887f4e5f8ea2858f79a629e684ea71b07359d49d2a0bf31b5f993cb367e547",
+    "backends.py": "e0613ebb8e05fd22673e0502a131dcc33a5af563998cf5a12143878131518f93",
+    "bench.py": "7f158bcb4dcd90395c21535527abe4283d6ca5dab92ded0be47cc352f022f0cd",
+    "calibration.json": "5b60ca8516b93982724145bc2227404c4c53a6f5657b4bf2155108c2f001685f",
+    "config.json": "0c84b933ac60e43394dbd748da21cb27bf91fe640cac7674acc3d02b80e8de11",
+    "core.py": "fb4169ed1c6c9d69e406e9a33d69a71cc49aa48e0155d9544bb9f3bf41690a17",
+    "fit_report.json": "fc20f2f631c98f043f7c02d4cadbe8122580ee400377eef5420d93621111248b",
+    "item_deadline.py": "8d3d637ab3db1be9769e26900b4fc46964e48425a05bb92402a2b504ae66ff3a",
+    "launch.py": "d8f966264dfa5d977ef6444364e1eb4128cf34ed1b60bc94a45f2649f6fce2d3",
+    "model_setup.py": "6c7cd992039327178b72247b8b0f44214a5a00e43abcc0f583130dfe91586dd7",
+    "pkg_check.py": "0e8bf9d4c723f6205e25244a28cae31f4534d8356e1cc1a1390dad199b2904c0",
+    "post_calibration.py": "62a89b2b87cee77322333774a6e081f28646804c926af0f0c0e798a2018b3ffe",
+    "prepare_model.sh": "454a0090ce0187bf6f4b4825a537183cf095c503592a7f673f7f57329258dc44",
+    "quota.py": "8b039bc30c13e6c5a63668f3064fbb311358f9295922a817ec26f09893936f8d",
+    "reconcile_usage.py": "a381dca414201652f254f5101f08d4cbba27b69c3abeb50e6dad86e0e151aba7",
+    "requirements.lock": "4536b444d83518a5d696ebf262aa9d3da156fe2d898f96582e4ab52d70cb4e4d",
+    "requirements.measured.lock": "888ec6b0d3fad1a17e9323c1ab9321040303c79df3c9fe43e6f755cf2ab24b91",
+    "selftest.py": "08d5cfceb703748b407ce0223dee2807bf49f27b4f9bbd28c43e9738673699e6",
+    "selftest.sh": "d0cc8cc9068667aec3525861e0d61ac8fc9bb617b90f7fee7818af9dc0af154e",
+    "serve.sh": "13008367a9b37988de9c7acfa1d2340ca7dea502fd352887fc61783749d279c3",
+    "server.py": "dbce4f70c2c684ac8ea5069a9dfa1d30aa33a8e1c33d274d91fd3c2a953c6951"
+  },
+  "replaced_documents": {},
+  "repo_id": "MeTask-Rain/metask-jev-rain-31B-trained",
+  "revision": "9392335f5a17395ecd70049f85b1909d53500144",
+  "schema": "metadecision_31b_freeze_v1",
+  "serving_selection": {
+    "freeze_sha256": "6e5325eb9713ed73ed9fc94992b67b1179ea60d6969757af61d5c8860d2bb22e",
+    "protocol_sha256": "1c279f3d780a0a522a217c38f74892e9c7019be5012409a9c83bbd9dbc179771",
+    "readout_sha256": "8cc627afcf00b3b9ba9edafe004adc8294d45c62bec8a2dc5b3f593cac4ea0a2",
+    "selected": "serving rules selected on our own development set",
+    "selection_sha256": "cc1070b2d79a088983cdb5c300adc7c1ed5c25232003830a9d1a8611e4df6820"
+  },
+  "status": "FROZEN"
+}

service/LICENSE

@@ -0,0 +1,25 @@
+MIT License
+
+Copyright (c) 2026 MeTask (Hangzhou MeTask Technology Co., Ltd.)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Model weights are not part of this repository. This service loads Google's
+official Gemma 4 31B W4A16 base and our unmerged LoRA adapter. See MODEL_CARD.md
+and DISCLOSURE.md for the model sources and applicable weight licensing.

service/MANIFEST.sha256

@@ -0,0 +1,32 @@
+705fd4d6451a31d36b3df7de96f83f30ac976c9b4a6d1e51671d8e2f33e2d0da  .gitattributes
+4254e460a0ff807e7fd68e104efa8935a769377f09876b8b4fec0fb0b544a0e6  .gitignore
+6c40a7534f5d2b87cc8591748f85c7958f0259821c98659a60e25308904aa430  ADAPTER_MANIFEST.json
+f50f14b50c9399ebf1cbf8867a8f807374448e0fbc9218dd3f3cd4a350a36135  DISCLOSURE.md
+e2cfe584c132b9b3fd666bbabae51dccca455aaad541aad649ccb92741f75d1c  FREEZE.json
+199f78d6c6e3b98c1601e4407aa092af57e42842d195409569c199b0393afac9  LICENSE
+311bf55c3eda8fb9e464c0870d662c87fc99ae8f9d1a38fa64e12b9cfed8b59f  MODEL_CARD.md
+d82f71ba20a912a3d5fe1ef8a5a2f8e004be16e7c67df2ee5308864b6832eb5f  MODEL_MANIFEST.json
+b069a1fdbe6b56867a84ecaad10ce7521cfe135cead029a002c83383f8bf6f44  PIPELINE_PROVENANCE.json
+91995c8d1a63816e14ec73a3d2c9d25993072592c98ddbbd073ac211d1f445c5  README.md
+f777656d2cd1d40d1df1bb8a9d55661175663564f5ad5c1de1db57d9be72af3b  SERVICE_LOCK.json
+9c887f4e5f8ea2858f79a629e684ea71b07359d49d2a0bf31b5f993cb367e547  attention.py
+e0613ebb8e05fd22673e0502a131dcc33a5af563998cf5a12143878131518f93  backends.py
+7f158bcb4dcd90395c21535527abe4283d6ca5dab92ded0be47cc352f022f0cd  bench.py
+5b60ca8516b93982724145bc2227404c4c53a6f5657b4bf2155108c2f001685f  calibration.json
+94d5765076019628a0dc736f79e470eefeea72661895599ea2670ef391b0fed0  config.json
+fb4169ed1c6c9d69e406e9a33d69a71cc49aa48e0155d9544bb9f3bf41690a17  core.py
+fc20f2f631c98f043f7c02d4cadbe8122580ee400377eef5420d93621111248b  fit_report.json
+8d3d637ab3db1be9769e26900b4fc46964e48425a05bb92402a2b504ae66ff3a  item_deadline.py
+d8f966264dfa5d977ef6444364e1eb4128cf34ed1b60bc94a45f2649f6fce2d3  launch.py
+6c7cd992039327178b72247b8b0f44214a5a00e43abcc0f583130dfe91586dd7  model_setup.py
+0e8bf9d4c723f6205e25244a28cae31f4534d8356e1cc1a1390dad199b2904c0  pkg_check.py
+62a89b2b87cee77322333774a6e081f28646804c926af0f0c0e798a2018b3ffe  post_calibration.py
+454a0090ce0187bf6f4b4825a537183cf095c503592a7f673f7f57329258dc44  prepare_model.sh
+8b039bc30c13e6c5a63668f3064fbb311358f9295922a817ec26f09893936f8d  quota.py
+a381dca414201652f254f5101f08d4cbba27b69c3abeb50e6dad86e0e151aba7  reconcile_usage.py
+4536b444d83518a5d696ebf262aa9d3da156fe2d898f96582e4ab52d70cb4e4d  requirements.lock
+888ec6b0d3fad1a17e9323c1ab9321040303c79df3c9fe43e6f755cf2ab24b91  requirements.measured.lock
+08d5cfceb703748b407ce0223dee2807bf49f27b4f9bbd28c43e9738673699e6  selftest.py
+d0cc8cc9068667aec3525861e0d61ac8fc9bb617b90f7fee7818af9dc0af154e  selftest.sh
+13008367a9b37988de9c7acfa1d2340ca7dea502fd352887fc61783749d279c3  serve.sh
+dbce4f70c2c684ac8ea5069a9dfa1d30aa33a8e1c33d274d91fd3c2a953c6951  server.py

service/MODEL_CARD.md

@@ -0,0 +1,53 @@
+---
+license: apache-2.0
+license_link: https://ai.google.dev/gemma/docs/gemma_4_license
+base_model: google/gemma-4-31B-it
+pipeline_tag: text-generation
+tags:
+- gemma4
+- compressed-tensors
+- lora
+- peft
+- vllm
+- jevbench
+---
+
+# MetaDecision-31B-Rain
+
+Decision service for JevBench. Base model:
+[`google/gemma-4-31B-it`](https://huggingface.co/google/gemma-4-31B-it), used through Google's
+official quantization-aware-training (QAT) releases of it. The model has two parts that are loaded
+together and **not merged**; both are the files of
+[`MeTask-Rain/metask-jev-rain-31B-trained`](https://huggingface.co/MeTask-Rain/metask-jev-rain-31B-trained/tree/2feff076f07711fe2fefe7a6fdcedfe4132b773e),
+unchanged and pinned by SHA-256 in `MODEL_MANIFEST.json` of the service code. This entry uses the same weights as that
+entry and changes only the serving rules.
+
+| Part | Location in that model repository | What it is |
+|---|---|---|
+| Base | repository root | The files of [`google/gemma-4-31B-it-qat-w4a16-ct`](https://huggingface.co/google/gemma-4-31B-it-qat-w4a16-ct) at revision `52f3f65bc7a02d555763bc923bd1d9094898219d`, byte for byte: Google's official compressed-tensors W4A16 checkpoint (4-bit weights, group size 32, symmetric). |
+| Adapter | `adapter/` | Our LoRA (PEFT format): rank 16, alpha 32, on the language model's q/k/v/o/gate/up/down projections, trained by us in BF16 on [`google/gemma-4-31B-it-qat-q4_0-unquantized`](https://huggingface.co/google/gemma-4-31B-it-qat-q4_0-unquantized) at revision `1e4d8beecacb8b7590c1d8bedd7335f687bf311f`. |
+
+`adapter/adapter_model.safetensors` contains the trained checkpoint tensors unchanged plus, for the 10 full-attention layers (5, 11, …, 59) that compute values with the key projection, a v_proj LoRA that is a byte copy of the trained k_proj LoRA (20 tensors), so that vLLM applies the update to keys and values as in training.
+
+## Serving
+
+The service code (`service/` in the Hugging Face repository `MeTask-Rain/MetaDecision-31B-Rain`, mirrored at https://github.com/chenyuwang166-oss/MetaDecision-31B-Rain) starts vLLM 0.31.0 on a folder holding the files of that weight
+repository, with `enable_lora`, `max_lora_rank=16`, and the `adapter/` folder as the single LoRA,
+applied to every request. Each decision is read from the next-token log-probabilities of the answer
+forms in one prompt pass; up to 49% of decisions then take a second, short drafting pass
+(at most 192 tokens) under a 9 s deadline, a choice decision answered in one pass although its length allowed the drafting pass is read a second time with the options in reverse order when its first-pass margin is below 4, and the reported probabilities get a per-type post-calibration (DISCLOSURE.md).
+
+Equivalent vLLM usage of the model:
+
+```python
+from vllm import LLM
+from vllm.lora.request import LoRARequest
+llm = LLM(model=MODEL_DIR, dtype="bfloat16", enable_lora=True, max_lora_rank=16)
+adapter = LoRARequest("adapter", 1, MODEL_DIR + "/adapter")
+# llm.generate(prompts, sampling_params, lora_request=adapter)
+```
+
+Facts on data and the serving rules are in `DISCLOSURE.md` of the service code (`service/DISCLOSURE.md` in the
+Hugging Face repository).
+
+Contact: chenyuwang166@gmail.com

service/MODEL_MANIFEST.json

@@ -0,0 +1,117 @@
+{
+  "adapter": {
+    "adapter_manifest_sha256": "6c40a7534f5d2b87cc8591748f85c7958f0259821c98659a60e25308904aa430",
+    "alpha": 32,
+    "files": {
+      "adapter_config.json": {
+        "sha256": "140bb00c832d117f57d29650dabe50e992b2e5064e908aaa29230b3ce19ca2c4",
+        "size": 22953
+      },
+      "adapter_model.safetensors": {
+        "sha256": "2b9568998ffed5d133223ffdff03056be297285c643122db4d45365d0bd86970",
+        "size": 494595208
+      }
+    },
+    "format": "peft-lora",
+    "merged": false,
+    "rank": 16,
+    "stored_tensors": {
+      "dtypes": {
+        "F32": 840
+      },
+      "k_eq_v_layers_without_v_proj_lora": [],
+      "layers": 60,
+      "module_count": 420,
+      "module_prefix": "model.language_model.",
+      "rank": 16,
+      "targets": [
+        "down_proj",
+        "gate_proj",
+        "k_proj",
+        "o_proj",
+        "q_proj",
+        "up_proj",
+        "v_proj"
+      ],
+      "tensor_count": 840,
+      "v_proj_copied_from_k_proj_layers": [
+        5,
+        11,
+        17,
+        23,
+        29,
+        35,
+        41,
+        47,
+        53,
+        59
+      ],
+      "variant": "key_value_aligned"
+    },
+    "subdir": "adapter",
+    "target_modules": [
+      "down_proj",
+      "gate_proj",
+      "k_proj",
+      "o_proj",
+      "q_proj",
+      "up_proj",
+      "v_proj"
+    ],
+    "training_base": "google/gemma-4-31B-it-qat-q4_0-unquantized",
+    "training_revision": "1e4d8beecacb8b7590c1d8bedd7335f687bf311f",
+    "variant": "key_value_aligned"
+  },
+  "arm": "trained",
+  "excluded_repository_documents": [
+    "README.md",
+    ".gitattributes"
+  ],
+  "files": {
+    "chat_template.jinja": {
+      "sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4",
+      "size": 18683
+    },
+    "config.json": {
+      "sha256": "b100d85e571c25b688e82919d819253063a1defb0e6cee3b9b1c1fbad99bd0c2",
+      "size": 18711
+    },
+    "generation_config.json": {
+      "sha256": "70110aa886fa7ee25ce47346d102dd29ca5f967500490b4362fc72068e1db70a",
+      "size": 208
+    },
+    "model.safetensors": {
+      "sha256": "1b9b1d622a93f02c0d33f98e502f233b5d707443af6ddc464ed0bf5498506c20",
+      "size": 23265352448
+    },
+    "processor_config.json": {
+      "sha256": "32bdf45d2ad4cc29a0822ddd157a182de76644f0419a6228d151495256e9813c",
+      "size": 1689
+    },
+    "tokenizer.json": {
+      "sha256": "cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f",
+      "size": 32169626
+    },
+    "tokenizer_config.json": {
+      "sha256": "b8045a4576903e86903291d5cbdd4adfc8859e9ce3c98621bdbd957f73ed394b",
+      "size": 3728
+    }
+  },
+  "format": "compressed-tensors",
+  "identity_sha256": "0513d121db967ec1a6aaccf627d3c42149d092dd027298da5b53684b030821ea",
+  "pricing_base_model": "google/gemma-4-31B-it",
+  "publish_authority": "repository owner",
+  "repo_id": "MeTask-Rain/metask-jev-rain-31B-trained",
+  "revision": "9392335f5a17395ecd70049f85b1909d53500144",
+  "served_base": {
+    "files_byte_identical": true,
+    "model_sha256": "ca750c75ea690276947dfc6d3c775668282e9180e4c56f180bfad912fed606a5",
+    "repo_id": "google/gemma-4-31B-it-qat-w4a16-ct",
+    "revision": "52f3f65bc7a02d555763bc923bd1d9094898219d"
+  },
+  "status": "evaluated_service_mirror",
+  "training_base": {
+    "repo_id": "google/gemma-4-31B-it-qat-q4_0-unquantized",
+    "revision": "1e4d8beecacb8b7590c1d8bedd7335f687bf311f"
+  }
+}

service/PIPELINE_PROVENANCE.json

@@ -0,0 +1,145 @@
+{
+  "adapter_reference": "model-subdir:adapter",
+  "arm": "trained",
+  "core_patch": null,
+  "input_budget": 6653,
+  "measured_environment": {
+    "entries": 201,
+    "equal_to_source_package_measured_lock": false,
+    "files": {
+      "requirements.lock": "4536b444d83518a5d696ebf262aa9d3da156fe2d898f96582e4ab52d70cb4e4d",
+      "requirements.measured.lock": "888ec6b0d3fad1a17e9323c1ab9321040303c79df3c9fe43e6f755cf2ab24b91"
+    },
+    "freeze_sha256": "888ec6b0d3fad1a17e9323c1ab9321040303c79df3c9fe43e6f755cf2ab24b91",
+    "source": "pip freeze of the measurement service environment"
+  },
+  "model_key": "metask-gemma4-31b-lora-w4a16",
+  "model_name": "MetaDecision-31B-Rain",
+  "notes": [
+    "fit_report.json, calibration.json and service_policy.base_fit / fit_n describe the first-step calibration of the earlier service of these weights (unchanged); the serving rules of this entry are config.json routing / generation / item_deadline / item_rules / post_calibration",
+    "weights.revision is the model manifest revision; evaluated_service_revision pins the same model files"
+  ],
+  "protocol": "adapter_arm_31b_v3",
+  "quota": 0.49,
+  "reference_source_files": {
+    "attention.py": "9c887f4e5f8ea2858f79a629e684ea71b07359d49d2a0bf31b5f993cb367e547",
+    "backends.py": "e0613ebb8e05fd22673e0502a131dcc33a5af563998cf5a12143878131518f93",
+    "bench.py": "7f158bcb4dcd90395c21535527abe4283d6ca5dab92ded0be47cc352f022f0cd",
+    "core.py": "fb4169ed1c6c9d69e406e9a33d69a71cc49aa48e0155d9544bb9f3bf41690a17",
+    "model_setup.py": "6c7cd992039327178b72247b8b0f44214a5a00e43abcc0f583130dfe91586dd7",
+    "quota.py": "8b039bc30c13e6c5a63668f3064fbb311358f9295922a817ec26f09893936f8d",
+    "reconcile_usage.py": "a381dca414201652f254f5101f08d4cbba27b69c3abeb50e6dad86e0e151aba7",
+    "server.py": "2cb085c1bf0f68758097afee55ecad2ef7ecd76dcced00c1b3f927b49151e213"
+  },
+  "route": "unmerged LoRA adapter (adapter/) on the official W4A16 compressed-tensors base",
+  "served_base": {
+    "repo_id": "google/gemma-4-31B-it-qat-w4a16-ct",
+    "revision": "52f3f65bc7a02d555763bc923bd1d9094898219d"
+  },
+  "served_mode": "routed",
+  "server_patch": {
+    "change": "install_extensions(): item_deadline with item_rules, then post_calibration",
+    "patched_server_sha256": "dbce4f70c2c684ac8ea5069a9dfa1d30aa33a8e1c33d274d91fd3c2a953c6951",
+    "reference_server_sha256": "2cb085c1bf0f68758097afee55ecad2ef7ecd76dcced00c1b3f927b49151e213"
+  },
+  "service_policy": {
+    "base_fit": {
+      "draft": 256,
+      "q": 0.056
+    },
+    "core_patch": null,
+    "cost_fuse": {
+      "arm": 0.9,
+      "enabled": true,
+      "release": 0.85,
+      "warmup": 100
+    },
+    "cost_fuse_prices": {
+      "input_per_million": 0.059819178473256074,
+      "output_per_million": 0.059819178473256074
+    },
+    "draft": {
+      "max_new_tokens": 192,
+      "reasoning_words": 50
+    },
+    "fit_n": 231,
+    "fit_protocol": "31b-adapter-arm-v3-routed",
+    "fuse": 0.49,
+    "item_deadline": {
+      "chunk_tokens": 32,
+      "ewma_alpha": 0.3,
+      "item_seconds": 9.0,
+      "min_chunk_tokens": 8,
+      "readout_reserve_init_s": 1.0,
+      "readout_reserve_min_s": 0.3,
+      "readout_safety": 1.5,
+      "response_reserve_s": 0.35,
+      "token_safety": 1.25,
+      "warmup_repeats": 3,
+      "warmup_tokens": 33
+    },
+    "item_rules": {
+      "reread_tau": 4.0,
+      "slow_types": [
+        "choice",
+        "noul",
+        "score"
+      ]
+    },
+    "margin_threshold": 6.625023671062799,
+    "mode": "routed",
+    "noul_p_cal": 0.898,
+    "post_calibration": {
+      "choice": {
+        "config": {
+          "debias": false,
+          "temperature": false
+        },
+        "params": {}
+      },
+      "noul": {
+        "config": {
+          "floor": null,
+          "temperature": "type"
+        },
+        "params": {
+          "T": 0.5
+        }
+      },
+      "score": {
+        "config": {
+          "mode_weight": 1.0,
+          "temperature": false
+        },
+        "params": {}
+      }
+    },
+    "post_calibration_active": true,
+    "protocol": "adapter_arm_31b_v3",
+    "quota": 0.49,
+    "slow_excluded_types": [],
+    "slow_max_prompt_tokens": 840,
+    "threshold_rule": "eligible_items",
+    "unparsed_policy": "force_answer",
+    "v3_params_sha256": "b28ed73bb5a36e99a280a47a6b846e0c5b2de54f34a96efa4a10754b71ea7573"
+  },
+  "source_files": {
+    "attention.py": "9c887f4e5f8ea2858f79a629e684ea71b07359d49d2a0bf31b5f993cb367e547",
+    "backends.py": "e0613ebb8e05fd22673e0502a131dcc33a5af563998cf5a12143878131518f93",
+    "bench.py": "7f158bcb4dcd90395c21535527abe4283d6ca5dab92ded0be47cc352f022f0cd",
+    "core.py": "fb4169ed1c6c9d69e406e9a33d69a71cc49aa48e0155d9544bb9f3bf41690a17",
+    "item_deadline.py": "8d3d637ab3db1be9769e26900b4fc46964e48425a05bb92402a2b504ae66ff3a",
+    "model_setup.py": "6c7cd992039327178b72247b8b0f44214a5a00e43abcc0f583130dfe91586dd7",
+    "post_calibration.py": "62a89b2b87cee77322333774a6e081f28646804c926af0f0c0e798a2018b3ffe",
+    "quota.py": "8b039bc30c13e6c5a63668f3064fbb311358f9295922a817ec26f09893936f8d",
+    "reconcile_usage.py": "a381dca414201652f254f5101f08d4cbba27b69c3abeb50e6dad86e0e151aba7",
+    "server.py": "dbce4f70c2c684ac8ea5069a9dfa1d30aa33a8e1c33d274d91fd3c2a953c6951"
+  },
+  "weights": {
+    "evaluated_service_revision": "2feff076f07711fe2fefe7a6fdcedfe4132b773e",
+    "identity_sha256": "0513d121db967ec1a6aaccf627d3c42149d092dd027298da5b53684b030821ea",
+    "note": "same model files as the earlier service of these weights; not copied",
+    "repo_id": "MeTask-Rain/metask-jev-rain-31B-trained",
+    "revision": "9392335f5a17395ecd70049f85b1909d53500144"
+  }
+}

service/README.md

@@ -0,0 +1,164 @@
+# MetaDecision-31B-Rain
+
+A self-hosted decision service for JevBench. It serves `POST /v1/systemone` for the unchanged
+`typesafe` adapter of the official harness, and `GET /health`.
+
+The model weights are exactly those of
+[`MeTask-Rain/metask-jev-rain-31B-trained`](https://huggingface.co/MeTask-Rain/metask-jev-rain-31B-trained/tree/2feff076f07711fe2fefe7a6fdcedfe4132b773e)
+(the same files, pinned by SHA-256 in `MODEL_MANIFEST.json`): Google's official W4A16 checkpoint
+`google/gemma-4-31B-it-qat-w4a16-ct` (unchanged) and our LoRA adapter, loaded together and never
+merged. `MODEL_MANIFEST.json` names that repository's earlier revision `9392335f5a17395ecd70049f85b1909d53500144`, which
+holds the same model files. This entry changes only how the service uses them:
+
+- a larger share of decisions may take a short drafting pass (slow-path quota 49%, at most 192 drafted tokens);
+- a decision that takes the drafting pass has a deadline of 9 s: the draft is stopped in time for its answer to be read, and a drafting pass that cannot be read in time returns the first-pass answer;
+- a choice decision answered in one pass although its length allowed the drafting pass, whose first-pass margin is below 4, is read once more with the options in reverse order, and the two answer distributions are averaged;
+- the reported probabilities get a per-type post-calibration fitted on our own development set (no model call);
+- the cost fuse prices engine tokens at the leaderboard's GPU-time rate for this architecture (USD 0.0598 per million tokens).
+
+Facts and serving rules are in [DISCLOSURE.md](DISCLOSURE.md).
+
+## Requirements
+
+- One NVIDIA GPU used only by this service: compute capability 8.0 or newer, NVIDIA driver 580 or
+  newer, 80 GB or more of GPU memory. Validated: steps 2-6 below, run with this package and the model files downloaded in step 2, completed the 231 public items on one NVIDIA A100 80GB (231/231, zero failed). The service starts vLLM with
+  `gpu_memory_utilization=0.94` (it reserves 94% of the GPU memory), `max_model_len=6656`,
+  `enable_lora` and `max_lora_rank=16`. The model files take 23.3 GB (base) plus 495 MB (adapter).
+  The commands use the host's GPU 0 (`--gpus '"device=0"'`).
+- Docker with the NVIDIA Container Toolkit.
+- On the host: Git and Python with pip; step 2 installs the Hugging Face command line tool `hf`.
+- Runtime image `vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b`
+  (Python 3.12.3, vLLM 0.31.0, torch 2.13.0+cu130), the linux/amd64 image of vLLM release
+  `v0.31.0`. Nothing is installed and the service has no network access.
+- Model folder: the model files of https://huggingface.co/MeTask-Rain/metask-jev-rain-31B-trained/tree/2feff076f07711fe2fefe7a6fdcedfe4132b773e
+  as regular files (step 2). The repository root holds the official base files; `adapter/` holds
+  the adapter (`adapter_config.json`, `adapter_model.safetensors`). A Hugging Face cache
+  `snapshots/<revision>` directory holds links and cannot be mounted on its own. At startup the
+  service checks the SHA-256 of the small model files, the size of the weight file and the full
+  SHA-256 of both adapter files (`MODEL_MANIFEST.json`), and refuses other weight, tokenizer,
+  processor or template files in the folder; it never writes to the folder.
+- Code: this repository (download command in step 1).
+- Harness: https://github.com/fstandhartinger/jevbench/tree/bb05a335bc809e61b20c0f745d25499a82b326fc
+
+## Run
+
+Copy the commands as they are, in the same Bash terminal.
+
+1. Obtain the code and harness, and create a fresh work directory. Run from a writable parent
+   directory; the new directories must not already contain another run:
+
+   ```bash
+   git clone https://github.com/chenyuwang166-oss/MetaDecision-31B-Rain.git
+   PKG="$(pwd)/MetaDecision-31B-Rain"
+   git clone https://github.com/fstandhartinger/jevbench.git jevbench
+   git -C jevbench checkout bb05a335bc809e61b20c0f745d25499a82b326fc
+   JEVBENCH="$(pwd)/jevbench"
+   MODEL="$(pwd)/model-31b"
+   WORK="$(pwd)/work-31b-run1"
+   mkdir "$MODEL" "$WORK"
+   TASKS=/jevbench/datasets/public/easy.jsonl,/jevbench/datasets/public/original.jsonl,/jevbench/datasets/public/hard.jsonl
+   ```
+
+2. Install the download tool, fetch the pinned weights, and pull the pinned runtime image:
+
+   ```bash
+   python3 -m pip install huggingface_hub
+   hf download MeTask-Rain/metask-jev-rain-31B-trained --revision 2feff076f07711fe2fefe7a6fdcedfe4132b773e --local-dir "$MODEL" --exclude 'service/*'
+   docker pull vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b
+   ```
+
+3. Start the service:
+
+   ```bash
+   docker run -d --name metadecision31b --init --gpus '"device=0"' --network none --read-only --shm-size 8g \
+     -v "$PKG":/pkg:ro -v "$MODEL":/model:ro -v "$WORK":/work -v "$JEVBENCH":/jevbench:ro \
+     --entrypoint bash vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b \
+     /pkg/serve.sh --model /model --state-dir /work/state --run-id run1
+   ```
+
+4. Follow the log until the service is ready:
+
+   ```bash
+   docker logs -f metadecision31b
+   ```
+
+   When a line starting with `{"status": "ok"` appears, press Ctrl-C (this ends only the log
+   display) and check health before step 5:
+
+   ```bash
+   docker exec metadecision31b python3 -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health").read().decode())'
+   ```
+
+   If the log ends before that line, the container has exited: the reason is in
+   `docker logs metadecision31b` and in `$WORK/state/run1-startup/attempt-*.log`; go to step 6.
+   After correcting the cause, use a new `WORK` and restart from step 3.
+
+5. Run the official harness in the same container, then summarize:
+
+   ```bash
+   docker exec -e PYTHONPATH=/jevbench metadecision31b python3 -B -m jevbench.cli run --adapter typesafe \
+     --endpoint http://localhost:8000 --model MetaDecision-31B-Rain --key-env '' --tasks "$TASKS" \
+     --results /work/run1/results.jsonl --ledger /work/run1/ledger.jsonl --raw-dir /work/run1/raw \
+     --manifest /work/run1/manifest.json --cap-usd 15 --price-in-per-m 0.09 --price-out-per-m 0.34
+   docker exec -e PYTHONPATH=/jevbench metadecision31b python3 -B -m jevbench.cli summarize --tasks "$TASKS" \
+     --results /work/run1/results.jsonl --ledger /work/run1/ledger.jsonl
+   ```
+
+6. Stop and remove the container:
+
+   ```bash
+   docker stop -t 40 metadecision31b && docker rm metadecision31b
+   ```
+
+Each evaluation uses a new `WORK` and a new container. A run name that already exists below the
+state directory is refused at startup ("use a new --state-dir"); with these commands that means a
+new `WORK`.
+
+## What to expect
+
+- Health line: `"status": "ok"`, `"warnings": []`, `"model_key": "metask-gemma4-31b-lora-w4a16"`,
+  `"mode": "routed"`, `"quantization_kernel": "compressed-tensors"`, `"prefix_caching": true`,
+  `"supports_slow_session": true`, and `threshold_identity.adapter` equal to `/model/adapter`, the
+  verified adapter folder.
+- Startup: `launch.py` verifies the model folder and the adapter, writes the run configuration
+  (with the adapter folder filled in) to `$WORK/state/run1-startup/config.runtime.json`, and starts
+  vLLM with `enable_lora` and `max_lora_rank=16`; it sets `VLLM_BATCH_INVARIANT=1`,
+  `VLLM_USE_FLASHINFER_SAMPLER=0` and `VLLM_WORKER_MULTIPROC_METHOD=spawn`. Each startup attempt
+  waits at most 1800 s for readiness: the first lets vLLM select the attention backend; only if the
+  engine fails to start does a second attempt request `TRITON_ATTN`. Before serving, the service
+  times a few short synthetic drafts to size its drafting chunks.
+- Each decision is first read in one prompt pass. At most 49% of decisions take a second,
+  drafting pass (at most 192 generated tokens) before the answer is read again; such a
+  decision is planned to finish within its 9 s deadline, counted from the moment the
+  service starts answering the request (an engine call in progress is completed, never abandoned;
+  DISCLOSURE.md states the rules and the cost fuse). Inputs longer than 6,653 tokens are answered in
+  one pass from a context shortened to at most 6,653 tokens by the service's extraction rule.
+- Writes: the service writes files only below `$WORK/state` (plus shared memory in `/dev/shm`) and
+  the harness only below `$WORK/run1`. The package and model mounts are read-only. Cache and
+  configuration path variables (`HF_HOME`, `XDG_*`, `TRITON_*`, `TORCH_HOME` and similar) must not
+  be passed to the container: a value outside `/work/state` is refused at startup. No script of
+  this package deletes directories recursively.
+- Responses: an item the service cannot process returns HTTP 422 without answer and without usage;
+  after an engine failure, and from the third consecutive item failure on, the service answers
+  every `POST /v1/systemone` with HTTP 503 and `/health` reports `"status": "error"`. No substitute
+  answer is ever returned. A decision whose drafting pass could not be read before its deadline is
+  answered (HTTP 200) with its valid first-pass answer and carries a `warning` note, also listed in
+  `partial_errors`.
+
+The health response keeps `threshold_identity.quant: "awq"` as a historical configuration alias;
+the actual backend is **compressed-tensors W4A16**, reported by `quantization_kernel`.
+
+## Package checks (offline, no GPU or network)
+
+```bash
+cd "$PKG"
+python3 -B pkg_check.py             # package integrity, configuration, adapter contract, serving policy
+python3 -B selftest.py --offline    # synthetic software tests, including long-input compression
+```
+
+`MODEL_MANIFEST.json` pins the weight repository, revision and every model and adapter file;
+`ADAPTER_MANIFEST.json` records the adapter's origin by SHA-256; `FREEZE.json` records the freeze
+of this configuration; `MANIFEST.sha256` binds all package files. `MODEL_CARD.md` describes the
+model.
+
+Contact: chenyuwang166@gmail.com

service/SERVICE_LOCK.json

@@ -0,0 +1,48 @@
+{
+  "adapter_files": {
+    "adapter_config.json": {
+      "sha256": "140bb00c832d117f57d29650dabe50e992b2e5064e908aaa29230b3ce19ca2c4",
+      "size": 22953
+    },
+    "adapter_model.safetensors": {
+      "sha256": "2b9568998ffed5d133223ffdff03056be297285c643122db4d45365d0bd86970",
+      "size": 494595208
+    }
+  },
+  "arm": "trained",
+  "calibration_sha256": "5b60ca8516b93982724145bc2227404c4c53a6f5657b4bf2155108c2f001685f",
+  "complete": true,
+  "config_file_sha256": "0c84b933ac60e43394dbd748da21cb27bf91fe640cac7674acc3d02b80e8de11",
+  "fit_report_sha256": "fc20f2f631c98f043f7c02d4cadbe8122580ee400377eef5420d93621111248b",
+  "protocol": "adapter_arm_31b_v3",
+  "q": 0.49,
+  "serving": {
+    "draft": {
+      "max_new_tokens": 192,
+      "reasoning_words": 50
+    },
+    "fuse": 0.49,
+    "item_deadline_sha256": "3633ad26b201ba9200d6bb606852a80cef33a0c58adc0709f15f4bc8a49d9148",
+    "item_rules": {
+      "reread_tau": 4.0,
+      "slow_types": [
+        "choice",
+        "noul",
+        "score"
+      ]
+    },
+    "post_calibration_sha256": "02cbf05b38bb627acd416f64e66a5cea6a70ce3a7055f1976fcf54e965396951",
+    "quota": 0.49,
+    "selection": {
+      "freeze_sha256": "6e5325eb9713ed73ed9fc94992b67b1179ea60d6969757af61d5c8860d2bb22e",
+      "protocol_sha256": "1c279f3d780a0a522a217c38f74892e9c7019be5012409a9c83bbd9dbc179771",
+      "readout_sha256": "8cc627afcf00b3b9ba9edafe004adc8294d45c62bec8a2dc5b3f593cac4ea0a2",
+      "selected": "serving rules selected on our own development set",
+      "selection_sha256": "cc1070b2d79a088983cdb5c300adc7c1ed5c25232003830a9d1a8611e4df6820"
+    },
+    "threshold": 6.625023671062799
+  },
+  "serving_identity_sha256": "0513d121db967ec1a6aaccf627d3c42149d092dd027298da5b53684b030821ea",
+  "status": "frozen",
+  "variant": "key_value_aligned"
+}

service/attention.py

@@ -0,0 +1,37 @@
+"""Report the attention backend named in the engine log; the requested setting is reported separately."""
+import os
+from pathlib import Path
+import re
+
+# vLLM 0.31 logs "Using AttentionBackendEnum.<NAME> backend."; older formats name the backend in prose.
+_PATTERN = re.compile(r'\bUsing\s+(?:AttentionBackendEnum\.([A-Za-z0-9_]+)|([A-Za-z][A-Za-z0-9_ -]*?))\s+backend\b', re.I)
+_ALIASES = {'FLASH_ATTENTION': 'FLASH_ATTN', 'TRITON_ATTENTION': 'TRITON_ATTN',
+            'FLASH_INFER': 'FLASHINFER', 'FLASHINFER_ATTENTION': 'FLASHINFER'}
+
+
+def attention_health():
+    """Backend names found in the engine log of this service process (JEV_ATTENTION_LOG).
+
+    The requested value comes from JEV_ATTENTION_BACKEND; 'auto' means vLLM selected the backend.
+    A request is never reported as the active backend.
+    """
+    requested = os.environ.get('JEV_ATTENTION_BACKEND') or 'auto'
+    text = ''
+    path = os.environ.get('JEV_ATTENTION_LOG')
+    if path:
+        try:
+            text = Path(path).read_text(encoding='utf-8', errors='replace')
+        except OSError:
+            pass
+    text = re.sub(r'\x1b\[[0-9;]*m', '', text)
+    names = []
+    for enum_name, prose in _PATTERN.findall(text):
+        raw = (enum_name or prose).strip().upper().replace('-', '_').replace(' ', '_')
+        if not enum_name and 'ATTN' not in raw and 'ATTENTION' not in raw:
+            continue  # "Using X backend" lines about other kernels
+        names.append(_ALIASES.get(raw, raw.removesuffix('_ATTENTION')))
+    names = list(dict.fromkeys(names))
+    return {'attention_backend': names[-1] if names else None,
+            'attention_backends': names,
+            'attention_backend_requested': requested,
+            'attention_backend_source': 'engine_log' if names else 'unreported'}

service/backends.py

@@ -0,0 +1,691 @@
+"""Lazy model backends; importing this module never imports or loads a model.
+
+Both backends return unmasked next-token scores. vLLM returns raw log
+probabilities, which differ from logits by one common additive constant; the
+form-set logsumexp and subsequent label temperature are invariant to it.
+"""
+
+from __future__ import annotations
+
+import math
+import logging
+import json
+import hashlib
+import os
+import re
+import threading
+import time
+from pathlib import Path
+from typing import Any, Iterable
+
+
+class BackendError(RuntimeError):
+    """An unavailable or incompatible backend, with no approximation fallback."""
+
+
+def _candidate_ids(values: Iterable[int]) -> list[int]:
+    ids = list(dict.fromkeys(values))
+    if not ids or any(isinstance(i, bool) or not isinstance(i, int) or i < 0 for i in ids):
+        raise ValueError("candidate_ids must contain nonnegative integer token IDs")
+    return ids
+
+
+def _max_new_tokens(value: int) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        raise ValueError("max_new_tokens must be a positive integer")
+    return value
+
+
+def _check_scores(scores: dict[int, float]) -> dict[int, float]:
+    if any(math.isnan(value) or value == math.inf for value in scores.values()):
+        raise BackendError("backend returned NaN or positive infinite candidate scores")
+    return scores
+
+
+def generation_audit(config):
+    """Compare explicit service settings; never silently inherit checkpoint policy."""
+    report={'verified':False,'status':'unavailable','differences':{},'intended_overrides':{},'sha256':None}
+    model_path=config['model'].get('path')
+    path=Path(os.path.expanduser(model_path))/'generation_config.json' if model_path else None
+    if path is not None and path.is_file():
+        try:
+            data=path.read_bytes();reference=json.loads(data)
+            if not isinstance(reference,dict):raise ValueError('Expected mapping')
+            report['sha256']=hashlib.sha256(data).hexdigest()
+            def safe(v):
+                if v is None or isinstance(v,bool) or isinstance(v,(int,float)) and math.isfinite(v):return v
+                if isinstance(v,list):return [safe(x) for x in v]
+                return 'unsupported value'
+            for key,value in config['generation'].items():
+                if key not in ('seed','reasoning_words','max_new_tokens','wall_timeout_seconds') and key in reference and reference[key]!=value:
+                    difference={'checkpoint':safe(reference[key]),'configured':safe(value)}
+                    declared=config.get('generation_reference',{}).get('intended_overrides',{}).get(key,{})
+                    intended=(isinstance(declared,dict) and declared.get('reason')
+                              and 'value' in declared and type(declared['value']) is type(value)
+                              and declared['value']==value and config['generation'].get('do_sample') is False)
+                    target='intended_overrides' if intended else 'differences'
+                    if intended:difference['reason']=declared['reason']
+                    report[target][key]=difference
+            report.update(verified=not report['differences'],status='mismatch' if report['differences'] else 'matched_explicit_keys')
+        except (OSError,ValueError,TypeError):report['status']='unreadable'
+    if not report['verified']:logging.warning('Generation reference %s; explicit configured parameters remain active',report['status'])
+    return report
+
+
+def install_forward_counter(backend):
+    backend.forward_counters={'forward_tokens':0,'prefill_tokens':0,'uncounted_forwards':0}
+    def count_forward(module,args,kwargs):
+        try:
+            ids=kwargs.get('input_ids',args[0] if args else None)
+            if ids is None:ids=kwargs.get('inputs_embeds')
+            if ids is None:raise ValueError('No input tensor')
+            n=int(ids.shape[1]);past=kwargs.get('past_key_values')
+            empty=past is None or (callable(getattr(past,'get_seq_length',None)) and past.get_seq_length()==0)
+            backend.forward_counters['forward_tokens']+=n
+            if empty:backend.forward_counters['prefill_tokens']+=n
+        except Exception:
+            backend.forward_counters['uncounted_forwards']+=1
+            logging.warning('HF forward count unavailable; usage estimated')
+    target=backend.model.get_base_model() if callable(getattr(backend.model,'get_base_model',None)) else backend.model
+    hook=getattr(target,'register_forward_pre_hook',None)
+    backend.stats_available=callable(hook)
+    if backend.stats_available:backend.stats_hook=hook(count_forward,with_kwargs=True)
+    else:logging.warning('HF forward hook unavailable; usage estimated')
+
+
+def runtime_info():
+    """Versions on the numeric path and the visible GPU, reported in /health info.runtime.
+
+    The GPU name comes from vLLM's platform interface (NVML), so this process creates no CUDA
+    context. triton_module is the version of the triton module the process actually imported.
+    """
+    import importlib.metadata
+    import platform
+    import sys
+    info = {"python": platform.python_version()}
+    for name in ("vllm", "torch", "transformers", "tokenizers", "triton", "compressed-tensors"):
+        try:
+            info[name] = importlib.metadata.version(name)
+        except importlib.metadata.PackageNotFoundError:
+            info[name] = None
+    info["triton_module"] = getattr(sys.modules.get("triton"), "__version__", None)
+    try:
+        import torch
+        info["torch_cuda"] = torch.version.cuda
+    except Exception:
+        info["torch_cuda"] = None
+    try:
+        from vllm.platforms import current_platform
+        info["gpu"] = current_platform.get_device_name()
+    except Exception:
+        info["gpu"] = None
+    return info
+
+
+class TokenizerMixin:
+    tokenizer: Any
+
+    def scan_digit_tokens(self):
+        vocab=self.tokenizer.get_vocab() if hasattr(self.tokenizer,'get_vocab') else {}
+        self.digit_start_ids={token for token in set(vocab.values())
+                              if re.match(r'^\d',self.decode([token]).lstrip())}
+
+    def encode(self, text: str) -> list[int]:
+        return list(self.tokenizer.encode(text, add_special_tokens=False))
+
+    def decode(self, ids: Iterable[int]) -> str:
+        return self.tokenizer.decode(list(ids), skip_special_tokens=True)
+
+    def prompt_ids(self, system: str, text: str, enable_thinking: bool = False) -> list[int]:
+        # The instructions and the context share one user turn, as in all measured runs.
+        messages = [{"role": "user", "content": system + "\n\n" + text}]
+        try:
+            prompt = self.tokenizer.apply_chat_template(
+                messages, tokenize=False, add_generation_prompt=True,
+                enable_thinking=enable_thinking,
+            )
+        except TypeError as exc:
+            if "enable_thinking" not in str(exc):
+                raise
+            prompt = self.tokenizer.apply_chat_template(
+                messages, tokenize=False, add_generation_prompt=True,
+            )
+        return self.encode(prompt)
+
+
+class TransformersBackend(TokenizerMixin):
+    """Direct single-forward readout, and a separate bounded draft generation."""
+
+    readout_output_tokens = 0
+
+    def __init__(self, config: dict[str, Any]):
+        model_cfg = config["model"]
+        self.generation = dict(config["generation"])
+        self.generation_reference=generation_audit(config)
+        self.marker = config.get("prompt", {}).get("answer_marker", "ANSWER:")
+        if model_cfg["quant"] not in {"int8", "bf16"}:
+            raise ValueError("model.quant must be int8 or bf16")
+        if not model_cfg["path"]:
+            raise ValueError("model.path is required before loading a model")
+        try:
+            import torch
+            import transformers
+        except ImportError as exc:
+            raise BackendError("transformers backend requires torch and transformers") from exc
+        self.torch = torch
+        torch.manual_seed(self.generation["seed"])
+        # expanduser keeps a Hub identifier's slash intact on Windows.
+        path = os.path.expanduser(model_cfg["path"])
+        cfg = transformers.AutoConfig.from_pretrained(path)
+        declared_quant = getattr(cfg, "quantization_config", None)
+        if declared_quant:
+            if not isinstance(declared_quant, dict):
+                declared_quant = declared_quant.to_dict()
+            if model_cfg["quant"] == "bf16":
+                raise BackendError("bf16 requires an unquantized checkpoint; base config declares quantization")
+            if (declared_quant.get("quant_method") != "bitsandbytes"
+                    or not declared_quant.get("load_in_8bit")
+                    or declared_quant.get("load_in_4bit")):
+                raise BackendError("int8 requires an unquantized base or a BitsAndBytes 8bit checkpoint")
+        archs = list(getattr(cfg, "architectures", None) or [])
+        if any("Qwen3_5ForConditionalGeneration" in name for name in archs):
+            model_cls = getattr(transformers, "Qwen3_5ForConditionalGeneration", None)
+            if model_cls is None:
+                raise BackendError("installed transformers lacks Qwen3_5ForConditionalGeneration")
+        else:
+            model_cls = transformers.AutoModelForCausalLM
+        kwargs = {
+            "dtype": torch.bfloat16,
+            "device_map": model_cfg["device_map"],
+            "low_cpu_mem_usage": True,
+            "attn_implementation": model_cfg["attn_implementation"],
+        }
+        if model_cfg["quant"] == "int8":
+            kwargs["quantization_config"] = transformers.BitsAndBytesConfig(load_in_8bit=True)
+        self.tokenizer = transformers.AutoTokenizer.from_pretrained(path)
+        self.scan_digit_tokens()
+        self.model = model_cls.from_pretrained(path, **kwargs).eval()
+        if model_cfg.get("adapter"):
+            try:
+                from peft import PeftModel
+            except ImportError as exc:
+                raise BackendError("--adapter requires peft for the transformers backend") from exc
+            self.model = PeftModel.from_pretrained(
+                self.model, str(Path(model_cfg["adapter"]).expanduser()),
+            ).eval()
+        # A fresh config prevents checkpoint-specific suppression, forced tokens,
+        # sampling filters or minimum-length policies from silently changing the
+        # configured draft. Preserve special-token metadata, including EOS lists.
+        special = {}
+        for name in ("bos_token_id", "eos_token_id", "pad_token_id"):
+            for source in (getattr(self.model, "generation_config", None),
+                           getattr(self.model, "config", None), self.tokenizer):
+                value = getattr(source, name, None)
+                if value is not None:
+                    special[name] = value
+                    break
+        self.prefix_caching=False
+        install_forward_counter(self)
+        self.draft_generation_config = transformers.GenerationConfig(**special, **{k:v for k,v in self.generation.items()
+            if k not in ("reasoning_words","seed","max_new_tokens","wall_timeout_seconds")})
+        eos = special.get("eos_token_id", [])
+        self.eos_ids = eos if isinstance(eos, list) else [eos]
+
+    def engine_stats(self):
+        return {'source':'hf_forward_hook','prefix_caching':False,
+                'counters':dict(self.forward_counters) if self.stats_available else {}}
+
+    def open_session(self, ids):
+        return CachedSession(self, ids)
+
+    def slow_session(self, ids, unparsed_policy, meter, max_new_tokens=None):
+        count=_max_new_tokens(max_new_tokens or self.generation['max_new_tokens'])
+        tensor=self._input(ids)
+        meter['prompt']+=len(ids); meter['engine_submitted']+=len(ids)
+        meter['engine_input']=meter.get('engine_input',0)+len(ids)
+        meter['engine_calls']=meter.get('engine_calls',0)+1
+        meter['hf_prefill_tokens']=meter.get('hf_prefill_tokens',0)+len(ids)
+        class MeterStreamer:
+            first=True
+            def put(inner,value):
+                if inner.first: inner.first=False; return
+                n=value.shape[-1]
+                meter['output']+=n; meter['draft']+=n
+                meter['engine_generated']=meter.get('engine_generated',0)+n
+                if not getattr(inner,'counted',False):
+                    meter['generate_nonempty_calls']=meter.get('generate_nonempty_calls',0)+1; inner.counted=True
+            def end(inner): pass
+        streamer=MeterStreamer()
+        g=self.generation
+        kwargs=dict(input_ids=tensor,attention_mask=self.torch.ones_like(tensor),
+            generation_config=self.draft_generation_config,max_new_tokens=count,
+            max_time=g.get("wall_timeout_seconds",60),
+            do_sample=g['do_sample'],num_beams=1,num_return_sequences=1,
+            repetition_penalty=g['repetition_penalty'],use_cache=True,
+            return_dict_in_generate=True,stop_strings=[self.marker],tokenizer=self.tokenizer,
+            streamer=streamer,pad_token_id=self.tokenizer.pad_token_id or self.tokenizer.eos_token_id)
+        if g['do_sample']: kwargs.update(temperature=g['temperature'],top_p=g['top_p'],top_k=g['top_k'])
+        check_deadline(meter)
+        try:
+            with self.torch.inference_mode(): output=self.model.generate(**kwargs)
+        except Exception:
+            meter['engine_usage_unknown']=True
+            raise
+        check_deadline(meter)
+        generated=output.sequences[0,len(ids):].tolist()
+        # Transformers streams sampled IDs even when the next forward raises.
+        if meter['draft']!=len(generated): raise BackendError('Generation streamer accounting mismatch')
+        parsed=bool(re.search(r'(?m)^[ \t]*'+re.escape(self.marker)+r'$',self.decode(generated)))
+        if not parsed and unparsed_policy=='fast': return None,False
+        cache=output.past_key_values
+        if cache is None or not generated: raise BackendError('Generation returned no reusable cache')
+        session=object.__new__(CachedSession)
+        session.backend,session.cache,session.length=self,cache,len(ids)+len(generated)-1
+        session._log_probs=None
+        session.meter=meter
+        # Standard generate cache is one sampled token behind sequences.
+        session.advance(generated[-1:])
+        if not parsed:
+            forced=self.encode('\n'+self.marker)
+            meter['output']+=len(forced);meter['forced']+=len(forced)
+            session.advance(forced)
+        return session,parsed
+
+    def _input(self, ids: Iterable[int]):
+        values = list(ids)
+        if not values:
+            raise ValueError("prompt token IDs cannot be empty")
+        embedding = self.model.get_input_embeddings()
+        device = embedding.weight.device
+        if getattr(device, "type", None) == "meta":
+            device = self.model.device
+        return self.torch.tensor([values], dtype=self.torch.long, device=device)
+
+    def next_logits(self, ids: Iterable[int], candidate_ids: Iterable[int]) -> dict[int, float]:
+        candidates = _candidate_ids(candidate_ids)
+        tensor = self._input(ids)
+        with self.torch.inference_mode():
+            try:
+                output = self.model(input_ids=tensor, use_cache=False, logits_to_keep=1)
+            except TypeError as exc:
+                if "logits_to_keep" not in str(exc):
+                    raise
+                output = self.model(input_ids=tensor, use_cache=False)
+            logits = output.logits[0, -1, :]
+            if max(candidates) >= logits.shape[0]:
+                raise BackendError("candidate token ID exceeds the model output vocabulary")
+            index = self.torch.tensor(candidates, dtype=self.torch.long, device=logits.device)
+            selected = logits.index_select(0, index).float().cpu().tolist()
+        return _check_scores(dict(zip(candidates, selected)))
+
+    def generate(self, ids: Iterable[int], max_new_tokens: int) -> list[int]:
+        count = _max_new_tokens(max_new_tokens)
+        tensor = self._input(ids)
+        generation = self.generation
+        pad_id = self.tokenizer.pad_token_id
+        if pad_id is None:
+            pad_id = self.tokenizer.eos_token_id
+        kwargs = {
+            "input_ids": tensor,
+            "attention_mask": self.torch.ones_like(tensor),
+            "generation_config": self.draft_generation_config,
+            "max_new_tokens": count,
+            "do_sample": generation["do_sample"],
+            "num_beams": 1,
+            "num_return_sequences": 1,
+            "repetition_penalty": generation["repetition_penalty"],
+            "use_cache": True,
+            "return_dict_in_generate": False,
+            "pad_token_id": pad_id,
+        }
+        if generation["do_sample"]:
+            kwargs.update(temperature=generation["temperature"],
+                          top_p=generation["top_p"], top_k=generation["top_k"])
+        with self.torch.inference_mode():
+            output = self.model.generate(**kwargs)
+        return output[0, tensor.shape[1]:].tolist()
+
+
+class VLLMBackend(TokenizerMixin):
+    """Exact candidate readout through explicit raw logprob token IDs.
+
+    The engine produces one probe token to expose the next-token distribution;
+    callers must include readout_output_tokens in measured output-token usage.
+    Normal probes gather only explicit candidate IDs. Full-vocabulary output
+    is reserved for one logged recovery probe if a backend omits candidates.
+    This implementation supports bf16 and an explicitly checked AWQ checkpoint.
+    It rejects int8 because vLLM's
+    BnB loader settings do not establish equivalence to the reference int8 base.
+    """
+
+    readout_output_tokens = 1
+    engine_metering = True
+
+    @staticmethod
+    def _awq_quant_arg(model_cfg):
+        """AWQ profile: pass quantization='awq' for classic AWQ checkpoints; let vLLM auto-detect
+        compressed-tensors (W4A16) checkpoints, whose config.json declares quant_method."""
+        if model_cfg["quant"]!="awq": return None
+        try:
+            import json as _json
+            cfg=_json.loads(Path(os.path.join(os.path.expanduser(model_cfg["path"]),"config.json")).read_text(encoding="utf-8"))
+            qm=((cfg.get("quantization_config") or {}).get("quant_method") or "").lower()
+            if qm in ("compressed-tensors","compressed_tensors"): return None
+        except Exception:
+            pass
+        return "awq"
+
+    def __init__(self, config: dict[str, Any]):
+        model_cfg = config["model"]
+        if model_cfg["quant"] not in ("bf16","awq"):
+            raise BackendError("vllm backend supports bf16 or AWQ; use transformers for int8")
+        if not model_cfg["path"]:
+            raise ValueError("model.path is required before loading a model")
+        options = dict(model_cfg["vllm_options"])
+        if config.get("limits"):
+            options.setdefault("max_model_len",config["limits"]["physical_max_prompt_tokens"]+2048)
+        reserved = {
+            "model", "tokenizer", "dtype", "quantization", "quantization_config",
+            "max_logprobs", "logprobs_mode", "seed", "enable_lora",
+            "generation_config", "override_generation_config", "logits_processors",
+            "skip_tokenizer_init", "speculative_config", "enable_prefix_caching", "disable_log_stats",
+            "attention_backend",
+        }
+        overlap = sorted(reserved.intersection(options))
+        if overlap:
+            raise ValueError("model.vllm_options cannot override readout contract: " + ", ".join(overlap))
+        try:
+            from vllm import LLM, SamplingParams
+        except ImportError as exc:
+            raise BackendError("vllm backend requires an importable compatible vllm installation") from exc
+        self.SamplingParams = SamplingParams
+        self.marker=config.get("prompt",{}).get("answer_marker","ANSWER:")
+        self.lock=threading.RLock()
+        self.generation = dict(config["generation"])
+        self.generation_reference=generation_audit(config)
+        self.lora_request = None
+        if model_cfg.get("adapter"):
+            try:
+                from vllm.lora.request import LoRARequest
+                self.lora_request = LoRARequest(
+                    "adapter", 1, str(Path(model_cfg["adapter"]).expanduser()),
+                )
+            except (ImportError, TypeError, ValueError) as exc:
+                raise BackendError("installed vllm does not support the requested LoRA adapter API") from exc
+        # Unset means vLLM selects the attention backend (auto). The supervisor sets
+        # JEV_ATTENTION_BACKEND=TRITON_ATTN only for its second startup attempt.
+        requested_attention = os.environ.get("JEV_ATTENTION_BACKEND")
+        if requested_attention:
+            options["attention_backend"] = requested_attention
+        try:
+            # These are correctness invariants, not configurable routing policy.
+            self.engine = LLM(
+                model=os.path.expanduser(model_cfg["path"]), dtype="bfloat16",
+                quantization=self._awq_quant_arg(model_cfg), seed=self.generation["seed"],
+                max_logprobs=-1, logprobs_mode="raw_logprobs",
+                generation_config="vllm", enable_lora=bool(self.lora_request), disable_log_stats=False,
+                enable_prefix_caching=config.get("model",{}).get("enable_prefix_caching",True), **options,
+            )
+            # quantization=None permits checkpoint auto-detection in vLLM. Do
+            # not label that run bf16 if the checkpoint activates quantization.
+            active_model_cfg = self.engine.llm_engine.model_config
+            if model_cfg["quant"]=="bf16" and active_model_cfg.quantization is not None:
+                raise BackendError("bf16 requires an unquantized checkpoint; vllm activated quantization")
+            if model_cfg['quant']=='awq' and active_model_cfg.quantization not in ('awq','awq_marlin','compressed-tensors'):
+                raise BackendError('AWQ profile requires active AWQ quantization')
+            self.quantization_kernel=active_model_cfg.quantization
+            cache_cfg=getattr(getattr(self.engine.llm_engine,'vllm_config',None),'cache_config',None)
+            self.prefix_caching=getattr(cache_cfg,'enable_prefix_caching',None)
+            self.runtime_info=runtime_info()
+            self.tokenizer = self.engine.get_tokenizer()
+            self.scan_digit_tokens()
+            self.probe_params = SamplingParams(
+                max_tokens=1, temperature=1.0, top_p=1.0, top_k=-1,
+                min_p=0.0, presence_penalty=0.0, frequency_penalty=0.0,
+                repetition_penalty=1.0, logprobs=-1, detokenize=False,
+                seed=self.generation["seed"],
+            )
+        except Exception as exc:
+            raise BackendError(
+                "vllm initialization failed; selected precision/architecture/adapter and full-vocabulary "
+                "raw_logprobs support are required. No approximate fallback is used."
+            ) from exc
+
+    def engine_stats(self):
+        counters={}
+        accepted={'prompt_tokens','generation_tokens','prefix_cache_queries','prefix_cache_hits','num_preemptions','request_success'}
+        for metric in self.engine.get_metrics():
+            name=metric.name.removeprefix('vllm:').removesuffix('_total')
+            if name not in accepted: continue
+            value=getattr(metric,'value',None)
+            if isinstance(value,(int,float)) and not isinstance(value,bool):
+                counters[name]=counters.get(name,0)+value
+        return {'source':'vllm_metrics','prefix_caching':self.prefix_caching,'counters':counters}
+
+    def _completion(self, ids: Iterable[int], sampling_params: Any, meter=None, kind="probes"):
+        check_deadline(meter)
+        values = list(ids)
+        if not values:
+            raise ValueError("prompt token IDs cannot be empty")
+        kwargs = {"use_tqdm": False}
+        if self.lora_request is not None:
+            kwargs["lora_request"] = self.lora_request
+        try:
+            if meter is not None: meter['engine_submitted']+=len(values)
+            # TokensPrompt is a TypedDict; its runtime representation is this dict.
+            with self.lock:
+                check_deadline(meter)
+                result = self.engine.generate(
+                    [{"prompt_token_ids": values}], sampling_params, **kwargs,
+                )
+        except TimeoutError:
+            raise
+        except Exception as exc:
+            if meter is not None: meter['engine_usage_unknown']=True
+            raise BackendError("vllm request failed; verify its architecture, adapter and logprobs support") from exc
+        if len(result) != 1 or len(result[0].outputs) != 1:
+            raise BackendError("vllm did not return exactly one completion")
+        if meter is not None:
+            n=len(result[0].outputs[0].token_ids)
+            meter['output']+=n;meter[kind]+=n
+            cached=getattr(result[0],'num_cached_tokens',None)
+            if cached is None: meter['cached_unknown']=True
+            else: meter['cached']+=int(cached)
+            meter['engine_calls']=meter.get('engine_calls',0)+1
+            meter['engine_generated']=meter.get('engine_generated',0)+n
+            meter['engine_input']=meter.get('engine_input',0)+len(values)-int(cached or 0)
+        check_deadline(meter)
+        return result[0].outputs[0]
+
+    def next_logits(self, ids: Iterable[int], candidate_ids: Iterable[int], meter=None) -> dict[int, float]:
+        candidates = _candidate_ids(candidate_ids)
+        values = list(ids)
+        scores = {}
+        # v0.31 supports explicit gather from RAW logprobs (before masks).
+        # Chunking preserves the same full-vocabulary normalizer for all IDs.
+        for start in range(0, len(candidates), 128):
+            wanted = candidates[start:start + 128]
+            kwargs = dict(max_tokens=1, temperature=1.0, top_p=1.0, top_k=-1,
+                          repetition_penalty=1.0, detokenize=False,
+                          seed=self.generation['seed'])
+            try:
+                params = self.SamplingParams(**kwargs, allowed_token_ids=wanted,
+                    logprob_token_ids=wanted, logprobs=len(wanted))
+            except TypeError:
+                params = self.SamplingParams(**kwargs, logprobs=min(50, max(len(wanted), 1)))
+            output = self._completion(values, params, meter)
+            distribution = output.logprobs[0] if output.logprobs else None
+            if len(output.token_ids) != 1:
+                raise BackendError('Probe must produce exactly one token')
+            if distribution is None or any(t not in distribution for t in wanted):
+                logging.warning('Candidate readout incomplete; one full-vocabulary recovery probe')
+                if meter is not None:
+                    meter.setdefault('warnings', []).append('candidate_full_vocab_recovery')
+                output = self._completion(values, self.probe_params, meter)
+                distribution = output.logprobs[0] if output.logprobs else None
+                if len(output.token_ids) != 1 or distribution is None or any(t not in distribution for t in candidates):
+                    raise BackendError('vllm omitted candidate token logprobs after recovery')
+                return _check_scores({t: float(distribution[t].logprob) for t in candidates})
+            scores.update({t: float(distribution[t].logprob) for t in wanted})
+        return _check_scores(scores)
+
+    def open_session(self, ids, meter=None):
+        return VLLMPrefixSession(self,ids,meter if meter is not None else new_meter())
+
+    def slow_session(self, ids, unparsed_policy, meter, max_new_tokens=None):
+        count=_max_new_tokens(max_new_tokens or self.generation['max_new_tokens'])
+        g=self.generation; sampling=g['do_sample']
+        params=self.SamplingParams(max_tokens=count,temperature=g['temperature'] if sampling else 0.0,
+            top_p=g['top_p'] if sampling else 1.0,top_k=(g['top_k'] or -1) if sampling else -1,
+            repetition_penalty=g['repetition_penalty'],seed=g['seed'],
+            stop=[self.marker],include_stop_str_in_output=True)
+        meter['prompt']+=len(ids)
+        out=self._completion(ids,params,meter,'draft')
+        gen=list(out.token_ids)
+        parsed=(getattr(out,'stop_reason',None)==self.marker and
+                getattr(out,'finish_reason',None)=='stop' and
+                bool(re.search(r'(?m)^[ \t]*'+re.escape(self.marker)+r'$',self.decode(gen))))
+        if not parsed:
+            if unparsed_policy=='fast': return None,False
+            forced=self.encode('\n'+self.marker);gen+=forced
+            meter['output']+=len(forced);meter['forced']+=len(forced)
+        return VLLMPrefixSession(self,list(ids)+gen,meter),parsed
+
+    def generate(self, ids: Iterable[int], max_new_tokens: int) -> list[int]:
+        count = _max_new_tokens(max_new_tokens)
+        generation = self.generation
+        sampling = generation["do_sample"]
+        # HF uses top_k=0 for unrestricted sampling; vLLM uses top_k=-1.
+        top_k = generation["top_k"] if generation["top_k"] > 0 else -1
+        params = self.SamplingParams(
+            max_tokens=count, temperature=generation["temperature"] if sampling else 0.0,
+            top_p=generation["top_p"] if sampling else 1.0,
+            top_k=top_k if sampling else -1,
+            repetition_penalty=generation["repetition_penalty"],
+            seed=generation["seed"], detokenize=False,
+        )
+        output = self._completion(ids, params)
+        if len(output.token_ids) > count:
+            raise BackendError("vllm generated more tokens than max_new_tokens")
+        return list(output.token_ids)
+
+
+def create_backend(config: dict[str, Any]):
+    name = config["model"]["backend"]
+    if name == "transformers":
+        return TransformersBackend(config)
+    if name == "vllm":
+        return VLLMBackend(config)
+    raise ValueError("model.backend must be transformers or vllm")
+
+class CachedSession:
+    """Single prefill followed exclusively by KV-backed token continuations.
+
+    Branches own their cache, including recurrent attention state. Models that
+    cannot copy/cache fail into the service's fast path rather than re-prefill.
+    """
+    def __init__(self, backend, ids):
+        self.backend = backend
+        self.cache = None
+        self.length = 0
+        self.advance(list(ids))
+
+    def advance(self, ids):
+        if not ids:
+            return
+        backend = self.backend
+        meter=getattr(self,'meter',None)
+        if meter is not None:
+            check_deadline(meter)
+            meter['engine_input']+=len(ids);meter['engine_submitted']+=len(ids)
+        kwargs = dict(input_ids=backend._input(ids), use_cache=True, logits_to_keep=1)
+        if self.cache is not None:
+            kwargs['past_key_values'] = self.cache
+        try:
+            with backend.torch.inference_mode():
+                output = backend.model(**kwargs)
+        except Exception:
+            if meter is not None:meter['engine_usage_unknown']=True
+            raise
+        self.cache = output.past_key_values
+        if self.cache is None:
+            raise BackendError('Model did not return a reusable cache')
+        self.length += len(ids)
+        self._logits = output.logits[0, -1, :].float()
+        self._log_probs = None
+
+    def top_token(self):
+        return int(self._logits.argmax().item())
+
+    def log_probs(self, ids):
+        if self._log_probs is None:
+            self._log_probs=self._logits.log_softmax(dim=-1)
+        index=self.backend.torch.tensor(list(ids),dtype=self.backend.torch.long,device=self._logits.device)
+        return _check_scores(dict(zip(ids,self._log_probs.index_select(0,index).cpu().tolist())))
+
+    def non_digit_logprob(self, digit_ids):
+        if self._log_probs is None: self._log_probs=self._logits.log_softmax(dim=-1)
+        ids=[i for i in digit_ids if i<self._logits.shape[0]]
+        if not ids: return 0.0
+        torch=self.backend.torch
+        index=torch.tensor(ids,dtype=torch.long,device=self._logits.device)
+        # Sum excluded mass on device; return one scalar, never vocab logits.
+        mass=self._log_probs.index_select(0,index).exp().sum().clamp(max=1.0)
+        return float(torch.log1p(-mass).item())
+
+    def fork(self):
+        import copy
+        branch = object.__new__(type(self))
+        branch.backend, branch.length = self.backend, self.length
+        branch.meter=getattr(self,'meter',None)
+        branch.cache = copy.deepcopy(self.cache)
+        branch._logits = self._logits
+        branch._log_probs = self._log_probs
+        return branch
+
+
+def check_deadline(meter):
+    if meter and (meter.get('cancel_event') is not None and meter['cancel_event'].is_set() or
+                  meter.get('deadline') is not None and time.monotonic()>=meter['deadline']):
+        raise TimeoutError('Slow deadline exceeded')
+
+
+def new_meter():
+    return dict(prompt=0,output=0,cached=0,engine_submitted=0,draft=0,forced=0,probes=0,
+                engine_calls=0,engine_generated=0,engine_input=0,generate_nonempty_calls=0,hf_prefill_tokens=0)
+
+
+class VLLMPrefixSession:
+    """Immutable prefix branches; APC owns KV storage and recomputation."""
+    handles_meter=True
+    def __init__(self,backend,ids,meter):
+        self.backend,self.ids,self.meter=backend,list(ids),meter
+        self._logits=None
+
+    @property
+    def logits(self):
+        if self._logits is None:
+            out=self.backend._completion(self.ids,self.backend.probe_params,self.meter)
+            if not out.logprobs or out.logprobs[0] is None:
+                raise BackendError('Missing full-vocabulary probe distribution')
+            self._logits=_check_scores({t:float(lp.logprob) for t,lp in out.logprobs[0].items()})
+        return self._logits
+
+    def log_probs(self,ids):
+        return self.backend.next_logits(self.ids, ids, self.meter)
+
+    def non_digit_logprob(self, digit_ids):
+        if not digit_ids: return 0.0
+        scores = self.log_probs(sorted(digit_ids))
+        mass = min(1.0, math.fsum(math.exp(v) for v in scores.values()))
+        return math.log1p(-mass) if mass < 1 else -math.inf
+
+    def fork(self):
+        branch=VLLMPrefixSession(self.backend,self.ids,self.meter)
+        branch._logits=self._logits
+        return branch
+
+    def advance(self,ids):
+        # These are candidate continuation tokens, separate from engine probes.
+        self.ids+=list(ids);self._logits=None
+        self.meter['output']+=len(ids);self.meter['probes']+=len(ids)

service/bench.py

@@ -0,0 +1,795 @@
+"""Sequential, model-free HTTP benchmark client for the /v1/systemone service (maintainer tool).
+
+All outputs are created with exclusive mode at an absolute --output-prefix outside the package.
+"""
+
+import argparse
+import json
+import math
+import statistics
+import time
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent
+PUBLIC_FILES = (("easy.jsonl", "easy"), ("original.jsonl", "standard"), ("hard.jsonl", "hard"))
+TYPES = ("noul", "choice", "score")
+ALIASES = {"boolean": "noul", "enum": "choice"}
+QUESTION_NAME = "decision"
+# Local service calls never go through an HTTP proxy.
+_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+
+
+def canonical_type(value):
+    value = ALIASES.get(value, value)
+    if value not in TYPES:
+        raise ValueError("unsupported question type")
+    return value
+
+
+def _read_rows(path):
+    with Path(path).expanduser().open(encoding="utf-8-sig") as source:
+        for line_number, line in enumerate(source, 1):
+            if not line.strip():
+                continue
+            try:
+                row = json.loads(line)
+            except (ValueError, TypeError) as exc:
+                raise ValueError(f"invalid JSONL at line {line_number}") from exc
+            if not isinstance(row, dict):
+                raise ValueError(f"JSONL line {line_number} must be an object")
+            yield row
+
+
+def normalize_task(task, tier=None, require_expected=False):
+    """Keep the source's label order, including choice tie-breaking order."""
+    if not isinstance(task.get("id"), str) or not task["id"]:
+        raise ValueError("task id must be a nonempty string")
+    if "state" not in task or not isinstance(task.get("question"), dict):
+        raise ValueError("task needs state and question")
+    question = dict(task["question"])
+    qtype = canonical_type(question.get("type"))
+    question["request_type"] = question.get("request_type",question["type"])
+    question["type"] = qtype
+    instructions = question.get("instructions", question.get("description"))
+    if not isinstance(instructions, str) or not instructions.strip():
+        raise ValueError("question needs nonempty instructions or description")
+    question["instructions"] = instructions
+    question.pop("description", None)
+    criteria = question.get("criteria")
+    if qtype == "noul":
+        if criteria is not None and not isinstance(criteria, dict):
+            raise ValueError("noul criteria must be an object")
+        labels = ["no", "yes"]
+    elif qtype == "score":
+        if not isinstance(criteria, list) or not criteria:
+            raise ValueError("score criteria must be a nonempty list")
+        labels = [str(index) for index in range(len(criteria))]
+    else:
+        labels = task.get("labels", question.get("labels"))
+        if labels is None and isinstance(criteria, dict):
+            labels = list(criteria)
+        if labels is None and isinstance(criteria, list):
+            labels = criteria
+        if not isinstance(labels, list) or not 1 <= len(labels) <= 676:
+            raise ValueError("choice requires between 1 and 676 labels")
+        labels = list(labels)
+        if any(not isinstance(label, str) or not label for label in labels):
+            raise ValueError("choice labels must be nonempty strings")
+        if len(set(labels)) != len(labels):
+            raise ValueError("choice labels must be distinct")
+        question["labels"] = labels
+    expected = task.get("expected")
+    if expected is not None:
+        if qtype == "noul" and isinstance(expected, bool):
+            expected = "yes" if expected else "no"
+        elif qtype == "score":
+            expected = str(expected)
+        if expected not in labels:
+            raise ValueError("expected answer is not a valid label")
+    elif require_expected:
+        raise ValueError("public source task is missing expected answer")
+    result = dict(task)
+    result.update(question=question, type=qtype, labels=labels, expected=expected)
+    result["tier"] = tier if tier is not None else task.get("tier", "custom")
+    return result
+
+
+def load_jsonl_tasks(path):
+    tasks = [normalize_task(row) for row in _read_rows(path)]
+    _require_unique(tasks)
+    if not tasks:
+        raise ValueError("input contains no tasks")
+    return tasks
+
+
+def _require_unique(rows):
+    seen = set()
+    for row in rows:
+        if row["id"] in seen:
+            raise ValueError("duplicate task id")
+        seen.add(row["id"])
+
+
+def load_public_tasks(index, public, expected_count):
+    """Use only id/type/tier from the index; all task content comes from public."""
+    metadata = []
+    for cached in _read_rows(index):
+        # Do not use cached expected labels, logits, predictions or prompt content.
+        item = {key: cached.get(key) for key in ("id", "type", "tier")}
+        if not isinstance(item["id"], str) or not item["id"]:
+            raise ValueError("index id must be a nonempty string")
+        item["type"] = canonical_type(item["type"])
+        if item["tier"] not in {tier for _, tier in PUBLIC_FILES}:
+            raise ValueError("index tier is not a public tier")
+        metadata.append(item)
+    _require_unique(metadata)
+    if len(metadata) != expected_count:
+        raise ValueError("index task count does not match expected_public_count")
+    by_id = {}
+    for filename, tier in PUBLIC_FILES:
+        for row in _read_rows(Path(public).expanduser() / filename):
+            if "tier" in row and row["tier"] != tier:
+                raise ValueError("public task tier conflicts with its source file")
+            task = normalize_task(row, tier=tier, require_expected=True)
+            if task["id"] in by_id:
+                raise ValueError("duplicate public task id")
+            by_id[task["id"]] = task
+    selected = []
+    for meta in metadata:
+        task = by_id.get(meta["id"])
+        if task is None:
+            raise ValueError("indexed task missing from public sources")
+        if task["type"] != meta["type"] or task["tier"] != meta["tier"]:
+            raise ValueError("indexed type or tier does not match public source")
+        selected.append(task)
+    return selected
+
+
+def request_payload(task, original_types=False, model=None, label_order=False):
+    question=task['question']
+    q={'type':question['type'],'instructions':question['instructions']}
+    if question.get('criteria') is not None: q['criteria']=question['criteria']
+    if original_types:q['type']=question.get('request_type',question['type'])
+    if label_order and task['type']=='choice': q['labels']=list(task['labels'])
+    return {'state':task['state'],'model':model,'questions':{QUESTION_NAME:q}}
+
+
+def endpoint_url(endpoint):
+    parts = urllib.parse.urlsplit(endpoint)
+    if parts.scheme not in {"http", "https"} or not parts.netloc or parts.query or parts.fragment:
+        raise ValueError("endpoint must be an HTTP base URL or the systemone route")
+    if parts.username or parts.password:
+        raise ValueError("endpoint must not contain credentials")
+    path = parts.path.rstrip("/")
+    if path not in {"", "/v1/systemone"}:
+        raise ValueError("endpoint path must be empty or /v1/systemone")
+    return urllib.parse.urlunsplit((parts.scheme, parts.netloc, "/v1/systemone", "", ""))
+
+
+def post_json(endpoint, payload, timeout):
+    request = urllib.request.Request(endpoint_url(endpoint),
+                                     data=json.dumps(payload).encode("utf-8"),
+                                     headers={"Content-Type": "application/json"}, method="POST")
+    with _OPENER.open(request, timeout=timeout) as response:
+        return json.load(response)
+
+
+def parse_prediction(response, task):
+    if not isinstance(response, dict) or not isinstance(response.get("answers"), dict):
+        raise ValueError("response must contain answers")
+    if not isinstance(response.get("model"), str) or not response["model"]:
+        raise ValueError("response must identify its model")
+    answer = response["answers"].get(QUESTION_NAME)
+    if not isinstance(answer, dict) or canonical_type(answer.get("type")) != task["type"]:
+        raise ValueError("response question type mismatch")
+    if answer.get("warning") or response.get("partial_errors"):
+        raise ValueError("degraded API response")
+    probabilities = answer.get("probabilities")
+    if task["type"]=="noul" and isinstance(probabilities,dict) and set(probabilities)=={"true","false"}:
+        probabilities={"yes":probabilities["true"],"no":probabilities["false"]}
+    if not isinstance(probabilities, dict) or set(probabilities) != set(task["labels"]):
+        raise ValueError("response probabilities must have exactly the requested labels")
+    values = list(probabilities.values())
+    if any(isinstance(value, bool) or not isinstance(value, (int, float))
+           or not math.isfinite(value) or value < 0 or value > 1 for value in values):
+        raise ValueError("response contains invalid probabilities")
+    if not math.isclose(sum(values), 1.0, rel_tol=1e-6, abs_tol=1e-6):
+        raise ValueError("response probabilities must sum to one")
+    # Match official score_task: lexicographically smallest label wins ties.
+    return max(sorted(task["labels"]), key=lambda label: probabilities[label]), probabilities
+
+
+def _token_count(value):
+    return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None
+
+
+def _safe_usage_json(value):
+    """Keep malformed optional metadata from making result JSON unwritable."""
+    if value is None or isinstance(value, (str, bool, int)):
+        return value, False
+    if isinstance(value, float):
+        return (value, False) if math.isfinite(value) else (None, True)
+    if isinstance(value, list):
+        parts = [_safe_usage_json(item) for item in value]
+        return [item for item, _ in parts], any(changed for _, changed in parts)
+    if isinstance(value, dict):
+        result, changed = {}, False
+        for key, item in value.items():
+            if not isinstance(key, str):
+                changed = True
+                continue
+            result[key], item_changed = _safe_usage_json(item)
+            changed = changed or item_changed
+        return result, changed
+    return None, True
+
+
+def extract_usage(response):
+    usage = response.get("usage") if isinstance(response, dict) else None
+    usage = usage if isinstance(usage, dict) else {}
+    usage, sanitized = _safe_usage_json(usage)
+    questions = usage.get("questions", {})
+    detail = questions.get(QUESTION_NAME, {}) if isinstance(questions, dict) else {}
+    detail = detail if isinstance(detail, dict) else {}
+    prompt = _token_count(usage.get("input_tokens", usage.get("prompt_tokens")))
+    completion = _token_count(usage.get("output_tokens", usage.get("completion_tokens")))
+    total = _token_count(usage.get("total_tokens"))
+    n_new_tokens = _token_count(detail.get("n_new_tokens"))
+    complete = prompt is not None and completion is not None and total is not None
+    complete = complete and total == prompt + completion
+    complete = complete and n_new_tokens is not None
+    if usage.get("accounting") is None:complete=complete and n_new_tokens<=completion if complete else False
+    complete = complete and not sanitized and not detail.get("usage_estimated", False) and not usage.get("usage_estimated",False)
+    path = detail.get("path")
+    if not isinstance(path, str) or path not in {"fast", "slow"}:
+        path = None
+    router_score = detail.get("router_score")
+    if isinstance(router_score, bool) or not isinstance(router_score, (int, float)) or not math.isfinite(router_score):
+        router_score = None
+    return {"usage": usage, "prompt_tokens": prompt, "completion_tokens": completion,
+            "total_tokens": total, "usage_complete": bool(complete), "usage_sanitized": sanitized,
+            "n_new_tokens": n_new_tokens, "path": path, "router_score": router_score,
+            "fast_prompt_tokens":detail.get("fast_prompt_tokens"), "prefix_caching":response.get("prefix_caching"),
+            "fast_scores":detail.get("fast_scores",detail.get("raw_label_scores")),
+            "slow_scores":detail.get("slow_scores",detail.get("slow_raw_label_scores")),
+            "slow_raw_label_scores":detail.get("slow_raw_label_scores"),
+            "calibration_block":detail.get("calibration_block"),"threshold_identity":detail.get("threshold_identity"),
+            "budget_tier":detail.get("budget_tier"), "cost_fuse":detail.get("cost_fuse"),
+            "raw_label_scores":detail.get("raw_label_scores"), "fast_margin":detail.get("fast_margin"), "margin_kind":detail.get("margin_kind")}
+
+
+def run_benchmark(tasks, endpoint, config, client=post_json, clock=time.perf_counter, writer=None, concurrency=1):
+    """One request per task, no retries; ordered rows, latency includes each call."""
+    if type(concurrency) is not int or concurrency<1: raise ValueError('concurrency must be positive')
+    if concurrency>1:
+        from concurrent.futures import ThreadPoolExecutor
+        def one(task): return run_benchmark([task],endpoint,config,client,clock)[0]
+        with ThreadPoolExecutor(max_workers=concurrency) as pool:
+            rows=list(pool.map(one,tasks))
+        for row in rows: row["concurrency"]=concurrency
+        if writer is not None:
+            for row in rows: writer.write(json.dumps(row,ensure_ascii=False,allow_nan=False)+'\n')
+            writer.flush()
+        return rows
+    rows = []
+    timeout = config["bench"]["timeout_seconds"]
+    for task in tasks:
+        row = {"request_order":"labels" if config.get("bench",{}).get("label_order",False) else "criteria", "id": task["id"], "type": task["type"], "tier": task.get("tier"),
+               "benchmark_set":task.get("benchmark_set"), "concurrency":concurrency, "expected": task.get("expected"), "labels": task["labels"], "pred": None,
+               "probabilities": None, "ok": False, "error": None, "model": None}
+        row.update(extract_usage({}))
+        start = clock()
+        try:
+            response = client(endpoint, request_payload(task,config.get("bench",{}).get("original_types",False),config["model"]["name"],config.get("bench",{}).get("label_order",False)), timeout)
+        except Exception as exc:
+            # Exception text can include an endpoint or credentials; never save it.
+            row["error"] = {"type": type(exc).__name__, "message": "request failed"}
+            row["latency_seconds"] = clock() - start
+        else:
+            row["latency_seconds"] = clock() - start
+            row.update(extract_usage(response))
+            if isinstance(response, dict) and isinstance(response.get("model"), str):
+                row["model"] = response["model"]
+            try:
+                row["pred"], row["probabilities"] = parse_prediction(response, task)
+                row["ok"] = True
+            except (ValueError, KeyError, TypeError) as exc:
+                row["error"] = {"type": type(exc).__name__, "message": "invalid API response"}
+        row["correct"] = bool(row["ok"] and row["pred"] == row["expected"]) if row["expected"] is not None else None
+        rows.append(row)
+        if writer is not None:
+            writer.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n")
+            writer.flush()
+    return rows
+
+
+def percentile(values, fraction):
+    if not values:
+        return None
+    ordered = sorted(values)
+    rank = (len(ordered) - 1) * fraction
+    low, high = math.floor(rank), math.ceil(rank)
+    return ordered[low] + (ordered[high] - ordered[low]) * (rank - low)
+
+
+def summarize(rows, config, expected_count=None):
+    count = len(rows)
+    expected_count = count if expected_count is None else expected_count
+    successes = sum(bool(row["ok"]) for row in rows)
+    complete = count > 0 and count == expected_count and successes == count
+    usage_complete = count > 0 and all(row["usage_complete"] for row in rows)
+    routing_complete = count > 0 and all(row["path"] in {"fast", "slow"} for row in rows)
+    new_tokens_complete = count > 0 and all(row["n_new_tokens"] is not None for row in rows)
+    latencies = [row["latency_seconds"] for row in rows]
+    p50 = statistics.median(latencies) if latencies else None
+    price_key = config["pricing"]["model_key"]
+    observed_models = sorted({row["model"] for row in rows if isinstance(row.get("model"), str) and row["model"]})
+    price_model_matches = count > 0 and all(config["pricing"].get("submission_models",{}).get(row.get("model"),row.get("model")) == price_key for row in rows)
+    prices = config["pricing"]["models"].get(price_key, {})
+    input_price = prices.get("input_per_million")
+    output_price = prices.get("output_per_million")
+    price_known = all(isinstance(value, (int, float)) and not isinstance(value, bool)
+                      and math.isfinite(value) and value >= 0 for value in (input_price, output_price))
+    total_prompt = sum(row["prompt_tokens"] for row in rows) if usage_complete else None
+    total_completion = sum(row["completion_tokens"] for row in rows) if usage_complete else None
+    cost = (total_prompt * input_price + total_completion * output_price) / 1_000_000 if usage_complete and price_known and price_model_matches else None
+    cost_per_1000 = cost / count * 1000 if cost is not None else None
+    by_type = {}
+    for qtype in TYPES:
+        subset = [row for row in rows if row["type"] == qtype]
+        labeled = [row for row in subset if row["expected"] is not None]
+        correct = sum(row["correct"] is True for row in labeled)
+        by_type[qtype] = {"count": len(subset), "labeled_count": len(labeled), "correct": correct,
+                          "accuracy": correct / len(subset) if subset and len(labeled) == len(subset) else None}
+    limits = config["limits"]
+    scale=config.get('bench',{}).get('latency_scale',1)
+    offset=config.get('bench',{}).get('latency_offset_seconds',0)
+    scaled_p50=p50*scale+offset if p50 is not None else None
+    margin=config.get('pricing',{}).get('margin',0)
+    if not isinstance(margin,(float,int)) or not 0<=margin<1: raise ValueError('pricing.margin must be in [0,1)')
+    cost_limit=limits['cost_per_1000_decisions_usd']*(1-margin)
+    eligible = None
+    if complete and usage_complete and routing_complete and new_tokens_complete and cost_per_1000 is not None:
+        eligible = scaled_p50 <= limits["median_latency_seconds"] and cost_per_1000 <= cost_limit
+    slow_share = sum(row["path"] == "slow" for row in rows) / count if routing_complete else None
+    fast_q = 0.5 / (1 - slow_share) if slow_share is not None and slow_share < 1 else None
+    fast_latencies = [row["latency_seconds"] for row in rows if row["path"] == "fast" and row["ok"]]
+    path_stats = {}
+    for path in ("fast", "slow"):
+        subset = [row for row in rows if row["path"] == path and row["ok"]]
+        times = [row["latency_seconds"] for row in subset]
+        path_stats[path] = {"count":len(subset), "p50":percentile(times, .5),
+                            "p75":percentile(times,.75),"mean":statistics.mean(times) if times else None,
+                            "scaled_p75":percentile(times,.75)*scale+offset if times else None,
+                            "scaled_mean":statistics.mean(times)*scale+offset if times else None,
+                            "p90":percentile(times, .9), "p99":percentile(times, .99),
+                            'average_prompt_tokens':statistics.mean(r['prompt_tokens'] for r in subset) if subset and all(r['prompt_tokens'] is not None for r in subset) else None,
+                            'average_completion_tokens':statistics.mean(r['completion_tokens'] for r in subset) if subset and all(r['completion_tokens'] is not None for r in subset) else None}
+    tail = config.get("bench", {}).get("long_tail", {"count":23, "prompt_tokens":79000, "total_decisions":1624})
+    tail_kept = min(tail["prompt_tokens"], limits.get("max_prompt_tokens", 16000))
+    tail_cost = tail["count"] * tail_kept * input_price / 1e6 if price_known else None
+    projection = {"count":tail["count"], "original_prompt_tokens_each":tail["prompt_tokens"],
+                  "kept_prompt_tokens_each":tail_kept, "path":"fast", "output_tokens_each":0,
+                  "estimated_cost_usd":tail_cost,
+                  "untruncated_fast_cost_usd":tail["count"] * tail["prompt_tokens"] * input_price / 1e6 if price_known else None,
+                  "assumption":"fast-only long-tail projection; final physical guard can discard protected content",
+                  "projected_cost_per_1000_decisions_usd":
+                      ((tail["total_decisions"]-tail["count"]) * cost/count + tail_cost) / tail["total_decisions"] * 1000
+                      if cost is not None and count and tail_cost is not None else None}
+    cost_basis = {'A':cost_per_1000,'B':None,'C':None}
+    accounting_details = [row.get('usage',{}).get('prompt_tokens_details') for row in rows]
+    if cost is not None and count and all(isinstance(d,dict) and
+            all(type(d.get(k)) is int and d[k]>=0 for k in ('cached_tokens','engine_submitted_tokens'))
+            and d['cached_tokens']<=d['engine_submitted_tokens'] for d in accounting_details):
+        submitted=sum(d['engine_submitted_tokens'] for d in accounting_details)
+        cached=sum(d['cached_tokens'] for d in accounting_details)
+        cost_basis['B']=(submitted*input_price+total_completion*output_price)/1e6/count*1000
+        if all(d.get('cached_tokens_known',True) for d in accounting_details):
+            cost_basis['C']=((submitted-cached)*input_price+total_completion*output_price)/1e6/count*1000
+    for name,legacy in [('engine','C'),('each_once','A'),('submitted','B')]:
+        values=[r.get('usage',{}).get('alternatives',{}).get(name) for r in rows]
+        if cost is not None and count and all(isinstance(v,dict) for v in values):
+            cost_basis[legacy]=sum(v['input_tokens']*input_price+v['output_tokens']*output_price for v in values)/1e6/count*1000
+        cost_basis[name]=cost_basis[legacy]
+    tail_tiers={}
+    for budget in tail.get('projection_budgets',[]):
+        extra=tail['count']*budget*input_price/1e6 if price_known else None
+        tail_tiers[str(budget)]={'count':tail['count'],'prompt_tokens_each':budget,
+            'fast_only_estimated_cost_usd':extra,
+            'projected_cost_per_1000_by_basis':{k:((tail['total_decisions']-tail['count'])*v/1000+extra)
+                /tail['total_decisions']*1000 if v is not None and extra is not None else None for k,v in cost_basis.items()}}
+    speed_rows=[r for r in rows if r.get('benchmark_set')=='open-set' and r.get('tier') in ('standard','judge') and r.get('concurrency',1)==1]
+    speed={'count':len(speed_rows),'scope':'explicit open-set standard+judge serial',
+           'scaled_p50':None,'scaled_p95':None,'complete':bool(speed_rows) and all(r['ok'] for r in speed_rows)}
+    if speed['complete']:
+        speed.update(scaled_p50=percentile([r['latency_seconds'] for r in speed_rows],.5)*scale+offset,
+                     scaled_p95=percentile([r['latency_seconds'] for r in speed_rows],.95)*scale+offset)
+    q_latency=percentile(fast_latencies,fast_q) if fast_q is not None and fast_q<=1 else None
+    budget_stats={}
+    for tier in ('normal','hard','long'):
+        subset=[r for r in rows if r.get('budget_tier')==tier]
+        labeled=[r for r in subset if r.get('correct') is not None]
+        known=bool(subset) and price_known and all(r['usage_complete'] for r in subset)
+        budget_stats[tier]={'count':len(subset),'accuracy':sum(r['correct'] for r in labeled)/len(labeled) if labeled else None,
+            'cost_usd':sum(r['prompt_tokens']*input_price+r['completion_tokens']*output_price for r in subset)/1e6 if known else None}
+    by_tier={}
+    for tier in sorted({str(r.get('tier')) for r in rows}):
+        subset=[r for r in rows if str(r.get('tier'))==tier];labeled=[r for r in subset if r.get('correct') is not None]
+        by_tier[tier]={'count':len(subset),'accuracy':sum(r['correct'] for r in labeled)/len(labeled) if labeled else None}
+    p_projection=project_cost(rows,config,config.get('bench',{}).get('projection_prompt_tokens'))
+    accuracy_only=config.get('bench',{}).get('accuracy_only',False)
+    if accuracy_only:eligible=None
+    return {"accuracy_only":accuracy_only,"by_tier":by_tier,"P_projection":p_projection,
+            "cost_gate":('UNKNOWN' if cost_per_1000 is None or accuracy_only else 'PASS' if cost_per_1000<=.85*cost_limit else 'WARN' if cost_per_1000<=cost_limit else 'FAIL'),
+            "p75_latency_seconds":percentile(latencies,.75),"mean_latency_seconds":statistics.mean(latencies) if latencies else None,
+            "scaled_p75_latency_seconds":percentile(latencies,.75)*scale+offset if latencies else None,
+            "scaled_mean_latency_seconds":statistics.mean(latencies)*scale+offset if latencies else None,
+            "by_budget_tier":budget_stats,"official_speed_subset":speed,"fast_required_quantile_meets_raw_limit":q_latency<=limits.get('raw_median_latency_seconds',.54) if q_latency is not None else None,
+            "count": count, "expected_count": expected_count, "successful_count": successes,
+            "failed_count": count - successes, "complete": complete,
+            "by_path":path_stats, "long_tail_projection":projection,
+            "cost_per_1000_by_basis":cost_basis, "long_tail_by_budget":tail_tiers,
+            "fast_required_quantile":fast_q,
+            "fast_required_quantile_latency_seconds":percentile(fast_latencies, fast_q) if fast_q is not None and fast_q <= 1 else None,
+            "fast_quantile_assumption":"all slow decisions exceed median target; successful fast samples only",
+            "successful_p50_latency_seconds":percentile([r["latency_seconds"] for r in rows if r["ok"]], .5),
+            "latency_scale":scale,"latency_offset_seconds":offset,"scaled_p50_latency_seconds":scaled_p50,
+            "pricing_margin":margin,"cost_limit_after_margin":cost_limit,
+            "cost_headroom_usd_per_1000":cost_limit-cost_per_1000 if cost_per_1000 is not None else None,
+            "latency_scope": "all_attempts_including_failures", "p50_latency_seconds": p50,
+            "p99_latency_seconds": percentile(latencies, 0.99),
+            "slow_count": sum(row["path"] == "slow" for row in rows),
+            "slow_share": sum(row["path"] == "slow" for row in rows) / count if routing_complete else None,
+            "usage_complete": usage_complete, "routing_complete": routing_complete,
+            "average_prompt_tokens": total_prompt / count if usage_complete else None,
+            "average_completion_tokens": total_completion / count if usage_complete else None,
+            "average_total_tokens": (total_prompt + total_completion) / count if usage_complete else None,
+            "average_new_tokens": sum(row["n_new_tokens"] for row in rows) / count if new_tokens_complete else None,
+            "pricing_model_key": price_key, "observed_models": observed_models,
+            "price_model_matches": price_model_matches, "reference_prices": prices,
+            "estimated_total_cost_usd": cost, "estimated_cost_per_1000_decisions_usd": cost_per_1000,
+            "by_type": by_type, "limits": limits, "local_reference_limits_met": eligible,
+            "qualification_note": "Local measurements only; evaluator hardware measurements determine admission."}
+
+
+def project_cost(rows,config,P):
+    if P is None:return None
+    if isinstance(P,bool) or not isinstance(P,(int,float)) or not math.isfinite(P) or P<=0:raise ValueError('P must be positive')
+    prices=config['pricing']['models'][config['pricing']['model_key']]
+    known=all(isinstance(prices.get(k),(int,float)) and prices[k]>=0 for k in ('input_per_million','output_per_million'))
+    result={'P':P,'by_basis':{},'assumption':'APC: additive input overhead; no APC: proportional input; output unchanged; long tail added'}
+    cap=config['limits']['cost_per_1000_decisions_usd']*(1-config['pricing']['margin'])
+    tail=config['bench'].get('long_tail',{'count':23,'total_decisions':1624})
+    for basis in ('engine','each_once','submitted'):
+        values=[];outputs=[];valid=bool(rows) and known and all(r.get('usage_complete') for r in rows)
+        modes=set()
+        if valid:
+            for r in rows:
+                usage=r.get('usage',{}).get('alternatives',{}).get(basis);f=r.get('fast_prompt_tokens')
+                if not usage or not isinstance(f,(int,float)) or f<=0:valid=False;break
+                apc=r.get('prefix_caching')
+                if basis=='engine' and apc is None:valid=False;break
+                additive=basis=='engine' and apc is True
+                values.append(max(0,P+usage['input_tokens']-f) if additive else usage['input_tokens']*P/f)
+                outputs.append(usage['output_tokens']);modes.add('additive' if additive else 'ratio')
+        cost=None
+        if valid:
+            regular=(statistics.mean(values)*prices['input_per_million']+statistics.mean(outputs)*prices['output_per_million'])/1e6
+            long_input=config['limits']['max_prompt_tokens'];probe=1 if config['model']['backend']=='vllm' else 0
+            long_cost=(long_input*prices['input_per_million']+probe*prices['output_per_million'])/1e6
+            cost=((tail['total_decisions']-tail['count'])*regular+tail['count']*long_cost)/tail['total_decisions']*1000
+        result['by_basis'][basis]={'cost_per_1000':cost,'method':sorted(modes),
+            'gate':'UNKNOWN' if cost is None else 'PASS' if cost<=.85*cap else 'WARN' if cost<=cap else 'FAIL'}
+    return result
+
+
+def margin_threshold_for_share(margins,target):
+    if not margins or not 0<target<1 or any(not math.isfinite(v) or v<0 for v in margins):
+        raise ValueError('Need finite nonnegative margins and target in (0,1)')
+    ordered=sorted(margins); k=math.ceil(target*len(ordered)); lower=ordered[k-1]
+    higher=next((v for v in ordered[k:] if v>lower),None)
+    # JSON-safe equivalent of infinity when all remaining margins are tied.
+    threshold=lower+(higher-lower)/2 if higher is not None else math.nextafter(lower,math.inf)
+    if not math.isfinite(threshold): raise ValueError("No finite threshold can include the boundary tie")
+    return threshold,sum(v<threshold for v in margins)/len(margins)
+
+
+def binomial_ci(successes,n):
+    if not n: return None
+    z=1.959963984540054; p=successes/n; d=1+z*z/n
+    center=(p+z*z/(2*n))/d
+    radius=z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))/d
+    return [max(0,center-radius),min(1,center+radius)]
+
+
+def margin_auroc(rows):
+    positive=[-r['fast_margin'] for r in rows if r.get('correct') is False]
+    negative=[-r['fast_margin'] for r in rows if r.get('correct') is True]
+    if not positive or not negative: return None
+    return sum((a>b)+.5*(a==b) for a in positive for b in negative)/(len(positive)*len(negative))
+
+
+def calibrate_margin(rows,target,health):
+    if health.get('mode')!='fast_only': raise ValueError('Margin calibration requires fast_only service')
+    if not rows or any(r.get('request_order')!='criteria' or not r['ok'] or not r['usage_complete'] or r.get('path')!='fast'
+            or isinstance(r.get('fast_margin'),bool) or not isinstance(r.get('fast_margin'),(int,float))
+            or not math.isfinite(r['fast_margin']) for r in rows):
+        raise ValueError('Every calibration task must succeed with complete margin and usage')
+    kinds={r.get('margin_kind') for r in rows}
+    if len(kinds)!=1 or next(iter(kinds)) not in ('logratio','probdiff'):
+        raise ValueError('Inconsistent margin kind')
+    threshold,achieved=margin_threshold_for_share([r['fast_margin'] for r in rows],target)
+    def stats(subset):
+        n=len(subset); selected=sum(r['fast_margin']<threshold for r in subset)
+        result={'n':n,'selected':selected,'share':selected/n if n else None,
+                'binomial_ci95':binomial_ci(selected,n),'auroc_fast_error':margin_auroc(subset)}
+        for side in ('below','above'):
+            labeled=[r for r in subset if r.get('correct') is not None and (r['fast_margin']<threshold)==(side=='below')]
+            result[side+'_fast_accuracy']=sum(r['correct'] for r in labeled)/len(labeled) if labeled else None
+        return result
+    identity=health.get('threshold_identity',{})
+    if any(k not in identity for k in ('model_key','quant','adapter','prompt_sha','backend','margin_kind','max_new_tokens','max_prompt_tokens','request_contract')):
+        raise ValueError('Health lacks threshold identity')
+    return dict(identity,request_contract='typesafe.criteria.v1',margin_kind=next(iter(kinds)),target=target,threshold=threshold,
+        achieved=achieved,n=len(rows),placeholder=False,overall=stats(rows),
+        by_type={k:stats([r for r in rows if r['type']==k]) for k in sorted({r['type'] for r in rows})},
+        by_tier={str(k):stats([r for r in rows if r['tier']==k]) for k in sorted({r['tier'] for r in rows},key=str)})
+
+
+def compare_order_accuracy(criteria_rows,labels_rows):
+    left={r['id']:r for r in criteria_rows};right={r['id']:r for r in labels_rows}
+    ids=[i for i in left if i in right and left[i]['type']=='choice' and left[i].get('expected') is not None]
+    if not ids or any(not left[i]['ok'] or not right[i]['ok'] for i in ids):
+        return {'count':len(ids),'criteria_accuracy':None,'labels_accuracy':None,'labels_minus_criteria':None}
+    a=sum(left[i]['correct'] for i in ids)/len(ids);b=sum(right[i]['correct'] for i in ids)/len(ids)
+    return {'count':len(ids),'criteria_accuracy':a,'labels_accuracy':b,'labels_minus_criteria':b-a}
+
+
+def calibrate_temperature(rows):
+    # Only replay raw scores from the official request order; never calibrate
+    # already floored/temperature-scaled probabilities.
+    if not rows or any(not r['ok'] or r.get('request_order')!='criteria' or not isinstance(r.get('raw_label_scores'),dict) or r.get('expected') is None for r in rows):
+        raise ValueError('Temperature fit needs successful labeled criteria-order raw scores')
+    result={}
+    grid=[2**(i/8) for i in range(-32,41)]
+    for kind in TYPES:
+        subset=[r for r in rows if r['type']==kind]
+        if not subset: result[kind]={'count':0,'fitted':False};continue
+        if kind=='noul':
+            correct=sum(max(r['raw_label_scores'],key=r['raw_label_scores'].get)==r['expected'] for r in subset)
+            result[kind]={'count':len(subset),'p_cal':max(.5,min(1-1e-6,correct/len(subset))),'fitted':True};continue
+        def loss(t):
+            total=0
+            for r in subset:
+                scores=r['raw_label_scores'];top=max(scores.values())
+                total+=math.log(sum(math.exp((v-top)/t) for v in scores.values()))-(scores[r['expected']]-top)/t
+            return total/len(subset)
+        t=min(grid,key=loss);result[kind]={'count':len(subset),'temperature':t,'nll':loss(t),'fitted':True}
+    return {'request_contract':'typesafe.criteria.v1','by_type':result,'status':'candidate; held-out validation required'}
+
+
+def fit_calibration(fast_rows, routed_rows, model_key):
+    import copy
+    if not fast_rows or not routed_rows:raise ValueError('Both fast and routed rows are required')
+    for rows in (fast_rows,routed_rows):
+        if len({r.get('id') for r in rows})!=len(rows):raise ValueError('Duplicate calibration IDs')
+        for r in rows:
+            identity=r.get('threshold_identity') or {}
+            if identity.get('model_key')!=model_key:raise ValueError('Calibration model identity mismatch')
+            if not r.get('ok') or r.get('request_order')!='criteria' or r.get('expected') is None:
+                raise ValueError('Calibration requires successful labeled criteria-order rows')
+    keys=('model_key','quant','adapter','backend','prompt_sha','margin_kind','max_prompt_tokens','request_contract')
+    identity=fast_rows[0]['threshold_identity']
+    if any(any(r['threshold_identity'].get(k)!=identity.get(k) for k in keys) for r in fast_rows+routed_rows):
+        raise ValueError('Mixed calibration identities')
+    if any(r.get('path')!='fast' or r.get('calibration_block') not in ('fast',None) for r in fast_rows):
+        raise ValueError('fast-run must contain only fast decisions')
+    def prepared(rows,field):
+        result=[]
+        for r in rows:
+            scores=r.get(field)
+            if scores is None and field=='fast_scores':scores=r.get('raw_label_scores')
+            if not isinstance(scores,dict) or r['expected'] not in scores or any(isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) for v in scores.values()):
+                raise ValueError('Missing or invalid pre-temperature label scores')
+            result.append(dict(r,raw_label_scores=scores))
+        return result
+    sources={'fast':prepared(fast_rows,'fast_scores'),
+             'slow':prepared([r for r in routed_rows if r.get('calibration_block')=='slow'],'slow_scores'),
+             'fallback':prepared([r for r in routed_rows if r.get('calibration_block')=='fallback'],'fast_scores')}
+    blocks={}
+    for name,rows in sources.items():
+        fit=calibrate_temperature(rows)['by_type'] if rows else {}
+        block={'placeholder':False,'status':'candidate; held-out validation required',
+               'temperature_by_kind':{'choice':1.,'score':1.,'noul':'one_bin'},'noul_calibration':{'p_cal':.5},'by_type':{}}
+        for kind in TYPES:
+            item=fit.get(kind,{'count':0,'fitted':False});n=item['count']
+            meta=dict(item,status='insufficient' if n<20 else 'candidate')
+            if name=='fallback' and n<20:
+                source=blocks['fast'];meta['inherited_from']='fast'
+                if kind=='noul':block['noul_calibration']=copy.deepcopy(source['noul_calibration'])
+                else:block['temperature_by_kind'][kind]=source['temperature_by_kind'][kind]
+            elif item.get('fitted'):
+                if kind=='noul':block['noul_calibration']['p_cal']=item['p_cal']
+                else:block['temperature_by_kind'][kind]=item['temperature']
+            if n<20:block['placeholder']=True
+            block['by_type'][kind]=meta
+        blocks[name]=block
+    return {'status':'candidate; no production overwrite; validate held-out data',
+            'models':{model_key:dict(blocks,placeholder=any(b['placeholder'] for b in blocks.values()),
+                                   fitted_for=identity)}}
+
+
+def read_run_rows(prefix):
+    if __package__:from .core import external_path
+    else:from core import external_path
+    path=external_path(prefix)
+    if path.suffix!='.jsonl':path=path.with_name(path.name+'.jsonl')
+    return [json.loads(line) for line in path.read_text(encoding='utf-8-sig').splitlines() if line.strip()]
+
+
+def write_candidate(path,value):
+    if __package__:from .core import external_output
+    else:from core import external_output
+    target=external_output(path)
+    with target.open('x',encoding='utf-8') as out:json.dump(value,out,ensure_ascii=False,indent=2,allow_nan=False)
+    return target
+
+
+def apply_margin_candidate(config, report, tier=None):
+    import copy
+    if __package__:from .core import apply_profile,threshold_identity,apply_budget_tier
+    else:from core import apply_profile,threshold_identity,apply_budget_tier
+    result=copy.deepcopy(config);key=report.get('model_key')
+    result['model']['model_key']=key;apply_profile(result);apply_budget_tier(result,tier)
+    identity=threshold_identity(result)
+    keys=('model_key','quant','adapter','backend','prompt_sha','margin_kind','max_prompt_tokens','request_contract')
+    if report.get('placeholder',True) or any(report.get(k)!=identity[k] for k in keys):raise ValueError('Margin candidate identity mismatch')
+    value=report['threshold']
+    if not isinstance(value,(int,float)) or not math.isfinite(value) or value<0:raise ValueError('Invalid threshold')
+    fit={k:report[k] for k in keys};fit.update(placeholder=False,max_new_tokens=identity['max_new_tokens'],reasoning_words=identity['reasoning_words'])
+    target=result['profiles'][key]['routing']
+    target.update(margin_threshold=value,margin_threshold_fitted_for=fit)
+    result['routing'].update(margin_threshold=value,margin_threshold_fitted_for=fit)
+    return result
+
+
+def get_health(endpoint,timeout,path="/health"):
+    parts=urllib.parse.urlsplit(endpoint)
+    url=urllib.parse.urlunsplit((parts.scheme,parts.netloc,path,'',''))
+    with _OPENER.open(url,timeout=timeout) as response:
+        return json.load(response)
+
+
+def output_paths(prefix, calibration_only=False):
+    if __package__:from .core import external_output
+    else:from core import external_output
+    candidate = external_output(prefix)
+    if calibration_only:
+        target=candidate.with_name(candidate.name+'.margin_calibration.json')
+        if target.exists(): raise FileExistsError('Calibration output exists')
+        return target
+    rows_path = candidate.with_name(candidate.name + ".jsonl")
+    summary_path = candidate.with_name(candidate.name + ".summary.json")
+    if rows_path.exists() or summary_path.exists():
+        raise FileExistsError("output already exists; choose a new prefix")
+    return rows_path, summary_path
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", type=Path, default=ROOT / "config.json")
+    source = parser.add_mutually_exclusive_group(required=False)
+    source.add_argument("--public", type=Path)
+    source.add_argument("--data", type=Path)
+    parser.add_argument("--index", type=Path)
+    parser.add_argument("--endpoint", help="Runtime HTTP endpoint; never written to output")
+    parser.add_argument("--output-prefix", help="Absolute path prefix outside the package (required for runs)")
+    parser.add_argument("--price-model", help="Key in config.pricing.models")
+    parser.add_argument('--original-types',action='store_true')
+    parser.add_argument('--concurrency',type=int,nargs='+')
+    parser.add_argument('--calibrate-margin',action='store_true')
+    parser.add_argument('--engine-stats',action='store_true')
+    parser.add_argument('--compare-label-order',action='store_true')
+    parser.add_argument('--target-slow-share',type=float)
+    parser.add_argument('--P',type=float)
+    parser.add_argument('--fit-calibration',action='store_true')
+    parser.add_argument('--fast-run',type=Path)
+    parser.add_argument('--routed-run',type=Path)
+    parser.add_argument('--output',type=Path)
+    parser.add_argument('--model-key')
+    parser.add_argument('--apply-margin',type=Path)
+    parser.add_argument('--tier',choices=('normal','hard','long'))
+    args = parser.parse_args(argv)
+    config_path = args.config.expanduser().resolve()
+    config = json.loads(config_path.read_text(encoding="utf-8-sig"))
+    if args.fit_calibration:
+        if not all((args.fast_run,args.routed_run,args.output,args.model_key)):parser.error('Fit needs fast-run, routed-run, output and model-key')
+        write_candidate(args.output,fit_calibration(read_run_rows(args.fast_run),read_run_rows(args.routed_run),args.model_key))
+        return 0
+    if args.apply_margin:
+        if not args.output:parser.error('Apply margin needs a new output config')
+        write_candidate(args.output,apply_margin_candidate(config,json.loads(args.apply_margin.read_text(encoding='utf-8')),args.tier))
+        return 0
+    if not args.endpoint or not (args.public or args.data):parser.error('Benchmark needs endpoint and public or data')
+    if not args.output_prefix:parser.error('Benchmark needs --output-prefix (absolute, outside the package)')
+    if args.P is not None:config['bench']['projection_prompt_tokens']=args.P
+    if args.original_types: config['bench']['original_types']=True
+    if args.price_model:
+        config["pricing"]["model_key"] = args.price_model
+    if args.public:
+        index = args.index or Path(config["bench"]["index"])
+        if not index.is_absolute():
+            index = config_path.parent / index
+        tasks = load_public_tasks(index, args.public, config["bench"]["expected_public_count"])
+    else:
+        if args.index:
+            parser.error("--index requires --public")
+        tasks = load_jsonl_tasks(args.data)
+    levels=args.concurrency or config['bench']['concurrency']
+    if any(n<1 for n in levels): parser.error('Concurrency must be positive')
+    if args.calibrate_margin and config['bench'].get('original_types'):parser.error('Calibration requires the official request types')
+    if args.calibrate_margin and levels!=[1]: parser.error('Margin calibration is serial')
+    if args.calibrate_margin and args.engine_stats:parser.error('Use a separate serial run for engine reconciliation')
+    endpoint = endpoint_url(args.endpoint)
+    paths = output_paths(args.output_prefix,calibration_only=args.calibrate_margin)
+    health=get_health(endpoint,config['bench']['timeout_seconds'])
+    if __package__: from .core import apply_profile
+    else: from core import apply_profile
+    config['model']['model_key']=health['model_key']
+    apply_profile(config)
+    if args.tier:
+        if __package__:from .core import apply_budget_tier
+        else:from core import apply_budget_tier
+        apply_budget_tier(config,args.tier)
+    if args.price_model: config['pricing']['model_key']=args.price_model
+    if args.calibrate_margin:
+        if health.get('mode')!='fast_only': parser.error('Requires fast_only service')
+        target=args.target_slow_share if args.target_slow_share is not None else config['bench']['target_slow_share']
+        calibration_path=paths
+        # Existing-file check precedes inference; exclusive creation prevents races.
+        rows=run_benchmark(tasks,endpoint,config)
+        report=calibrate_margin(rows,target,health)
+        report['temperature_fit']=calibrate_temperature(rows)
+        if args.compare_label_order:
+            import copy
+            other=copy.deepcopy(config);other['bench']['label_order']=True
+            report['order_accuracy']=compare_order_accuracy(rows,run_benchmark(tasks,endpoint,other))
+        with calibration_path.open('x',encoding='utf-8') as output:
+            json.dump(report,output,ensure_ascii=False,indent=2,allow_nan=False)
+        print(json.dumps(report,ensure_ascii=False,indent=2,allow_nan=False))
+        return 0
+    stats_path=paths[0].with_name(paths[0].stem+'.engine_stats.json')
+    if args.engine_stats and stats_path.exists():raise FileExistsError('Engine snapshot exists')
+    # Claim both paths before requests, using exclusive creation for preservation.
+    with paths[0].open("x", encoding="utf-8") as writer, paths[1].open("x", encoding="utf-8") as summary_writer:
+        summaries={}
+        snapshots={"health":health,"by_concurrency":{}}
+        stats_path=paths[0].with_name(paths[0].stem+".engine_stats.json")
+        if args.engine_stats and stats_path.exists(): raise FileExistsError("Engine snapshot exists")
+        for n in dict.fromkeys(levels):
+            if args.engine_stats: before=get_health(endpoint,config['bench']['timeout_seconds'],'/engine_stats')
+            rows=run_benchmark(tasks,endpoint,config,concurrency=n)
+            if args.engine_stats: snapshots['by_concurrency'][str(n)]={'before':before,'after':get_health(endpoint,config['bench']['timeout_seconds'],'/engine_stats')}
+            for row in rows:
+                row['concurrency']=n
+                row['prefix_caching']=health.get('prefix_caching')
+                writer.write(json.dumps(row,ensure_ascii=False,allow_nan=False)+'\n')
+            writer.flush()
+            summaries[str(n)]=summarize(rows,config,expected_count=len(tasks))
+            if args.compare_label_order:
+                import copy
+                other=copy.deepcopy(config);other['bench']['label_order']=True
+                summaries[str(n)]['order_accuracy']=compare_order_accuracy(rows,run_benchmark(tasks,endpoint,other,concurrency=n))
+        if args.engine_stats:
+            with stats_path.open('x',encoding='utf-8') as output: json.dump(snapshots,output,indent=2,allow_nan=False)
+        summary=dict(summaries[str(levels[0])],by_concurrency=summaries)
+        summary_writer.write(json.dumps(summary, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
+    print(json.dumps(summary, ensure_ascii=False, indent=2, allow_nan=False))
+    return 0 if all(v["complete"] for v in summary["by_concurrency"].values()) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

service/calibration.json

@@ -0,0 +1,136 @@
+{
+  "models": {
+    "metask-gemma4-31b-lora-w4a16": {
+      "fallback": {
+        "by_type": {
+          "choice": {
+            "count": 139,
+            "fitted": true,
+            "inherited": true,
+            "inherited_from": "fast",
+            "nll": 0.268809886282054,
+            "temperature": 1.2968395546510096
+          },
+          "noul": {
+            "constant_source": "service parameter (fit_report.json noul_constant)",
+            "count": 74,
+            "fitted": true,
+            "fitted_p_cal": 0.9459459459459459,
+            "inherited": true,
+            "inherited_from": "fast",
+            "p_cal": 0.898
+          },
+          "score": {
+            "count": 18,
+            "fitted": true,
+            "inherited": true,
+            "inherited_from": "fast",
+            "nll": 0.26993267192031306,
+            "temperature": 1.189207115002721
+          }
+        },
+        "noul_calibration": {
+          "p_cal": 0.898
+        },
+        "placeholder": false,
+        "status": "temperatures: public231 in-sample fast fit (unchanged); yes/no constant: service parameter, see fit_report.json noul_constant",
+        "temperature_by_kind": {
+          "choice": 1.2968395546510096,
+          "noul": "one_bin",
+          "score": 1.189207115002721
+        }
+      },
+      "fast": {
+        "by_type": {
+          "choice": {
+            "count": 139,
+            "fitted": true,
+            "nll": 0.268809886282054,
+            "temperature": 1.2968395546510096
+          },
+          "noul": {
+            "constant_source": "service parameter (fit_report.json noul_constant)",
+            "count": 74,
+            "fitted": true,
+            "fitted_p_cal": 0.9459459459459459,
+            "p_cal": 0.898
+          },
+          "score": {
+            "count": 18,
+            "fitted": true,
+            "nll": 0.26993267192031306,
+            "temperature": 1.189207115002721
+          }
+        },
+        "noul_calibration": {
+          "p_cal": 0.898
+        },
+        "placeholder": false,
+        "status": "temperatures: public231 in-sample fast fit (unchanged); yes/no constant: service parameter, see fit_report.json noul_constant",
+        "temperature_by_kind": {
+          "choice": 1.2968395546510096,
+          "noul": "one_bin",
+          "score": 1.189207115002721
+        }
+      },
+      "fitted_for": {
+        "adapter": "model-subdir:adapter",
+        "backend": "vllm",
+        "margin_kind": "logratio",
+        "max_new_tokens": 256,
+        "max_prompt_tokens": 6653,
+        "model_key": "metask-gemma4-31b-lora-w4a16",
+        "prompt_sha": "e18ed7a06eb52f1b351458f384710a2c240aaf2b4f64d916b776c6d18ca3f938",
+        "quant": "awq",
+        "reasoning_words": 160,
+        "request_contract": "typesafe.criteria.v1"
+      },
+      "placeholder": false,
+      "slow": {
+        "by_type": {
+          "choice": {
+            "applied_rule": "slow samples of a type < 30: the fast parameter of that type (temperature or yes/no confidence) is copied unchanged",
+            "count": 0,
+            "fast_count": 139,
+            "fitted": false,
+            "inherited": true,
+            "inherited_from": "fast",
+            "status": "inherited"
+          },
+          "noul": {
+            "applied_rule": "slow samples of a type < 30: the fast parameter of that type (temperature or yes/no confidence) is copied unchanged",
+            "constant_source": "service parameter (fit_report.json noul_constant)",
+            "count": 0,
+            "fast_count": 74,
+            "fitted": false,
+            "fitted_p_cal": 0.9459459459459459,
+            "inherited": true,
+            "inherited_from": "fast",
+            "p_cal": 0.898,
+            "status": "inherited"
+          },
+          "score": {
+            "applied_rule": "slow samples of a type < 30: the fast parameter of that type (temperature or yes/no confidence) is copied unchanged",
+            "count": 0,
+            "fast_count": 18,
+            "fitted": false,
+            "inherited": true,
+            "inherited_from": "fast",
+            "status": "inherited"
+          }
+        },
+        "held_out_validated": false,
+        "noul_calibration": {
+          "p_cal": 0.898
+        },
+        "placeholder": false,
+        "status": "temperatures: public231 in-sample; slow types with fewer than 30 slow samples inherit fast (unchanged); yes/no constant: service parameter, see fit_report.json noul_constant",
+        "temperature_by_kind": {
+          "choice": 1.2968395546510096,
+          "noul": "one_bin",
+          "score": 1.189207115002721
+        }
+      }
+    }
+  }
+}

service/config.json

@@ -0,0 +1,346 @@
+{
+  "bench": {
+    "concurrency": [
+      1
+    ],
+    "expected_public_count": 231,
+    "latency_offset_seconds": 0.15,
+    "latency_scale": 2.0,
+    "long_tail": {
+      "count": 23,
+      "projection_budgets": [
+        8192
+      ],
+      "prompt_tokens": 79000,
+      "total_decisions": 1624
+    },
+    "original_types": false,
+    "output_prefix": "31b-exploration",
+    "target_slow_share": 0.49,
+    "timeout_seconds": 120
+  },
+  "calibration_file": "calibration.json",
+  "generation": {
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "do_sample": false,
+    "forced_bos_token_id": null,
+    "forced_decoder_ids": null,
+    "forced_eos_token_id": null,
+    "length_penalty": 1.0,
+    "max_new_tokens": 192,
+    "min_length": 0,
+    "min_new_tokens": 0,
+    "no_repeat_ngram_size": 0,
+    "reasoning_words": 50,
+    "renormalize_logits": false,
+    "repetition_penalty": 1.0,
+    "seed": 0,
+    "suppress_tokens": null,
+    "temperature": 1.0,
+    "top_k": 0,
+    "top_p": 1.0,
+    "wall_timeout_seconds": 60
+  },
+  "generation_reference": {
+    "intended_overrides": {
+      "do_sample": {
+        "reason": "Frozen greedy decoding; checkpoint sampling policy is intentionally overridden.",
+        "value": false
+      },
+      "suppress_tokens": {
+        "reason": "All reported measurements were taken without checkpoint token suppression; drafts only, fast-path candidate scoring unaffected.",
+        "value": null
+      },
+      "temperature": {
+        "reason": "Frozen greedy decoding; checkpoint sampling policy is intentionally overridden.",
+        "value": 1.0
+      },
+      "top_k": {
+        "reason": "Frozen greedy decoding; checkpoint sampling policy is intentionally overridden.",
+        "value": 0
+      },
+      "top_p": {
+        "reason": "Frozen greedy decoding; checkpoint sampling policy is intentionally overridden.",
+        "value": 1.0
+      }
+    }
+  },
+  "interface": {
+    "boolean_keys": "request"
+  },
+  "item_deadline": {
+    "chunk_tokens": 32,
+    "ewma_alpha": 0.3,
+    "item_seconds": 9.0,
+    "min_chunk_tokens": 8,
+    "readout_reserve_init_s": 1.0,
+    "readout_reserve_min_s": 0.3,
+    "readout_safety": 1.5,
+    "response_reserve_s": 0.35,
+    "token_safety": 1.25,
+    "warmup_repeats": 3,
+    "warmup_tokens": 33
+  },
+  "item_rules": {
+    "reread_tau": 4.0,
+    "slow_types": [
+      "choice",
+      "noul",
+      "score"
+    ]
+  },
+  "limits": {
+    "compression_reserve_tokens": 32,
+    "cost_per_1000_decisions_usd": 0.0646,
+    "estimated_chars_per_token": 4,
+    "max_prompt_tokens": 6653,
+    "median_latency_seconds": 1.0,
+    "oom_retry_prompt_tokens": 4096,
+    "physical_max_prompt_tokens": 6653,
+    "precompression_trigger_chars_per_token": 8,
+    "raw_median_latency_seconds": 0.425,
+    "state_chunk_tokens": 200,
+    "state_head_tokens": 512,
+    "state_tail_tokens": 512,
+    "truncation_head_frac": 0.6
+  },
+  "model": {
+    "adapter": "model-subdir:adapter",
+    "attn_implementation": "sdpa",
+    "backend": "vllm",
+    "device_map": {
+      "": 0
+    },
+    "enable_prefix_caching": true,
+    "model_key": "metask-gemma4-31b-lora-w4a16",
+    "name": "MetaDecision-31B-Rain",
+    "path": "",
+    "quant": "awq",
+    "version": "MetaDecision-31B-Rain",
+    "vllm_expected_version": "0.31.0",
+    "vllm_gpu_memory_utilization": 0.94,
+    "vllm_max_model_len": 6656,
+    "vllm_options": {
+      "gpu_memory_utilization": 0.94,
+      "max_lora_rank": 16,
+      "max_model_len": 6656,
+      "max_num_seqs": 1
+    }
+  },
+  "package": {
+    "finalized": true,
+    "nonformal_fitted": true,
+    "sample_only": false,
+    "version": "MetaDecision-31B-Rain"
+  },
+  "pipeline": {
+    "arm": "trained",
+    "model_identity_sha256": "0513d121db967ec1a6aaccf627d3c42149d092dd027298da5b53684b030821ea",
+    "protocol": "adapter_arm_31b_v3"
+  },
+  "post_calibration": {
+    "choice": {
+      "config": {
+        "debias": false,
+        "temperature": false
+      },
+      "params": {}
+    },
+    "noul": {
+      "config": {
+        "floor": null,
+        "temperature": "type"
+      },
+      "params": {
+        "T": 0.5
+      }
+    },
+    "score": {
+      "config": {
+        "mode_weight": 1.0,
+        "temperature": false
+      },
+      "params": {}
+    }
+  },
+  "pricing": {
+    "margin": 0.0,
+    "model_key": "31B",
+    "models": {
+      "31B": {
+        "input_per_million": 0.059819178473256074,
+        "output_per_million": 0.059819178473256074
+      }
+    },
+    "submission_models": {
+      "MetaDecision-31B-Rain": "31B",
+      "metask-gemma4-31b-lora-w4a16": "31B",
+      "metask-jev-rain-31B-trained": "31B"
+    }
+  },
+  "profiles": {
+    "metask-gemma4-31b-lora-w4a16": {
+      "bench": {
+        "target_slow_share": 0.49
+      },
+      "generation": {
+        "bad_words_ids": null,
+        "begin_suppress_tokens": null,
+        "do_sample": false,
+        "forced_bos_token_id": null,
+        "forced_decoder_ids": null,
+        "forced_eos_token_id": null,
+        "length_penalty": 1.0,
+        "max_new_tokens": 192,
+        "min_length": 0,
+        "min_new_tokens": 0,
+        "no_repeat_ngram_size": 0,
+        "reasoning_words": 50,
+        "renormalize_logits": false,
+        "repetition_penalty": 1.0,
+        "seed": 0,
+        "suppress_tokens": null,
+        "temperature": 1.0,
+        "top_k": 0,
+        "top_p": 1.0,
+        "wall_timeout_seconds": 60
+      },
+      "limits": {
+        "max_prompt_tokens": 6653,
+        "physical_max_prompt_tokens": 6653
+      },
+      "model": {
+        "backend": "vllm",
+        "name": "MetaDecision-31B-Rain",
+        "quant": "awq"
+      },
+      "pricing": {
+        "model_key": "31B"
+      },
+      "routing": {
+        "cost_fuse": {
+          "arm": 0.9,
+          "enabled": true,
+          "release": 0.85,
+          "warmup": 100
+        },
+        "fuse": 0.49,
+        "long_item_slow_policy": "fast_only",
+        "margin_kind": "logratio",
+        "margin_threshold": 6.625023671062799,
+        "margin_threshold_fitted_for": {
+          "achieved_share": 0.5207547169811321,
+          "adapter": "model-subdir:adapter",
+          "backend": "vllm",
+          "fitted_on": "our own independently built development set (fit split): fast-path margins of the earlier service routing, eligible_items rule (the lowest-margin share of the decisions that can take the drafting pass)",
+          "margin_kind": "logratio",
+          "max_new_tokens": 192,
+          "max_prompt_tokens": 6653,
+          "model_key": "metask-gemma4-31b-lora-w4a16",
+          "n": 530,
+          "placeholder": false,
+          "prompt_sha": "e18ed7a06eb52f1b351458f384710a2c240aaf2b4f64d916b776c6d18ca3f938",
+          "quant": "awq",
+          "reasoning_words": 50,
+          "request_contract": "typesafe.criteria.v1"
+        },
+        "mode": "routed",
+        "quota": 0.49,
+        "slow_max_prompt_tokens": 840,
+        "target_slow_share": 0.49,
+        "tier": "normal",
+        "tiers": {
+          "hard": {
+            "fast_only": false,
+            "max_new_tokens": 192,
+            "reasoning_words": 50
+          },
+          "long": {
+            "fast_only": true,
+            "max_prompt_tokens": 6653
+          },
+          "normal": {
+            "fast_only": false,
+            "max_prompt_tokens": 6653
+          }
+        },
+        "unparsed_policy": "force_answer"
+      }
+    }
+  },
+  "prompt": {
+    "answer_marker": "ANSWER:",
+    "cot_instructions": {
+      "choice": "First reason step by step in at most {reasoning_words} words, tracking any intermediate state explicitly. Then output a final line `ANSWER: <letter>`.",
+      "noul": "First reason step by step in at most {reasoning_words} words, tracking any intermediate state explicitly. Then output a final line of the form `ANSWER: yes` or `ANSWER: no`.",
+      "score": "First reason step by step in at most {reasoning_words} words, tracking any intermediate state explicitly. Then output a final line `ANSWER: <number>`."
+    },
+    "enable_thinking": false,
+    "system": "You are a careful, decisive judge. Read the context, apply every rule, definition, threshold, exception and date it contains, and answer the question. Reply with exactly one word and nothing else."
+  },
+  "routing": {
+    "cost_fuse": {
+      "arm": 0.9,
+      "enabled": true,
+      "release": 0.85,
+      "warmup": 100
+    },
+    "fuse": 0.49,
+    "long_item_slow_policy": "fast_only",
+    "margin_kind": "logratio",
+    "margin_threshold": 6.625023671062799,
+    "margin_threshold_fitted_for": {
+      "achieved_share": 0.5207547169811321,
+      "adapter": "model-subdir:adapter",
+      "backend": "vllm",
+      "fitted_on": "our own independently built development set (fit split): fast-path margins of the earlier service routing, eligible_items rule (the lowest-margin share of the decisions that can take the drafting pass)",
+      "margin_kind": "logratio",
+      "max_new_tokens": 192,
+      "max_prompt_tokens": 6653,
+      "model_key": "metask-gemma4-31b-lora-w4a16",
+      "n": 530,
+      "placeholder": false,
+      "prompt_sha": "e18ed7a06eb52f1b351458f384710a2c240aaf2b4f64d916b776c6d18ca3f938",
+      "quant": "awq",
+      "reasoning_words": 50,
+      "request_contract": "typesafe.criteria.v1"
+    },
+    "mode": "routed",
+    "quota": 0.49,
+    "resume": false,
+    "run_id": null,
+    "slow_max_prompt_tokens": 840,
+    "state_dir": null,
+    "target_slow_share": 0.49,
+    "tier": "normal",
+    "tiers": {
+      "hard": {
+        "fast_only": false,
+        "max_new_tokens": 192,
+        "reasoning_words": 50
+      },
+      "long": {
+        "fast_only": true,
+        "max_prompt_tokens": 6653
+      },
+      "normal": {
+        "fast_only": false,
+        "max_prompt_tokens": 6653
+      }
+    },
+    "unparsed_policy": "force_answer"
+  },
+  "server": {
+    "bind": "127.0.0.1",
+    "max_questions": 64,
+    "max_request_bytes": 16777216,
+    "port": 8000,
+    "socket_timeout_seconds": 120,
+    "warmup": true,
+    "warmup_slow_tokens": 2
+  },
+  "usage": {
+    "basis": "engine"
+  }
+}

service/core.py

@@ -0,0 +1,1197 @@
+"""Rendering, candidate aggregation and service logic, without model imports."""
+from __future__ import annotations
+
+import json
+import math
+import re
+import string
+from pathlib import Path
+
+import threading
+if __package__:
+    from .backends import new_meter, check_deadline
+    from .quota import QuotaStateError
+else:
+    from backends import new_meter, check_deadline
+    from quota import QuotaStateError
+
+ROOT = Path(__file__).resolve().parent
+YES_FORMS = ["yes", "Yes", "YES", " yes", " Yes", " YES", "true", "True", " true", " True", "Y", " Y"]
+NO_FORMS = ["no", "No", "NO", " no", " No", " NO", "false", "False", " false", " False", "N", " N"]
+ALIASES = {"boolean": "noul", "enum": "choice", "noul": "noul", "choice": "choice", "score": "score"}
+
+
+class ItemFailure(RuntimeError):
+    """Inference for one item failed; the service answers HTTP 422 and keeps running."""
+
+
+class ServiceFailure(RuntimeError):
+    """Service-level failure; the service answers HTTP 503."""
+
+
+# The third consecutive item failure is treated as service-level (HTTP 503, service
+# marked failed), so the harness stop rule for consecutive infrastructure errors applies.
+ITEM_FAILURE_LIMIT = 3
+
+
+def unrecoverable_engine_error(exc):
+    """True if the __cause__/__context__ graph of exc holds an unrecoverable vLLM engine error.
+
+    Classes are matched by name, so the check needs no vLLM import: EngineDeadError, or a
+    vLLM VLLMServerError subclass other than the per-request EngineGenerateError.
+    """
+    pending, seen = [exc], set()
+    while pending:
+        current = pending.pop()
+        if current is None or id(current) in seen:
+            continue
+        seen.add(id(current))
+        classes = type(current).__mro__
+        if isinstance(current, ServiceFailure) or any(cls.__name__ == 'EngineDeadError' for cls in classes):
+            return True
+        vllm_classes = {cls.__name__ for cls in classes
+                        if cls.__module__ == 'vllm' or cls.__module__.startswith('vllm.')}
+        if 'VLLMServerError' in vllm_classes and 'EngineGenerateError' not in vllm_classes:
+            return True
+        pending.extend((current.__cause__, current.__context__))
+    return False
+
+
+def external_path(path):
+    """Resolve an absolute path that is neither inside the package nor an ancestor of it."""
+    candidate = Path(path)
+    if not candidate.is_absolute():
+        raise ValueError("Path must be absolute: " + str(path))
+    result = candidate.resolve()
+    if result == ROOT or result.is_relative_to(ROOT) or ROOT.is_relative_to(result):
+        raise ValueError("Path must be outside the package and must not contain it: " + str(result))
+    return result
+
+
+def external_output(path):
+    """Validated output location outside the package; callers create files with mode 'x'."""
+    result = external_path(path)
+    result.parent.mkdir(parents=True, exist_ok=True)
+    return result
+
+
+def load_json(path):
+    def reject(value):
+        raise ValueError("Non-finite JSON value")
+    return json.loads(Path(path).read_text(encoding="utf-8"), parse_constant=reject)
+
+
+def load_config(path=None):
+    config = load_json(path or ROOT / "config.json")
+    apply_profile(config)
+    validate_config(config)
+    return config
+
+
+def apply_profile(config):
+    import copy
+    profile=config.get('profiles',{}).get(config['model']['model_key'])
+    if profile is None: raise ValueError('No profile for model_key')
+    for section,values in profile.items():
+        config.setdefault(section,{}).update(copy.deepcopy(values))
+    return apply_budget_tier(config)
+
+
+def apply_budget_tier(config, tier=None):
+    r=config['routing'];tier=tier or r.get('tier','normal')
+    if tier not in ('normal','hard','long'):raise ValueError('Unknown quota tier')
+    r['tier']=tier
+    tiers=r.get('tiers',{})
+    values=tiers.get(tier,{})
+    for key in ('reasoning_words','max_new_tokens'):
+        if key in tiers.get('hard',{}):config['generation'][key]=tiers['hard'][key]
+    if 'max_prompt_tokens' in values:config['limits']['max_prompt_tokens']=values['max_prompt_tokens']
+    for key in ('reasoning_words','max_new_tokens'):
+        if key in values:config['generation'][key]=values[key]
+    # Forced long is a runtime admission switch, preserving fast identity.
+    return config
+
+
+def finite_number(value, name, *, low=None, high=None, positive=False):
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
+        raise ValueError(f"{name} must be a finite number")
+    if (low is not None and value < low) or (high is not None and value > high) or (positive and value <= 0):
+        raise ValueError(f"{name} is outside its allowed range")
+    return value
+
+
+def validate_config(c):
+    if __package__: from .quota import cost_settings
+    else: from quota import cost_settings
+    cost_settings(c)
+    if c['model']['model_key'] not in c.get('profiles',{}):raise ValueError('No profile for model_key')
+    if c.get('usage',{}).get('basis','engine') not in ('engine','each_once','submitted'):
+        raise ValueError('Invalid usage basis')
+    m, r, g, p, s = (c[k] for k in ("model", "routing", "generation", "prompt", "server"))
+    if m["backend"] not in ("transformers", "vllm") or m["quant"] not in ("int8", "bf16", "awq"):
+        raise ValueError("Unsupported backend or precision")
+    if m['backend']=='vllm':
+        neutral={'suppress_tokens':None,'begin_suppress_tokens':None,'bad_words_ids':None,
+            'forced_bos_token_id':None,'forced_eos_token_id':None,'forced_decoder_ids':None,
+            'no_repeat_ngram_size':0,'min_new_tokens':0,'min_length':0,'length_penalty':1.0,'renormalize_logits':False}
+        if any(g.get(k,v)!=v for k,v in neutral.items()):raise ValueError('Unsupported vLLM decode override')
+    if r["mode"] not in ("routed", "fast_only", "uniform"):
+        raise ValueError("Unsupported routing mode")
+    tiers=r.get('tiers',{})
+    if tiers:
+        if set(tiers)!={'normal','hard','long'} or r.get('tier','normal') not in tiers:raise ValueError('Require normal/hard/long tiers')
+        if tiers['long'].get('fast_only') is not True:raise ValueError('Long tier must be fast only')
+        for tier in ('normal','long'):
+            value=tiers[tier].get('max_prompt_tokens')
+            if type(value) is not int or not 0<value<=c['limits']['physical_max_prompt_tokens']:raise ValueError('Invalid tier input budget')
+        for key in ('reasoning_words','max_new_tokens'):
+            if type(tiers['hard'].get(key)) is not int or tiers['hard'][key]<=0:raise ValueError('Invalid hard draft budget')
+    finite_number(r["margin_threshold"], "margin_threshold", low=0)
+    if r["margin_kind"]=="probdiff" and r["margin_threshold"]>1:
+        raise ValueError("probdiff threshold must be <=1")
+    if r["margin_kind"] not in ("logratio", "probdiff"):
+        raise ValueError("Invalid margin_kind")
+    if r.get('long_item_slow_policy','fast_only') not in ('fast_only','allow'):
+        raise ValueError('Invalid long item policy')
+    if r["unparsed_policy"] not in ("fast", "force_answer"):
+        raise ValueError("Invalid unparsed_policy")
+    finite_number(r["quota"], "quota", low=0, high=1)
+    finite_number(r["fuse"], "fuse", low=0, high=1)
+    if r["quota"] > r["fuse"]:
+        raise ValueError("quota must not exceed fuse")
+    if r["mode"] == "uniform" and (r["quota"] != 1 or r["fuse"] != 1):
+        raise ValueError("uniform requires explicit quota=1 and fuse=1; it is an unbounded draft experiment")
+    for key in ("reasoning_words", "max_new_tokens"):
+        if type(g[key]) is not int or g[key] <= 0:
+            raise ValueError(f"{key} must be a positive integer")
+    if type(g["seed"]) is not int or type(g["do_sample"]) is not bool:
+        raise ValueError("Invalid generation seed or do_sample")
+    finite_number(g.get("wall_timeout_seconds",60),"slow wall timeout",positive=True)
+    finite_number(g["temperature"], "generation temperature", positive=True)
+    finite_number(g["top_p"], "top_p", positive=True, high=1)
+    finite_number(g["repetition_penalty"], "repetition_penalty", positive=True)
+    if type(g["top_k"]) is not int or g["top_k"] < 0:
+        raise ValueError("top_k must be a nonnegative integer")
+    for key in ("max_request_bytes", "max_questions", "port"):
+        if type(s[key]) is not int or s[key] <= 0:
+            raise ValueError(f"{key} must be a positive integer")
+    if s["port"] > 65535:
+        raise ValueError("Invalid port")
+    finite_number(s["socket_timeout_seconds"], "socket timeout", positive=True)
+    if not isinstance(p["answer_marker"], str) or not p["answer_marker"].strip() or "\n" in p["answer_marker"]:
+        raise ValueError("answer_marker must be a nonempty single line")
+    if type(p["enable_thinking"]) is not bool or not isinstance(p["system"], str):
+        raise ValueError("Invalid prompt settings")
+    for template in p["cot_instructions"].values():
+        template.format(reasoning_words=g["reasoning_words"])
+    for value in (c["limits"]["max_prompt_tokens"], r["slow_max_prompt_tokens"]):
+        if type(value) is not int or value <= 0:
+            raise ValueError("Token limits must be positive integers")
+    limits=c['limits']
+    for name in ('physical_max_prompt_tokens','oom_retry_prompt_tokens','state_chunk_tokens','precompression_trigger_chars_per_token','estimated_chars_per_token'):
+        if type(limits[name]) is not int or limits[name]<=0: raise ValueError('Invalid compression limit: '+name)
+    for name in ('state_head_tokens','state_tail_tokens','compression_reserve_tokens'):
+        if type(limits[name]) is not int or limits[name]<0: raise ValueError('Invalid compression limit: '+name)
+    if limits['physical_max_prompt_tokens']<limits['max_prompt_tokens']:
+        raise ValueError('Physical maximum must cover soft prompt budget')
+    finite_number(c["limits"]["truncation_head_frac"], "head fraction", low=0, high=1)
+    if c.get("interface", {}).get("boolean_keys", "request") not in ("request", "true_false", "yes_no"):
+        raise ValueError("Unknown boolean key style")
+
+
+class UnsupportedRequest(ValueError):
+    """Structurally valid request outside the supported semantic domain (422)."""
+
+
+def choice_codes(count):
+    if count<=26:return list(string.ascii_uppercase[:count])
+    if count>676:raise UnsupportedRequest('At most 676 choices supported')
+    return [a+b for a in string.ascii_uppercase for b in string.ascii_uppercase][:count]
+
+
+def normalize_question(question):
+    if not isinstance(question, dict):
+        raise ValueError("Each question must be an object")
+    qtype = ALIASES.get(question.get("type"))
+    if qtype is None:
+        raise UnsupportedRequest("Unsupported question type")
+    instructions = question.get("instructions", question.get("description"))
+    if not isinstance(instructions, str) or not instructions.strip():
+        raise ValueError("Question needs instructions or description")
+    criteria = question.get("criteria")
+    q = {"request_type": question["type"], "type": qtype, "instructions": instructions, "criteria": criteria}
+    labels = None
+    if qtype == "noul":
+        if criteria is not None and not isinstance(criteria, dict):
+            raise ValueError("noul criteria must be an object")
+    elif qtype == "score":
+        if not isinstance(criteria,list): raise ValueError("score criteria must be an ordered list")
+        if not criteria: raise UnsupportedRequest("score criteria is empty")
+    else:
+        labels = question.get("labels")
+        if labels is None:
+            labels = list(criteria) if isinstance(criteria, (dict, list)) else None
+        if not isinstance(labels,list): raise ValueError("choice needs labels or ordered criteria")
+        if not 1<=len(labels)<=676: raise UnsupportedRequest("choice needs 1..676 labels")
+        if any(not isinstance(x, str) or not x for x in labels) or len(set(labels)) != len(labels):
+            raise ValueError("choice labels must be distinct nonempty strings")
+        if criteria is not None and not isinstance(criteria, (dict, list)):
+            raise ValueError("choice criteria must be an object or list")
+    return q, labels
+
+
+def normalize_request(payload, max_questions):
+    if not isinstance(payload, dict) or "state" not in payload:
+        raise ValueError("Request needs state and questions")
+    questions = payload.get("questions")
+    if not isinstance(questions, dict) or not 1 <= len(questions) <= max_questions:
+        raise ValueError("questions must be a nonempty object within max_questions")
+    tasks = []
+    for name, question in questions.items():
+        if not isinstance(name, str) or not name:
+            raise ValueError("Question names must be nonempty strings")
+        q, labels = normalize_question(question)
+        tasks.append((name, {"state": payload["state"], "question": q, "labels": labels}))
+    return tasks
+
+
+def letter_forms(c):
+    return [c, " " + c, c.lower(), " " + c.lower(), c + ".", "(" + c]
+
+
+def digit_forms(d):
+    return [d, " " + d, d + "."]
+
+
+def render(task):
+    # Source-faithful copy of cache_hidden_v2.render; do not deduplicate forms.
+    q = task["question"]; qtype = q["type"]; crit = q.get("criteria")
+    state = task["state"] if isinstance(task["state"], str) else json.dumps(task["state"], ensure_ascii=False)
+    cq = "Context:\n" + state + "\n\nQuestion: " + q["instructions"]
+    if qtype == "noul":
+        labels = ["no", "yes"]; c = crit or {}; groups = {"yes": YES_FORMS, "no": NO_FORMS}
+        parts = [cq] + (["Answer yes if: " + str(c.get("true", "Yes")) + "\nAnswer no if: " + str(c.get("false", "No"))] if (c.get("true") or c.get("false")) else []) + ["Answer with one word: yes or no."]
+    elif qtype == "score":
+        labels = [str(i) for i in range(len(crit))]; groups = {l: digit_forms(l) for l in labels}
+        parts = [cq, "Levels:\n" + "\n".join(str(i) + ": " + str(t) for i, t in enumerate(crit)), "Answer with one number from 0 to " + str(len(crit) - 1) + "."]
+    else:
+        labels = list(task["labels"]); rubric = crit if isinstance(crit, dict) else {}; codes = choice_codes(len(labels))
+        groups = {l: letter_forms(c) for c, l in zip(codes, labels)}
+        parts = [cq, "Options:\n" + "\n".join(c + ". " + str(l) + ((": " + str(rubric[l])) if rubric.get(l) else "") for c, l in zip(codes, labels)), "Answer with the letter of the best option."]
+    return labels, "\n\n".join(parts), groups
+
+
+def parse_answer(generation, qtype="noul", labels=None):
+    """Audit only: keep the explicit-last and standalone fallbacks of the measured parser."""
+    if qtype == "noul":
+        matches = re.findall(r"ANSWER:\s*(yes|no)", generation, re.I) or re.findall(r"\b(yes|no)\b", generation, re.I)
+        return matches[-1].lower() if matches else None
+    if qtype == "choice":
+        mapping = dict(zip(choice_codes(len(labels or [])), labels or []))
+        explicit, fallback = r"ANSWER:\s*([A-Z]{1,2})\b", r"\b([A-Z]{1,2})\b"
+    elif qtype == "score":
+        mapping = {str(label): str(label) for label in (labels or [])}
+        explicit, fallback = r"ANSWER:\s*(\d+)", r"(?<![\w.+-])(\d+)(?!\w|\.\d)"
+    else:
+        raise ValueError("Unsupported question type")
+    matches = re.findall(explicit, generation)
+    if matches and matches[-1] in mapping:
+        return mapping[matches[-1]]
+    matches = [symbol for symbol in re.findall(fallback, generation) if symbol in mapping]
+    return mapping[matches[-1]] if matches else None
+
+
+def readout_prefix(generation, marker):
+    matches = list(re.finditer(r"(?m)^[ \t]*" + re.escape(marker), generation))
+    if matches:
+        return generation[:matches[-1].end()], False
+    return generation.rstrip() + "\n" + marker, True
+
+
+def logsumexp(values):
+    top = max(values)
+    if top == -math.inf: return top
+    return top + math.log(sum(math.exp(x - top) for x in values))
+
+
+def candidate_ids(backend, groups):
+    # Keep multiplicity even when two textual forms map to the same token id.
+    result = {}
+    for label, forms in groups.items():
+        tokens = [backend.encode(form) for form in forms]
+        result[label] = [ids[0] for ids in tokens if len(ids) == 1]
+    if not any(result.values()):
+        raise ValueError("Tokenizer has no single-token candidate forms")
+    return result
+
+
+def aggregate(logits, groups):
+    for token in {t for ids in groups.values() for t in ids}:
+        if token not in logits or not math.isfinite(logits[token]):
+            raise RuntimeError("Backend omitted a candidate or returned a non-finite logit")
+    return {label: logsumexp([logits[t] for t in ids]) if ids else -1e9 for label, ids in groups.items()}
+
+
+def validate_calibration(calibration, model_key, path="fast"):
+    model = calibration.get("models", {}).get(model_key)
+    if model is None:
+        raise ValueError("Calibration lacks selected model_key")
+    block = model.get(path)
+    if block is None:
+        raise ValueError("Calibration lacks selected route")
+    for kind in ("choice", "score"):
+        finite_number(block["temperature_by_kind"][kind], kind + " temperature", positive=True)
+    if block["temperature_by_kind"]["noul"] != "one_bin":
+        raise ValueError("noul calibration must be one_bin")
+    finite_number(block["noul_calibration"]["p_cal"], "p_cal", low=0, high=1)
+    return block
+
+
+def probabilities(labels, logits, qtype, block):
+    if qtype == "noul":
+        # Deterministic tie: earliest in render's label order, namely no.
+        winner = max(labels, key=logits.__getitem__)
+        p_cal = min(1 - 1e-6, max(0.5, block["noul_calibration"]["p_cal"]))
+        return {label: p_cal if label == winner else 1 - p_cal for label in labels}
+    temperature = block["temperature_by_kind"][qtype]
+    # Subtract before dividing to avoid overflow from small positive T.
+    top = max(logits.values())
+    weights = {label: math.exp((logits[label] - top) / temperature) for label in labels}
+    total = sum(weights.values())
+    floored = {label: max(1e-6, weights[label] / total) for label in labels}
+    return {label: value / sum(floored.values()) for label, value in floored.items()}
+
+
+# Rule sections are protected conservatively. Unknown text outside a recognized
+# section is narrative; explicit rule/policy/definition/criteria subtrees are sacred.
+_RULE = re.compile(r'(?im)^\s*(?:R\d+\s*[:：]|(?:rules?|polic(?:y|ies)|definitions?|criteria|规则|規則|定义|定義|政策|判定标准)\s*[:：])')
+
+_RULE_KEY = re.compile(r'^\s*(?:rules?|polic(?:y|ies)|definitions?|criteria|application_rule_set|规则|規則|定义|定義|政策|标准|判定标准)\s*$', re.I)
+
+def contains_rule_subtree(value):
+    if isinstance(value,str):return bool(_RULE.search(value))
+    if isinstance(value,dict):return any(_RULE_KEY.fullmatch(str(k)) or contains_rule_subtree(v) for k,v in value.items())
+    if isinstance(value,list):return any(contains_rule_subtree(v) for v in value)
+    return False
+
+
+OMISSION = '\n[... state middle omitted ...]\n'
+
+
+def lexical_terms(text):
+    words = re.findall(r'[a-zA-Z_][\w-]*|\d+(?:\.\d+)?|[\u4e00-\u9fff]', text.lower())
+    # Character bigrams preserve Chinese query signal without a segmenter.
+    words += re.findall(r'(?=([\u4e00-\u9fff]{2}))', text)
+    return set(words)
+
+
+def shrink_state(state, keep_chars, head_fraction=0.6, *, backend=None, query='',
+                 head_tokens=512, tail_tokens=512, chunk_tokens=200):
+    """Extract query-relevant chunks in original order, protecting explicit rules.
+
+    The budget is tokens when a backend is supplied, otherwise characters.
+    A contiguous omitted range produces exactly one marker.
+    """
+    encode = backend.encode if backend else lambda x: list(x)
+    decode = backend.decode if backend else lambda x: ''.join(x)
+    query_terms = lexical_terms(query)
+    important = {w for w in query_terms if any(c.isdigit() for c in w)}
+    important |= {w.lower() for w in re.findall(r'\b(?:[A-Z][a-z]+|[A-Z][A-Z0-9_]+)\b',query)}
+    def text_shrink(text, budget):
+        if len(encode(text)) <= budget:
+            return text
+        # Slice original paragraphs including their delimiters; never make a
+        # delimiter into an independent compressible paragraph.
+        spans=[]; rule_section=False
+        for match in re.finditer(r'.+?(?:\n[ \t]*\n|\Z)',text,re.S):
+            part=match.group()
+            heading=re.match(r'^\s*(?:#{1,6}\s*)?([^\n:：]{1,60})[:：]\s*(?:\n|$)',part)
+            if heading:
+                title=heading.group(1)
+                if _RULE_KEY.search(title): rule_section=True
+                elif re.fullmatch(r'(?i)narrative|events?|history|state|context|data|observations?|背景|叙述|事件|状态|记录',title.strip()): rule_section=False
+            protected=rule_section or bool(_RULE.search(part))
+            if spans and spans[-1][1]==protected: spans[-1]=(spans[-1][0]+part,protected)
+            else: spans.append((part,protected))
+        merged=spans
+        encoded=[(part,keep,encode(part)) for part,keep in merged]
+        total=sum(len(ids) for _,_,ids in encoded)
+        head_end=min(head_tokens,total);tail_start=max(head_end,total-tail_tokens)
+        blocks=[];cursor=0
+        for part,protected,ids in encoded:
+            if protected:
+                blocks.append((part,len(ids),True));cursor+=len(ids)
+                continue
+            # Sentence/newline boundaries first, word boundaries for oversized
+            # sentences. Never decode an arbitrary token slice into a block.
+            sentences=re.findall(r'.*?(?:[.!?。！？](?=\s|$)|\n|$)',part,re.S)
+            pieces=[]
+            for sentence in sentences:
+                if not sentence:continue
+                if len(encode(sentence))<=chunk_tokens:pieces.append(sentence)
+                else:
+                    words=re.findall(r'\s+|\S+\s*',sentence)
+                    chunk=''
+                    for word in words:
+                        if chunk and len(encode(chunk+word))>chunk_tokens:
+                            pieces.append(chunk);chunk=''
+                        chunk+=word
+                    if chunk:pieces.append(chunk)
+            chunk=''
+            def add(value):
+                nonlocal cursor
+                size=len(encode(value))
+                edge=cursor<head_end or cursor+size>tail_start
+                blocks.append((value,size,edge));cursor+=size
+            for piece in pieces:
+                if chunk and len(encode(chunk+piece))>chunk_tokens:
+                    add(chunk);chunk=''
+                chunk+=piece
+            if chunk:add(chunk)
+        selected={i for i,(_,_,keep) in enumerate(blocks) if keep}
+        marker_cost=len(encode(OMISSION))
+        used=sum(blocks[i][1] for i in selected)+marker_cost*(len(selected)+1)
+        ranked=[]
+        for i,(part,size,keep) in enumerate(blocks):
+            terms=lexical_terms(part); overlap=query_terms & terms
+            score=(len(overlap)+2*len(important & terms))/max(1,math.sqrt(len(terms)))
+            if not keep: ranked.append((-score,i))
+        for _,i in sorted(ranked):
+            if used+blocks[i][1]+marker_cost <= budget:
+                selected.add(i); used+=blocks[i][1]+marker_cost
+        def join(indices):
+            out=[]; omitted=False
+            for i,(part,_,_) in enumerate(blocks):
+                if i in indices:
+                    if omitted: out.append(OMISSION)
+                    out.append(part); omitted=False
+                else: omitted=True
+            if omitted: out.append(OMISSION)
+            return ''.join(out)
+        result=join(selected)
+        # Recover conservative omission-marker reserves using adjacent blocks.
+        while len(encode(result))<.8*budget:
+            neighbors=sorted({j for i in selected for j in (i-1,i+1) if 0<=j<len(blocks) and j not in selected})
+            added=False
+            for j in neighbors:
+                candidate=join(selected|{j})
+                if len(encode(candidate))<=budget:
+                    selected.add(j);result=candidate;added=True
+            if not added:break
+        if len(encode(result)) <= budget: return result
+        # Protected material survives the soft budget. For an unusually small
+        # budget shrink only narrative edges globally, retaining the rules.
+        if any(keep for _,keep in merged):
+            protected_text=''.join(part for part,keep in merged if keep)
+            allowance=max(0,budget-len(encode(protected_text))-marker_cost*(len(merged)+1))
+            if not allowance: return protected_text
+            n=sum(len(encode(part)) for part,keep in merged if not keep)
+            h=int(allowance*head_fraction);t=allowance-h
+            out=[];offset=0;omitted=False
+            for part,keep in merged:
+                if keep:
+                    if omitted:out.append(OMISSION);omitted=False
+                    out.append(part);continue
+                ids=encode(part)
+                head=ids[:max(0,min(len(ids),h-offset))]
+                tail=ids[max(0,n-t-offset):] if offset+len(ids)>n-t else []
+                if head:
+                    if omitted:out.append(OMISSION);omitted=False
+                    out.append(decode(head))
+                if len(head)+len(tail)<len(ids):omitted=True
+                if tail:
+                    if omitted:out.append(OMISSION);omitted=False
+                    out.append(decode(tail))
+                offset+=len(ids)
+            if omitted:out.append(OMISSION)
+            return ''.join(out)
+        ids=encode(text); available=max(0,budget-marker_cost)
+        h=int(available*head_fraction); t=available-h
+        return decode(ids[:h])+OMISSION+(decode(ids[-t:]) if t else '')
+
+    if isinstance(state,str): return text_shrink(state,max(0,keep_chars))
+    if isinstance(state,dict):
+        protected={k:v for k,v in state.items() if _RULE_KEY.search(str(k)) or
+                   (not isinstance(v,(dict,list)) and len(str(v))<=200)}
+        rest=[k for k in state if k not in protected]
+        remaining=max(0,keep_chars-len(encode(json.dumps(protected,ensure_ascii=False))))
+        sizes={k:len(encode(json.dumps(state[k],ensure_ascii=False))) for k in rest}
+        total=max(1,sum(sizes.values()))
+        return {k:v if k in protected else shrink_state(v,int(remaining*sizes[k]/total),head_fraction,
+                backend=backend,query=query,head_tokens=head_tokens,tail_tokens=tail_tokens,chunk_tokens=chunk_tokens)
+                for k,v in state.items()}
+    if isinstance(state,list):
+        if len(encode(json.dumps(state,ensure_ascii=False)))<=keep_chars: return state
+        if not state: return state
+        # Budget whole entries, protect explicit rule entries; preserve both ends.
+        selected={0,len(state)-1}|{i for i,v in enumerate(state) if contains_rule_subtree(v)}
+        ranked=sorted(range(1,len(state)-1),key=lambda i:(-len(lexical_terms(str(state[i])) & query_terms),i))
+        used=sum(len(encode(json.dumps(state[i],ensure_ascii=False))) for i in selected)
+        for i in ranked:
+            size=len(encode(json.dumps(state[i],ensure_ascii=False)))
+            if i not in selected and used+size+len(OMISSION)<keep_chars: selected.add(i); used+=size
+        protected_entries={i for i in selected if isinstance(state[i],str) and _RULE.search(state[i])}
+        protected_size=sum(len(encode(json.dumps(state[i],ensure_ascii=False))) for i in protected_entries)
+        narrative_size=max(1,sum(len(encode(json.dumps(state[i],ensure_ascii=False))) for i in selected-protected_entries))
+        free=max(0,keep_chars-protected_size-len(selected)*len(OMISSION))
+        result=[]; omitted=False
+        for i,v in enumerate(state):
+            if i in selected:
+                if omitted: result.append(OMISSION.strip())
+                if i not in protected_entries and len(str(v))>200:
+                    size=len(encode(json.dumps(v,ensure_ascii=False)))
+                    v=shrink_state(v,int(free*size/narrative_size),head_fraction,backend=backend,
+                        query=query,head_tokens=head_tokens,tail_tokens=tail_tokens,chunk_tokens=chunk_tokens)
+                result.append(v);omitted=False
+            else: omitted=True
+        return result
+    return state
+
+
+def prepare_task(task, backend, config):
+    def prompts(value):
+        labels,text,groups=render(value)
+        instruction=config['prompt']['cot_instructions'][value['question']['type']].format(
+            reasoning_words=config['generation']['reasoning_words']).replace('ANSWER:',config['prompt']['answer_marker'])
+        system,think=config['prompt']['system'],config['prompt']['enable_thinking']
+        return labels,groups,backend.prompt_ids(system,text,think),backend.prompt_ids(system,text+'\n'+instruction,think)
+    limits=config['limits']; budget=limits['max_prompt_tokens'];normal_budget=budget
+    long_budget=min(budget,config['routing'].get('tiers',{}).get('long',{}).get('max_prompt_tokens',budget))
+    query=task['question']['instructions']+' '+json.dumps(task['question'].get('criteria'),ensure_ascii=False)+' '+str(task.get('labels') or '')
+    raw=task['state'] if isinstance(task['state'],str) else json.dumps(task['state'],ensure_ascii=False)
+    precompressed=len(raw)>limits['precompression_trigger_chars_per_token']*budget
+    source=task
+    if precompressed:budget=long_budget
+    if precompressed:
+        ratio=limits['estimated_chars_per_token']
+        source=dict(task,state=shrink_state(task['state'],budget*ratio,limits['truncation_head_frac'],query=query,
+            head_tokens=limits['state_head_tokens']*ratio,tail_tokens=limits['state_tail_tokens']*ratio,
+            chunk_tokens=limits['state_chunk_tokens']*ratio))
+    original=prompts(source); original_count=len(original[2])
+    long_input=precompressed or original_count>normal_budget
+    if long_input:budget=long_budget
+    current,prepared=source,original
+    refilled=False
+    if precompressed and original_count<.8*budget:
+        fixed=len(prompts(dict(task,state=''))[2])
+        refill=shrink_state(task['state'],max(0,budget-fixed-limits['compression_reserve_tokens']),
+            limits['truncation_head_frac'],backend=backend,query=query,
+            head_tokens=limits['state_head_tokens'],tail_tokens=limits['state_tail_tokens'],chunk_tokens=limits['state_chunk_tokens'])
+        candidate=dict(task,state=refill);candidate_prompts=prompts(candidate)
+        if len(candidate_prompts[2])<=budget:
+            current,prepared=candidate,candidate_prompts;refilled=True
+    if original_count>budget:
+        fixed=len(prompts(dict(task,state=''))[2])
+        query=task['question']['instructions']+' '+json.dumps(task['question'].get('criteria'),ensure_ascii=False)+' '+str(task.get('labels') or '')
+        allowance=max(0,budget-fixed-limits['compression_reserve_tokens'])
+        for _ in range(3):
+            current=dict(task,state=shrink_state(source['state'],allowance,limits['truncation_head_frac'],
+                backend=backend,query=query,head_tokens=limits['state_head_tokens'],
+                tail_tokens=limits['state_tail_tokens'],chunk_tokens=limits['state_chunk_tokens']))
+            prepared=prompts(current); size=len(prepared[2])
+            if size<=budget: break
+            allowance=max(0,allowance-(size-budget)-limits['compression_reserve_tokens'])
+        # Compression cannot remove sacred rule/question/option material. The
+        # final physical guard always bounds tensors, with explicit metadata.
+    kept=len(prepared[2]); physical=limits['physical_max_prompt_tokens']
+    hard_cut=kept>physical
+    if hard_cut:
+        def cut(ids):
+            h=int(physical*limits['truncation_head_frac']);t=physical-h
+            return ids if len(ids)<=physical else ids[:h]+ids[-t:]
+        prepared=prepared[:2]+(cut(prepared[2]),cut(prepared[3]))
+    return current,prepared,{'original_prompt_tokens':original_count if not precompressed else None,
+        'original_prompt_tokens_estimate':math.ceil(len(raw)/limits['estimated_chars_per_token']) if precompressed else original_count,
+        'long_input':long_input,'character_precompressed':precompressed,'adjacent_refill':refilled,
+        'kept_prompt_tokens':len(prepared[2]),
+        'slow_prompt_exceeds_budget':len(prepared[3])>min(budget,config['routing']['slow_max_prompt_tokens']),
+        'truncated':current['state']!=task['state'] or hard_cut,
+        'protected_content_exceeds_budget':kept>budget,'physical_truncated':hard_cut}
+
+
+def sequence_scores(session, backend, groups, meter=None):
+    """Full-vocabulary chain probabilities; copies branch KV, never prompt prefill.
+
+    Keep every form (including repeated tokenizations) with its original weight.
+    """
+    cache = {(): session}
+    continuation_tokens = 0
+    normalized = {}
+    path_scores = {(): 0.0}
+    def score(ids):
+        nonlocal continuation_tokens
+        prefix = ()
+        for token in ids:
+            check_deadline(meter)
+            child = prefix + (token,)
+            if child not in path_scores:
+                parent = cache[prefix]
+                if prefix not in normalized:
+                    wanted=sorted({encoded[len(prefix)] for forms in groups.values() for form in forms
+                        if (encoded:=backend.encode(form))[:len(prefix)]==list(prefix) and len(encoded)>len(prefix)})
+                    if hasattr(parent,'log_probs'):
+                        normalized[prefix]=parent.log_probs(wanted)
+                    else:
+                        logits=parent.logits; lse=logsumexp(list(logits.values()))
+                        normalized[prefix]={t:logits[t]-lse for t in wanted if t in logits}
+                value=normalized[prefix].get(token)
+                if value is None or not math.isfinite(value): raise RuntimeError('Missing candidate token')
+                path_scores[child]=path_scores[prefix]+value
+            prefix = child
+            if prefix not in cache and len(prefix) < len(ids):
+                branch = cache[prefix[:-1]].fork()
+                if meter is not None and not getattr(branch, 'handles_meter', False):
+                    meter['output'] += 1
+                    meter['probes'] += 1
+                    if getattr(branch,'meter',None) is None:
+                        meter['engine_input']=meter.get('engine_input',0)+1; meter['engine_submitted']+=1
+                branch.advance([token])
+                continuation_tokens += 1
+                cache[prefix] = branch
+        return path_scores[prefix]
+    digit_ids=getattr(backend,'digit_start_ids',None)
+    if digit_ids is None:
+        # Synthetic tokenizer fallback only; real backends scan the vocabulary.
+        digit_ids={ids[0] for d in string.digits if len(ids:=backend.encode(d))==1}
+    terminals={}
+    def termination(ids):
+        nonlocal continuation_tokens
+        key=tuple(ids)
+        if key not in terminals:
+            if key not in cache:
+                branch=cache[key[:-1]].fork()
+                if meter is not None and not getattr(branch,'handles_meter',False):
+                    meter['output']+=1;meter['probes']+=1
+                    if getattr(branch,'meter',None) is None:
+                        meter['engine_input']=meter.get('engine_input',0)+1;meter['engine_submitted']+=1
+                branch.advance([key[-1]]);continuation_tokens+=1;cache[key]=branch
+            se=cache[key]
+            if hasattr(se,'non_digit_logprob'):
+                value=se.non_digit_logprob(digit_ids)
+            else:
+                logits=se.logits; z=logsumexp(list(logits.values()))
+                allowed=[v for t,v in logits.items() if t not in digit_ids]
+                value=logsumexp(allowed)-z if allowed else -math.inf
+            terminals[key]=value
+        return terminals[key]
+    result={}
+    for label,forms in groups.items():
+        is_prefix=label.isdigit() and any(other!=label and other.startswith(label) for other in groups)
+        values=[]
+        for form in forms:
+            ids=backend.encode(form)
+            if not ids: continue
+            value=score(ids)
+            if is_prefix and form.strip().isdigit(): value+=termination(ids)
+            values.append(value)
+        result[label]=logsumexp(values)
+    return result, continuation_tokens
+
+
+def slow_readout(backend, ids, groups, config, meter):
+    """One autoregressive session: one prompt prefill, cached draft/marker/readout."""
+    if hasattr(backend,'slow_session'):
+        session,parsed=backend.slow_session(ids,config['routing']['unparsed_policy'],meter)
+        if session is None: return None,meter['draft'],False,False,0
+        # Marginalize textual whitespace forms; no full-vocabulary CPU copy.
+        scores,probes=sequence_scores(session,backend,groups,meter)
+        return scores,meter['draft']+meter['forced'],not parsed,False,probes
+    meter['prompt'] += len(ids)
+    meter['engine_submitted'] += len(ids)
+    meter['engine_input']=meter.get('engine_input',0)+len(ids)
+    meter['engine_calls']=meter.get('engine_calls',0)+1
+    session = backend.open_session(ids)
+    import random
+    rng = random.Random(config['generation']['seed'])
+    generated = []
+    marker = config['prompt']['answer_marker']
+    found = False
+    for _ in range(config['generation']['max_new_tokens']):
+        check_deadline(meter)
+        logits = dict(session.logits)
+        g = config['generation']
+        for seen in set(generated):
+            value = logits[seen]
+            logits[seen] = value * g['repetition_penalty'] if value < 0 else value / g['repetition_penalty']
+        token = max(logits, key=logits.get)
+        if g['do_sample']:
+            ordered = sorted(logits, key=logits.get, reverse=True)
+            if g['top_k']:
+                ordered = ordered[:g['top_k']]
+            weights = [math.exp((logits[t]-logits[token])/g['temperature']) for t in ordered]
+            mass, cutoff = 0, sum(weights)*g['top_p']
+            end = 0
+            while end < len(weights) and mass < cutoff:
+                mass += weights[end]
+                end += 1
+            token = rng.choices(ordered[:end], weights=weights[:end])[0]
+        generated.append(token)
+        meter['output'] += 1
+        meter['draft'] += 1
+        meter['engine_generated']=meter.get('engine_generated',0)+1
+        session.advance([token])
+        text = backend.decode(generated)
+        if re.search(r'(?m)^[ \t]*' + re.escape(marker) + r'$', text):
+            found = True
+            break
+        if token in getattr(backend, 'eos_ids', ()):
+            break
+    if not found and config['routing']['unparsed_policy'] == 'fast':
+        return None, len(generated), False, False, 0
+    if not found:
+        suffix = backend.encode('\n' + marker)
+        meter['output'] += len(suffix)
+        meter['forced'] += len(suffix)
+        meter['engine_input']=meter.get('engine_input',0)+len(suffix);meter['engine_submitted']+=len(suffix)
+        session.advance(suffix)
+        generated.extend(suffix)
+    # Whitespace is a form branch, never a top-1 destructive advance.
+    whitespace = False
+    scores, probes = sequence_scores(session, backend, groups, meter)
+    return scores, len(generated), not found, whitespace, probes
+
+
+def label_margin(labels, scores, kind="logratio"):
+    """Aggregated label distribution at T=1, before any calibration/floor."""
+    if len(labels) < 2:
+        return math.inf
+    values = sorted((scores[label] for label in labels), reverse=True)
+    if kind == 'logratio':
+        return values[0] - values[1]
+    if kind != 'probdiff':
+        raise ValueError('Invalid margin kind')
+    weights = [math.exp(v-values[0]) for v in values]
+    return (weights[0]-weights[1])/sum(weights)
+
+
+def fast_readout(backend, ids, labels, groups, meter):
+    meter["prompt"]+=len(ids)
+    if not getattr(backend,"engine_metering",False):
+        meter["engine_submitted"]+=len(ids);meter['engine_input']+=len(ids)
+        meter['hf_prefill_tokens']+=len(ids)
+    try:
+        tokens = candidate_ids(backend, groups)
+    except ValueError:
+        tokens = {label: [] for label in labels}
+    probes = 0
+    if any(not values for values in tokens.values()):
+        session=backend.open_session(ids,meter=meter) if getattr(backend,'engine_metering',False) else backend.open_session(ids)
+        if hasattr(session,'cache'): session.meter=meter
+        scores, probes = sequence_scores(session, backend, groups, meter)
+    else:
+        wanted = sorted({t for values in tokens.values() for t in values})
+        logits=backend.next_logits(ids,wanted,meter=meter) if getattr(backend,'engine_metering',False) else backend.next_logits(ids,wanted)
+        scores = aggregate(logits, tokens)
+        if not getattr(backend,'engine_metering',False):
+            meter['output']+=backend.readout_output_tokens;meter['probes']+=backend.readout_output_tokens
+            meter['engine_generated']+=backend.readout_output_tokens
+    return scores, probes + backend.readout_output_tokens
+
+
+def release_failed_session(backend):
+    import gc
+    gc.collect()
+    torch=getattr(backend,'torch',None)
+    cuda=getattr(torch,'cuda',None)
+    if cuda is not None:
+        try: cuda.empty_cache()
+        except Exception: pass
+
+
+def fast_attempt_with_retry(backend,task,config,ids,labels,groups,meter):
+    try:
+        scores,out=fast_readout(backend,ids,labels,groups,meter)
+        return scores,out,None
+    except Exception as exc:
+        if unrecoverable_engine_error(exc):
+            raise
+        cuda=getattr(getattr(backend,'torch',None),'cuda',None)
+        oom_type=getattr(cuda,'OutOfMemoryError',MemoryError)
+        if not isinstance(exc,(MemoryError,oom_type)) and 'out of memory' not in str(exc).lower():
+            raise
+    # Leave the failed forward traceback before clearing allocator state.
+    release_failed_session(backend)
+    import copy
+    reduced=copy.deepcopy(config)
+    budget=min(config['limits']['oom_retry_prompt_tokens'],max(1,config['limits']['max_prompt_tokens']//2))
+    reduced['limits']['max_prompt_tokens']=budget
+    reduced['limits']['physical_max_prompt_tokens']=budget
+    _,(_,new_groups,new_ids,_),length=prepare_task(task,backend,reduced)
+    scores,out=fast_readout(backend,new_ids,labels,new_groups,meter)
+    return scores,out,length
+
+
+def slow_attempt(backend,ids,groups,config,meter):
+    # Catch in this frame, then leave except before collecting. The failed
+    # readout/generate traceback (and its KV references) no longer exists.
+    try:
+        return slow_readout(backend,ids,groups,config,meter)
+    except Exception as exc:
+        if unrecoverable_engine_error(exc):
+            raise ServiceFailure('unrecoverable engine failure') from exc
+        timed_out=isinstance(exc,TimeoutError)
+    release_failed_session(backend)
+    if timed_out: raise TimeoutError('Slow deadline exceeded')
+    raise RuntimeError('Slow session failed')
+
+
+def bounded_slow_attempt(backend,ids,groups,config,meter,model_lock, *, on_service_failure=None):
+    """Bound response wait; an in-flight synchronous engine call drains under lock.
+
+    No worker may start another engine call after cancellation. Timeout usage is
+    explicitly estimated and reconciliation rejects the run. An unrecoverable engine
+    error is reported through on_service_failure even after the response timed out.
+    """
+    import queue,time
+    result=queue.Queue(maxsize=1)
+    timeout=config['generation'].get('wall_timeout_seconds',60)
+    meter['cancel_event']=threading.Event();meter['deadline']=time.monotonic()+timeout
+    def work():
+        try:
+            with model_lock:
+                check_deadline(meter)
+                value=slow_attempt(backend,ids,groups,config,meter)
+                check_deadline(meter)
+            result.put((True,value))
+        except Exception as exc:
+            fatal=unrecoverable_engine_error(exc)
+            if fatal and on_service_failure is not None:
+                on_service_failure()
+            result.put((False,ServiceFailure if fatal else type(exc)))
+    worker=threading.Thread(target=work,daemon=True);worker.start()
+    try: ok,value=result.get(timeout=timeout)
+    except queue.Empty:
+        meter['cancel_event'].set();meter['engine_usage_unknown']=True
+        raise TimeoutError('Slow wall timeout; in-flight call draining')
+    if not ok:
+        if value is ServiceFailure: raise ServiceFailure('unrecoverable engine failure')
+        if value is TimeoutError: raise TimeoutError('Slow wall timeout')
+        raise RuntimeError('Slow session failed')
+    return value
+
+
+def threshold_identity(config):
+    import hashlib
+    import inspect
+    # The renderer source binds label layout, and settings bind both prompts.
+    canonical=json.dumps({'prompt':{k:config['prompt'][k] for k in ('system','enable_thinking')},'render':inspect.getsource(render),'choice_codes':inspect.getsource(choice_codes),
+        'generation':{k:v for k,v in config['generation'].items() if k not in ('max_new_tokens','reasoning_words','wall_timeout_seconds')}},sort_keys=True,ensure_ascii=False)
+    m=config['model']
+    return dict(model_key=m['model_key'],quant=m['quant'],adapter=m.get('adapter'),
+                prompt_sha=hashlib.sha256(canonical.encode('utf-8')).hexdigest(),backend=m['backend'],
+                margin_kind=config['routing']['margin_kind'],max_new_tokens=config['generation']['max_new_tokens'],
+                reasoning_words=config['generation']['reasoning_words'],
+                max_prompt_tokens=config['limits']['max_prompt_tokens'],request_contract='typesafe.criteria.v1')
+
+
+def validate_threshold(config,allow_placeholder=False):
+    identity=threshold_identity(config)
+    fit=config['routing'].get('margin_threshold_fitted_for',{})
+    reasons=[]
+    if fit.get('placeholder',True): reasons.append('placeholder')
+    if any(fit.get(k)!=identity[k] for k in ('model_key','quant','adapter','prompt_sha','margin_kind','max_prompt_tokens','request_contract')):
+        reasons.append('identity mismatch')
+    if fit.get('backend')!=identity['backend']: reasons.append('backend mismatch')
+    if reasons and config['routing']['mode']=='routed' and not allow_placeholder:
+        raise ValueError('Unfitted margin threshold: '+', '.join(reasons))
+    return ['margin threshold degraded: '+', '.join(reasons)] if reasons else []
+
+
+class Service:
+    def __init__(self, config, calibration, backend, counter, *, allow_placeholder_threshold=False):
+        validate_config(config)
+        threshold_warnings=validate_threshold(config,allow_placeholder_threshold)
+        self.model_lock = threading.RLock()
+        self._failure_lock = threading.Lock()
+        self.item_failure_streak = 0
+        self.dead = False
+        self.config, self.backend, self.counter = config, backend, counter
+        counter.configure_cost(config)
+        key = config['model']['model_key']
+        self.calibration = {path: validate_calibration(calibration, key, path) for path in ('fast', 'slow')}
+        model = calibration['models'][key]
+        self.warnings = ['calibration placeholder: refit required'] if (
+            model.get('placeholder') or any(b.get('placeholder') for b in self.calibration.values())) else []
+        self.warnings.extend(threshold_warnings)
+        if not config['routing'].get('cost_fuse',{}).get('enabled',True):
+            self.warnings.append('cost fuse explicitly disabled')
+        if 'fallback' in model:
+            self.calibration['fallback']=validate_calibration(calibration,key,'fallback')
+        else:
+            self.calibration['fallback']=self.calibration['fast']
+            self.warnings.append('fallback calibration missing: using fast block')
+
+    def _mark_dead(self):
+        with self._failure_lock:
+            self.dead = True
+
+    def is_dead(self):
+        """Service failed: an unrecoverable engine error, the engine's own dead flag, or the
+        third consecutive item failure. Reading the flag performs no engine call."""
+        if self.dead:
+            return True
+        resource = self.backend
+        for name in ('engine', 'llm_engine', 'engine_core', 'resources'):
+            resource = getattr(resource, name, None)
+        if getattr(resource, 'engine_dead', False) is True:
+            self._mark_dead()
+        return self.dead
+
+    def _raise_if_service_failure(self, exc):
+        if unrecoverable_engine_error(exc) or self.is_dead():
+            self._mark_dead()
+            raise ServiceFailure('unrecoverable engine failure') from exc
+
+    def _item_failed(self):
+        with self._failure_lock:
+            self.item_failure_streak += 1
+            if self.item_failure_streak >= ITEM_FAILURE_LIMIT:
+                self.dead = True
+            return self.item_failure_streak
+
+    def _item_succeeded(self):
+        with self._failure_lock:
+            self.item_failure_streak = 0
+
+    def health(self):
+        dead = self.is_dead()
+        warnings = list(self.warnings)
+        if dead:
+            warnings.append('service failure')
+        reference=getattr(self.backend,'generation_reference',None)
+        if reference and not reference['verified']:warnings.append('generation reference '+reference['status'])
+        if hasattr(self.backend,'forward_counters') and not getattr(self.backend,'stats_available',False):warnings.append('HF forward hook unavailable; usage estimated')
+        if getattr(self.backend,'forward_counters',{}).get('uncounted_forwards',0):warnings.append('HF uncounted forwards; usage estimated')
+        if getattr(self.counter, 'memory_only', False):
+            warnings.append('quota persistence unavailable: memory counts, fast only')
+        return {'status': 'error' if dead else 'warning' if warnings else 'ok', 'warnings': warnings,
+                'info': {'intended_overrides': (reference or {}).get('intended_overrides',{})},
+                'model_key': self.config['model']['model_key'], 'mode':self.config['routing']['mode'],
+                'quota_tier':self.config['routing'].get('tier','normal'), 'budget_tiers':self.config['routing'].get('tiers',{}),
+                'threshold_identity':threshold_identity(self.config),
+                'cost_fuse':self.counter.cost_fuse.snapshot(self.config['routing']['quota']),
+                'accuracy_only':self.config.get('bench',{}).get('accuracy_only',False),
+                'generation_reference':({k:v for k,v in reference.items() if k!='intended_overrides'} if reference else reference),'quantization_kernel':getattr(self.backend,'quantization_kernel',None),
+                'prefix_caching':getattr(self.backend,'prefix_caching',None),
+                'supports_slow_session':callable(getattr(self.backend,'slow_session',None)) or callable(getattr(self.backend,'open_session',None))}
+
+    def engine_stats(self):
+        with self.model_lock:
+            return self.backend.engine_stats()
+
+    def warmup(self):
+        ids = self.backend.prompt_ids(self.config['prompt']['system'], 'Ready?', False)
+        self.backend.next_logits(ids, [self.backend.encode('yes')[0]])
+        self.backend.generate(ids, 1)
+        meter=new_meter()
+        with self.model_lock:
+            if callable(getattr(self.backend,'slow_session',None)):
+                session,_=self.backend.slow_session(ids,'force_answer',meter,
+                    max_new_tokens=self.config['server']['warmup_slow_tokens'])
+                if session is not None:
+                    branch=session.fork();branch.advance(self.backend.encode('yes'))
+                    if hasattr(branch,'log_probs'): branch.log_probs([self.backend.encode('yes')[0]])
+                    else: _=branch.logits
+                    del branch,session
+            elif callable(getattr(self.backend,'open_session',None)):
+                import copy
+                warm=copy.deepcopy(self.config)
+                warm['generation']['max_new_tokens']=self.config['server']['warmup_slow_tokens']
+                warm['routing']['unparsed_policy']='force_answer'
+                slow_attempt(self.backend,ids,{'yes':['yes',' yes']},warm,meter)
+
+    def _one(self, task):
+        with self.counter.accounting():
+            if self.is_dead():
+                raise ServiceFailure('service failure')
+            try:
+                answer,detail=self._one_impl(task)
+                if self.is_dead():
+                    raise ServiceFailure('service failure')
+                detail['threshold_identity']=threshold_identity(self.config)
+                detail['cost_fuse']=self.counter.record_cost(detail['usage_bases'][detail['accounting']],detail['path']=='slow')
+                try:
+                    json.dumps([answer,detail],allow_nan=False)
+                except ValueError as exc:
+                    raise ItemFailure('non-finite value in response') from exc
+            except ItemFailure as exc:
+                if self._item_failed() >= ITEM_FAILURE_LIMIT:
+                    raise ServiceFailure('consecutive item failures') from exc
+                raise
+            except ServiceFailure:
+                self._mark_dead()
+                raise
+            self._item_succeeded()
+            return answer,detail
+
+    def _one_impl(self, task):
+        c, backend = self.config, self.backend
+        uncounted_before=getattr(backend,'forward_counters',{}).get('uncounted_forwards',0)
+        labels, _, groups = render(task)
+        qtype = task['question']['type']
+        detail = {'path':'fast', 'route_reason':'error', 'router_score':0.0,
+                  'prompt_tokens':0, 'completion_tokens':0, 'n_new_tokens':0,
+                  'prompt_tokens_details':{'cached_tokens':0,'engine_submitted_tokens':0},
+                  'fast_scores':None,'slow_scores':None,'budget_tier':'normal',
+                  'readout_output_tokens':0, 'forced_answer':False, 'draft_prediction':None}
+        warning = None
+        meters = []
+        try:
+            kept_task, (_, groups, fast_ids, slow_ids), length = prepare_task(task, backend, c)
+            detail.update(length)
+            long_item=length.get('long_input',False) or c['routing'].get('tier')=='long' or length.get('slow_prompt_exceeds_budget',False) or any(length[k] for k in ('truncated','character_precompressed','protected_content_exceeds_budget','physical_truncated'))
+            hard=long_item if c['routing'].get('long_item_slow_policy','fast_only')=='fast_only' else (
+                length['character_precompressed'] or (length['original_prompt_tokens'] or 0)>c['routing']['slow_max_prompt_tokens'] or length['protected_content_exceeds_budget'])
+            hard=hard or c['routing'].get('tier')=='long' or length.get('slow_prompt_exceeds_budget',False)
+            detail['long_item']=long_item
+            r = c['routing']
+            fast_scores = None
+            fast_meter=new_meter()
+            meters.append(fast_meter)
+            try:
+                with self.model_lock:
+                    fast_scores, out, retry_length = fast_attempt_with_retry(backend,task,c,fast_ids,labels,groups,fast_meter)
+            except Exception as exc:
+                self._raise_if_service_failure(exc)
+                raise ItemFailure('item could not be processed') from exc
+            if retry_length is not None:
+                detail.update(retry_length)
+                detail['fast_oom_retry']=True
+                hard=True
+            detail['prompt_tokens']+=fast_meter['prompt']
+            detail['completion_tokens']+=fast_meter['output']
+            detail['readout_output_tokens']+=fast_meter['probes']
+            detail['prompt_tokens_details'].update(cached_tokens=fast_meter['cached'],engine_submitted_tokens=fast_meter['engine_submitted'])
+            if fast_meter.get('engine_usage_unknown'): detail['usage_estimated']=True
+            if fast_meter.get('cached_unknown'): detail['prompt_tokens_details']['cached_tokens_known']=False
+            detail['fast_prompt_tokens']=fast_meter['prompt']
+            detail['raw_label_scores']=detail['fast_scores']=fast_scores
+            margin = label_margin(labels, fast_scores, r['margin_kind']) if fast_scores is not None else None
+            detail.update(fast_margin=margin if margin is None or math.isfinite(margin) else None,
+                          fast_margin_infinite=margin == math.inf, margin_kind=r['margin_kind'])
+            wants_slow = fast_scores is not None and not hard and (
+                r['mode'] == 'uniform' or (r['mode'] == 'routed' and margin < r['margin_threshold']))
+            wants_slow = wants_slow and (callable(getattr(backend,'slow_session',None)) or callable(getattr(backend,'open_session',None)))
+            detail['budget_tier']='long' if hard else 'hard' if wants_slow else 'normal'
+            detail['admission_quota'],detail['admission_quota_source']=self.counter.cost_fuse.quota(r['quota'])
+            with self.counter.decision(wants_slow, quota=detail['admission_quota']) as decision:
+                detail.update(path=decision.path, route_reason='oom_retry' if detail.get('fast_oom_retry') else 'length' if hard else decision.reason,
+                              quota_at_reservation=decision.snapshot)
+                scores, block = fast_scores, 'fast'
+                if decision.path == 'slow':
+                    meter = new_meter()
+                    meters.append(meter)
+                    try:
+                        scores, count, forced, whitespace, probes = bounded_slow_attempt(backend,slow_ids,groups,c,meter,self.model_lock,on_service_failure=self._mark_dead)
+                        detail['slow_raw_label_scores']=detail['slow_scores']=scores
+                        block = 'slow' if scores is not None else 'fallback'
+                        detail['route'] = ('slow_forced_answer' if forced else 'slow') if scores is not None else 'slow_unparsed_fast'
+                        if scores is None:
+                            scores = fast_scores
+                        detail.update(n_new_tokens=count, forced_answer=forced, whitespace_step=whitespace)
+                    except Exception as exc:
+                        self._raise_if_service_failure(exc)
+                        scores, block = fast_scores, 'fallback'
+                        warning = 'slow_failed: fast answer kept'
+                        detail.update(fallback='slow_to_fast', route='slow_timeout_fast' if isinstance(exc,TimeoutError) else 'slow_failed_fast')
+                        if isinstance(exc,TimeoutError):
+                            warning='slow_timeout: fast answer kept'
+                            detail['usage_estimated']=True
+                            # Detach response accounting from the draining worker.
+                            meter=dict(meter);meters[-1]=meter
+                    finally:
+                        if meter.get('engine_usage_unknown'): detail['usage_estimated']=True
+                        if meter.get('cached_unknown'): detail['prompt_tokens_details']['cached_tokens_known']=False
+                        detail['prompt_tokens'] += meter['prompt']
+                        detail['completion_tokens'] += meter['output']
+                        detail['n_new_tokens'] = meter['draft'] + meter['forced']
+                        detail['readout_output_tokens'] += meter['probes']
+                        detail['prompt_tokens_details']['cached_tokens'] += meter['cached']
+                        detail['prompt_tokens_details']['engine_submitted_tokens'] += meter['engine_submitted']
+                detail['calibration_block']=block
+                if scores is None:
+                    raise ItemFailure('no label scores')
+                probs = probabilities(labels, scores, qtype, self.calibration.get(block, self.calibration['fast']))
+            detail['quota_after_completion'] = decision.snapshot
+        except (ItemFailure, ServiceFailure, QuotaStateError):
+            raise
+        except Exception as exc:
+            # No substitute answer: an engine failure is service-level (HTTP 503); any other
+            # failure fails this item (HTTP 422), which the harness scores as wrong.
+            self._raise_if_service_failure(exc)
+            raise ItemFailure('item could not be processed') from exc
+        if (getattr(backend,'forward_counters',{}).get('uncounted_forwards',0)>uncounted_before
+                or hasattr(backend,'forward_counters') and not getattr(backend,'stats_available',False)):
+            detail['usage_estimated']=True
+            warning='HF count unavailable: usage estimated'
+        detail['backend_warnings']=[w for m in meters for w in m.get('warnings',[])]
+        if any(m.get('cached_unknown') for m in meters):detail['usage_estimated']=True
+        submitted=sum(m['engine_submitted'] for m in meters)
+        cached=sum(m['cached'] for m in meters)
+        generated=sum(m.get('engine_generated',m['output']) for m in meters)
+        engine_input=sum(m.get('engine_input',m['engine_submitted']-m['cached']) for m in meters)
+        if detail.get('usage_estimated'):
+            engine_input=max(engine_input,submitted)+c['generation']['max_new_tokens']
+            generated+=c['generation']['max_new_tokens']
+        detail['input_tokens_details']=dict(cached_tokens=cached,engine_submitted_tokens=submitted,
+            engine_calls=sum(m.get('engine_calls',0) for m in meters),engine_generated_tokens=generated,
+            cached_tokens_known=not any(m.get('cached_unknown') for m in meters),
+            generate_nonempty_calls=sum(m.get('generate_nonempty_calls',0) for m in meters),
+            hf_prefill_tokens=sum(m.get('hf_prefill_tokens',0) for m in meters))
+        detail['usage_bases']={
+            'engine':dict(input_tokens=engine_input,output_tokens=generated),
+            'each_once':dict(input_tokens=detail['prompt_tokens'],output_tokens=detail['completion_tokens']),
+            'submitted':dict(input_tokens=submitted,output_tokens=detail['completion_tokens'])}
+        if detail.get('usage_estimated'):
+            for values in detail['usage_bases'].values():
+                values['input_tokens']=max(values['input_tokens'],engine_input)
+                values['output_tokens']=max(values['output_tokens'],generated)
+        detail['accounting']=c.get('usage',{}).get('basis','engine')
+        detail['kept_prompt_tokens_max'] = detail.get('kept_prompt_tokens', 0)
+        detail['total_tokens'] = detail['prompt_tokens'] + detail['completion_tokens']
+        detail['prediction'] = max(labels, key=probs.get)
+        request_type = task['question']['request_type']
+        if qtype == 'noul':
+            style = c.get('interface', {}).get('boolean_keys', 'request')
+            if style == 'true_false' or (style == 'request' and request_type == 'boolean'):
+                probs = {'false':probs['no'], 'true':probs['yes']}
+        answer = {'type':request_type, 'probabilities':probs}
+        if qtype == 'noul': answer['noul'] = float(probs.get('yes', probs.get('true')))
+        elif qtype == 'choice': answer['choice'] = detail['prediction']
+        if warning:
+            answer['warning'] = warning
+        return answer, detail
+
+    def answer(self, payload):
+        if self.is_dead():
+            raise ServiceFailure('service failure')
+        tasks = normalize_request(payload, self.config['server']['max_questions'])
+        answers, details, errors, noul, choice = {}, {}, {}, {}, {}
+        for name, task in tasks:
+            answer, detail = self._one(task)
+            answers[name], details[name] = answer, detail
+            if 'warning' in answer:
+                errors[name] = answer['warning']
+            if task['question']['type'] == 'noul':
+                noul[name] = answer['probabilities'].get('true', answer['probabilities'].get('yes'))
+            else:
+                choice[name] = detail['prediction']
+        basis=self.config.get('usage',{}).get('basis','engine')
+        alternatives={b:{k:sum(d['usage_bases'][b][k] for d in details.values())
+            for k in ('input_tokens','output_tokens')} for b in ('engine','each_once','submitted')}
+        p,o=alternatives[basis]['input_tokens'],alternatives[basis]['output_tokens']
+        token_details={k:sum(d['input_tokens_details'][k] for d in details.values()) for k in
+            ('cached_tokens','engine_submitted_tokens','engine_calls','engine_generated_tokens','generate_nonempty_calls','hf_prefill_tokens')}
+        token_details['cached_tokens_known']=all(d['input_tokens_details']['cached_tokens_known'] for d in details.values())
+        result={'answers':answers,'model':self.config['model']['name'],
+            'usage':dict(input_tokens=p,output_tokens=o,prompt_tokens=p,completion_tokens=o,total_tokens=p+o,
+                cost_fuse=next(reversed(details.values()))['cost_fuse'],
+                accounting=basis,usage_estimated=any(d.get('usage_estimated',False) for d in details.values()),
+                alternatives=alternatives,questions=details,input_tokens_details=token_details,
+                prompt_tokens_details=token_details,token_accounting=basis)}
+        for key, values in (('noul',noul),('choice',choice)):
+            if values:
+                result[key] = next(iter(values.values())) if len(tasks) == 1 else values
+        if errors:
+            result['partial_errors'] = errors
+        return result

service/fit_report.json

@@ -0,0 +1,246 @@
+{
+  "adapter_files": {
+    "adapter_config.json": {
+      "sha256": "140bb00c832d117f57d29650dabe50e992b2e5064e908aaa29230b3ce19ca2c4",
+      "size": 22953
+    },
+    "adapter_model.safetensors": {
+      "sha256": "2b9568998ffed5d133223ffdff03056be297285c643122db4d45365d0bd86970",
+      "size": 494595208
+    }
+  },
+  "arm": "trained",
+  "complete": true,
+  "config_sha256": "5cebe511ef103d5104539d3ffa34435274c6a3bc08dbf40d5f80f9ec0ce3455a",
+  "draft": 256,
+  "elapsed_seconds": 0.0,
+  "engine_lora": {
+    "lora_dtype": "torch.bfloat16",
+    "max_lora_rank": 16,
+    "max_loras": 1
+  },
+  "fast_rows_source": {
+    "fit_report_sha256": "252cb2baf2ecb21a7892cb34b9334c1bad5f59fe7b924dd7f868addf3f27081e",
+    "protocol": "adapter_arm_31b_v1",
+    "records_sha256": "5e118133ce686b2d12b8d8a58b6e68560221c106eec54438dcfe1110e7ecfe7e",
+    "role": "q0 public231 fast-only collection of this arm",
+    "service_lock_sha256": "50a4c56c5c9fbfbde9c127ad0247736fa16355bc8deae90b5311f3ba19a03d44"
+  },
+  "fit_protocol": "31b-adapter-arm-v3-routed",
+  "input_budget": 6653,
+  "isolation_sha256": "802bfec07914c4c10dbdd35fdfdced0ef88945b06717252038da35b0b0d9bf99",
+  "model_sha256": "ddb0a8178194de6d3a3ed6e5a00ea01d6d89c143d43d169a68ad9561636cf4ff",
+  "n": 231,
+  "noul_constant": {
+    "applied_to": [
+      "fast",
+      "slow",
+      "fallback"
+    ],
+    "fitted_on": "the 74 yes/no items of the 231 public items, in-sample fast fit",
+    "fitted_values": {
+      "fallback": 0.9459459459459459,
+      "fast": 0.9459459459459459,
+      "slow": 0.9459459459459459
+    },
+    "outputs": [
+      0.898,
+      0.102
+    ],
+    "params_sha256": "b28ed73bb5a36e99a280a47a6b846e0c5b2de54f34a96efa4a10754b71ea7573",
+    "provenance": {
+      "development": {
+        "accuracy": 0.897815362931642,
+        "n": 1419,
+        "standard_error": 0.010166783957967924,
+        "standard_error_kind": "clustered by source group (663 groups)"
+      },
+      "result_sha256": "6cd7105b25050ece8ab7e99cd437db38368f6169bb452908785e413a3cdbe26a",
+      "rule": "Rule (v3, own development data only): the yes/no accuracy of our own held-out development data, rounded to 3 decimals. No leaderboard result of any kind was used for this constant.",
+      "rule_sha256": "be93d0ba00dd1a268f8722955111622ab71f0035ba66c60db86c4e88d35f0dbc"
+    },
+    "rule": "Rule (v3, own development data only): the yes/no accuracy of our own held-out development data, rounded to 3 decimals. No leaderboard result of any kind was used for this constant.",
+    "value": 0.898
+  },
+  "numeric_config": {
+    "NVIDIA_TF32_OVERRIDE": "0",
+    "VLLM_BATCH_INVARIANT": "1",
+    "cuda_matmul_allow_tf32": false,
+    "cudnn_allow_tf32": false,
+    "dtype": "bfloat16",
+    "engine_lora": {
+      "lora_dtype": "torch.bfloat16",
+      "max_lora_rank": 16,
+      "max_loras": 1
+    },
+    "postload_verified": {
+      "NVIDIA_TF32_OVERRIDE": "0",
+      "TORCH_ALLOW_TF32_CUBLAS_OVERRIDE": null,
+      "cuda_matmul_allow_tf32": false,
+      "cudnn_allow_tf32": false,
+      "float32_matmul_precision": "highest"
+    },
+    "process_environment": {
+      "DO_NOT_TRACK": "1",
+      "HF_DATASETS_OFFLINE": "1",
+      "HF_HUB_DISABLE_TELEMETRY": "1",
+      "HF_HUB_OFFLINE": "1",
+      "NVIDIA_TF32_OVERRIDE": "0",
+      "TRANSFORMERS_OFFLINE": "1",
+      "VLLM_BATCH_INVARIANT": "1",
+      "VLLM_NO_USAGE_STATS": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_WORKER_MULTIPROC_METHOD": "spawn"
+    },
+    "startup": {
+      "applied_environment": {
+        "DO_NOT_TRACK": "1",
+        "PYTHONUNBUFFERED": "1",
+        "VLLM_HOST_IP": "127.0.0.1",
+        "VLLM_NO_USAGE_STATS": "1",
+        "VLLM_USE_FLASHINFER_SAMPLER": "0"
+      },
+      "removed_attention_environment": {
+        "JEV_ATTENTION_BACKEND": null,
+        "VLLM_ATTENTION_BACKEND": null
+      },
+      "startup_environment_policy": {
+        "operator_cache_environment_preserved": true,
+        "removed_attention_variables": [
+          "VLLM_ATTENTION_BACKEND",
+          "JEV_ATTENTION_BACKEND"
+        ],
+        "reused_function": "child_environment",
+        "source": "reference launch.py",
+        "source_sha256": "2136bcb3330a80ad3f7e73ee5c99633a41e26fa922f5c8aef6f7df696fb7ce11"
+      }
+    },
+    "torch_version": "2.13.0+cu130",
+    "versions": {
+      "compressed-tensors": "0.17.0",
+      "tokenizers": "0.23.2",
+      "torch": "2.13.0+cu130",
+      "transformers": "5.17.0",
+      "triton": "3.7.1",
+      "vllm": "0.31.0"
+    }
+  },
+  "phases": [
+    "q0 public231 fast-only rows of the arm (reused, hash-bound)",
+    "offline routed fit of the v3 slow setting",
+    "v3 yes/no constant in the fast, slow and fallback blocks"
+  ],
+  "protocol": "adapter_arm_31b_v3",
+  "public_data_sha256": "d7ac378b817b9c32c2128a83bb8699f5770439503f7204c79572e2749319b5d4",
+  "public_fit_only": true,
+  "public_identity": {
+    "contains_public_benchmark": true,
+    "contains_sealed_benchmark": false,
+    "data_sha256": "8e7b2fec1275413649d2894670cbe3ea4a734c768a01ff833697123b2b44562c",
+    "n": 231,
+    "order_sha256": "3cac6d401ca18059628f0f48c8d7f974cbbea020f39f28109eaf84bd1a0f22e1",
+    "origin": "public231_crlf",
+    "public_sha256": {
+      "easy.jsonl": "314cd493a4c080fca5ad90e1c8a7c2b3f24137e66dc06f8fceaf51ea54f38467",
+      "hard.jsonl": "8c8f1efb5a04b7016f9c8cd8a70a485f58a70a918aac3e3c1107f41172ca91b6",
+      "original.jsonl": "b2abe9ac8cabd953a3bc452abd4b229da42ca6d0e647bf93f7638076f42db19f"
+    },
+    "public_source_verified": true,
+    "role": "service_fit_public",
+    "usage_scope": "service_fit_only"
+  },
+  "public_selection_score_exported": false,
+  "q": 0.056,
+  "records_sha256": "5e118133ce686b2d12b8d8a58b6e68560221c106eec54438dcfe1110e7ecfe7e",
+  "role": "service_fit_public",
+  "routing": {
+    "cost_fuse": {
+      "arm": 0.9,
+      "enabled": true,
+      "release": 0.85,
+      "warmup": 100
+    },
+    "excluded_types": [],
+    "mode": "routed",
+    "quota": 0.056,
+    "slow_draft": {
+      "max_new_tokens": 256,
+      "reasoning_words": 160
+    },
+    "slow_max_prompt_tokens": 840,
+    "slow_share_fuse": 0.2,
+    "target_slow_share": 0.056,
+    "threshold_rule": "bench.calibrate_margin(public231 rows that can draft, ceil(q x 231) of them)",
+    "threshold_rule_name": "eligible_items",
+    "unparsed_policy": "force_answer"
+  },
+  "selection_sha256": "621fde9159b24e773298511418d3373dcd2da45bed535327c5be01facf218994",
+  "serving_identity_sha256": "0513d121db967ec1a6aaccf627d3c42149d092dd027298da5b53684b030821ea",
+  "slow_calibration": {
+    "by_type": {
+      "choice": {
+        "fast_value": 1.2968395546510096,
+        "inherited": true,
+        "slow_samples": 0,
+        "value": 1.2968395546510096
+      },
+      "noul": {
+        "constant": "service parameter",
+        "fast_value": 0.898,
+        "fitted_fast_value": 0.9459459459459459,
+        "inherited": true,
+        "slow_samples": 0,
+        "value": 0.898
+      },
+      "score": {
+        "fast_value": 1.189207115002721,
+        "inherited": true,
+        "slow_samples": 0,
+        "value": 1.189207115002721
+      }
+    },
+    "routed_collection": {
+      "calibration_routed_quota": 0.1,
+      "collected": false,
+      "max_possible_slow_samples": 23,
+      "reason": "at the calibration-routed quota 0.10 the quota counter admits at most 23 of 231 public decisions to the slow path, fewer than 30 for any type, so every slow type inherits the fast parameter whatever such a collection shows; it is therefore not run"
+    },
+    "rule": "slow samples of a type < 30: the fast parameter of that type (temperature or yes/no confidence) is copied unchanged"
+  },
+  "source_code_sha256": "51269609aa077981a1a5e0bc5775a3d3aad5c1df04f6329b4c199fa2c5477d05",
+  "threshold": {
+    "achieved": 0.05627705627705628,
+    "eligible_below": 13,
+    "eligible_n": 181,
+    "n": 231,
+    "rule": "eligible_items",
+    "target": 0.056,
+    "threshold": 2.250016468674351
+  },
+  "v3_params": {
+    "cost_fuse_arm": 0.9,
+    "cost_fuse_release": 0.85,
+    "draft_max_new_tokens": 256,
+    "draft_reasoning_words": 160,
+    "exclude_noul": false,
+    "noul_p_cal": 0.898,
+    "noul_p_cal_provenance": {
+      "development": {
+        "accuracy": 0.897815362931642,
+        "n": 1419,
+        "standard_error": 0.010166783957967924,
+        "standard_error_kind": "clustered by source group (663 groups)"
+      },
+      "result_sha256": "6cd7105b25050ece8ab7e99cd437db38368f6169bb452908785e413a3cdbe26a",
+      "rule": "Rule (v3, own development data only): the yes/no accuracy of our own held-out development data, rounded to 3 decimals. No leaderboard result of any kind was used for this constant.",
+      "rule_sha256": "be93d0ba00dd1a268f8722955111622ab71f0035ba66c60db86c4e88d35f0dbc"
+    },
+    "schema": "adapter_arm_31b_v3_params_v1",
+    "slow_max_prompt_tokens": 840,
+    "slow_quota": 0.056,
+    "threshold_rule": "eligible_items",
+    "unparsed_policy": "force_answer"
+  },
+  "v3_params_sha256": "b28ed73bb5a36e99a280a47a6b846e0c5b2de54f34a96efa4a10754b71ea7573",
+  "variant": "key_value_aligned"
+}

service/item_deadline.py

@@ -0,0 +1,551 @@
+"""Per-decision deadline for the drafting pass, plus two item rules (installed by server.py when
+config.json has an `item_deadline` block, with the `item_rules` block; no package file is modified at
+runtime).
+
+One hard deadline per request (the harness sends one decision per request): the moment the service
+starts answering the request + item_seconds. The draft is generated greedily in chunks of at most
+`chunk_tokens`; each chunk is sized by a conservative speed model (a short synthetic warm-up at
+startup, updated online) so that it ends before deadline - response reserve - readout reserve.
+No engine call is abandoned in flight, so every call is metered exactly, and the model lock is
+released when the decision returns.
+- Draft budget used up without the answer marker: the marker is appended, exactly as when
+  max_new_tokens is reached.
+- Readout that cannot finish before the deadline: the first-pass answer is returned (route
+  slow_timeout_fast) and the decision's usage is reconciled exactly from its meters.
+- An answer marker that straddles a chunk boundary ends the draft at the token that completes it,
+  where one-shot generation would have stopped.
+Decisions that do not take the drafting pass are unaffected by the deadline.
+
+Item rules (wrap_service):
+- slow_types: question types that may take the drafting pass (None = all). For any other type the
+  routing threshold is 0 for that decision: it never requests the drafting pass and the quota
+  counter never counts it.
+- reread_tau: a choice decision answered in one pass although its length allowed the drafting pass
+  (budget tier not 'long', not a long item), whose first-pass margin is below tau, is read once
+  more, with the options in reverse order (each label keeps its criterion); the two answer
+  distributions are averaged by label. The second read is metered into the
+  decision's usage (so the cost fuse sees it) and is not a quota decision. If it fails (not an
+  engine failure) the first-pass answer is kept and `reread_failed` is recorded.
+Controller.truncated_readout is a diagnostic used only by offline measurement; serving never calls it.
+"""
+import math
+import re
+import threading
+import time
+from contextlib import contextmanager
+
+DEFAULTS = dict(item_seconds=9.0, response_reserve_s=0.35, readout_reserve_init_s=1.0,
+                readout_reserve_min_s=0.3, readout_safety=1.5, chunk_tokens=32, min_chunk_tokens=8,
+                token_safety=1.25, ewma_alpha=0.3, warmup_tokens=33, warmup_repeats=3)
+
+_local = threading.local()
+
+
+def need(ok, code):
+    if not ok:
+        raise RuntimeError(code)
+
+
+def settings_from(values=None):
+    s = dict(DEFAULTS)
+    s.update(values or {})
+    need(set(s) == set(DEFAULTS), 'DEADLINE_SETTINGS_KEYS')
+    for key in ('item_seconds', 'response_reserve_s', 'readout_reserve_init_s', 'readout_reserve_min_s',
+                'readout_safety', 'token_safety', 'ewma_alpha'):
+        x = s[key]
+        need(isinstance(x, (int, float)) and not isinstance(x, bool) and math.isfinite(x) and x > 0,
+             'DEADLINE_SETTING_' + key.upper())
+    for key in ('chunk_tokens', 'min_chunk_tokens', 'warmup_tokens', 'warmup_repeats'):
+        need(type(s[key]) is int and s[key] > 0, 'DEADLINE_SETTING_' + key.upper())
+    need(s['token_safety'] >= 1 and s['readout_safety'] >= 1 and s['ewma_alpha'] <= 1, 'DEADLINE_SAFETY')
+    need(s['min_chunk_tokens'] <= s['chunk_tokens'] and s['warmup_tokens'] >= 2 and s['warmup_repeats'] >= 2,
+         'DEADLINE_TOKEN_LIMITS')
+    need(s['item_seconds'] > s['response_reserve_s'] + s['readout_reserve_init_s'], 'DEADLINE_RESERVES')
+    return s
+
+
+class SpeedModel:
+    """Conservative chunk planner: seconds(n) = call_s + n * token_s * token_safety."""
+
+    def __init__(self, token_s, call_s, settings):
+        need(all(isinstance(x, float) and math.isfinite(x) and x > 0 for x in (token_s, call_s)),
+             'DEADLINE_SPEED')
+        self.s = settings
+        self.lock = threading.Lock()
+        self.warm_token_s = token_s
+        self.token_s = token_s
+        self.call_s = call_s
+        self.ewma = None
+        self.readout = {}
+        self.chunks_observed = 0
+
+    def fit_tokens(self, seconds):
+        if not math.isfinite(seconds) or seconds <= self.call_s:
+            return 0
+        return int(math.floor((seconds - self.call_s) / (self.token_s * self.s['token_safety'])))
+
+    def observe_chunk(self, generated, seconds):
+        if generated < 4 or not math.isfinite(seconds) or seconds <= 0:
+            return
+        per = max(0., seconds - self.call_s) / generated
+        with self.lock:
+            a = self.s['ewma_alpha']
+            self.ewma = per if self.ewma is None else a * per + (1 - a) * self.ewma
+            self.token_s = max(self.warm_token_s, self.ewma)
+            self.chunks_observed += 1
+
+    def readout_reserve(self, kind):
+        v = self.readout.get(kind)
+        if v is None:
+            return self.s['readout_reserve_init_s']
+        return max(self.s['readout_reserve_min_s'], self.s['readout_safety'] * v)
+
+    def observe_readout(self, kind, seconds):
+        if not math.isfinite(seconds) or seconds < 0:
+            return
+        with self.lock:
+            a = self.s['ewma_alpha']
+            old = self.readout.get(kind)
+            self.readout[kind] = seconds if old is None else a * seconds + (1 - a) * old
+
+    def snapshot(self):
+        return dict(token_s=self.token_s, warm_token_s=self.warm_token_s, call_s=self.call_s,
+                    chunks_observed=self.chunks_observed, readout={k: v for k, v in sorted(self.readout.items())})
+
+
+class ItemPlan:
+    def __init__(self, controller, start, max_new_tokens, kind):
+        s = controller.settings
+        self.c = controller
+        self.start = start
+        self.kind = kind
+        self.max_new_tokens = max_new_tokens
+        self.hard = start + s['item_seconds']
+        self.meter_deadline = self.hard - s['response_reserve_s']
+        self.meters = []
+        self.chunks = []
+        self.budget_stop = False
+        self.straddle = False
+        self.forced = False
+        self.draft_tokens = 0
+        self.slow_started = None
+        self.draft_end_at = None
+        self.readout_start = None
+        self.readout_s = None
+        self.timeout_phase = None
+        self.reconciled = False
+        # draft bookkeeping (memory only; never serialised)
+        self._prompt_ids = None
+        self._draft_ids = None
+        self._by_marker = False
+        self._groups = None
+        self._config = None
+        self._slow_meter = None
+
+    def draft_deadline(self):
+        return self.meter_deadline - self.c.speed.readout_reserve(self.kind)
+
+    def exact(self):
+        return all(not m.get('engine_usage_unknown') and not m.get('cached_unknown') for m in self.meters)
+
+    def audit(self, now):
+        def rel(t):
+            return None if t is None else round(t - self.start, 6)
+        speed = self.c.speed
+        return dict(schema='item-deadline-v1', item_seconds=self.c.settings['item_seconds'], elapsed_s=rel(now),
+                    slow_started_s=rel(self.slow_started), draft_end_s=rel(self.draft_end_at),
+                    readout_s=None if self.readout_s is None else round(self.readout_s, 6),
+                    draft_tokens=self.draft_tokens, chunks=len(self.chunks),
+                    chunk_tokens=[c['generated'] for c in self.chunks][:64],
+                    chunk_seconds=[round(c['seconds'], 6) for c in self.chunks][:64],
+                    chunk_end_s=[round(c['end'] - self.start, 6) for c in self.chunks][:64],
+                    chunk_cum_tokens=[c['cum_tokens'] for c in self.chunks][:64],
+                    chunk_cum_input=[c['cum_input'] for c in self.chunks][:64],
+                    chunk_cum_generated=[c['cum_generated'] for c in self.chunks][:64],
+                    draft_by_marker=bool(self._by_marker),
+                    budget_stop=self.budget_stop, straddle=self.straddle, forced=self.forced,
+                    timeout_phase=self.timeout_phase, reconciled=self.reconciled, exact=self.exact(),
+                    max_new_tokens=self.max_new_tokens, token_s=round(speed.token_s, 6),
+                    call_s=round(speed.call_s, 6), readout_reserve_s=round(speed.readout_reserve(self.kind), 6))
+
+
+def reconcile(detail, meters):
+    """Usage arithmetic of core.py _one_impl, without the timeout estimate."""
+    submitted = sum(m['engine_submitted'] for m in meters)
+    cached = sum(m['cached'] for m in meters)
+    generated = sum(m.get('engine_generated', m['output']) for m in meters)
+    engine_input = sum(m.get('engine_input', m['engine_submitted'] - m['cached']) for m in meters)
+    detail['usage_estimated'] = False
+    detail['input_tokens_details'].update(
+        cached_tokens=cached, engine_submitted_tokens=submitted, engine_generated_tokens=generated,
+        engine_calls=sum(m.get('engine_calls', 0) for m in meters), cached_tokens_known=True)
+    detail['usage_bases'] = {
+        'engine': dict(input_tokens=engine_input, output_tokens=generated),
+        'each_once': dict(input_tokens=detail['prompt_tokens'], output_tokens=detail['completion_tokens']),
+        'submitted': dict(input_tokens=submitted, output_tokens=detail['completion_tokens'])}
+
+
+def marker_completion(decode, previous, new, marker):
+    """Smallest j such that the marker is complete in decode(previous+new[:j]); None if absent."""
+    if marker not in decode(list(previous) + list(new)):
+        return None
+    for j in range(1, len(new) + 1):
+        if marker in decode(list(previous) + list(new[:j])):
+            return j
+    return None
+
+
+@contextmanager
+def request_clock(start=None):
+    previous = getattr(_local, 'request_start', None)
+    _local.request_start = time.monotonic() if start is None else start
+    try:
+        yield
+    finally:
+        _local.request_start = previous
+
+
+def active_plan():
+    return getattr(_local, 'plan', None)
+
+
+def _install_core(core):
+    if getattr(core, '_deadline_core', None):
+        return
+    original_bounded = core.bounded_slow_attempt
+    original_new_meter = core.new_meter
+
+    def bounded_slow_attempt(backend, ids, groups, config, meter, model_lock, *, on_service_failure=None):
+        plan = getattr(_local, 'plan', None)
+        if plan is None:
+            return original_bounded(backend, ids, groups, config, meter, model_lock,
+                                    on_service_failure=on_service_failure)
+        return plan.c.bounded(plan, backend, ids, groups, config, meter, model_lock, on_service_failure)
+
+    def new_meter():
+        m = original_new_meter()
+        plan = getattr(_local, 'plan', None)
+        if plan is not None:
+            plan.meters.append(m)
+        return m
+
+    core.bounded_slow_attempt = bounded_slow_attempt
+    core.new_meter = new_meter
+    core._deadline_core = dict(bounded_slow_attempt=original_bounded, new_meter=original_new_meter)
+
+
+def _install_backend(backend):
+    if getattr(backend, '_deadline_backend', None):
+        return
+    original = backend.slow_session
+
+    def slow_session(ids, unparsed_policy, meter, max_new_tokens=None):
+        plan = getattr(_local, 'plan', None)
+        if plan is None:
+            return original(ids, unparsed_policy, meter, max_new_tokens=max_new_tokens)
+        need(plan.c.backend is backend, 'DEADLINE_BACKEND_MISMATCH')
+        return plan.c.chunked_session(plan, ids, unparsed_policy, meter, max_new_tokens)
+
+    backend.slow_session = slow_session
+    backend._deadline_backend = dict(slow_session=original)
+
+
+def measure_speed(backend, settings, system_prompt, clock=time.monotonic):
+    """Synthetic prompt only. Paired 1/N exact-length greedy calls; conservative maxima."""
+    g = backend.generation
+    ids = backend.prompt_ids(system_prompt, 'Ready?', False)
+    n_tokens = settings['warmup_tokens']
+    pairs = []
+
+    def run(n):
+        params = backend.SamplingParams(max_tokens=n, min_tokens=n, temperature=0.0, top_p=1.0, top_k=-1,
+                                        repetition_penalty=g['repetition_penalty'], seed=g['seed'],
+                                        detokenize=False)
+        t0 = clock()
+        out = backend._completion(ids, params, None, 'draft')
+        dt = clock() - t0
+        need(len(out.token_ids) == n, 'DEADLINE_WARMUP_LENGTH')
+        return dt
+
+    for _ in range(settings['warmup_repeats']):
+        one = run(1)
+        many = run(n_tokens)
+        pairs.append((one, many))
+    call_s = float(max(x for x, _ in pairs))
+    token_s = float(max(max((y - x) / (n_tokens - 1), y / n_tokens) for x, y in pairs))
+    token_s = max(token_s, 1e-6)
+    call_s = max(call_s, 1e-6)
+    return SpeedModel(token_s, call_s, settings), dict(pairs_s=pairs, tokens=n_tokens)
+
+
+class Controller:
+    def __init__(self, core, backends, backend, settings, speed, clock=time.monotonic, warmup=None):
+        self.core, self.backends, self.backend = core, backends, backend
+        self.settings, self.speed, self.clock = settings, speed, clock
+        self.warmup = warmup
+
+    # ---- per item ---------------------------------------------------------------------------
+    def slow_attempt(self, backend, ids, groups, config, meter):
+        """core.slow_attempt semantics (same exception mapping), except that the deadline
+        path skips release_failed_session's gc.collect(): on vLLM there is no allocator state to
+        clear, and a full collection would only delay the fast answer past the item cap."""
+        core = self.core
+        failed = None
+        try:
+            return core.slow_readout(backend, ids, groups, config, meter)
+        except TimeoutError:
+            failed = 'timeout'
+        except Exception as exc:
+            if core.unrecoverable_engine_error(exc):
+                raise core.ServiceFailure('unrecoverable engine failure') from exc
+            failed = 'error'
+        if failed == 'timeout':
+            raise TimeoutError('Slow deadline exceeded')
+        core.release_failed_session(backend)
+        raise RuntimeError('Slow session failed')
+
+    def bounded(self, plan, backend, ids, groups, config, meter, model_lock, on_service_failure):
+        core = self.core
+        plan._groups, plan._config, plan._slow_meter = groups, config, meter
+        meter['deadline'] = plan.meter_deadline
+        plan.slow_started = self.clock()
+        try:
+            with model_lock:
+                core.check_deadline(meter)
+                value = self.slow_attempt(backend, ids, groups, config, meter)
+                core.check_deadline(meter)
+            if plan.readout_start is not None:
+                plan.readout_s = self.clock() - plan.readout_start
+                self.speed.observe_readout(plan.kind, plan.readout_s)
+            return value
+        except Exception as exc:
+            if core.unrecoverable_engine_error(exc) and on_service_failure is not None:
+                on_service_failure()
+            if isinstance(exc, TimeoutError):
+                plan.timeout_phase = 'readout' if plan.readout_start is not None else 'draft'
+            raise
+
+    def chunked_session(self, plan, ids, unparsed_policy, meter, max_new_tokens=None):
+        backend, s, speed = self.backend, self.settings, self.speed
+        g = backend.generation
+        need(g['do_sample'] is False and g['repetition_penalty'] == 1.0, 'DEADLINE_CHUNKS_REQUIRE_GREEDY')
+        count = self.backends._max_new_tokens(max_new_tokens or plan.max_new_tokens)
+        marker = backend.marker
+        prompt = list(ids)
+        gen = []
+        by_marker = False
+        counted_prompt = False
+        while len(gen) < count:
+            now = self.clock()
+            if now >= plan.meter_deadline:
+                raise TimeoutError('item deadline before chunk')
+            left = count - len(gen)
+            n = min(s['chunk_tokens'], left, speed.fit_tokens(plan.draft_deadline() - now))
+            if n < min(s['min_chunk_tokens'], left):
+                if not gen:
+                    raise TimeoutError('no draft budget')
+                plan.budget_stop = True
+                break
+            params = backend.SamplingParams(max_tokens=n, temperature=0.0, top_p=1.0, top_k=-1,
+                                            repetition_penalty=g['repetition_penalty'], seed=g['seed'],
+                                            stop=[marker], include_stop_str_in_output=True)
+            if not counted_prompt:
+                meter['prompt'] += len(prompt)   # the backend's slow_session counts the prompt once
+                counted_prompt = True
+            t0 = self.clock()
+            out = backend._completion(prompt + gen, params, meter, 'draft')
+            t1 = self.clock()
+            dt = t1 - t0
+            new = list(out.token_ids)
+            speed.observe_chunk(len(new), dt)
+            plan.chunks.append(dict(n=n, generated=len(new), seconds=dt, end=t1, cum_tokens=len(gen) + len(new),
+                                    cum_input=int(meter.get('engine_input', 0)),
+                                    cum_generated=int(meter.get('engine_generated', 0))))
+            if getattr(out, 'stop_reason', None) == marker and getattr(out, 'finish_reason', None) == 'stop':
+                gen += new
+                by_marker = True
+                break
+            j = marker_completion(backend.decode, gen, new, marker)
+            if j is not None:
+                # One-shot generation would have stopped at the token completing the marker. The
+                # extra tokens of this chunk were generated and stay metered (exact cost).
+                gen += new[:j]
+                by_marker = True
+                plan.straddle = True
+                break
+            gen += new
+            if getattr(out, 'finish_reason', None) == 'stop' or len(new) < n:
+                break                        # EOS: the draft ended without the marker
+        plan.draft_tokens = len(gen)
+        plan.draft_end_at = self.clock()
+        plan._prompt_ids, plan._draft_ids, plan._by_marker = list(prompt), list(gen), by_marker
+        parsed = by_marker and bool(re.search(r'(?m)^[ \t]*' + re.escape(marker) + r'$', backend.decode(gen)))
+        if not parsed:
+            if unparsed_policy == 'fast':
+                return None, False
+            forced = backend.encode('\n' + marker)
+            gen += forced
+            meter['output'] += len(forced)
+            meter['forced'] += len(forced)
+            plan.forced = True
+        plan.readout_start = self.clock()
+        return self.backends.VLLMPrefixSession(backend, prompt + gen, meter), parsed
+
+    # ---- truncated-draft readout (diagnostic for offline measurement; not used when serving) ----
+    def truncated_readout(self, plan, n_tokens, labels, qtype, calibration_block):
+        """Readout of this item's draft cut at n_tokens (no marker within them) + the forced answer
+        marker, on a fresh meter: what a one-shot cap of n_tokens would have read out (greedy drafts
+        are prefix-identical). Returns (probabilities dict, meter, seconds). Caller checks eligibility."""
+        need(plan._draft_ids is not None and len(plan._draft_ids) > n_tokens, 'DEADLINE_TRUNC_NOT_LONGER')
+        backend, core = self.backend, self.core
+        forced = backend.encode('\n' + backend.marker)
+        meter = self.backends.new_meter()
+        meter['output'] += len(forced)
+        meter['forced'] += len(forced)
+        t0 = self.clock()
+        session = self.backends.VLLMPrefixSession(backend, plan._prompt_ids + plan._draft_ids[:n_tokens] + forced, meter)
+        scores, _ = core.sequence_scores(session, backend, plan._groups, meter)
+        probs = core.probabilities(labels, scores, qtype, calibration_block)
+        return probs, meter, self.clock() - t0
+
+    # ---- low-margin choice re-read ------------------------------------------------------------
+    def reread_reading(self, service, task):
+        """One fast read of a choice task with the option order reversed (label -> criterion pairs kept;
+        render re-letters the options), the service's 'fast' calibration. Returns (probabilities by label, meter,
+        seconds). The meter is created through core.new_meter, so inside an item it joins the item's meters."""
+        core = self.core
+        need(task['question']['type'] == 'choice' and task.get('labels'), 'DEADLINE_REREAD_TASK')
+        t2 = dict(task, labels=list(reversed(task['labels'])))
+        meter = core.new_meter()
+        t0 = self.clock()
+        _, (labels2, groups2, fast_ids2, _), _ = core.prepare_task(t2, self.backend, service.config)
+        with service.model_lock:
+            scores2, _, _ = core.fast_attempt_with_retry(self.backend, t2, service.config, fast_ids2, labels2, groups2, meter)
+        need(scores2 is not None, 'DEADLINE_REREAD_SCORES')
+        probs = core.probabilities(labels2, scores2, 'choice', service.calibration['fast'])
+        return probs, meter, self.clock() - t0
+
+    @staticmethod
+    def reread_due(task, detail, tau):
+        m = detail.get('fast_margin')
+        return (tau is not None and tau > 0 and task['question']['type'] == 'choice' and detail.get('path') == 'fast'
+                and detail.get('budget_tier') != 'long' and not detail.get('long_item')
+                and isinstance(m, (int, float)) and math.isfinite(m) and m < tau)
+
+    def apply_reread(self, service, task, answer_value, detail):
+        """Online re-read: average the fast answer with the reversed-order read; add the read to the item's
+        usage (every basis) and token details; record it in detail['reread']."""
+        probs, meter, seconds = self.reread_reading(service, task)
+        first = answer_value['probabilities']
+        need(set(first) == set(probs), 'DEADLINE_REREAD_LABELS')
+        mixed = {k: (first[k] + probs[k]) / 2 for k in first}
+        winner = max(task['labels'], key=mixed.get)
+        engine_input = meter.get('engine_input', meter['engine_submitted'] - meter['cached'])
+        generated = meter.get('engine_generated', meter['output'])
+        bases = detail['usage_bases']
+        bases['engine']['input_tokens'] += engine_input
+        bases['engine']['output_tokens'] += generated
+        bases['each_once']['input_tokens'] += meter['prompt']
+        bases['each_once']['output_tokens'] += meter['output']
+        bases['submitted']['input_tokens'] += meter['engine_submitted']
+        bases['submitted']['output_tokens'] += meter['output']
+        d = detail['input_tokens_details']
+        d['cached_tokens'] += meter['cached']
+        d['engine_submitted_tokens'] += meter['engine_submitted']
+        d['engine_generated_tokens'] += generated
+        d['engine_calls'] += meter.get('engine_calls', 0)
+        if meter.get('cached_unknown'):
+            d['cached_tokens_known'] = False
+        if meter.get('engine_usage_unknown') or meter.get('cached_unknown'):
+            detail['usage_estimated'] = True
+        detail['prompt_tokens'] += meter['prompt']
+        detail['completion_tokens'] += meter['output']
+        detail['total_tokens'] = detail['prompt_tokens'] + detail['completion_tokens']
+        detail['readout_output_tokens'] += meter['probes']
+        detail['prediction'] = winner
+        detail['reread'] = dict(input_tokens=int(engine_input), output_tokens=int(generated), seconds=seconds,
+                                     order='reversed')
+        answer_value = dict(answer_value, probabilities=mixed)
+        answer_value['choice'] = winner
+        return answer_value, detail
+
+    # ---- per service ------------------------------------------------------------------------
+    def wrap_service(self, service, slow_types=None, reread_tau=None):
+        """slow_types: question types allowed on the slow path (None = all); reread_tau: low-margin choice
+        re-read threshold (None or 0 = off)."""
+        need(service.backend is self.backend, 'DEADLINE_SERVICE_BACKEND')
+        need(not getattr(service, '_deadline_wrapped', False), 'DEADLINE_ALREADY_WRAPPED')
+        need(slow_types is None or (set(slow_types) <= {'choice', 'noul', 'score'} and len(slow_types) > 0), 'DEADLINE_SLOW_TYPES')
+        need(reread_tau is None or (isinstance(reread_tau, (int, float)) and math.isfinite(reread_tau) and reread_tau >= 0),
+             'DEADLINE_REREAD_TAU')
+        original_one, original_answer = service._one_impl, service.answer
+        clock = self.clock
+        allowed = None if slow_types is None else frozenset(slow_types)
+
+        def answer(payload):
+            owned = getattr(_local, 'request_start', None) is None
+            if owned:
+                _local.request_start = clock()
+            try:
+                return original_answer(payload)
+            finally:
+                if owned:
+                    _local.request_start = None
+
+        def one(task):
+            need(getattr(_local, 'plan', None) is None, 'DEADLINE_NESTED_ITEM')
+            start = getattr(_local, 'request_start', None)
+            plan = ItemPlan(self, clock() if start is None else start,
+                            service.config['generation']['max_new_tokens'], task['question']['type'])
+            service._deadline_last = plan
+            _local.plan = plan
+            routing = service.config['routing']
+            switched_off = allowed is not None and task['question']['type'] not in allowed
+            saved = routing['margin_threshold']
+            try:
+                if switched_off:
+                    routing['margin_threshold'] = 0.0      # margins are >= 0: never requests slow, never counted
+                try:
+                    answer_value, detail = original_one(task)
+                finally:
+                    routing['margin_threshold'] = saved
+                if detail.get('route') == 'slow_timeout_fast' and plan.exact():
+                    reconcile(detail, plan.meters)
+                    plan.reconciled = True
+                if self.reread_due(task, detail, reread_tau):
+                    try:
+                        answer_value, detail = self.apply_reread(service, task, answer_value, detail)
+                    except Exception as exc:
+                        if self.core.unrecoverable_engine_error(exc):
+                            raise self.core.ServiceFailure('unrecoverable engine failure') from exc
+                        detail['reread_failed'] = type(exc).__name__     # fast answer kept, as without re-read
+                detail['item_deadline'] = plan.audit(clock())
+                detail['item_deadline']['slow_type_off'] = switched_off
+                return answer_value, detail
+            finally:
+                _local.plan = None
+
+        service._one_impl = one
+        service.answer = answer
+        service._deadline_wrapped = True
+        service._deadline_slow_types = None if allowed is None else sorted(allowed)
+        service._deadline_reread_tau = reread_tau
+        return service
+
+
+def install(core, backends, backend, settings=None, system_prompt=None, clock=time.monotonic, speed=None):
+    """Process-wide, idempotent dispatchers + one controller for this backend."""
+    s = settings_from(settings)
+    existing = getattr(backend, '_deadline_controller', None)
+    if existing is not None:
+        need(existing.settings == s, 'DEADLINE_SETTINGS_CHANGED')
+        return existing
+    _install_core(core)
+    warm = None
+    if speed is None:
+        need(isinstance(system_prompt, str), 'DEADLINE_SYSTEM_PROMPT')
+        speed, warm = measure_speed(backend, s, system_prompt, clock)
+    _install_backend(backend)
+    controller = Controller(core, backends, backend, s, speed, clock, warm)
+    backend._deadline_controller = controller
+    return controller

service/launch.py

@@ -0,0 +1,456 @@
+"""Foreground supervisor: runtime and model checks, startup attempts, readiness, termination.
+
+The service writes files only below --state-dir (plus shared memory in /dev/shm):
+  <state-dir>/<run-id>-startup/   attempts.jsonl, attempt-N.log, state-N/ (quota state), tmp/, home/
+  <state-dir>/cache/              compilation caches shared by runs that use the same state directory
+"""
+import argparse
+from contextlib import contextmanager
+import hashlib
+import importlib.metadata
+import json
+import os
+from pathlib import Path
+import re
+import signal
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from datetime import datetime, timezone
+import urllib.request
+
+ROOT=Path(__file__).resolve().parent
+IMAGE='vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b'
+# Model identity: official W4A16 CT base files + our unmerged adapter (MODEL_MANIFEST.json).
+_MANIFEST=json.loads((ROOT/'MODEL_MANIFEST.json').read_text(encoding='utf-8'))
+MODEL_KEY='metask-gemma4-31b-lora-w4a16'
+MODEL_REVISION=_MANIFEST['repo_id']+'@'+str(_MANIFEST.get('revision') or 'unpublished')
+# q=0.056: routed; the quota counter below --state-dir caps the share of slow (drafting) decisions.
+SERVED_MODE='routed'
+VERSIONS={'vllm':'0.31.0','torch':'2.13.0','transformers':'5.17.0','tokenizers':'0.23.2',
+          'triton':'3.7.1','compressed-tensors':'0.17.0'}
+MODEL_SHA256={n:v['sha256'] for n,v in _MANIFEST['files'].items() if not n.endswith('.safetensors')}
+SHARD_BYTES={n:v['size'] for n,v in _MANIFEST['files'].items() if n.endswith('.safetensors')}
+SHARD_SHA256={n:v['sha256'] for n,v in _MANIFEST['files'].items() if n.endswith('.safetensors')}
+# Unmerged LoRA: <model>/adapter holds exactly these files; configs name it by ADAPTER_REF.
+ADAPTER_REF='model-subdir:adapter'
+_ADAPTER=_MANIFEST.get('adapter')
+ADAPTER_SUBDIR=(_ADAPTER or {}).get('subdir') or 'adapter'
+ADAPTER_FILES={n:dict(v) for n,v in ((_ADAPTER or {}).get('files') or {}).items()}
+RUN_ID=re.compile(r'[A-Za-z0-9][A-Za-z0-9._-]{0,63}')
+BINDS=('127.0.0.1','0.0.0.0')
+STARTUP_TIMEOUT=1800
+# vLLM places engine IPC sockets in TMPDIR; Unix socket paths are limited to 107 bytes.
+MAX_TMP_PATH=70
+# Startup attempts: vLLM's own choice (auto), then TRITON_ATTN. This checkpoint mixes attention
+# head sizes 256 and 512; where FlashAttention 4 is unavailable (for example compute capability
+# 8.x, the measured RTX 4090) vLLM 0.31 resolves auto to TRITON_ATTN, and FLASH_ATTN cannot start.
+ATTEMPTS=(None,'TRITON_ATTN')
+# Model folder entries a loader may read; among them only the pinned files are accepted.
+LOADER_SUFFIXES=('.json','.safetensors','.bin','.pt','.pth','.gguf','.model','.tiktoken','.jinja','.py','.txt')
+# Inherited path variables must stay inside --state-dir; unset ones default below state/cache or HOME.
+PATH_VARIABLES=('XDG_CACHE_HOME','VLLM_CACHE_ROOT','TRITON_CACHE_DIR','TORCHINDUCTOR_CACHE_DIR','HF_HOME',
+                'CUDA_CACHE_PATH','NUMBA_CACHE_DIR','XDG_CONFIG_HOME','XDG_DATA_HOME','XDG_STATE_HOME',
+                'VLLM_CONFIG_ROOT','TRITON_HOME','FLASHINFER_WORKSPACE_BASE','TORCH_HOME','TORCH_EXTENSIONS_DIR',
+                'MPLCONFIGDIR','HF_HUB_CACHE','HUGGINGFACE_HUB_CACHE','TRANSFORMERS_CACHE','HF_ASSETS_CACHE')
+
+
+
+def check_preparation(config=None,allow_nonformal=False):
+    c=json.loads(Path(config or ROOT/'config.json').read_text(encoding='utf-8'))
+    if not c.get('package',{}).get('nonformal_fitted') or c['package'].get('sample_only'):
+        raise ValueError('Service fit pending; preparation placeholders cannot serve')
+    if not _MANIFEST.get('identity_sha256') or not _MANIFEST['files'] or not ADAPTER_FILES:
+        raise ValueError('Model identity pending')
+    if c.get('pipeline',{}).get('model_identity_sha256')!=_MANIFEST['identity_sha256']:
+        raise ValueError('Configuration and model manifest identities differ')
+    if c['model'].get('adapter')!=ADAPTER_REF:
+        raise ValueError('Configuration adapter reference differs from the model manifest')
+    if c.get('routing',{}).get('mode')!=SERVED_MODE:
+        raise ValueError('Configuration mode differs from the served fast-only mode')
+    if not allow_nonformal:
+        if not re.fullmatch('[0-9a-f]{40}',str(_MANIFEST.get('revision') or '')) or _MANIFEST['repo_id'].startswith('OWNER/'):
+            raise ValueError('The model manifest lacks the own Hugging Face repository and full revision')
+        if not c['package'].get('finalized') or not (ROOT/'FREEZE.json').is_file():
+            raise ValueError('Package has not been frozen (no FREEZE.json); use a frozen release package')
+
+
+def check_adapter(model):
+    """Read-only check of <model>/adapter: exactly the pinned files, regular, pinned size and full SHA-256."""
+    if _ADAPTER is None:
+        return None
+    if not ADAPTER_FILES:
+        raise ValueError('Adapter identity pending: rebuild with the staged model folder')
+    folder=Path(model)/ADAPTER_SUBDIR
+    if folder.is_symlink() or not folder.is_dir():
+        raise ValueError(f'Missing adapter folder {ADAPTER_SUBDIR}/ in the model folder; it must hold '+', '.join(sorted(ADAPTER_FILES)))
+    names={p.name for p in folder.iterdir()}
+    if names!=set(ADAPTER_FILES):
+        raise ValueError(f'{ADAPTER_SUBDIR}/ must contain exactly: '+', '.join(sorted(ADAPTER_FILES)))
+    for name,entry in sorted(ADAPTER_FILES.items()):
+        path=_model_file(folder,name)
+        if path.stat().st_size!=entry['size'] or sha256(path)!=entry['sha256']:
+            raise ValueError(f'Adapter file differs from {MODEL_REVISION}: {name}')
+    return folder.resolve()
+
+
+def materialize_adapter(config,adapter):
+    """Replace ADAPTER_REF by the verified adapter folder in model.adapter and the threshold identity."""
+    c=json.loads(json.dumps(config))
+    blocks=[c.get('routing',{})]+[p.get('routing',{}) for p in c.get('profiles',{}).values()]
+    if adapter is None:
+        if c['model'].get('adapter') is not None or any((p.get('model') or {}).get('adapter') is not None
+                                                        for p in c.get('profiles',{}).values()):
+            raise ValueError('Configuration names an adapter, but this arm serves none')
+        return c
+    if c['model'].get('adapter')!=ADAPTER_REF:
+        raise ValueError('Adapter configuration must use the model-relative adapter reference')
+    value=str(adapter)
+    c['model']['adapter']=value
+    for profile in c.get('profiles',{}).values():
+        model=profile.get('model',{})
+        if 'adapter' in model:
+            if model['adapter']!=ADAPTER_REF:raise ValueError('Profile names a different adapter')
+            model['adapter']=value
+    for block in blocks:
+        fit=block.get('margin_threshold_fitted_for')
+        if isinstance(fit,dict) and 'adapter' in fit:
+            if fit['adapter']!=ADAPTER_REF:raise ValueError('Threshold was fitted for a different adapter reference')
+            fit['adapter']=value
+    return c
+
+
+def write_runtime_config(startup,config):
+    """The materialized per-run configuration lives only below --state-dir."""
+    path=Path(startup)/'config.runtime.json'
+    with path.open('x',encoding='utf-8',newline='\n') as out:
+        json.dump(config,out,ensure_ascii=False,indent=2,allow_nan=False)
+        out.write('\n')
+    return path
+
+
+def check_python(version_info=None):
+    version=tuple((version_info or sys.version_info)[:2])
+    if version<(3,10):raise ValueError('Python >= 3.10 required')
+
+
+def check_versions():
+    found={}
+    for name,expected in VERSIONS.items():
+        try:actual=importlib.metadata.version(name)
+        except importlib.metadata.PackageNotFoundError:
+            raise ValueError(f'{name} is not installed; run in {IMAGE}') from None
+        if actual.split('+',1)[0]!=expected:
+            raise ValueError(f'{name} {actual} found, {expected} required; run in {IMAGE}')
+        found[name]=actual
+    return found
+
+
+def _absolute(value,flag):
+    path=Path(value)
+    if not path.is_absolute():raise ValueError(f'{flag} must be an absolute path')
+    return path.resolve()
+
+
+def _overlap(a,b):
+    return a==b or a.is_relative_to(b) or b.is_relative_to(a)
+
+
+def check_paths(model,state_dir):
+    model=_absolute(model,'--model');state=_absolute(state_dir,'--state-dir')
+    for path,flag in ((model,'--model'),(state,'--state-dir')):
+        if _overlap(path,ROOT):raise ValueError(f'{flag} must be outside the package and must not contain it')
+    if _overlap(model,state):raise ValueError('--model and --state-dir must not contain each other')
+    return model,state
+
+
+def sha256(path):
+    digest=hashlib.sha256()
+    with Path(path).open('rb') as source:
+        for chunk in iter(lambda:source.read(1<<20),b''):digest.update(chunk)
+    return digest.hexdigest()
+
+
+MODEL_FOLDER_RULE=('MODEL must contain regular files '
+                   '(a Hugging Face cache snapshots/<sha> directory cannot be mounted alone)')
+
+
+def _model_file(model,name):
+    path=model/name
+    if path.is_symlink():
+        try:path.resolve(strict=True)
+        except (OSError,RuntimeError):  # RuntimeError: a link loop before Python 3.13
+            raise ValueError(f'Model file is a dangling or looping symbolic link: {name}; {MODEL_FOLDER_RULE}') from None
+        raise ValueError(f'Model file is a symbolic link: {name}; {MODEL_FOLDER_RULE}')
+    if not path.is_file():raise ValueError('Missing model file: '+name)
+    return path
+
+
+def check_model(model,full=False):
+    """Read-only identity check of the model folder against MODEL_REVISION; nothing is written.
+
+    The pinned files must be regular files with the pinned SHA-256 (shards: size, or full hash).
+    Any other entry a loader may read (weights, tokenizer, processor or template files) is refused.
+    """
+    model=Path(model)
+    if not MODEL_SHA256 or not SHARD_BYTES:raise ValueError('Model identity pending')
+    if not model.is_dir():raise ValueError('Model folder not found: '+str(model))
+    pinned={*MODEL_SHA256,*SHARD_BYTES,'PREPARATION.json'}
+    for entry in sorted(model.iterdir()):
+        name=entry.name
+        if name not in pinned and (name.lower().endswith(LOADER_SUFFIXES) or name.lower().startswith('tokenizer')):
+            raise ValueError(f'Unexpected file in model folder: {name}; use the files of {MODEL_REVISION} only')
+    for name,expected in MODEL_SHA256.items():
+        path=_model_file(model,name)
+        if sha256(path)!=expected:raise ValueError(f'Model file differs from {MODEL_REVISION}: {name}')
+    for name,size in SHARD_BYTES.items():
+        path=_model_file(model,name)
+        if path.stat().st_size!=size:raise ValueError(f'Model shard size differs from {MODEL_REVISION}: {name}')
+        if full and sha256(path)!=SHARD_SHA256[name]:raise ValueError(f'Model shard differs from {MODEL_REVISION}: {name}')
+    check_adapter(model)
+
+
+EXISTING_RUN='run_id already exists in --state-dir; use a new --state-dir (with the README commands: a new WORK)'
+
+
+def prepare_state(state,run_id=None):
+    """Claim <state>/<run-id>-startup with exclusive creation; nothing is overwritten or removed."""
+    state=Path(state)
+    rid=run_id or datetime.now(timezone.utc).strftime('run-%Y%m%dT%H%M%S%fZ')
+    if not RUN_ID.fullmatch(rid):raise ValueError('run_id must match [A-Za-z0-9][A-Za-z0-9._-]{0,63}')
+    startup=state/(rid+'-startup')
+    if len(os.fsencode(startup/'tmp'))>MAX_TMP_PATH:
+        raise ValueError(f'--state-dir path is too long for engine sockets; keep {startup/"tmp"} within {MAX_TMP_PATH} bytes')
+    try:state.mkdir(parents=True,exist_ok=True)
+    except OSError:raise ValueError('State directory is not writable: '+str(state)) from None
+    if (state/rid).exists() or startup.exists():raise ValueError(EXISTING_RUN)
+    cache=state/'cache'
+    if cache.is_symlink() or (cache.exists() and not cache.is_dir()):
+        raise ValueError(f'{cache} must be a directory, not a link or a file')
+    try:
+        startup.mkdir()
+        for name in ('tmp','home'):(startup/name).mkdir()
+        cache.mkdir(exist_ok=True)
+        with (startup/'attempts.jsonl').open('x',encoding='utf-8') as probe:
+            probe.flush();os.fsync(probe.fileno())
+    except FileExistsError:raise ValueError(EXISTING_RUN) from None
+    except OSError:raise ValueError('State directory is not writable: '+str(state)) from None
+    if cache.is_symlink() or not cache.resolve().is_relative_to(state.resolve()):
+        raise ValueError(f'{cache} must be a directory inside --state-dir, not a link')
+    return rid,startup
+
+
+def check_environment(base,state):
+    """Refuse inherited cache or configuration paths outside --state-dir; nothing is written.
+
+    An inherited path variable is accepted only if it is absolute and inside --state-dir, so
+    writes cannot leave the state directory. start() runs this before claiming the run name.
+    """
+    root=Path(state).resolve()
+    for key in PATH_VARIABLES:
+        if key not in base:continue
+        value=Path(base[key])
+        if not value.is_absolute() or not value.resolve().is_relative_to(root):
+            raise ValueError(f'{key}={base[key]} is outside --state-dir; do not set cache or configuration paths')
+
+
+def child_environment(base,state,startup):
+    """Engine environment: offline, no telemetry, every write location below --state-dir."""
+    check_environment(base,state)
+    env=dict(base)
+    env['VLLM_BATCH_INVARIANT']='1'
+    env['VLLM_WORKER_MULTIPROC_METHOD']='spawn'
+    # vLLM 0.31 no longer reads the legacy attention variable; an inherited value would only log a warning.
+    env.pop('VLLM_ATTENTION_BACKEND',None);env.pop('JEV_ATTENTION_BACKEND',None)
+    env.update(VLLM_USE_FLASHINFER_SAMPLER='0',VLLM_NO_USAGE_STATS='1',DO_NOT_TRACK='1',
+               HF_HUB_OFFLINE='1',TRANSFORMERS_OFFLINE='1',PYTHONDONTWRITEBYTECODE='1',PYTHONUNBUFFERED='1',
+               VLLM_HOST_IP='127.0.0.1',HOME=str(startup/'home'),TMPDIR=str(startup/'tmp'))
+    cache=Path(state)/'cache'
+    for key,sub in (('XDG_CACHE_HOME',None),('VLLM_CACHE_ROOT','vllm'),('TRITON_CACHE_DIR','triton'),
+                    ('TORCHINDUCTOR_CACHE_DIR','inductor'),('HF_HOME','hf'),('CUDA_CACHE_PATH','nv'),
+                    ('NUMBA_CACHE_DIR','numba')):
+        env.setdefault(key,str(cache/sub if sub else cache))
+    return env
+
+
+def validate_health(health, no_cache=False):
+    bad=health.get('warnings',[])
+    if bad or health.get('status','ok')!='ok' or health.get('prefix_caching') is not (not no_cache) or health.get('supports_slow_session') is not True:
+        raise RuntimeError('Service readiness rejected: health warnings or cache/session capability; health='+json.dumps(health)[:1500])
+
+
+def start(model, port, mode='routed', config=None, no_cache=False, state_dir=None, run_id=None, bind='127.0.0.1', allow_nonformal=False):
+    check_preparation(config,allow_nonformal)
+    check_python()
+    check_versions()
+    if bind not in BINDS:raise ValueError('--bind must be one of '+', '.join(BINDS))
+    if state_dir is None:raise ValueError('--state-dir is required')
+    model,state=check_paths(model,state_dir)
+    check_environment(os.environ,state)  # an input check: refused before anything is written
+    check_model(model)
+    from model_setup import report_processor_configs
+    report_processor_configs(model)
+    cfg=Path(config or ROOT/'config.json').resolve()
+    c=json.loads(cfg.read_text(encoding='utf-8'))
+    if mode=='routed' and not c.get('package',{}).get('finalized') and not allow_nonformal:
+        raise ValueError('Real fitted parameters must be finalized before routed serving')
+    try:
+        with socket.socket() as probe:
+            probe.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
+            probe.bind((bind,port))
+    except OSError:raise ValueError(f'Port {port} on {bind} is not available') from None
+    runtime=materialize_adapter(c,check_adapter(model))
+    rid,startup=prepare_state(state,run_id)
+    cfg=write_runtime_config(startup,runtime)
+    tempfile.tempdir=str(startup/'tmp')
+    env=child_environment(os.environ,state,startup)
+    args=[sys.executable,'-B',str(ROOT/'server.py'),'--config',str(cfg),'--model',str(model),
+          '--bind',bind,'--port',str(port),'--backend','vllm','--quant','awq',
+          '--mode',mode,'--model-key',MODEL_KEY,'--state-dir',str(state),'--run-id',rid]
+    if no_cache:args.append('--no-prefix-caching')
+    return start_attempts(args,env,startup,port,mode,no_cache)
+
+
+def start_attempts(args,env,startup,port,mode='routed',no_cache=False,timeout=STARTUP_TIMEOUT):
+    """auto first; TRITON_ATTN only if the engine fails to start. Each attempt has its own deadline."""
+    startup=Path(startup)
+    opener=urllib.request.build_opener(urllib.request.ProxyHandler({}))  # readiness never uses a proxy
+    def event(attempt,backend,result,log=None):
+        # Best effort: a record that cannot be written or printed never blocks starting or stopping.
+        record=dict(attempt=attempt,requested_backend=backend or 'auto',result=result)
+        if log is not None:record['log']=str(log)
+        line=json.dumps(record)
+        try:
+            with (startup/'attempts.jsonl').open('a',encoding='utf-8',newline='\n') as out:
+                out.write(line+'\n')
+        except OSError:pass
+        try:print(line,flush=True)
+        except OSError:pass
+    for attempt,backend in enumerate(ATTEMPTS,1):
+        child_env=dict(env)
+        child_env.pop('VLLM_ATTENTION_BACKEND',None);child_env.pop('JEV_ATTENTION_BACKEND',None)
+        if backend:child_env['JEV_ATTENTION_BACKEND']=backend
+        log_path=startup/f'attempt-{attempt}.log'
+        child_env['JEV_ATTENTION_LOG']=str(log_path)
+        child_args=list(args)
+        # Separate quota state per attempt: a failed warmup never counts toward the served run.
+        child_args[child_args.index('--state-dir')+1]=str(startup/f'state-{attempt}')
+        event(attempt,backend,'starting',log_path)
+        proc=None
+        try:
+            with log_path.open('xb') as out:
+                proc=subprocess.Popen(child_args,cwd=ROOT,env=child_env,
+                                      stdout=out,stderr=subprocess.STDOUT,start_new_session=True)
+            deadline=time.monotonic()+timeout
+            while time.monotonic()<deadline:
+                logs=log_path.read_text(encoding='utf-8',errors='replace')
+                if 'EngineCore failed to start' in logs:
+                    event(attempt,backend,'failed: EngineCore failed to start',log_path)
+                    break
+                if proc.poll() is not None:
+                    event(attempt,backend,'failed: exited before readiness',log_path)
+                    break
+                try:
+                    with opener.open(f'http://127.0.0.1:{port}/health',timeout=2) as response:
+                        health=json.load(response)
+                        ready=response.status==200 and health['mode']==mode and health['model_key']==MODEL_KEY
+                except (OSError,ValueError,KeyError):
+                    ready=False
+                if ready:
+                    # Recheck after the HTTP round trip; failure takes precedence.
+                    if proc.poll() is not None or 'EngineCore failed to start' in log_path.read_text(encoding='utf-8',errors='replace'):
+                        event(attempt,backend,'failed: engine stopped during readiness',log_path)
+                        break
+                    validate_health(health,no_cache)
+                    event(attempt,backend,'ready',log_path)
+                    print(json.dumps(health,ensure_ascii=False),flush=True)
+                    return proc
+                time.sleep(1)
+            else:
+                raise TimeoutError(f'Health readiness timed out after {timeout} s; see {log_path}')
+        except BaseException as exc:
+            # Stop the server first; the record follows even if a stop request arrives meanwhile.
+            try:
+                if proc is not None:stop(proc)
+            finally:
+                event(attempt,backend,'aborted: '+type(exc).__name__,log_path)
+            raise
+        stop(proc)
+    raise RuntimeError(f'Both attention backend attempts (auto, TRITON_ATTN) failed before readiness; see {startup}')
+
+
+KILL_WAIT=60
+
+
+@contextmanager
+def cleanup_signals():
+    """Hold SIGTERM, SIGINT and SIGHUP while child processes are being stopped.
+
+    A signal that arrives meanwhile is recorded, not acted on; once the previous handlers are
+    restored it is raised as KeyboardInterrupt, so a stop request is never lost.
+    """
+    saved={};pending=[]
+    def hold(number,frame):pending.append(number)
+    try:
+        if threading.current_thread() is threading.main_thread():
+            for name in ('SIGTERM','SIGINT','SIGHUP'):
+                number=getattr(signal,name,None)
+                if number is not None:saved[number]=signal.signal(number,hold)
+        yield
+    finally:
+        for number,handler in saved.items():
+            if handler is not None:signal.signal(number,handler)
+    if pending:raise KeyboardInterrupt
+
+
+def stop(proc):
+    """SIGTERM to the server's process group, up to 30 s, then SIGKILL to the group and a wait of
+    at most KILL_WAIT s; a signal during this sequence is raised afterwards (cleanup_signals)."""
+    with cleanup_signals():
+        try:os.killpg(proc.pid,signal.SIGTERM)
+        except ProcessLookupError:pass
+        try:
+            try:proc.wait(timeout=30)
+            except subprocess.TimeoutExpired:pass
+        finally:
+            # Reap descendants left in the process group before rebinding on retry.
+            try:os.killpg(proc.pid,signal.SIGKILL)
+            except ProcessLookupError:pass
+            try:proc.wait(timeout=KILL_WAIT)
+            except subprocess.TimeoutExpired:
+                try:print(f'launch: warning: process group {proc.pid} still running {KILL_WAIT} s after SIGKILL',
+                          file=sys.stderr,flush=True)
+                except OSError:pass
+
+
+def main(argv=None):
+    ap=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument('--model',required=True,help='absolute path of the model folder (read only)')
+    ap.add_argument('--state-dir',required=True,help='absolute path; the service writes only below it')
+    ap.add_argument('--run-id',help='new name for this run; default run-<UTC timestamp>')
+    ap.add_argument('--port',type=int,default=8000)
+    ap.add_argument('--bind',choices=BINDS,default='127.0.0.1')
+    ap.add_argument('--allow-nonformal',action='store_true',help='Local fitted engineering package only; never signs release')
+    a=ap.parse_args(argv)
+    def interrupted(*_):raise KeyboardInterrupt
+    for name in ('SIGTERM','SIGINT','SIGHUP'):
+        number=getattr(signal,name,None)
+        if number is not None:signal.signal(number,interrupted)
+    proc=None
+    try:
+        proc=start(a.model,a.port,mode=SERVED_MODE,state_dir=a.state_dir,run_id=a.run_id,bind=a.bind,allow_nonformal=a.allow_nonformal)
+        return proc.wait()
+    except KeyboardInterrupt:return 130
+    except (ValueError,RuntimeError,OSError) as exc:
+        print('launch: error: '+str(exc),file=sys.stderr,flush=True)
+        return 1
+    finally:
+        if proc:
+            # A stop request during this final stop is already being carried out.
+            try:stop(proc)
+            except KeyboardInterrupt:pass
+
+if __name__=='__main__':raise SystemExit(main())

service/model_setup.py

@@ -0,0 +1,14 @@
+"""Report a missing processor config in the model folder; never downloads or copies."""
+from pathlib import Path
+
+# Gemma 4 keeps its processor settings in processor_config.json; the checkpoint has no
+# preprocessor_config.json or video_preprocessor_config.json and needs neither.
+NAMES=('processor_config.json',)
+
+def report_processor_configs(model_dir):
+    """Print nothing for a complete model folder; return the missing names."""
+    p=Path(model_dir)
+    missing=[n for n in NAMES if not (p/n).is_file()]
+    if missing:
+        print('[model_setup] processor config not in the model folder: '+', '.join(missing),flush=True)
+    return missing

service/pkg_check.py

@@ -0,0 +1,455 @@
+"""Offline package checks: integrity, model identity, configuration, adapter contract, serving
+policy, calibration and disclosure items. No model is executed and nothing is downloaded.
+
+A preparation package (config package.finalized false) and a frozen package
+(finalized true plus FREEZE.json) are both checked. The freeze may change only
+package.finalized in config.json and replace README.md, DISCLOSURE.md and
+MODEL_CARD.md; every other file keeps the bytes of the prepared package. A Git
+working copy of the package (a `.git` directory next to the files) is accepted.
+
+SERVICE_POLICY (written by the builder from the selected serving parameters) states what this
+package serves: the routing mode, threshold and quota, the drafting budget and its prompt limit, the
+per-decision deadline and the item rules (types that may draft, low-margin choice re-read), the
+post-calibration, the unparsed policy, the cost-fuse lines and prices and the yes/no confidence
+constant of the base calibration. Every check below compares the configuration, calibration and fit
+records with it, and the guards hold whatever it says.
+"""
+import argparse
+import ast
+import copy
+import hashlib
+import json
+from pathlib import Path
+import re
+
+ROOT=Path(__file__).resolve().parent
+MODEL_KEY='metask-gemma4-31b-lora-w4a16'
+MODEL_NAME='MetaDecision-31B-Rain'
+ADAPTER_REF='model-subdir:adapter'
+ADAPTER_SUBDIR='adapter'
+ADAPTER_FILES=('adapter_config.json','adapter_model.safetensors')
+TARGETS=('q_proj','k_proj','v_proj','o_proj','gate_proj','up_proj','down_proj')
+LORA=dict(rank=16,alpha=32,target_modules=sorted(TARGETS))
+PRICING_BASE='google/gemma-4-31B-it'
+TRAINING_BASE='google/gemma-4-31B-it-qat-q4_0-unquantized'
+TRAINING_REVISION='1e4d8beecacb8b7590c1d8bedd7335f687bf311f'
+SERVED_BASE='google/gemma-4-31B-it-qat-w4a16-ct'
+SERVED_REVISION='52f3f65bc7a02d555763bc923bd1d9094898219d'
+SERVICE_POLICY={'protocol': 'adapter_arm_31b_v3', 'mode': 'routed', 'quota': 0.49, 'fuse': 0.49, 'margin_threshold': 6.625023671062799, 'fit_protocol': '31b-adapter-arm-v3-routed', 'fit_n': 231, 'base_fit': {'q': 0.056, 'draft': 256}, 'noul_p_cal': 0.898, 'draft': {'max_new_tokens': 192, 'reasoning_words': 50}, 'slow_max_prompt_tokens': 840, 'unparsed_policy': 'force_answer', 'slow_excluded_types': [], 'cost_fuse': {'arm': 0.9, 'enabled': True, 'release': 0.85, 'warmup': 100}, 'cost_fuse_prices': {'input_per_million': 0.059819178473256074, 'output_per_million': 0.059819178473256074}, 'item_deadline': {'item_seconds': 9.0, 'response_reserve_s': 0.35, 'readout_reserve_init_s': 1.0, 'readout_reserve_min_s': 0.3, 'readout_safety': 1.5, 'chunk_tokens': 32, 'min_chunk_tokens': 8, 'token_safety': 1.25, 'ewma_alpha': 0.3, 'warmup_tokens': 33, 'warmup_repeats': 3}, 'item_rules': {'slow_types': ['choice', 'noul', 'score'], 'reread_tau': 4.0}, 'post_calibration': {'choice': {'config': {'debias': False, 'temperature': False}, 'params': {}}, 'noul': {'config': {'temperature': 'type', 'floor': None}, 'params': {'T': 0.5}}, 'score': {'config': {'temperature': False, 'mode_weight': 1.0}, 'params': {}}}, 'post_calibration_active': True, 'core_patch': None, 'v3_params_sha256': 'b28ed73bb5a36e99a280a47a6b846e0c5b2de54f34a96efa4a10754b71ea7573', 'threshold_rule': 'eligible_items'}
+INPUT_BUDGET=6653
+MAX_MODEL_LEN=6656
+GPU_MEMORY_UTILIZATION=0.94
+MAX_LORA_RANK=16
+PROTOCOL='adapter_arm_31b_v3'
+FIT_PROTOCOLS={'fast_only':'31b-adapter-arm-v3-q0','routed':'31b-adapter-arm-v3-routed'}
+# The reference cost fuse of the fast-only configuration; a routed package states its own lines in SERVICE_POLICY.
+COST_FUSE={'enabled':True,'warmup':100,'arm':0.85,'release':0.8}
+NOUL_P_CAL_RANGE=(0.81,0.95)
+SLOW_READOUT_MARGIN=32
+DRAFTING_WALL_TIMEOUT_SECONDS=60
+SLOW_QUOTA_MAX=0.49
+BASE_FUSE=0.2
+ITEM_SECONDS_MAX=9.5
+KINDS=('choice','noul','score')
+SERVED_MODE=(SERVICE_POLICY or {}).get('mode')
+REQUIRED=('core.py backends.py quota.py attention.py model_setup.py server.py bench.py item_deadline.py post_calibration.py '
+          'reconcile_usage.py launch.py serve.sh selftest.py selftest.sh pkg_check.py '
+          'prepare_model.sh config.json calibration.json MODEL_MANIFEST.json ADAPTER_MANIFEST.json '
+          'PIPELINE_PROVENANCE.json DISCLOSURE.md README.md MODEL_CARD.md '
+          'requirements.lock requirements.measured.lock LICENSE MANIFEST.sha256').split()
+RUNTIME=('core.py','backends.py','quota.py','attention.py','model_setup.py','server.py','launch.py','item_deadline.py','post_calibration.py')
+REFERENCE_CODE=('core.py','backends.py','quota.py','attention.py','model_setup.py','server.py','bench.py','reconcile_usage.py')
+FREEZE_SCHEMA='metadecision_31b_freeze_v1'
+FREEZE_DOCUMENTS=('README.md','DISCLOSURE.md','MODEL_CARD.md')
+CONFIG_CHANGE='package.finalized: false -> true (single textual replacement)'
+FROZEN_FLAG,PREPARED_FLAG=b'"finalized": true',b'"finalized": false'
+TEXT_SUFFIXES=('.py','.md','.json','.sh','.lock','.txt')
+def percent(value):
+    """'6%' for 0.06, '2.5%' for 0.025, '6.25%' for 0.0625: the service's quota and fuse lines as written in the
+    documents, exact (never rounded)."""
+    from decimal import Decimal
+    text=format((Decimal(repr(float(value)))*100).normalize(),'f')
+    return (text.rstrip('0').rstrip('.') if '.' in text else text)+'%'
+
+def policy_disclosure(policy):
+    """Policy facts DISCLOSURE.md must state: the yes/no constant; routed: the drafting pass, its quota, budget,
+    prompt limit, 60 s limit and the cost-fuse lines."""
+    if not policy:return ()
+    items=('%.3f'%policy['noul_p_cal'],)
+    if policy['mode']!='routed':return items
+    pct=percent(policy['quota'])
+    items+=('drafting pass','slow-path quota '+pct,'at most '+pct+' of all decisions',
+            'at most %d tokens'%policy['draft']['max_new_tokens'],
+            'longer than {:,} tokens'.format(policy['slow_max_prompt_tokens']),
+            'deadline of %s s'%format(policy['item_deadline']['item_seconds'],'g'),'Cost fuse',
+            percent(policy['cost_fuse']['arm']),percent(policy['cost_fuse']['release']))
+    rules=policy.get('item_rules') or {}
+    if rules.get('reread_tau'):items+=('reverse order','below %s'%format(rules['reread_tau'],'g'))
+    if set(rules.get('slow_types') or KINDS)!=set(KINDS):items+=('only these question types',)
+    if policy.get('post_calibration_active'):items+=('Post-calibration',)
+    return items
+
+# Necessary disclosure: base repo@revision, own weights, quantization, unmerged adapter,
+# input budget and the cost/latency measurement conditions, plus the policy facts.
+DISCLOSURE_REQUIRED=(PRICING_BASE,TRAINING_BASE,TRAINING_REVISION,SERVED_BASE,SERVED_REVISION,
+                     'rank 16','alpha 32','W4A16','not merged','max_lora_rank=16','6,653',
+                     'VLLM_BATCH_INVARIANT=1','engine token')+policy_disclosure(SERVICE_POLICY)
+# README requirements: tested GPU and memory, the adapter route's vLLM options, the download tool, the mode.
+README_REQUIRED=('NVIDIA A100 80GB','gpu_memory_utilization=0.94','max_model_len=6656','enable_lora',
+                 'max_lora_rank=16','The model files take','`hf`','hf download')+(
+                 ('`"mode": "%s"`'%SERVED_MODE,) if SERVED_MODE else ())
+
+def read(name):
+    return json.loads((ROOT/name).read_text(encoding='utf-8'))
+
+def sha(path):
+    h=hashlib.sha256()
+    with path.open('rb') as f:
+        for block in iter(lambda:f.read(1<<20),b''):h.update(block)
+    return h.hexdigest()
+
+def canonical(value):
+    return hashlib.sha256(json.dumps(value,ensure_ascii=False,sort_keys=True,separators=(',',':'),allow_nan=False).encode()).hexdigest()
+
+def identity_sha256(m):
+    """Identity of the model folder: official root files + the unmerged adapter/ files."""
+    identity={'format':m['format'],'files':m['files']}
+    if m.get('adapter') is not None:
+        a=m['adapter']
+        identity['adapter']={'format':a['format'],'subdir':a['subdir'],'files':a['files']}
+    return canonical(identity)
+
+def check_manifest(root=ROOT):
+    expected={}
+    for line in (root/'MANIFEST.sha256').read_text(encoding='utf-8').splitlines():
+        digest,name=line.split('  ',1)
+        if not re.fullmatch('[0-9a-f]{64}',digest) or Path(name).name!=name or name in expected:
+            raise ValueError('Unsafe or duplicate manifest entry')
+        p=root/name
+        if p.is_symlink() or not p.is_file() or sha(p)!=digest:
+            raise ValueError('Missing, linked or changed package member: '+name)
+        expected[name]=digest
+    actual={p.name for p in root.iterdir() if p.is_file() and p.name!='MANIFEST.sha256'}
+    if actual!=set(expected):raise ValueError('Manifest membership differs')
+    for p in root.iterdir():
+        if p.is_symlink() or (p.is_dir() and p.name not in ('__pycache__','.git')):
+            raise ValueError('Unexpected package directory or symlink')
+    return expected
+
+def front_matter(text):
+    """Minimal YAML front-matter reader for top-level 'key: value' lines."""
+    lines=text.splitlines()
+    if not lines or lines[0].strip()!='---':raise ValueError('Model card lacks YAML front matter')
+    values={}
+    for line in lines[1:]:
+        if line.strip()=='---':return values
+        match=re.fullmatch(r'([A-Za-z_][A-Za-z0-9_]*):\s*(\S.*)?',line)
+        if match:values[match[1]]=(match[2] or '').strip()
+    raise ValueError('Unterminated model card front matter')
+
+def check_adapter_contract(c,m):
+    adapter=m.get('adapter')
+    profiles=[(p.get('model') or {}) for p in c.get('profiles',{}).values()]
+    if (not isinstance(adapter,dict) or adapter.get('format')!='peft-lora' or adapter.get('subdir')!=ADAPTER_SUBDIR
+            or not isinstance(adapter.get('files'),dict)):
+        raise ValueError('The manifest must pin the unmerged adapter/ folder')
+    if adapter['files'] and (set(adapter['files'])!=set(ADAPTER_FILES) or adapter.get('merged') is not False
+            or adapter.get('rank')!=LORA['rank'] or adapter.get('alpha')!=LORA['alpha']
+            or adapter.get('target_modules')!=LORA['target_modules'] or adapter.get('variant') not in ('trained_original','key_value_aligned')
+            or adapter.get('training_base')!=TRAINING_BASE or adapter.get('training_revision')!=TRAINING_REVISION):
+        raise ValueError('Adapter manifest differs from the trained r16/alpha32 unmerged LoRA contract')
+    if c['model'].get('adapter')!=ADAPTER_REF or any('adapter' in p and p['adapter']!=ADAPTER_REF for p in profiles):
+        raise ValueError('Configuration must reference the model-relative adapter')
+    options=c['model'].get('vllm_options',{})
+    if options.get('max_lora_rank')!=MAX_LORA_RANK or 'enable_lora' in options:
+        raise ValueError('vLLM max_lora_rank=16 required; enable_lora is backend-owned')
+    record=read('ADAPTER_MANIFEST.json')
+    if (sha(ROOT/'ADAPTER_MANIFEST.json')!=adapter.get('adapter_manifest_sha256') or record.get('files')!=adapter['files']
+            or record.get('variant')!=adapter.get('variant') or (record.get('lora') or {}).get('merged') is not False
+            or (record.get('served_base') or {}).get('repo_id')!=SERVED_BASE
+            or (record.get('served_base') or {}).get('revision')!=SERVED_REVISION):
+        raise ValueError('ADAPTER_MANIFEST.json does not bind the pinned adapter files')
+
+def policy_problems(policy):
+    """Guards that hold whatever the builder wrote: mode, quota, yes/no constant, drafting budget and context."""
+    if not isinstance(policy,dict):return ['service policy pending']
+    problems=[]
+    p=policy.get('noul_p_cal')
+    if isinstance(p,bool) or not isinstance(p,float) or not NOUL_P_CAL_RANGE[0]<=p<=NOUL_P_CAL_RANGE[1]:
+        problems.append('yes/no constant outside [0.81, 0.95]')
+    if policy.get('protocol')!=PROTOCOL or policy.get('fit_protocol')!=FIT_PROTOCOLS.get(policy.get('mode')):
+        problems.append('protocol or fit protocol')
+    q=policy.get('quota')
+    if policy.get('mode')=='routed':
+        problems+=v4_policy_problems(policy)
+    if policy.get('mode')=='fast_only':
+        if q!=0 or policy.get('cost_fuse')!=COST_FUSE or policy.get('slow_excluded_types'):
+            problems.append('fast-only policy with a quota, other cost fuse or excluded types')
+    elif policy.get('mode')=='routed':
+        fuse,draft=policy.get('cost_fuse') or {},policy.get('draft') or {}
+        if isinstance(q,bool) or not isinstance(q,float) or not 0<q<=SLOW_QUOTA_MAX:problems.append('quota')
+        if (set(fuse)!={'enabled','warmup','arm','release'} or fuse['enabled'] is not True or fuse['warmup']!=100
+                or not 0.5<=fuse['release']<fuse['arm']<=0.9):problems.append('cost fuse lines')
+        length,prompt=draft.get('max_new_tokens'),policy.get('slow_max_prompt_tokens')
+        if (type(length) is not int or type(prompt) is not int or type(draft.get('reasoning_words')) is not int
+                or prompt+length+SLOW_READOUT_MARGIN>MAX_MODEL_LEN or prompt>INPUT_BUDGET):
+            problems.append('drafting budget outside the vLLM context')
+        if policy.get('unparsed_policy') not in ('fast','force_answer'):problems.append('unparsed policy')
+        if policy.get('slow_excluded_types') not in ([],['noul']):problems.append('excluded types')
+        if bool(policy.get('slow_excluded_types'))!=(policy.get('core_patch') is not None):
+            problems.append('excluded slow types need exactly the v3 core patch')
+    else:
+        problems.append('mode')
+    return problems
+
+def v4_policy_problems(policy):
+    """Fuse, cost-fuse prices, per-decision deadline, item rules and post-calibration of a routed package."""
+    problems=[]
+    q=policy.get('quota')
+    if not isinstance(q,float) or policy.get('fuse')!=max(BASE_FUSE,q):problems.append('fuse must be max(0.2, quota)')
+    t=policy.get('margin_threshold')
+    if isinstance(t,bool) or not isinstance(t,float) or not 0<=t<float('inf'):problems.append('margin threshold')
+    prices=policy.get('cost_fuse_prices') or {}
+    if (set(prices)!={'input_per_million','output_per_million'}
+            or any(isinstance(v,bool) or not isinstance(v,float) or not 0<v<float('inf') for v in prices.values())):
+        problems.append('cost-fuse prices')
+    try:
+        import item_deadline
+        s=item_deadline.settings_from(policy.get('item_deadline'))
+        if s!=policy.get('item_deadline') or not 0<s['item_seconds']<=ITEM_SECONDS_MAX:problems.append('item deadline')
+    except Exception:
+        problems.append('item deadline')
+    rules=policy.get('item_rules')
+    if (not isinstance(rules,dict) or set(rules)!={'slow_types','reread_tau'} or not isinstance(rules['slow_types'],list)
+            or not rules['slow_types'] or not set(rules['slow_types'])<=set(KINDS)
+            or len(set(rules['slow_types']))!=len(rules['slow_types'])
+            or isinstance(rules['reread_tau'],bool) or not isinstance(rules['reread_tau'],float)
+            or not 0<=rules['reread_tau']<float('inf')):
+        problems.append('item rules')
+    try:
+        import post_calibration
+        block=post_calibration.validate(policy.get('post_calibration'))
+        if bool(post_calibration.active(block))!=bool(policy.get('post_calibration_active')):problems.append('post-calibration flag')
+    except Exception:
+        problems.append('post-calibration')
+    return problems
+
+def check_serving_policy(c):
+    """SERVICE_POLICY's mode, quota, draft, prompt limit and cost fuse; 6653 input budget everywhere server.py can
+    apply it; 31B context/memory; the served configuration is server.py's fixed point."""
+    from core import apply_profile, apply_budget_tier
+    problems=policy_problems(SERVICE_POLICY)
+    if problems:raise ValueError('Service policy invalid: '+', '.join(problems))
+    policy=SERVICE_POLICY
+    r,limits,options,g=c['routing'],c['limits'],c['model'].get('vllm_options',{}),c['generation']
+    q,draft=policy['quota'],policy['draft']
+    if (r['mode']!=policy['mode'] or r['quota']!=q or r['target_slow_share']!=q or c['bench'].get('target_slow_share')!=q
+            or r['fuse']!=policy['fuse'] or c['usage']!={'basis':'engine'} or r.get('cost_fuse')!=policy['cost_fuse']):
+        raise ValueError('Changed serving mode, quota, usage or cost-fuse convention')
+    if (r['margin_threshold']!=policy['margin_threshold']
+            or c['pricing']['models'][c['pricing']['model_key']]!=policy['cost_fuse_prices']
+            or c.get('item_deadline')!=policy['item_deadline'] or c.get('item_rules')!=policy['item_rules']
+            or c.get('post_calibration')!=policy['post_calibration']):
+        raise ValueError('Changed threshold, cost-fuse prices, deadline, item rules or post-calibration')
+    if (r['margin_threshold']!=0 if policy['mode']=='fast_only' else not 0<r['margin_threshold']<float('inf')):
+        raise ValueError('Changed serving threshold')
+    if ((g['max_new_tokens'],g['reasoning_words'])!=(draft['max_new_tokens'],draft['reasoning_words'])
+            or (r['tiers']['hard'].get('max_new_tokens'),r['tiers']['hard'].get('reasoning_words'))!=(draft['max_new_tokens'],draft['reasoning_words'])
+            or r['slow_max_prompt_tokens']!=policy['slow_max_prompt_tokens'] or r['unparsed_policy']!=policy['unparsed_policy']
+            or r.get('slow_excluded_types',[])!=policy['slow_excluded_types']
+            or ('slow_excluded_types' in r and not r['slow_excluded_types'])):
+        raise ValueError('Changed drafting budget, prompt limit, unparsed policy or excluded types')
+    if policy['mode']=='routed' and (r.get('long_item_slow_policy')!='fast_only'
+                                     or g.get('wall_timeout_seconds')!=DRAFTING_WALL_TIMEOUT_SECONDS):
+        raise ValueError('Changed routed serving, drafting timeout or long-item policy')
+    if (limits['max_prompt_tokens'],limits['physical_max_prompt_tokens'])!=(INPUT_BUDGET,INPUT_BUDGET) \
+            or any(r['tiers'][t]['max_prompt_tokens']!=INPUT_BUDGET for t in ('normal','long')) \
+            or limits['oom_retry_prompt_tokens']>INPUT_BUDGET:
+        raise ValueError('Input budget must be 6653 for every prompt limit')
+    if options.get('max_model_len')!=MAX_MODEL_LEN or options.get('gpu_memory_utilization')!=GPU_MEMORY_UTILIZATION:
+        raise ValueError('vLLM context 6656 and GPU memory 0.94 required')
+    served=copy.deepcopy(c)
+    apply_profile(served)
+    apply_budget_tier(served,None)
+    served['routing']['mode']=SERVED_MODE
+    if served!=c:
+        raise ValueError('server.py would apply a different configuration than the measured one')
+    import core
+    if policy['slow_excluded_types'] and getattr(core,'SLOW_TYPE_EXCLUSION',None)!='routing.slow_excluded_types':
+        raise ValueError('Excluded slow types need the patched core.py')
+
+def check_calibration_policy(cal):
+    """The yes/no constant of SERVICE_POLICY in the fast, slow and fallback blocks; drafting-pass and fallback
+    answers use the first-pass temperatures (stated in DISCLOSURE.md)."""
+    blocks=cal['models'][MODEL_KEY]
+    p=SERVICE_POLICY['noul_p_cal']
+    for side in ('fast','slow','fallback'):
+        if blocks[side]['noul_calibration']['p_cal']!=p or blocks[side]['by_type']['noul'].get('p_cal')!=p:
+            raise ValueError('Calibration block does not carry the stated yes/no constant: '+side)
+        if blocks[side]['temperature_by_kind']!=blocks['fast']['temperature_by_kind']:
+            raise ValueError('DISCLOSURE.md states that drafting-pass answers use the first-pass calibration')
+
+def prepared_config_bytes(data):
+    """Undo the freeze's single textual change of config.json."""
+    if data.count(FROZEN_FLAG)!=1 or PREPARED_FLAG in data:
+        raise ValueError('Frozen config.json must differ from the prepared one only in package.finalized')
+    return data.replace(FROZEN_FLAG,PREPARED_FLAG)
+
+def check_freeze(c,m,members):
+    """None for a preparation package; the verified FREEZE.json record for a frozen one."""
+    path=ROOT/'FREEZE.json'
+    if not c['package'].get('finalized'):
+        if path.exists():raise ValueError('FREEZE.json in a package that is not finalized')
+        return None
+    if not path.is_file():raise ValueError('Finalized package without FREEZE.json')
+    f=read('FREEZE.json')
+    if f.get('schema')!=FREEZE_SCHEMA or f.get('status')!='FROZEN':
+        raise ValueError('Unknown freeze record')
+    if not c['package'].get('nonformal_fitted') or c['package'].get('sample_only'):
+        raise ValueError('Only the fitted package can be frozen')
+    if not re.fullmatch('[0-9a-f]{40}',str(m.get('revision') or '')) or m['repo_id'].startswith('OWNER/'):
+        raise ValueError('Frozen package lacks the own repository and full revision')
+    if (f.get('repo_id'),f.get('revision'),f.get('model_identity_sha256'))!=(m['repo_id'],m['revision'],m['identity_sha256']):
+        raise ValueError('Freeze record names another repository, revision or model')
+    prepared=f.get('prepared_files')
+    replaced=f.get('replaced_documents') or {}
+    if not isinstance(prepared,dict) or set(members)-{'FREEZE.json'}!=set(prepared) or set(replaced)-set(FREEZE_DOCUMENTS):
+        raise ValueError('Frozen package membership differs from the prepared package')
+    for name,digest in members.items():
+        if name=='FREEZE.json':continue
+        if name=='config.json':
+            before=hashlib.sha256(prepared_config_bytes((ROOT/name).read_bytes())).hexdigest()
+            if before!=prepared[name] or f.get('config')!={'prepared_sha256':before,'frozen_sha256':digest,'change':CONFIG_CHANGE}:
+                raise ValueError('Frozen config.json is not the recorded freeze of the prepared config')
+        elif name in replaced:
+            if replaced[name]!={'prepared_sha256':prepared[name],'frozen_sha256':digest}:
+                raise ValueError('Replaced document differs from the freeze record: '+name)
+        elif digest!=prepared[name]:
+            raise ValueError('Freeze changed a file other than config.json and the documents: '+name)
+    return f
+
+def disclosure_problems(root=ROOT):
+    """Necessary disclosure items that DISCLOSURE.md (and the model card base_model) must state."""
+    text=(root/'DISCLOSURE.md').read_text(encoding='utf-8',errors='replace')
+    problems=['DISCLOSURE.md lacks: '+item for item in DISCLOSURE_REQUIRED if item not in text]
+    readme=(root/'README.md').read_text(encoding='utf-8',errors='replace')
+    problems+=['README.md lacks: '+item for item in README_REQUIRED if item not in readme]
+    card=(root/'MODEL_CARD.md').read_text(encoding='utf-8',errors='replace')
+    try:
+        values=front_matter(card)
+    except ValueError as exc:
+        return problems+[str(exc)]
+    if values.get('base_model')!=PRICING_BASE:
+        problems.append('MODEL_CARD.md front matter base_model must be '+PRICING_BASE)
+    for item in (TRAINING_BASE,TRAINING_REVISION,SERVED_BASE,SERVED_REVISION):
+        if item not in card:problems.append('MODEL_CARD.md lacks: '+item)
+    return problems
+
+def checks():
+    for name in REQUIRED:
+        if not (ROOT/name).is_file():raise ValueError('Missing '+name)
+    members=check_manifest()
+    for p in ROOT.glob('*.py'):ast.parse(p.read_text(encoding='utf-8'))
+    c,m,prov=read('config.json'),read('MODEL_MANIFEST.json'),read('PIPELINE_PROVENANCE.json')
+    from core import validate_config, validate_threshold, validate_calibration
+    import launch
+    validate_config(c)
+    if c['model']['model_key']!=MODEL_KEY or c['model']['name']!=MODEL_NAME or set(c['profiles'])!={MODEL_KEY}:
+        raise ValueError('Wrong trained model key')
+    if c['pipeline']['arm']!='trained' or prov.get('arm')!='trained' or m.get('arm')!='trained':
+        raise ValueError('Arm differs between config, manifest and provenance')
+    if c['pipeline']['model_identity_sha256']!=m['identity_sha256']:
+        raise ValueError('Model identity mismatch')
+    if m.get('revision') is not None and not re.fullmatch('[0-9a-f]{40}',m['revision']):
+        raise ValueError('Abbreviated/mutable revision rejected')
+    if m['repo_id'].startswith('google/'):
+        raise ValueError('Own HF repository required')
+    if (m.get('served_base')or{}).get('repo_id')!=SERVED_BASE or (m.get('served_base')or{}).get('revision')!=SERVED_REVISION:
+        raise ValueError('Served base must be the official W4A16 CT revision')
+    check_adapter_contract(c,m)
+    check_serving_policy(c)
+    frozen=check_freeze(c,m,members)
+    complete=bool(m['files']) and bool(m['adapter']['files'])
+    if complete and identity_sha256(m)!=m['identity_sha256']:raise ValueError('Manifest identity hash differs')
+    if not complete and m['identity_sha256'] is not None:raise ValueError('Incomplete model has an identity')
+    cal=read('calibration.json')
+    if set(cal['models'])!={MODEL_KEY}:raise ValueError('Foreign calibration model')
+    for side in ('fast','slow','fallback'):validate_calibration(cal,MODEL_KEY,side)
+    check_calibration_policy(cal)
+    fitted=c['package'].get('nonformal_fitted',False)
+    if fitted:
+        if validate_threshold(c):raise ValueError('Threshold identity differs from the configuration')
+        # The served configuration is the materialized one; its threshold identity must still hold.
+        if validate_threshold(launch.materialize_adapter(c,Path('/materialized/adapter'))):
+            raise ValueError('Materialized threshold identity differs')
+        lock=read('SERVICE_LOCK.json')
+        report=read('fit_report.json')
+        if (lock['protocol']!=prov.get('protocol') or lock['protocol']!=PROTOCOL
+                or lock['serving_identity_sha256']!=m['identity_sha256'] or lock['arm']!='trained'
+                or lock.get('adapter_files')!=m['adapter']['files'] or lock.get('q')!=SERVICE_POLICY['quota']):
+            raise ValueError('Fit lock identity mismatch')
+        actual={'config_file':frozen['config']['prepared_sha256'] if frozen else sha(ROOT/'config.json'),
+                'calibration':sha(ROOT/'calibration.json'),'fit_report':sha(ROOT/'fit_report.json')}
+        for stem,digest in actual.items():
+            if lock[stem+'_sha256']!=digest:raise ValueError('Fit hash mismatch')
+        base=SERVICE_POLICY['base_fit']
+        if (report.get('fit_protocol')!=SERVICE_POLICY['fit_protocol'] or report.get('n')!=SERVICE_POLICY['fit_n']
+                or report.get('q')!=base['q']
+                or (report.get('noul_constant') or {}).get('value')!=SERVICE_POLICY['noul_p_cal']
+                or report.get('draft')!=base['draft']):
+            raise ValueError('Wrong fitting method')
+    else:
+        if not c['package']['sample_only'] or not cal['models'][MODEL_KEY].get('placeholder'):
+            raise ValueError('Unfitted package must clearly carry placeholders')
+    if (c['package']['finalized'] and frozen is None) or prov.get('release_ready') or prov.get('official_submission_authorized'):
+        raise ValueError('Only the recorded freeze may finalize; the builder never authorizes a release')
+    for name in REFERENCE_CODE+('item_deadline.py','post_calibration.py'):
+        if sha(ROOT/name)!=prov['source_files'][name]:raise ValueError('Request code changed: '+name)
+    patch_record=prov.get('core_patch')
+    if patch_record!=SERVICE_POLICY.get('core_patch'):raise ValueError('Core patch record differs from the policy')
+    if patch_record is None:
+        if prov['source_files']['core.py']!=prov['reference_source_files']['core.py']:raise ValueError('Reference core.py changed')
+    else:
+        raise ValueError('This package serves the unchanged core.py')
+    server_patch=prov.get('server_patch') or {}
+    if (sha(ROOT/'server.py')!=server_patch.get('patched_server_sha256')
+            or prov['reference_source_files']['server.py']!=server_patch.get('reference_server_sha256')):
+        raise ValueError('server.py is not the declared extension patch of the reference server.py')
+    if any(prov['source_files'][n]!=prov['reference_source_files'][n] for n in REFERENCE_CODE if n not in ('core.py','server.py')):
+        raise ValueError('Reference request code changed')
+    if 'install_extensions(service, c)' not in (ROOT/'server.py').read_text(encoding='utf-8'):
+        raise ValueError('server.py does not install the serving extensions')
+    for name in RUNTIME:
+        text=(ROOT/name).read_text(encoding='utf-8')
+        if re.search(r'\brmtree\s*\(|\b(?:snapshot_download|hf_hub_download)\s*\(',text):
+            raise ValueError('Runtime downloads or recursive deletion found')
+    launch_text=(ROOT/'launch.py').read_text(encoding='utf-8')
+    for token in ("HF_HUB_OFFLINE='1'","TRANSFORMERS_OFFLINE='1'","env['VLLM_BATCH_INVARIANT']='1'","env['VLLM_WORKER_MULTIPROC_METHOD']='spawn'",
+                  "VLLM_USE_FLASHINFER_SAMPLER='0'","def check_adapter(","def materialize_adapter(",
+                  "    check_adapter(model)\n","cfg=write_runtime_config(startup,runtime)","config.runtime.json",
+                  "SERVED_MODE='%s'"%SERVED_MODE,"mode=SERVED_MODE","(ROOT/'FREEZE.json').is_file()"):
+        if token not in launch_text:raise ValueError('Offline/invariant/adapter/mode/freeze startup setting missing: '+token.strip())
+    if launch.ADAPTER_REF!=ADAPTER_REF or launch.MODEL_KEY!=MODEL_KEY or launch.SERVED_MODE!=c['routing']['mode']:
+        raise ValueError('Launcher adapter reference, model key or mode differs')
+    for p in ROOT.glob('*.sh'):
+        if re.search(r'\brm\s+-[^\n]*r|\brmtree\b',p.read_text(encoding='utf-8')):
+            raise ValueError('Recursive deletion in shell')
+    serve=(ROOT/'serve.sh').read_text(encoding='utf-8')
+    if 'VLLM_USE_FLASHINFER_SAMPLER=0' not in serve or 'VLLM_BATCH_INVARIANT=1' not in serve or 'VLLM_WORKER_MULTIPROC_METHOD=spawn' not in serve:
+        raise ValueError('serve.sh must set VLLM_USE_FLASHINFER_SAMPLER=0, VLLM_BATCH_INVARIANT=1 and VLLM_WORKER_MULTIPROC_METHOD=spawn')
+    problems=disclosure_problems()
+    if problems:raise ValueError('Necessary disclosure incomplete: '+'; '.join(problems))
+    pins=dict(line.split('==',1) for line in (ROOT/'requirements.lock').read_text().splitlines() if '==' in line and not line.startswith('#'))
+    for name,version in launch.VERSIONS.items():
+        if pins.get(name,'').split('+')[0]!=version:raise ValueError('Runtime version pin mismatch')
+    return {'status':'FROZEN_PACKAGE_CHECK_PASS' if frozen else 'PREPARATION_CHECK_PASS','model_execution':False,'network':False,
+            'adapter':'unmerged adapter/ ('+m['adapter'].get('variant','pending')+')','served_mode':SERVED_MODE,
+            'input_budget':INPUT_BUDGET,'service_fit':'frozen' if frozen else 'fitted_not_frozen' if fitted else 'pending',
+            'serving_extensions':['item_deadline','item_rules','post_calibration']}
+
+def main():
+    argparse.ArgumentParser(description=__doc__).parse_args()
+    print(json.dumps(checks(),ensure_ascii=False,indent=2))
+    return 0
+
+if __name__=='__main__':raise SystemExit(main())

service/post_calibration.py

@@ -0,0 +1,212 @@
+"""Per-type post-calibration of the reported probabilities (no model call, no tokens).
+
+server.py installs it when config.json has a `post_calibration` block. For each question type the
+block holds the options chosen on the service's own fitting data and their fitted parameters:
+  choice: debias (divide out the mean probability of each option position, per option count, then
+          renormalise), then a temperature;
+  noul:   a temperature, then a decision-side floor (P(yes) is moved to at least `floor` on the
+          side it already favours, or to at most 1 - floor);
+  score:  a temperature, then a reading weight w (probability mass w moved onto the most probable
+          level; w = 1 puts all mass on it).
+The temperature option is off, one value per type ("type"), or one value per type and route cell
+("route": fast, slow, slow_timeout_fast, fast_reread; a cell without its own value uses the type
+value). A temperature t replaces the probabilities p by p^(1/t), renormalised. A type without an
+entry, or with every option off, and a question with a single option or level, are returned exactly
+as the service computed them. The answer label (`choice`, `noul`, the prediction) is recomputed from
+the calibrated probabilities.
+"""
+import math
+
+KINDS = ('choice', 'noul', 'score')
+OPTIONS = {'choice': ('debias', 'temperature'), 'noul': ('temperature', 'floor'),
+           'score': ('temperature', 'mode_weight')}
+ROUTE_BUCKETS = ('fast', 'slow', 'slow_timeout_fast', 'fast_reread')
+TEMPERATURE_MODES = (False, True, 'type', 'route')
+T_MIN, T_MAX = 0.5, 8.0
+
+
+def need(ok, code):
+    if not ok:
+        raise RuntimeError(code)
+
+
+def finite(x):
+    return isinstance(x, (int, float)) and not isinstance(x, bool) and math.isfinite(x)
+
+
+# ---------------------------------------------------------------- transforms
+def distribution(p):
+    need(len(p) >= 2 and all(type(v) in (float, int) and math.isfinite(v) and 0 <= v <= 1 for v in p), 'BAD_PROBS')
+    need(abs(sum(p)-1) < 1e-6, 'BAD_PROB_SUM')
+    return list(p)
+
+
+def softmax(scores):
+    need(scores and all(math.isfinite(x) for x in scores), 'BAD_LOGITS')
+    z = max(scores)
+    w = [math.exp(x-z) for x in scores]
+    return [x/sum(w) for x in w]
+
+
+def temperature(p, t):
+    distribution(p)
+    need(math.isfinite(t) and t >= .5, 'BAD_TEMPERATURE')
+    return list(p) if t == 1 else softmax([math.log(max(x, 1e-300))/t for x in p])
+
+
+def _argmax(p):
+    return max(range(len(p)), key=p.__getitem__)
+
+
+def debias(p, prior):
+    """Option-position prior divided out, renormalised."""
+    need(len(prior) == len(p) and all(w > 0 for w in prior), 'CAL_PRIOR')
+    q = [x / w for x, w in zip(p, prior)]
+    s = sum(q)
+    return [x / s for x in q]
+
+
+def noul_floor(p, floor):
+    """Labels no/yes: P(yes) is committed to >= floor on the favoured side (<= 1 - floor on the other)."""
+    need(len(p) == 2 and .8 < floor < 1, 'CAL_NOUL_FLOOR')
+    if p[1] >= floor or p[1] <= 1 - floor:
+        return list(p)
+    y = floor if p[1] >= .5 else 1 - floor
+    return [1 - y, y]
+
+
+def score_mode(p, weight):
+    """Weight 0 keeps the distribution; weight 1 puts all mass on the most probable level."""
+    need(0 <= weight <= 1, 'CAL_SCORE_WEIGHT')
+    if weight == 0:
+        return list(p)
+    m = _argmax(p)
+    return [(1 - weight) * x + (weight if i == m else 0.) for i, x in enumerate(p)]
+
+
+def n_active(cfg):
+    return sum(bool(v) for v in cfg.values())
+
+
+def apply_one(kind, cfg, params, p, bucket='fast'):
+    q = list(p)
+    if cfg.get('debias') and str(len(q)) in params.get('prior', {}):
+        q = debias(q, params['prior'][str(len(q))])
+    mode = cfg.get('temperature')
+    if mode:
+        need(mode in (True, 'type', 'route'), 'CAL_TEMPERATURE_MODE')
+        t = params['route_T'].get(bucket, params['T']) if mode == 'route' else params['T']
+        q = temperature(q, t)
+    if kind == 'noul' and cfg.get('floor'):
+        q = noul_floor(q, cfg['floor'])
+    if kind == 'score' and cfg.get('mode_weight'):
+        q = score_mode(q, cfg['mode_weight'])
+    return distribution(q)
+
+
+def route_bucket(detail):
+    """Route cell of one answered decision, from the service's own per-decision record:
+    slow_timeout_fast = first-pass answer served after an admitted drafting pass (deadline, failure or
+    unparsed draft; the 'fallback' calibration block); slow = answer read after the drafting pass
+    (appended answer line included); fast_reread = one-pass answer with the reversed-order re-read
+    averaged in; fast = everything else."""
+    if detail.get('route') in ('slow_timeout_fast', 'slow_failed_fast', 'slow_unparsed_fast') \
+            or detail.get('calibration_block') == 'fallback':
+        return 'slow_timeout_fast'
+    if detail.get('path') == 'slow' or detail.get('calibration_block') == 'slow' \
+            or detail.get('route') in ('slow', 'slow_forced_answer'):
+        return 'slow'
+    if isinstance(detail.get('reread'), dict):
+        return 'fast_reread'
+    return 'fast'
+
+
+# ---------------------------------------------------------------- configuration
+def _temperature_value(t):
+    return finite(t) and T_MIN <= t <= T_MAX
+
+
+def validate(block):
+    """The config.json block: {type: {"config": {option: value},
+    "params": {"T": float, "route_T": {cell: float}, "route_counts": {cell: int}, "prior": {...}}}}."""
+    need(isinstance(block, dict) and set(block) <= set(KINDS), 'POSTCAL_TYPES')
+    for kind, entry in block.items():
+        need(isinstance(entry, dict) and set(entry) == {'config', 'params'}, 'POSTCAL_ENTRY')
+        cfg, params = entry['config'], entry['params']
+        need(isinstance(cfg, dict) and set(cfg) <= set(OPTIONS[kind]), 'POSTCAL_OPTIONS')
+        need(isinstance(params, dict) and set(params) <= {'T', 'prior', 'route_T', 'route_counts'}, 'POSTCAL_PARAMS')
+        need('debias' not in cfg or type(cfg['debias']) is bool, 'POSTCAL_SWITCH')
+        mode = cfg.get('temperature', False)
+        need(any(mode is m or (isinstance(m, str) and mode == m) for m in TEMPERATURE_MODES), 'POSTCAL_TEMPERATURE_MODE')
+        if mode:
+            need(_temperature_value(params.get('T')), 'POSTCAL_TEMPERATURE')
+        if mode == 'route':
+            cells = params.get('route_T')
+            need(isinstance(cells, dict) and set(cells) <= set(ROUTE_BUCKETS)
+                 and all(_temperature_value(v) for v in cells.values()), 'POSTCAL_ROUTE_TEMPERATURE')
+        else:
+            need('route_T' not in params, 'POSTCAL_ROUTE_TEMPERATURE_UNUSED')
+        counts = params.get('route_counts')
+        need(counts is None or (isinstance(counts, dict) and set(counts) <= set(ROUTE_BUCKETS)
+                                and all(type(v) is int and v >= 0 for v in counts.values())), 'POSTCAL_ROUTE_COUNTS')
+        if cfg.get('debias'):
+            prior = params.get('prior')
+            need(isinstance(prior, dict) and all(
+                k.isdigit() and int(k) >= 2 and isinstance(v, list) and len(v) == int(k)
+                and all(finite(w) and w > 0 for w in v) for k, v in prior.items()), 'POSTCAL_PRIOR')
+        if cfg.get('floor') is not None:
+            need(finite(cfg['floor']) and .8 < cfg['floor'] < 1, 'POSTCAL_FLOOR')
+        if 'mode_weight' in cfg:
+            need(finite(cfg['mode_weight']) and 0 <= cfg['mode_weight'] <= 1, 'POSTCAL_MODE_WEIGHT')
+    return block
+
+
+def active(block):
+    """{type: [active option names]} for the types this block changes."""
+    return {kind: sorted(k for k, v in entry['config'].items() if v)
+            for kind, entry in sorted(block.items()) if n_active(entry['config'])}
+
+
+# ---------------------------------------------------------------- service hook
+def wrap_service(service, block, core):
+    """Calibrate each answer after the service computed it (same order as the measured setup:
+    after the deadline wrapper and its item rules, before the answer is serialised)."""
+    policy = validate(block)
+    need(not getattr(service, '_post_calibration_wrapped', False), 'POSTCAL_ALREADY_WRAPPED')
+    original = service._one_impl
+
+    def one(task):
+        answer, detail = original(task)
+        kind = task['question']['type']
+        entry = policy.get(kind)
+        if entry is None or not n_active(entry['config']):
+            return answer, detail
+        labels = core.render(task)[0]
+        if len(labels) < 2:
+            return answer, detail                        # one option or level: nothing to calibrate
+        probs = answer['probabilities']
+        if set(probs) == set(labels):
+            keys = list(labels)
+        elif kind == 'noul' and set(probs) == {'false', 'true'}:
+            keys = ['false', 'true']                     # the service's no/yes order
+        else:
+            raise core.ItemFailure('post-calibration label mismatch')
+        bucket = route_bucket(detail)
+        try:
+            q = apply_one(kind, entry['config'], entry['params'], [probs[k] for k in keys], bucket)
+        except (RuntimeError, ValueError, ZeroDivisionError, OverflowError) as exc:
+            raise core.ItemFailure('post-calibration failed') from exc
+        calibrated = dict(zip(keys, q))
+        answer['probabilities'] = {k: calibrated[k] for k in probs}
+        by_label = dict(zip(labels, q))
+        detail['prediction'] = max(labels, key=by_label.get)
+        if kind == 'noul':
+            answer['noul'] = float(calibrated.get('yes', calibrated.get('true')))
+        elif kind == 'choice':
+            answer['choice'] = detail['prediction']
+        detail['post_calibration'] = dict(options=sorted(k for k, v in entry['config'].items() if v), route=bucket)
+        return answer, detail
+
+    service._one_impl = one
+    service._post_calibration_wrapped = True
+    return service

service/prepare_model.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# NETWORKED maintainer preparation, never called by serve.sh.
+# Downloads the pinned revision: the official base files at the folder root and the unmerged LoRA in adapter/.
+set -euo pipefail
+PKG=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
+exec "${PYTHON:-python3}" -B - "$PKG" "${1:?new absolute external model directory}" <<'PY'
+import json,re,sys
+from pathlib import Path
+pkg=Path(sys.argv[1]).resolve(); dest=Path(sys.argv[2])
+m=json.loads((pkg/'MODEL_MANIFEST.json').read_text())
+if not dest.is_absolute():raise SystemExit('Use an absolute destination')
+dest=dest.resolve()
+if dest.exists() or dest.is_relative_to(pkg) or pkg.is_relative_to(dest):raise SystemExit('Use a new separate model directory')
+if m['repo_id'].startswith('OWNER/') or not re.fullmatch('[0-9a-f]{40}',str(m.get('revision') or '')):
+    raise SystemExit('Set the own Hugging Face repository and a complete 40-character revision in MODEL_MANIFEST.json')
+if not m['files'] or not m['adapter']['files']:raise SystemExit('Model hash manifest is pending')
+patterns=[*m['files'],*(m['adapter']['subdir']+'/'+name for name in m['adapter']['files'])]
+from huggingface_hub import snapshot_download
+snapshot_download(repo_id=m['repo_id'],revision=m['revision'],local_dir=dest,allow_patterns=patterns)
+sys.path.insert(0,str(pkg))
+from launch import check_model
+check_model(dest,full=True)
+print('Downloaded base and adapter files match the pinned manifest; no inference or submission performed.')
+PY

service/quota.py

@@ -0,0 +1,454 @@
+"""Durable, conservative per-decision slow-path quota.
+
+The lock spans inference: a slow attempt is charged before work starts, while
+the denominator grows only after successful work. A crash therefore cannot
+create quota credit. One counter file represents one global service history.
+"""
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass
+from decimal import Decimal
+import errno
+import json
+import math
+import os
+from pathlib import Path
+import tempfile
+import threading
+import time
+from typing import Iterator
+
+
+class QuotaStateError(RuntimeError):
+    """Counter storage is unreadable or invalid; inference must fail closed."""
+
+
+@dataclass
+class QuotaDecision:
+    path: str
+    reason: str
+    snapshot: dict
+
+
+class _PathLock:
+    def __init__(self) -> None:
+        self.mutex = threading.RLock()
+        self.local = threading.local()
+
+
+_REGISTRY_MUTEX = threading.Lock()
+_PATH_LOCKS: dict[str, _PathLock] = {}
+_ROOT = Path(__file__).resolve().parent
+
+
+@contextmanager
+def _os_lock(path: Path) -> Iterator[None]:
+    # Keep the lock file permanently: unlinking it would let another process
+    # lock a different inode while an existing process still holds this lock.
+    with path.open("a+b") as handle:
+        if os.name == "nt":
+            import msvcrt
+
+            handle.seek(0, os.SEEK_END)
+            if handle.tell() == 0:
+                handle.write(b"\0")
+                handle.flush()
+            while True:
+                handle.seek(0)
+                try:
+                    msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+                    break
+                except OSError as exc:
+                    if exc.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
+                        raise
+                    time.sleep(0.05)
+            try:
+                yield
+            finally:
+                handle.seek(0)
+                msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+        else:
+            import fcntl
+
+            fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
+            try:
+                yield
+            finally:
+                fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+
+
+def _fsync_directory(path: Path) -> None:
+    if os.name == "nt":
+        return  # Windows does not expose a directory fsync via os.open.
+    descriptor = os.open(str(path), os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
+    try:
+        try:
+            os.fsync(descriptor)
+        except OSError as exc:
+            if exc.errno not in (errno.EINVAL, errno.ENOTSUP):
+                raise
+    finally:
+        os.close(descriptor)
+
+
+def cost_settings(config):
+    settings = config['routing'].get('cost_fuse', {'enabled': False})
+    prices = config['pricing']['models'][config['pricing']['model_key']]
+    known = all(isinstance(prices.get(k), (int, float)) and not isinstance(prices[k], bool)
+                and math.isfinite(prices[k]) and prices[k] >= 0
+                for k in ('input_per_million', 'output_per_million'))
+    if settings.get('enabled', True) and not known and config['routing']['mode'] == 'routed':
+        raise ValueError('Routed cost fuse requires known profile prices; explicitly use --cost-fuse off')
+    return settings, prices, known
+
+
+class CostFuse:
+    def __init__(self, limit_per_1000, *, warmup=100, arm=.85, release=.80, state=None, enabled=True):
+        if not math.isfinite(limit_per_1000) or limit_per_1000 <= 0 or not 0 < release < arm <= 1:
+            raise ValueError('Invalid cost fuse limits')
+        if type(warmup) is not int or warmup < 100: raise ValueError('Cost warmup must be at least 100')
+        self.limit = limit_per_1000 / 1000
+        self.warmup, self.arm, self.release, self.enabled = warmup, arm, release, enabled
+        self.s = dict(n=0, cost=0., n_slow=0, cost_slow=0., engaged=False, transitions=0) if state is None else dict(state)
+        v=self.s
+        if (any(type(v.get(k)) is not int or v[k]<0 for k in ('n','n_slow','transitions'))
+            or v['n_slow']>v['n'] or type(v.get('engaged')) is not bool
+            or any(not isinstance(v.get(k),(int,float)) or not math.isfinite(v[k]) or v[k]<0 for k in ('cost','cost_slow'))
+            or v['cost_slow']>v['cost']+1e-12): raise QuotaStateError('Invalid cost fuse state')
+
+    def state(self):
+        return 'off' if not self.enabled else 'observe' if self.s['n'] < self.warmup else 'engaged' if self.s['engaged'] else 'idle'
+
+    def quota(self, tier_quota):
+        s=self.s
+        if self.state()!='engaged': return tier_quota,'tier'
+        n_fast=s['n']-s['n_slow']
+        fast=(s['cost']-s['cost_slow'])/n_fast if n_fast else 0.
+        slow=s['cost_slow']/s['n_slow'] if s['n_slow'] else None
+        # If even fast exceeds target, remove slow admissions; never alter fast.
+        q=0. if fast>=self.release*self.limit else (
+            max(0.,(self.release*self.limit-fast)/(slow-fast)) if slow is not None and slow>fast else tier_quota)
+        return (q,'cost') if q<tier_quota else (tier_quota,'tier')
+
+    def record(self, cost_usd, slow):
+        if not math.isfinite(cost_usd) or cost_usd<0: raise ValueError('Invalid observed cost')
+        s=self.s;s['n']+=1;s['cost']+=cost_usd
+        if slow:s['n_slow']+=1;s['cost_slow']+=cost_usd
+        if not self.enabled or s['n']<self.warmup:return
+        running=s['cost']/s['n'];old=s['engaged']
+        if not old and running>=self.arm*self.limit:s['engaged']=True
+        elif old and running<=self.release*self.limit:s['engaged']=False
+        if old!=s['engaged']:s['transitions']+=1
+
+    def snapshot(self, quota):
+        q,source=self.quota(quota)
+        return dict(state=self.state(),n=self.s['n'],running_cost_per_1000=self.s['cost']/self.s['n']*1000 if self.s['n'] else 0.,
+                    effective_quota=q,quota_source=source,transitions=self.s['transitions'],cap_per_1000=self.limit*1000)
+
+
+def configure_cost(counter, config):
+    settings,prices,known=cost_settings(config)
+    counter.cost_prices=prices if known else None
+    counter.cost_options=dict(limit_per_1000=config['limits']['cost_per_1000_decisions_usd']*(1-config['pricing']['margin']),
+        warmup=settings.get('warmup',100),arm=settings.get('arm',.85),release=settings.get('release',.80),
+        enabled=settings.get('enabled',True) and known and config['routing']['mode']=='routed')
+    counter.cost_fuse=CostFuse(**counter.cost_options)
+    counter.tier_quota=config['routing']['quota']
+
+
+class QuotaCounter:
+    """Serialize decisions across threads/processes sharing ``state_path``.
+
+    ``quota`` and ``fuse`` must come from the caller's configuration. Uniform
+    experiments can explicitly use quota=fuse=1; an ordinary routed service
+    should use its configured limits. Persisted slow attempts include failures.
+    """
+
+    def __init__(self, state_path: str | Path, *, quota: float, fuse: float):
+        if (
+            isinstance(quota, bool)
+            or isinstance(fuse, bool)
+            or not math.isfinite(quota)
+            or not math.isfinite(fuse)
+            or not 0 <= quota <= fuse <= 1
+        ):
+            raise ValueError("Require finite 0 <= quota <= fuse <= 1")
+        self.quota = float(quota)
+        self.fuse = float(fuse)
+        self._quota = Decimal(str(quota))
+        self._fuse = Decimal(str(fuse))
+        target = Path(state_path)
+        if not target.is_absolute():
+            raise QuotaStateError("Quota state path must be absolute")
+        # Symbolic links are resolved first; state and lock must stay beside each other, outside the package.
+        self.state_path = target.resolve()
+        if self.state_path.is_relative_to(_ROOT) or _ROOT.is_relative_to(self.state_path):
+            raise QuotaStateError("Quota state must be separate from the package")
+        self.lock_path = self.state_path.with_name(self.state_path.name + ".lock")
+        if self.lock_path.resolve().parent != self.state_path.parent:
+            raise QuotaStateError("Quota lock must stay inside its state directory")
+        key = os.path.normcase(str(self.state_path))
+        with _REGISTRY_MUTEX:
+            self._lock = _PATH_LOCKS.setdefault(key, _PathLock())
+        self.state_path.parent.mkdir(parents=True, exist_ok=True)
+        with self._exclusive():
+            if self.state_path.exists():
+                self._read()
+            else:
+                self._write({"version": 2, "total": 0, "slow": 0})
+
+    @contextmanager
+    def _exclusive(self) -> Iterator[None]:
+        with self._lock.mutex:
+            depth = getattr(self._lock.local, "depth", 0)
+            if depth:
+                self._lock.local.depth = depth + 1
+                try:
+                    yield
+                finally:
+                    self._lock.local.depth = depth
+            else:
+                with _os_lock(self.lock_path):
+                    self._lock.local.depth = 1
+                    try:
+                        yield
+                    finally:
+                        self._lock.local.depth = 0
+
+    def _read(self) -> dict:
+        try:
+            with self.state_path.open("r", encoding="utf-8") as handle:
+                state = json.load(handle)
+        except (OSError, ValueError) as exc:
+            raise QuotaStateError("Cannot read quota state; refusing to reset it") from exc
+        if (
+            not isinstance(state, dict)
+            or type(state.get("version")) is not int
+            or state["version"] not in (1, 2)
+            or any(type(state.get(key)) is not int or state[key] < 0 for key in ("total", "slow"))
+        ):
+            raise QuotaStateError("Invalid quota state; refusing to reset it")
+        # slow may exceed total after a failed or interrupted slow attempt.
+        return state
+
+    def _write(self, state: dict) -> None:
+        temporary: str | None = None
+        try:
+            with tempfile.NamedTemporaryFile(
+                mode="w", encoding="utf-8", dir=self.state_path.parent,
+                prefix=self.state_path.name + ".", suffix=".tmp", delete=False,
+            ) as handle:
+                temporary = handle.name
+                json.dump(state, handle, sort_keys=True, allow_nan=False)
+                handle.write("\n")
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary, self.state_path)
+            temporary = None
+            _fsync_directory(self.state_path.parent)
+        except OSError as exc:
+            raise QuotaStateError("Cannot persist quota state; refusing inference") from exc
+        finally:
+            if temporary is not None:
+                try:
+                    Path(temporary).unlink(missing_ok=True)
+                except OSError:
+                    pass
+
+    @staticmethod
+    def _snapshot(state: dict) -> dict:
+        total, slow = state["total"], state["slow"]
+        return {
+            "total": total,
+            "slow": slow,
+            "slow_fraction": slow / total if total else float(slow > 0),
+        }
+
+    def snapshot(self) -> dict:
+        with self._exclusive():
+            return self._snapshot(self._read())
+
+    def configure_cost(self, config):
+        configure_cost(self, config)
+        with self._exclusive():self.cost_fuse=CostFuse(**self.cost_options,state=self._read().get('cost_fuse'))
+
+    @contextmanager
+    def accounting(self):
+        with self._exclusive():
+            state=self._read()
+            self.cost_fuse=CostFuse(**self.cost_options,state=state.get('cost_fuse'))
+            yield
+
+    def record_cost(self, usage, slow):
+        with self._exclusive():
+            before=self.cost_fuse.snapshot(self.tier_quota)
+            if self.cost_prices is not None:
+                p=self.cost_prices
+                self.cost_fuse.record((usage['input_tokens']*p['input_per_million']+usage['output_tokens']*p['output_per_million'])/1e6,slow)
+            after=self.cost_fuse.snapshot(self.tier_quota)
+            state=self._read();state.update(version=2,cost_fuse=self.cost_fuse.s);self._write(state)
+            if any(before[k]!=after[k] for k in ('state','effective_quota')):
+                print('cost_fuse '+json.dumps(after,sort_keys=True),flush=True)
+            return after
+
+    @contextmanager
+    def decision(self, wants_slow: bool, quota=None) -> Iterator[QuotaDecision]:
+        """Reserve, run caller work, then commit only on normal context exit.
+
+        Use ``with counter.decision(wants_slow) as decision`` and run inference
+        inside that context. Propagate failures out of the block: swallowing an
+        inference failure would incorrectly record a successful decision.
+        """
+        with self._exclusive():
+            if getattr(self._lock.local, "active_decision", False):
+                raise RuntimeError("Nested quota decisions are not supported")
+            self._lock.local.active_decision = True
+            try:
+                state = self._read()
+                total, slow = state["total"], state["slow"]
+                if Decimal(slow) > self._fuse * total:
+                    path, reason = "fast", "fuse"
+                elif not wants_slow:
+                    path, reason = "fast", "router"
+                elif Decimal(slow + 1) > self._fuse * (total + 1):
+                    path, reason = "fast", "fuse"
+                elif Decimal(slow + 1) > min(self._quota, Decimal(str(quota)) if quota is not None else self._quota) * (total + 1):
+                    path, reason = "fast", "quota"
+                else:
+                    path, reason = "slow", "eligible"
+                    state["slow"] += 1
+                    self._write(state)
+                decision = QuotaDecision(path, reason, self._snapshot(state))
+                yield decision
+                state["total"] += 1
+                self._write(state)
+                decision.snapshot = self._snapshot(state)
+            finally:
+                self._lock.local.active_decision = False
+
+class RuntimeCounter:
+    """Run-scoped durable counter: startup IO failures abort; later IO failures use fast only."""
+    def __init__(self, config):
+        import logging
+        import uuid
+        routing = config['routing']
+        self.run_id = routing.get('run_id') or uuid.uuid4().hex
+        if Path(self.run_id).name != self.run_id or self.run_id in ('.', '..'):
+            raise ValueError('run_id must be a single directory name')
+        self.memory_only = False
+        self._mutex = threading.RLock()
+        self._state = {'total':0, 'slow':0}
+        self.durable = None
+        if not routing.get('state_dir') or not Path(routing['state_dir']).is_absolute():
+            raise QuotaStateError('An absolute quota state directory is required; refusing startup')
+        state = Path(routing['state_dir']).resolve()
+        if state.is_relative_to(_ROOT) or _ROOT.is_relative_to(state):
+            raise QuotaStateError('Quota state directory must be separate from the package')
+        directory = (state / self.run_id).resolve()
+        if directory.parent != state:
+            raise QuotaStateError('Run directory must stay inside the state directory')
+        for name in ('run.json', 'quota.json', 'quota.json.lock'):
+            if (directory / name).resolve().parent != directory:
+                raise QuotaStateError('Run files must stay inside the run directory')
+        fingerprint = {k:routing[k] for k in ('mode','quota','fuse')}
+        fingerprint.update(model_key=config['model']['model_key'], run_id=self.run_id,
+            tier=routing.get('tier'), tiers=routing.get('tiers'), cost_fuse=routing.get('cost_fuse'),
+            pricing=config.get('pricing'), cost_cap=config['limits']['cost_per_1000_decisions_usd'],
+            basis=config.get('usage',{}).get('basis','engine'))
+        try:
+            if routing.get('resume'):
+                if not routing.get('run_id'):
+                    raise ValueError('resume requires an explicit run_id')
+                if not (directory/'quota.json').is_file() or not (directory/'run.json').is_file():
+                    raise ValueError('Resume state is missing')
+                if json.loads((directory/'run.json').read_text(encoding='utf-8')) != fingerprint:
+                    raise ValueError('Run fingerprint mismatch')
+            else:
+                if directory.exists():
+                    raise ValueError('run_id already exists; select a new run_id or explicit resume')
+                directory.mkdir(parents=True, exist_ok=False)
+                with (directory/'run.json').open('x', encoding='utf-8') as handle:
+                    handle.write(json.dumps(fingerprint))
+            self.durable = QuotaCounter(directory/'quota.json', quota=routing['quota'], fuse=routing['fuse'])
+            self._state = self.durable.snapshot()
+        except (OSError, QuotaStateError):
+            if routing.get('resume'):
+                raise ValueError('Resume state cannot be read or validated') from None
+            raise QuotaStateError('Quota persistence unavailable; refusing startup') from None
+
+    def configure_cost(self, config):
+        configure_cost(self, config)
+        if self.durable is not None:
+            self.durable.configure_cost(config)
+            self.cost_fuse=self.durable.cost_fuse
+
+    @contextmanager
+    def accounting(self):
+        import logging
+        with self._mutex:
+            context=None
+            if not self.memory_only:
+                try:
+                    context=self.durable.accounting();context.__enter__()
+                    self.cost_fuse=self.durable.cost_fuse
+                except (OSError,QuotaStateError):
+                    context=None;self.memory_only=True
+                    logging.warning('Cost state unavailable; memory counts and fast only')
+            try:yield
+            except BaseException as exc:
+                if context is not None:context.__exit__(type(exc),exc,exc.__traceback__)
+                raise
+            else:
+                if context is not None:context.__exit__(None,None,None)
+
+    def record_cost(self, usage, slow):
+        import logging
+        before=self.cost_fuse.s['n']
+        if not self.memory_only:
+            try:return self.durable.record_cost(usage,slow)
+            except (OSError,QuotaStateError):
+                self.memory_only=True
+                logging.warning('Cost commit unavailable; memory counts and fast only')
+        # A failed disk commit may already have updated this shared fuse object.
+        if self.cost_prices is not None and self.cost_fuse.s['n']==before:
+            p=self.cost_prices
+            self.cost_fuse.record((usage['input_tokens']*p['input_per_million']+usage['output_tokens']*p['output_per_million'])/1e6,slow)
+        return self.cost_fuse.snapshot(self.tier_quota)
+
+    def snapshot(self):
+        with self._mutex:
+            return dict(self._state)
+
+    @contextmanager
+    def decision(self, wants_slow, quota=None):
+        import logging
+        with self._mutex:
+            context = None
+            if not self.memory_only:
+                try:
+                    context = self.durable.decision(wants_slow, quota=quota)
+                    decision = context.__enter__()
+                    self._state = dict(decision.snapshot)
+                except (OSError, QuotaStateError):
+                    context = None
+                    self.memory_only = True
+                    logging.warning('Quota persistence failed; using memory counts and fast only')
+            if context is None:
+                decision = QuotaDecision('fast', 'memory_only', dict(self._state))
+            try:
+                yield decision
+            except BaseException as exc:
+                if context is not None:
+                    context.__exit__(type(exc), exc, exc.__traceback__)
+                raise
+            else:
+                self._state['total'] += 1
+                if context is not None:
+                    try:
+                        context.__exit__(None, None, None)
+                        self._state = dict(decision.snapshot)
+                    except (OSError, QuotaStateError):
+                        self.memory_only = True
+                        logging.warning('Quota commit failed; using memory counts and fast only')
+                decision.snapshot = dict(self._state)

service/reconcile_usage.py

@@ -0,0 +1,156 @@
+"""Usage reconciliation before release (maintainer tool): reads bench outputs and engine counter
+snapshots only; sends no inference requests and reads no request content.
+
+python3 -B reconcile_usage.py --run /abs/out/fast [--run-off /abs/out/routed_off] [--tol 0.001]
+Exit code: 0 passed; 1 an identity does not hold; 2 a precondition is not met (missing data or counters).
+"""
+import argparse, json, sys, math
+from pathlib import Path
+
+from core import external_path
+
+COUNTERS = ('prompt_tokens', 'generation_tokens', 'prefix_cache_queries', 'prefix_cache_hits',
+            'num_preemptions', 'request_success')
+FIELDS = ('input', 'output', 'total', 'cached', 'submitted', 'calls', 'generated')
+
+
+def resolve(prefix):
+    """Run prefixes are absolute paths outside the package (see bench.py --output-prefix)."""
+    return external_path(prefix)
+
+
+def load(prefix):
+    p = resolve(prefix)
+    rows = [json.loads(x) for x in p.with_name(p.name + '.jsonl').read_text(encoding='utf-8').splitlines() if x.strip()]
+    stats = json.loads(p.with_name(p.name + '.engine_stats.json').read_text(encoding='utf-8'))
+    rows = [r for r in rows if r.get('concurrency', 1) == 1]     # reconcile the serial level only
+    if not rows:
+        sys.exit(2)
+    return p, rows, stats
+
+
+def delta(stats):
+    level = stats['by_concurrency']['1']
+    before, after = level['before']['counters'], level['after']['counters']
+    missing = [k for k in COUNTERS if k not in before or k not in after]
+    if missing:
+        raise ValueError('engine counters missing: '+','.join(missing))
+    if any(isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) for d in (before,after) for v in (d[k] for k in COUNTERS)):raise ValueError('Invalid counter')
+    return {k: after[k] - before[k] for k in COUNTERS}
+
+
+def row_usage(row):
+    u = row.get('usage') or {}
+    if not isinstance(u,dict):return False,{}
+    d = u.get('input_tokens_details') or {}
+    if not isinstance(d,dict):return False,{}
+    v = dict(input=u.get('input_tokens'), output=u.get('output_tokens'), total=u.get('total_tokens'),
+             cached=d.get('cached_tokens'), submitted=d.get('engine_submitted_tokens'),
+             calls=d.get('engine_calls'), generated=d.get('engine_generated_tokens'))
+    ok = (row.get('ok') and u.get('accounting') == 'engine' and not u.get('usage_estimated')
+          and d.get('cached_tokens_known', True) and all(type(v[k]) is int and v[k] >= 0 for k in FIELDS))
+    return bool(ok), v
+
+
+def close(x, y, tol):
+    return abs(x - y) <= tol * max(1, abs(y))
+
+
+def reconcile_one(rows, stats, tol):
+    fails, parsed = [], [row_usage(r) for r in rows]
+    if not rows or len({r.get('id') for r in rows})!=len(rows) or any(r.get('concurrency',1)!=1 for r in rows) or not all(ok for ok, _ in parsed):
+        return {'precondition': 'all rows ok, engine basis, exact usage'}, ['precondition']
+    U = [v for _, v in parsed]
+    S = {k: sum(v[k] for v in U) for k in FIELDS}
+    if S['generated']!=S['output']:fails.append('generated/output identity broken')
+    level=stats.get('by_concurrency',{}).get('1',{})
+    source=level.get('after',{}).get('source')
+    if source=='hf_forward_hook':
+        try:
+            before,after=level['before']['counters'],level['after']['counters']
+            if after.get('uncounted_forwards',0)>before.get('uncounted_forwards',0):
+                return {'precondition':'HF uncounted forwards'},['precondition']
+            D={k:after[k]-before[k] for k in ('prefill_tokens','forward_tokens')}
+            ds=[r['usage']['input_tokens_details'] for r in rows]
+            prefill=sum(d['hf_prefill_tokens'] for d in ds)
+            calls=sum(d['generate_nonempty_calls'] for d in ds)
+        except (KeyError,TypeError): return {'precondition':'HF counters missing'},['precondition']
+        if S['cached'] or any(v['input']!=v['submitted'] or v['total']!=v['input']+v['output'] for v in U): fails.append('HF row identity broken')
+        if D['prefill_tokens']!=prefill: fails.append('HF initial prefill mismatch')
+        if D['forward_tokens']!=S['input']+S['output']-calls: fails.append('HF forward identity broken')
+        return dict(sums=S,engine_delta=D,counter_mode='hf_forward'),fails
+    try: D = delta(stats)
+    except (KeyError,ValueError,TypeError): return {'precondition':'engine counters missing'},['precondition']
+    if any(v<0 for v in D.values()): return {'precondition':'counter reset'},['precondition']
+    if D['num_preemptions']:
+        return {'precondition':'preemption during run'},['precondition']
+    if D['request_success'] != S['calls']:
+        return {'precondition':'stray, warmup or missed engine calls'},['precondition']
+    bad = [r['id'] for r, v in zip(rows, U) if v['input'] != v['submitted'] - v['cached'] or v['total'] != v['input'] + v['output']]
+    if bad:
+        fails.append('row identity broken: %s' % bad[:5])
+    if close(S['submitted'], D['prompt_tokens'], tol):
+        mode, prefill = 'full_prompt', D['prompt_tokens'] - D['prefix_cache_hits']
+    elif close(S['submitted'] - S['cached'], D['prompt_tokens'], tol):
+        mode, prefill = 'computed_only', D['prompt_tokens']
+    else:
+        mode, prefill = 'unknown', None
+        fails.append('sum(engine_submitted) matches neither engine prompt-counter definition')
+    if not close(S['cached'], D['prefix_cache_hits'], tol):
+        fails.append('sum(cached_tokens) != delta prefix_cache_hits')
+    if prefill is not None and not close(S['input'], prefill, tol):            # identity E1
+        fails.append('E1: sum(usage.input_tokens) != engine prefill')
+    if not close(S['output'], D['generation_tokens'], tol):
+        fails.append('sum(usage.output_tokens) != delta generation_tokens')
+    return dict(sums=S, engine_delta=D, counter_mode=mode, engine_prefill=prefill), fails
+
+
+def reconcile_pair(on_rows, off_rows, off_stats, tol):
+    fails = []
+    if off_stats.get('health', {}).get('prefix_caching') is not False:
+        fails.append('off run did not disable prefix caching')
+    if not all(row_usage(r)[0] for r in on_rows+off_rows): return {},['precondition']
+    if len({r['id'] for r in on_rows})!=len(on_rows) or len({r['id'] for r in off_rows})!=len(off_rows): return {},['precondition']
+    on = {r['id']: row_usage(r)[1] for r in on_rows}
+    off = {r['id']: row_usage(r)[1] for r in off_rows}
+    if any(v['cached'] for v in off.values()):
+        fails.append('off run reports cached tokens')
+    matched = [i for i in on if i in off and on[i]['submitted'] == off[i]['submitted'] and on[i]['output'] == off[i]['output']]
+    coverage = len(matched) / max(1, len(on),len(off))
+    if coverage < 0.99:
+        fails.append('decode drift: only %.1f%% rows identical between runs' % (100 * coverage))
+    diff = sum(off[i]['input'] - on[i]['input'] for i in matched)
+    cached = sum(on[i]['cached'] for i in matched)
+    if not close(diff, cached, tol):                                             # identity E2
+        fails.append('E2: sum(input_off) - sum(input_on) != sum(cached_on)')
+    return dict(matched=len(matched), coverage=coverage, input_diff=diff, cached_on=cached), fails
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('--run', required=True)
+    ap.add_argument('--run-off')
+    ap.add_argument('--tol', type=float, default=0.001)
+    a = ap.parse_args(argv)
+    if not 0<=a.tol<=0.001: ap.error('tol must be within [0, 0.001]')
+    try: path, rows, stats = load(a.run)
+    except (OSError,ValueError,KeyError) as exc:
+        print('precondition: '+str(exc), file=sys.stderr); return 2
+    report, fails = reconcile_one(rows, stats, a.tol)
+    if a.run_off:
+        try: _, off_rows, off_stats = load(a.run_off)
+        except (OSError,ValueError,KeyError) as exc:
+            print('precondition: '+str(exc), file=sys.stderr); return 2
+        off_report, off_fails = reconcile_one(off_rows, off_stats, a.tol)
+        pair, pair_fails = reconcile_pair(rows, off_rows, off_stats, a.tol)
+        report.update(off=off_report, pair=pair)
+        fails += ['off: ' + f for f in off_fails] + pair_fails
+    report.update(tolerance=a.tol, passed=not fails, submission_ready=bool(a.run_off) and not fails, failures=fails)
+    with path.with_name(path.name + '.reconcile.json').open('x', encoding='utf-8') as out:   # exclusive create, never overwrite
+        json.dump(report, out, ensure_ascii=False, indent=2, allow_nan=False)
+    print(json.dumps({'passed': not fails, 'failures': fails}, ensure_ascii=False))
+    return 0 if not fails else (2 if any('precondition' in f for f in fails) else 1)
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

service/requirements.lock

@@ -0,0 +1,204 @@
+# Measured version lock; versions only, no artifact hashes.
+# All measured entries retained; torch CUDA build made explicit.
+--extra-index-url https://download.pytorch.org/whl/cu130
+agent-detector==2.0.0
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anthropic==1.11.0
+anyio==4.15.1
+apache-tvm-ffi==0.1.11
+astor==0.8.1
+async-timeout==5.0.1
+attrs==26.1.0
+backports.strenum==1.3.1
+blake3==1.0.10
+cachetools==7.2.0
+cbor2==6.1.5
+certifi==2026.7.22
+cffi==2.1.1
+charset-normalizer==3.5.2
+click==8.5.0
+cloudpickle==3.1.2
+compressed-tensors==0.17.0
+cryptography==50.0.2
+cuda-bindings==13.4.3
+cuda-core==1.2.1
+cuda-pathfinder==1.8.3
+cuda-python==13.4.1
+cuda-tile==1.6.0
+cuda-toolkit==13.0.3.0
+depyf==0.20.0
+detect-installer==0.2.1
+dill==0.4.1
+dnspython==2.8.0
+docstring_parser==0.18.0
+einops==0.8.2
+email-validator==2.3.0
+exceptiongroup==1.3.1
+fastapi==0.136.3
+fastapi-cli==0.0.32
+fastapi-cloud-cli==0.26.0
+fastar==0.12.0
+fastsafetensors==0.4.0
+filelock==4.0.12
+flashinfer-python==0.7.0.post1
+frozenlist==1.8.0
+fsspec==2026.9.0
+googleapis-common-protos==1.75.5
+grpcio==1.84.0
+h11==0.16.0
+hf-xet==1.6.0
+httpcore==1.0.9
+httpcore2==2.13.1
+httptools==0.8.0
+httpx==0.28.1
+httpx2==2.13.1
+huggingface_hub==1.33.0
+humming-kernels==0.1.16
+idna==3.20
+ijson==3.5.1
+instanttensor==0.2.0
+interegular==0.3.3
+Jinja2==3.1.6
+jiter==0.17.0
+jmespath==1.1.0
+jsonschema==4.26.0
+jsonschema-specifications==2025.9.1
+lark==1.2.2
+llguidance==1.7.6
+llvmlite==0.47.0
+lm-format-enforcer==0.11.3
+loguru==0.7.3
+markdown-it-py==4.2.0
+MarkupSafe==3.0.4
+mcp==2.3.0
+mcp-types==2.3.0
+mdurl==0.1.2
+mistral_common==1.12.0
+ml_dtypes==0.6.0
+model-hosting-container-standards==0.1.16
+mpmath==1.3.0
+msgspec==0.22.0
+multidict==6.9.1
+nccl-extensions==0.1.0
+nccl4py==0.6.0
+networkx==3.4.2
+ninja==1.13.2
+numba==0.65.0
+numpy==2.2.6
+nvidia-cublas==13.1.1.3
+nvidia-cuda-cccl==13.3.4.3.1
+nvidia-cuda-crt==13.4.92
+nvidia-cuda-cupti==13.0.85
+nvidia-cuda-nvcc==13.4.92
+nvidia-cuda-nvdisasm==13.4.92
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-runtime==13.0.96
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cudnn-frontend==1.30.0
+nvidia-cufft==12.0.0.61
+nvidia-cufile==1.15.1.6
+nvidia-curand==10.4.0.35
+nvidia-cusolver==12.0.4.66
+nvidia-cusparse==12.6.3.3
+nvidia-cusparselt-cu13==0.8.1
+nvidia-cutlass-dsl==4.7.1
+nvidia-cutlass-dsl-libs-base==4.7.1
+nvidia-cutlass-dsl-libs-core==4.7.1
+nvidia-cutlass-dsl-libs-cu12==4.7.1
+nvidia-cutlass-dsl-libs-cu13==4.7.1
+nvidia-ml-py==13.615.71
+nvidia-nccl-cu13==2.29.7
+nvidia-nvjitlink==13.4.92
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvtx==13.0.85
+nvidia-nvvm==13.4.92
+nvtx==0.2.15
+openai==3.24.0
+opencv-python-headless==5.0.0.93
+opentelemetry-api==1.45.0
+opentelemetry-exporter-http-transport==0.66b0
+opentelemetry-exporter-otlp==1.45.0
+opentelemetry-exporter-otlp-common==0.66b0
+opentelemetry-exporter-otlp-proto-common==1.45.0
+opentelemetry-exporter-otlp-proto-grpc==1.45.0
+opentelemetry-exporter-otlp-proto-http==1.45.0
+opentelemetry-proto==1.45.0
+opentelemetry-sdk==1.45.0
+opentelemetry-semantic-conventions==0.66b0
+opentelemetry-semantic-conventions-ai==0.5.1
+oss-harmony==0.0.11
+outlines_core==0.2.14
+packaging==26.3
+partial-json-parser==0.2.1.1.post7
+pillow==12.3.0
+prometheus-fastapi-instrumentator==8.1.0
+prometheus_client==0.26.0
+propcache==0.5.4
+protobuf==7.36.2
+psutil==7.2.2
+py-cpuinfo==9.0.0
+pybase64==1.5.1
+pycountry==26.2.16
+pycparser==3.0
+pydantic==2.13.5
+pydantic-extra-types==2.11.1
+pydantic-settings==2.15.0
+pydantic_core==2.46.5
+Pygments==2.21.0
+PyJWT==2.15.1
+pynvvideocodec==2.0.4
+python-dotenv==1.2.4
+python-json-logger==4.2.0
+python-multipart==0.0.32
+PyYAML==6.0.3
+pyzmq==27.2.0
+quack-kernels==0.6.5
+referencing==0.37.0
+regex==2026.9.29
+requests==2.34.2
+rich==15.0.0
+rich-toolkit==0.20.5
+rignore==0.8.1
+rpds-py==0.30.0
+safetensors==0.8.0
+sentencepiece==0.2.2
+sentry-sdk==2.71.0
+setproctitle==1.3.8
+shellingham==1.5.4
+sniffio==1.3.1
+sse-starlette==3.5.0
+starlette==1.7.0
+supervisor==4.3.0
+sympy==1.14.0
+tabulate==0.10.0
+tiktoken==0.14.0
+tilelang==0.1.12
+tokenizers==0.23.2
+tokenspeed-mla==0.1.8
+tokenspeed-triton==3.8.10.post20260920
+tomli==2.4.1
+torch==2.13.0+cu130
+torch_c_dlpack_ext==0.1.5
+torchaudio==2.11.0+cu130
+torchcodec==0.17.0+cu130
+torchvision==0.28.0+cu130
+tqdm==4.70.1
+transformers==5.17.0
+triton==3.7.1
+truststore==0.10.4
+typer==0.27.2
+typing-inspection==0.4.4
+typing_extensions==4.16.0
+urllib3==2.8.0
+uvicorn==0.54.0
+uvloop==0.23.0
+vllm==0.31.0
+watchfiles==1.3.0
+websockets==16.1.1
+xgrammar==0.2.7
+yarl==1.25.1
+z3-solver==4.15.4.0

service/requirements.measured.lock

@@ -0,0 +1,201 @@
+agent-detector==2.0.0
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anthropic==1.11.0
+anyio==4.15.1
+apache-tvm-ffi==0.1.11
+astor==0.8.1
+async-timeout==5.0.1
+attrs==26.1.0
+backports.strenum==1.3.1
+blake3==1.0.10
+cachetools==7.2.0
+cbor2==6.1.5
+certifi==2026.7.22
+cffi==2.1.1
+charset-normalizer==3.5.2
+click==8.5.0
+cloudpickle==3.1.2
+compressed-tensors==0.17.0
+cryptography==50.0.2
+cuda-bindings==13.4.3
+cuda-core==1.2.1
+cuda-pathfinder==1.8.3
+cuda-python==13.4.1
+cuda-tile==1.6.0
+cuda-toolkit==13.0.3.0
+depyf==0.20.0
+detect-installer==0.2.1
+dill==0.4.1
+dnspython==2.8.0
+docstring_parser==0.18.0
+einops==0.8.2
+email-validator==2.3.0
+exceptiongroup==1.3.1
+fastapi==0.136.3
+fastapi-cli==0.0.32
+fastapi-cloud-cli==0.26.0
+fastar==0.12.0
+fastsafetensors==0.4.0
+filelock==4.0.12
+flashinfer-python==0.7.0.post1
+frozenlist==1.8.0
+fsspec==2026.9.0
+googleapis-common-protos==1.75.5
+grpcio==1.84.0
+h11==0.16.0
+hf-xet==1.6.0
+httpcore==1.0.9
+httpcore2==2.13.1
+httptools==0.8.0
+httpx==0.28.1
+httpx2==2.13.1
+huggingface_hub==1.33.0
+humming-kernels==0.1.16
+idna==3.20
+ijson==3.5.1
+instanttensor==0.2.0
+interegular==0.3.3
+Jinja2==3.1.6
+jiter==0.17.0
+jmespath==1.1.0
+jsonschema==4.26.0
+jsonschema-specifications==2025.9.1
+lark==1.2.2
+llguidance==1.7.6
+llvmlite==0.47.0
+lm-format-enforcer==0.11.3
+loguru==0.7.3
+markdown-it-py==4.2.0
+MarkupSafe==3.0.4
+mcp==2.3.0
+mcp-types==2.3.0
+mdurl==0.1.2
+mistral_common==1.12.0
+ml_dtypes==0.6.0
+model-hosting-container-standards==0.1.16
+mpmath==1.3.0
+msgspec==0.22.0
+multidict==6.9.1
+nccl-extensions==0.1.0
+nccl4py==0.6.0
+networkx==3.4.2
+ninja==1.13.2
+numba==0.65.0
+numpy==2.2.6
+nvidia-cublas==13.1.1.3
+nvidia-cuda-cccl==13.3.4.3.1
+nvidia-cuda-crt==13.4.92
+nvidia-cuda-cupti==13.0.85
+nvidia-cuda-nvcc==13.4.92
+nvidia-cuda-nvdisasm==13.4.92
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-runtime==13.0.96
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cudnn-frontend==1.30.0
+nvidia-cufft==12.0.0.61
+nvidia-cufile==1.15.1.6
+nvidia-curand==10.4.0.35
+nvidia-cusolver==12.0.4.66
+nvidia-cusparse==12.6.3.3
+nvidia-cusparselt-cu13==0.8.1
+nvidia-cutlass-dsl==4.7.1
+nvidia-cutlass-dsl-libs-base==4.7.1
+nvidia-cutlass-dsl-libs-core==4.7.1
+nvidia-cutlass-dsl-libs-cu12==4.7.1
+nvidia-cutlass-dsl-libs-cu13==4.7.1
+nvidia-ml-py==13.615.71
+nvidia-nccl-cu13==2.29.7
+nvidia-nvjitlink==13.4.92
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvtx==13.0.85
+nvidia-nvvm==13.4.92
+nvtx==0.2.15
+openai==3.24.0
+opencv-python-headless==5.0.0.93
+opentelemetry-api==1.45.0
+opentelemetry-exporter-http-transport==0.66b0
+opentelemetry-exporter-otlp==1.45.0
+opentelemetry-exporter-otlp-common==0.66b0
+opentelemetry-exporter-otlp-proto-common==1.45.0
+opentelemetry-exporter-otlp-proto-grpc==1.45.0
+opentelemetry-exporter-otlp-proto-http==1.45.0
+opentelemetry-proto==1.45.0
+opentelemetry-sdk==1.45.0
+opentelemetry-semantic-conventions==0.66b0
+opentelemetry-semantic-conventions-ai==0.5.1
+oss-harmony==0.0.11
+outlines_core==0.2.14
+packaging==26.3
+partial-json-parser==0.2.1.1.post7
+pillow==12.3.0
+prometheus-fastapi-instrumentator==8.1.0
+prometheus_client==0.26.0
+propcache==0.5.4
+protobuf==7.36.2
+psutil==7.2.2
+py-cpuinfo==9.0.0
+pybase64==1.5.1
+pycountry==26.2.16
+pycparser==3.0
+pydantic==2.13.5
+pydantic-extra-types==2.11.1
+pydantic-settings==2.15.0
+pydantic_core==2.46.5
+Pygments==2.21.0
+PyJWT==2.15.1
+pynvvideocodec==2.0.4
+python-dotenv==1.2.4
+python-json-logger==4.2.0
+python-multipart==0.0.32
+PyYAML==6.0.3
+pyzmq==27.2.0
+quack-kernels==0.6.5
+referencing==0.37.0
+regex==2026.9.29
+requests==2.34.2
+rich==15.0.0
+rich-toolkit==0.20.5
+rignore==0.8.1
+rpds-py==0.30.0
+safetensors==0.8.0
+sentencepiece==0.2.2
+sentry-sdk==2.71.0
+setproctitle==1.3.8
+shellingham==1.5.4
+sniffio==1.3.1
+sse-starlette==3.5.0
+starlette==1.7.0
+supervisor==4.3.0
+sympy==1.14.0
+tabulate==0.10.0
+tiktoken==0.14.0
+tilelang==0.1.12
+tokenizers==0.23.2
+tokenspeed-mla==0.1.8
+tokenspeed-triton==3.8.10.post20260920
+tomli==2.4.1
+torch==2.13.0+cu130
+torch_c_dlpack_ext==0.1.5
+torchaudio==2.11.0+cu130
+torchcodec==0.17.0+cu130
+torchvision==0.28.0+cu130
+tqdm==4.70.1
+transformers==5.17.0
+triton==3.7.1
+truststore==0.10.4
+typer==0.27.2
+typing-inspection==0.4.4
+typing_extensions==4.16.0
+urllib3==2.8.0
+uvicorn==0.54.0
+uvloop==0.23.0
+vllm==0.31.0
+watchfiles==1.3.0
+websockets==16.1.1
+xgrammar==0.2.7
+yarl==1.25.1
+z3-solver==4.15.4.0

service/selftest.py

@@ -0,0 +1,286 @@
+"""Synthetic offline selftest. Never imports a model runtime or opens a socket."""
+import argparse
+import copy
+import hashlib
+import json
+import math
+import re
+import socket
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+import backends
+import core
+import launch
+import pkg_check
+
+
+class WordTokenizer:
+    """Reversible synthetic tokenizer: one token per whitespace-prefixed word."""
+    def __init__(self):
+        self.ids, self.pieces = {}, []
+
+    def _id(self, piece):
+        if piece not in self.ids:
+            self.ids[piece] = len(self.pieces)
+            self.pieces.append(piece)
+        return self.ids[piece]
+
+    def encode(self, text, add_special_tokens=False):
+        return [self._id(p) for p in re.findall(r'\s*\S+|\s+', text)]
+
+    def decode(self, ids, skip_special_tokens=True):
+        return ''.join(self.pieces[i] for i in ids)
+
+    def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=True, **_):
+        return '<turn>user ' + messages[0]['content'] + ' <turn>model'
+
+
+def synthetic_task(words):
+    """Invented rules + narrative of about `words` tokens; a yes/no question about the rules."""
+    rules = 'Rules:\nR1: A crate is late if it reaches dock B after day 30.\nR2: Late crates are refunded.\n\n'
+    sentences, day = [], 0
+    while sum(len(s.split()) for s in sentences) < words:
+        day += 1
+        sentences.append('Day %d: the courier moved crate C%d from dock A to dock B without delay.' % (day, day))
+    state = rules + 'Events:\n' + ' '.join(sentences)
+    payload = {'state': state, 'questions': {'q': {'type': 'noul', 'instructions': 'Is crate C7 refunded?'}}}
+    return core.normalize_request(payload, 1)[0][1]
+
+
+class SelfTest(unittest.TestCase):
+    def test_package_integrity(self):
+        self.assertIn(pkg_check.checks()['status'], ('PREPARATION_CHECK_PASS', 'FROZEN_PACKAGE_CHECK_PASS'))
+
+    def test_pending_cannot_serve(self):
+        config = pkg_check.read('config.json')
+        if not config['package'].get('nonformal_fitted'):
+            with self.assertRaisesRegex(ValueError, 'fit pending'):
+                launch.check_preparation(allow_nonformal=True)
+        if config['package'].get('finalized'):
+            launch.check_preparation()  # a frozen package starts formally, without any extra switch
+        else:
+            with self.assertRaises(ValueError):
+                launch.check_preparation()
+
+    def test_offline_environment(self):
+        state = Path(Path.cwd().anchor) / 'jev-31b-offline-state'
+        env = launch.child_environment({}, state, state / 'startup')
+        for key in ('HF_HUB_OFFLINE', 'TRANSFORMERS_OFFLINE', 'VLLM_BATCH_INVARIANT'):
+            self.assertEqual(env[key], '1')
+        self.assertEqual(env['VLLM_USE_FLASHINFER_SAMPLER'], '0')
+        self.assertTrue(Path(env['TMPDIR']).is_relative_to(state))
+
+    def test_served_mode_is_the_stated_policy(self):
+        policy = pkg_check.SERVICE_POLICY
+        self.assertEqual(launch.SERVED_MODE, policy['mode'])
+        c = pkg_check.read('config.json')
+        self.assertEqual((c['routing']['mode'], c['routing']['quota'], c['routing']['target_slow_share']),
+                         (policy['mode'], policy['quota'], policy['quota']))
+        self.assertEqual(pkg_check.policy_problems(policy), [])
+
+    def test_yes_no_constant(self):
+        p = pkg_check.SERVICE_POLICY['noul_p_cal']
+        self.assertTrue(0.81 <= p <= 0.95)
+        cal = pkg_check.read('calibration.json')['models'][pkg_check.MODEL_KEY]
+        for side in ('fast', 'slow', 'fallback'):
+            block = core.validate_calibration({'models': {'m': cal}}, 'm', side)
+            self.assertEqual(block['noul_calibration']['p_cal'], p)
+            out = core.probabilities(['no', 'yes'], {'no': 0.2, 'yes': 1.5}, 'noul', block)
+            self.assertEqual(out, {'no': 1 - p, 'yes': p})
+            out = core.probabilities(['no', 'yes'], {'no': 1.5, 'yes': 0.2}, 'noul', block)
+            self.assertEqual(out, {'no': p, 'yes': 1 - p})
+        for p_bad in (0.80, 0.951):
+            policy = dict(pkg_check.SERVICE_POLICY, noul_p_cal=p_bad)
+            self.assertIn('yes/no constant outside [0.81, 0.95]', pkg_check.policy_problems(policy))
+
+    def test_drafting_budget_inside_the_context(self):
+        policy = pkg_check.SERVICE_POLICY
+        c = pkg_check.read('config.json')
+        served = copy.deepcopy(c)
+        core.apply_profile(served)
+        core.apply_budget_tier(served, None)
+        identity = core.threshold_identity(served)
+        self.assertEqual((identity['max_new_tokens'], identity['reasoning_words']),
+                         (policy['draft']['max_new_tokens'], policy['draft']['reasoning_words']))
+        self.assertEqual(served['generation']['max_new_tokens'], policy['draft']['max_new_tokens'])
+        self.assertEqual(served['routing']['slow_max_prompt_tokens'], policy['slow_max_prompt_tokens'])
+        if policy['mode'] == 'routed':
+            self.assertLessEqual(policy['slow_max_prompt_tokens'] + policy['draft']['max_new_tokens']
+                                 + pkg_check.SLOW_READOUT_MARGIN, c['model']['vllm_options']['max_model_len'])
+            wide = dict(policy, slow_max_prompt_tokens=6653)
+            self.assertIn('drafting budget outside the vLLM context', pkg_check.policy_problems(wide))
+
+    def test_slow_type_exclusion(self):
+        """Excluded types (v3 core patch) never draft; the other types do (uniform mode, synthetic readout)."""
+        policy = pkg_check.SERVICE_POLICY
+        c = pkg_check.read('config.json')
+        excluded = policy['slow_excluded_types']
+        self.assertEqual(c['routing'].get('slow_excluded_types', []), excluded)
+        if not excluded:
+            return
+        self.assertEqual(getattr(core, 'SLOW_TYPE_EXCLUSION', None), 'routing.slow_excluded_types')
+
+        class DraftingBackend:
+            def slow_session(self, ids, unparsed_policy, meter, max_new_tokens=None):
+                return None, False  # a draft without answer marker: the first-pass answer is kept
+
+        def fast(backend, task, config, ids, labels, groups, meter):
+            return {label: -float(index) for index, label in enumerate(labels)}, 0, None
+        length = {key: False for key in ('truncated', 'character_precompressed', 'protected_content_exceeds_budget',
+                                         'physical_truncated', 'long_input', 'slow_prompt_exceeds_budget')}
+        length['original_prompt_tokens'] = 1
+        cal = pkg_check.read('calibration.json')
+        with tempfile.TemporaryDirectory() as tmp:
+            config = copy.deepcopy(c)
+            core.apply_profile(config)
+            config['routing'].update(state_dir=str(Path(tmp).resolve()), run_id='exclusion', resume=False,
+                                     mode='uniform', quota=1., fuse=1.)
+            import quota
+            service = core.Service(config, cal, DraftingBackend(), quota.RuntimeCounter(config))
+            routes = {}
+            with patch('core.prepare_task', return_value=(None, (None, {}, [1], [1]), length)), \
+                    patch('core.fast_attempt_with_retry', side_effect=fast):
+                for kind, question in (('noul', {'type': 'noul', 'instructions': 'Is it late?'}),
+                                       ('choice', {'type': 'choice', 'instructions': 'Which one?',
+                                                   'criteria': {'red': 'r', 'blue': 'b'}})):
+                    response = service.answer({'state': 'synthetic', 'questions': {'q': question}})
+                    routes[kind] = response['usage']['questions']['q']
+            for kind in ('choice', 'noul', 'score'):
+                if kind not in routes:
+                    continue
+                with self.subTest(kind=kind):
+                    if kind in excluded:
+                        self.assertEqual((routes[kind]['path'], routes[kind]['route_reason']), ('fast', 'type_excluded'))
+                    else:
+                        self.assertEqual(routes[kind]['path'], 'slow')
+        bad = copy.deepcopy(c)
+        bad['routing']['slow_excluded_types'] = ['noul', 'noul']
+        with self.assertRaisesRegex(ValueError, 'slow_excluded_types'):
+            core.validate_config(bad)
+
+    def test_threshold_identity(self):
+        c = pkg_check.read('config.json')
+        expected = core.threshold_identity(c)
+        self.assertEqual(expected['model_key'], pkg_check.MODEL_KEY)
+        self.assertEqual(expected['max_prompt_tokens'], pkg_check.INPUT_BUDGET)
+        c['routing']['margin_threshold_fitted_for'] = {'placeholder': False, **expected}
+        self.assertEqual(core.validate_threshold(c), [])
+        c['routing']['margin_threshold_fitted_for']['model_key'] = 'wrong'
+        if c['routing']['mode'] == 'routed':
+            with self.assertRaisesRegex(ValueError, 'identity mismatch'):
+                core.validate_threshold(c)  # routed serving refuses a threshold fitted for another identity
+        else:
+            self.assertTrue(core.validate_threshold(c))
+
+    def test_adapter_reference_materializes_into_served_identity(self):
+        c = pkg_check.read('config.json')
+        identity = core.threshold_identity(c)
+        c['routing']['margin_threshold_fitted_for'] = {'placeholder': False, **identity}
+        c['profiles'][pkg_check.MODEL_KEY]['routing'] = copy.deepcopy(c['routing'])
+        folder = Path(tempfile.gettempdir()) / 'synthetic-model' / 'adapter'
+        self.assertEqual(c['model']['adapter'], launch.ADAPTER_REF)
+        served = launch.materialize_adapter(c, folder)
+        self.assertEqual(served['model']['adapter'], str(folder))
+        self.assertEqual(served['routing']['margin_threshold_fitted_for']['adapter'], str(folder))
+        self.assertEqual(core.validate_threshold(served), [])
+        self.assertEqual(c['model']['adapter'], launch.ADAPTER_REF)
+        with self.assertRaises(ValueError):
+            launch.materialize_adapter(c, None)
+        raw = copy.deepcopy(c)
+        raw['model']['adapter'] = str(folder)
+        with self.assertRaises(ValueError):
+            launch.materialize_adapter(raw, folder)
+
+    def test_adapter_folder_is_exactly_pinned(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            model = Path(tmp)
+            folder = model / 'adapter'
+            folder.mkdir()
+            payload = {'adapter_config.json': b'{"r": 16}\n', 'adapter_model.safetensors': b'synthetic weights'}
+            pinned = {}
+            for name, data in payload.items():
+                (folder / name).write_bytes(data)
+                pinned[name] = {'sha256': hashlib.sha256(data).hexdigest(), 'size': len(data)}
+            with patch.object(launch, 'ADAPTER_FILES', pinned):
+                self.assertEqual(launch.check_adapter(model), folder.resolve())
+                (folder / 'extra.json').write_text('{}')
+                with self.assertRaisesRegex(ValueError, 'exactly'):
+                    launch.check_adapter(model)
+                (folder / 'extra.json').unlink()
+                (folder / 'adapter_model.safetensors').write_bytes(b'tampered weights!')
+                with self.assertRaisesRegex(ValueError, 'differs'):
+                    launch.check_adapter(model)
+            with patch.object(launch, 'ADAPTER_FILES', {}):
+                with self.assertRaisesRegex(ValueError, 'pending'):
+                    launch.check_adapter(model)
+
+    def test_input_budget_6653_everywhere(self):
+        c = pkg_check.read('config.json')
+        served = copy.deepcopy(c)
+        core.apply_profile(served)
+        core.apply_budget_tier(served, None)
+        self.assertEqual(served['limits']['max_prompt_tokens'], 6653)
+        self.assertEqual(served['limits']['physical_max_prompt_tokens'], 6653)
+        for tier in ('normal', 'long'):
+            self.assertEqual(served['routing']['tiers'][tier]['max_prompt_tokens'], 6653)
+        self.assertLessEqual(6653 + 3, c['model']['vllm_options']['max_model_len'])
+
+    def test_long_inputs_are_compressed_not_refused(self):
+        """80k- and 7k-token synthetic inputs are answered from at most 6653 prompt tokens."""
+        c = pkg_check.read('config.json')
+        core.apply_profile(c)
+        backend = backends.TokenizerMixin()
+        backend.tokenizer = WordTokenizer()
+        for words, precompressed in ((80000, True), (7000, False)):
+            task = synthetic_task(words)
+            _, prepared, length = core.prepare_task(task, backend, c)
+            self.assertTrue(length['long_input'])
+            self.assertEqual(length['character_precompressed'], precompressed)
+            if not precompressed:
+                self.assertGreater(length['original_prompt_tokens'], 6653)
+            self.assertLessEqual(length['kept_prompt_tokens'], 6653)
+            self.assertLessEqual(len(prepared[2]), 6653)
+            self.assertFalse(length['physical_truncated'])
+            self.assertFalse(length['protected_content_exceeds_budget'])
+        task = synthetic_task(3000)
+        _, prepared, length = core.prepare_task(task, backend, c)
+        self.assertFalse(length['long_input'])
+        self.assertFalse(length['truncated'])
+
+    def test_readout_probability_contract(self):
+        block = {'temperature_by_kind': {'choice': 1., 'score': 1., 'noul': 'one_bin'}, 'noul_calibration': {'p_cal': .81}}
+        for kind, labels in [('choice', ['A', 'B', 'C']), ('score', ['1', '2', '3']), ('noul', ['no', 'yes'])]:
+            logits = {label: float(i) for i, label in enumerate(labels)}
+            p = core.probabilities(labels, logits, kind, block)
+            self.assertEqual(set(p), set(labels))
+            self.assertAlmostEqual(sum(p.values()), 1.)
+            self.assertTrue(all(math.isfinite(v) and 0 <= v <= 1 for v in p.values()))
+            self.assertEqual(max(labels, key=p.__getitem__), labels[-1])
+
+    def test_readout_rejects_missing_and_nonfinite(self):
+        for logits in ({0: 0.}, {0: 0., 1: float('nan')}, {0: 0., 1: float('inf')}):
+            with self.assertRaises(RuntimeError):
+                core.aggregate(logits, {'a': [0], 'b': [1]})
+
+    def test_config_does_not_accept_invalid_quota(self):
+        c = copy.deepcopy(pkg_check.read('config.json'))
+        c['routing']['quota'] = .9
+        with self.assertRaises(ValueError):
+            core.validate_config(c)
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--offline', action='store_true', help='default; compatibility flag')
+    p.parse_args()
+    with patch.object(socket, 'socket', side_effect=AssertionError('network forbidden in offline selftest')):
+        result = unittest.TextTestRunner(verbosity=2).run(unittest.defaultTestLoader.loadTestsFromTestCase(SelfTest))
+    print(json.dumps({'status': 'PASS' if result.wasSuccessful() else 'FAIL', 'tests': result.testsRun,
+                      'synthetic': True, 'model_called': False, 'network': False}, ensure_ascii=False))
+    return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

service/selftest.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+PKG=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
+export VLLM_USE_FLASHINFER_SAMPLER=0 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1
+export PYTHONDONTWRITEBYTECODE=1
+exec "${PYTHON:-python3}" -B "$PKG/selftest.py" --offline "$@"

service/serve.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Start the service in the foreground. MODEL must hold the base files and adapter/ (see README.md).
+#   bash serve.sh --model /abs/model --state-dir /abs/state --run-id NAME [--port 8000] [--bind 127.0.0.1]
+set -euo pipefail
+PKG=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
+export VLLM_USE_FLASHINFER_SAMPLER=0 VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1
+export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1
+export VLLM_BATCH_INVARIANT=1 VLLM_WORKER_MULTIPROC_METHOD=spawn
+exec "${PYTHON:-python3}" -B "$PKG/launch.py" "$@"

service/server.py

@@ -0,0 +1,242 @@
+"""Standard-library HTTP adapter. No model is loaded until main()."""
+from __future__ import annotations
+
+import argparse
+import json
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+if __package__:
+    from .core import ItemFailure, ServiceFailure, UnsupportedRequest, Service, ROOT, load_json, validate_config, apply_profile, validate_calibration, validate_threshold
+else:
+    from core import ItemFailure, ServiceFailure, UnsupportedRequest, Service, ROOT, load_json, validate_config, apply_profile, validate_calibration, validate_threshold
+
+BINDS = ("127.0.0.1", "0.0.0.0")
+
+
+def make_handler(service):
+    def service_dead():
+        check = getattr(service, 'is_dead', None)
+        return check() if callable(check) else getattr(service, 'dead', False)
+
+    class Handler(BaseHTTPRequestHandler):
+        protocol_version='HTTP/1.1'
+
+        def handle_expect_100(self):
+            if self.command == 'POST' and service_dead():
+                self.close_connection=True
+                self.respond(503, {'error':'service failure'})
+                return False
+            try:
+                n=int(self.headers.get('Content-Length','0'))
+                if n<=0 or n>service.config['server']['max_request_bytes'] or self.headers.get('Transfer-Encoding'):
+                    raise ValueError
+            except (ValueError,TypeError):
+                self.close_connection=True
+                self.respond(400,{'error':'Invalid request size'})
+                return False
+            self.send_response_only(100)
+            self.end_headers()
+            return True
+
+        def setup(self):
+            super().setup()
+            self.connection.settimeout(service.config["server"]["socket_timeout_seconds"])
+
+        def log_message(self, format, *args):
+            # Avoid logging request contents, model paths or network endpoints.
+            pass
+
+        def respond(self, status, value):
+            data = json.dumps(value, ensure_ascii=False, allow_nan=False).encode("utf-8")
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+
+        def do_GET(self):
+            if self.path.split('?', 1)[0].rstrip('/') == '/health':
+                if __package__:
+                    from .attention import attention_health
+                else:
+                    from attention import attention_health
+                health=service.health()
+                # Attention backend and runtime versions are information, never warnings.
+                info=health.setdefault('info',{})
+                info.update(attention_health())
+                info['runtime']=getattr(service.backend,'runtime_info',None)
+                self.respond(200, health)
+            elif self.path.split('?',1)[0].rstrip('/') == '/engine_stats':
+                # Read-only engine counters for maintainer usage reconciliation; not used by evaluation.
+                try: self.respond(200,service.engine_stats())
+                except Exception: self.respond(503,{'error':'Engine counters unavailable'})
+            else:
+                self.respond(404, {'error':'Unknown route'})
+
+        def do_POST(self):
+            # After an unrecoverable engine error or the third consecutive item failure,
+            # every POST is answered 503 without reading or processing the request.
+            if service_dead():
+                self.close_connection=True
+                self.respond(503, {'error':'service failure'})
+                return
+            if self.path.split("?", 1)[0].rstrip("/") != "/v1/systemone":
+                self.close_connection=True
+                self.respond(404, {"error": "Unknown route"})
+                return
+            try:
+                if self.headers.get("Transfer-Encoding"):
+                    raise ValueError("Transfer-Encoding is unsupported")
+                length = int(self.headers.get("Content-Length", "0"))
+                if length <= 0 or length > service.config["server"]["max_request_bytes"]:
+                    raise ValueError("Invalid request size")
+                def reject(value):
+                    raise ValueError("Non-finite JSON value")
+                raw = self.rfile.read(length)
+                if len(raw) != length:
+                    raise ValueError("Incomplete request body")
+                payload = json.loads(raw.decode("utf-8"), parse_constant=reject)
+                # Validate before inference so model ValueError is never a client error.
+                if __package__:
+                    from .core import normalize_request
+                else:
+                    from core import normalize_request
+                normalize_request(payload, service.config["server"]["max_questions"])
+            except UnsupportedRequest:
+                self.close_connection=True
+                self.respond(422,{"error":"Unsupported systemone request"})
+                return
+            except (ValueError, TypeError, UnicodeError, OSError):
+                self.close_connection=True
+                self.respond(400, {"error": "Invalid systemone request"})
+                return
+            # Failures never produce a substitute answer: 422 for one item, 503 for the service.
+            try:
+                result = service.answer(payload)
+                json.dumps(result, allow_nan=False)
+            except ItemFailure as exc:
+                print("item failure: "+type(exc.__cause__ or exc).__name__, flush=True)
+                self.respond(422, {"error": "item could not be processed"})
+                return
+            except ServiceFailure as exc:
+                print("service failure: "+str(exc), flush=True)
+                self.respond(503, {"error": "service failure"})
+                return
+            except Exception as exc:
+                print("service failure: "+type(exc).__name__, flush=True)
+                self.respond(503, {"error": "service failure"})
+                return
+            self.respond(200, result)
+
+    return Handler
+
+
+def install_extensions(service, config):
+    """Serving extensions named in config.json, installed on the running service in this order:
+    item_deadline (per-decision deadline of the drafting pass, with the item rules of the
+    `item_rules` block: the question types that may take the drafting pass and the low-margin choice
+    re-read), then post_calibration (per-type calibration of the reported probabilities). A
+    configuration without these blocks serves the unchanged service."""
+    if __package__:
+        from . import backends, core, item_deadline, post_calibration
+    else:
+        import backends, core, item_deadline, post_calibration
+    if config.get('item_deadline') is not None:
+        rules = config.get('item_rules') or {}
+        types = rules.get('slow_types')
+        service.warmup()   # the drafting speed model is timed on a warmed engine, as in the measured setup
+        item_deadline.install(core, backends, service.backend, config['item_deadline'],
+                              system_prompt=config['prompt']['system']).wrap_service(
+            service, slow_types=None if types is None or set(types) == {'choice', 'noul', 'score'} else types,
+            reread_tau=rules.get('reread_tau') or None)
+    if config.get('post_calibration') is not None:
+        post_calibration.wrap_service(service, config['post_calibration'], core)
+    return service
+
+
+class QuietHTTPServer(ThreadingHTTPServer):
+    # Request workers must not keep a stopped service alive while draining.
+    daemon_threads = True
+
+    def handle_error(self, request, client_address):
+        # The base implementation prints a peer endpoint on disconnected clients.
+        print("HTTP connection failed", flush=True)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", type=Path, default=ROOT / "config.json")
+    parser.add_argument("--model")
+    parser.add_argument("--quant", choices=("int8", "bf16", "awq"))
+    parser.add_argument("--adapter")
+    parser.add_argument("--backend", choices=("transformers", "vllm"))
+    parser.add_argument("--bind", choices=BINDS, help="Listening address; defaults to server.bind in config")
+    parser.add_argument("--port", type=int)
+    parser.add_argument("--mode", choices=("routed", "uniform", "fast_only"))
+    parser.add_argument("--calibration", type=Path)
+    parser.add_argument('--state-dir', type=Path, help='Absolute directory for run state (required)')
+    parser.add_argument('--run-id')
+    parser.add_argument('--resume', action='store_true')
+    parser.add_argument('--model-key')
+    parser.add_argument('--quota-tier',choices=('normal','hard','long'))
+    parser.add_argument('--cost-fuse',choices=('on','off'))
+    parser.add_argument('--no-prefix-caching',action='store_true')
+    parser.add_argument('--allow-placeholder-threshold',action='store_true')
+    args = parser.parse_args(argv)
+    if args.state_dir is None or not args.state_dir.is_absolute():
+        parser.error("--state-dir must be given as an absolute path")
+    c = load_json(args.config)
+    for arg, field in (("model", "path"), ("quant", "quant"), ("adapter", "adapter"), ("backend", "backend")):
+        if getattr(args, arg) is not None:
+            c["model"][field] = getattr(args, arg)
+    for key in ("bind", "port"):
+        if getattr(args, key) is not None:
+            c["server"][key] = getattr(args, key)
+    if args.mode is not None:
+        c["routing"]["mode"] = args.mode
+    if args.state_dir is not None:
+        c['routing']['state_dir'] = str(args.state_dir)
+    if args.run_id:
+        c['routing']['run_id'] = args.run_id
+    c['routing']['resume'] = args.resume
+    if args.model_key:
+        c['model']['model_key'] = args.model_key
+    apply_profile(c)
+    if __package__: from .core import apply_budget_tier
+    else: from core import apply_budget_tier
+    apply_budget_tier(c,args.quota_tier)
+    if args.mode is not None:c["routing"]["mode"]=args.mode
+    if args.quant is not None and args.quant!=c['model']['quant']:parser.error('Precision conflicts with model profile')
+    if args.backend is not None and args.backend!=c['model']['backend']:parser.error('Backend conflicts with model profile')
+    if args.no_prefix_caching: c["model"]["enable_prefix_caching"]=False
+    if args.cost_fuse:c['routing']['cost_fuse']['enabled']=args.cost_fuse=='on'
+    validate_config(c)
+    validate_threshold(c,args.allow_placeholder_threshold)
+    if not c["model"]["path"] or not c["server"]["bind"]:
+        parser.error("Set --model and --bind, or their config fields")
+    calibration = load_json(args.calibration or ROOT / c["calibration_file"])
+    validate_calibration(calibration, c["model"]["model_key"])
+    if __package__:
+        from .backends import create_backend
+        from .quota import RuntimeCounter
+    else:
+        from backends import create_backend
+        from quota import RuntimeCounter
+    counter = RuntimeCounter(c)
+    service = Service(c, calibration, create_backend(c), counter, allow_placeholder_threshold=args.allow_placeholder_threshold)
+    install_extensions(service, c)
+    if c["server"].get("warmup", True):
+        service.warmup()
+    httpd = QuietHTTPServer((c["server"]["bind"], c["server"]["port"]), make_handler(service))
+    print("systemone service ready", flush=True)
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        httpd.server_close()
+
+
+if __name__ == "__main__":
+    main()