Publish standalone Kev-4B ternary MLX v0.1.0 experimental

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Jared Palmer
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,7 @@
+Kev source and original decision model: Copyright 2026 Jared Palmer. Apache-2.0.
+Base backbone: Qwen/Qwen3.5-4B-Base, Apache-2.0.
+Standalone mixed ternary/4-bit compression export and packaging: Jakevin, 2026-10-06.
+This distribution modifies kev/checkpoint.py and kev/mlx_model.py and adds kev/packed_model.py.
+Upstream source: https://github.com/jaredpalmer/kev
+Upstream Kev checkpoint: jaredpalmer/kev-4b@6cfce5c2fa4b4bd64026336ab649c5ca78857d52
+Base checkpoint: Qwen/Qwen3.5-4B-Base@1001bb4d826a52d1f399e183466143f4da7b741b

README.md

@@ -0,0 +1,176 @@
+---
+license: apache-2.0
+base_model: jaredpalmer/kev-4b
+base_model_relation: quantized
+library_name: mlx
+tags:
+  - kev
+  - decision-model
+  - ternary
+  - mixed-precision
+  - post-training-quantization
+  - mlx
+  - experimental
+---
+
+# Kev-4B Ternary + CLoQ16 for MLX
+
+An experimental, standalone compression of **Jared Palmer's Kev-4B decision
+model**, published by Jakevin. The entire distribution is approximately
+**1.71 GB**, including the compressed backbone, compensation adapter, pointer
+head, configuration, tokenizer, and the required Kev runtime source.
+
+On **decision-v7 development**, 1,264 scored clean questions, native packed
+inference reaches **84.41% accuracy**, retaining **96.82%** of the archived
+bf16-MLX accuracy. These are development diagnostics for a suite related to
+Kev's training recipe, not a fresh blind test or a general retention guarantee.
+
+This is a **decision scorer**, not a text-generating chat model. It implements
+Kev's existing `state` / `questions` -> `answers` / `usage` System One API.
+**Apple Silicon and the included custom MLX runtime are required.** Stock
+Transformers, stock mlx-lm text generation, and llama.cpp are not supported
+loading paths for this artifact.
+
+## Download and Run
+
+Download this repository, including `runtime/`. It contains the packed-aware
+Kev loader and serving source; the upstream Kev release alone does not include
+this loader. Python 3.12 or 3.13 is required.
+
+```sh
+python3 -m pip install huggingface_hub
+python3 -c "from huggingface_hub import snapshot_download; snapshot_download('Jakevin/kev-4b-ternary-mlx', revision='v0.1.0-experimental', local_dir='kev-4b-ternary-mlx')"
+cd kev-4b-ternary-mlx
+python3.13 -m venv .venv
+.venv/bin/python -m pip install './runtime[serve]'
+
+HF_HUB_OFFLINE=1 HF_HOME=/tmp/kev-empty-hf-cache KEV_BACKEND=mlx \
+KEV_PREFIX_CACHE=1 KEV_PREFIX_MAX_TOKENS=4096 \
+.venv/bin/python -m kev.serve --run . --host 127.0.0.1 --port 8010
+```
+
+```sh
+curl http://127.0.0.1:8010/v1/systemone \
+  -H 'Content-Type: application/json' \
+  -d '{"state":"I was charged twice for order 8812.","questions":{"team":{"type":"choice","instructions":"Which team should handle this ticket?","criteria":{"billing":"Charges and payments","shipping":"Delivery and delays"}}}}'
+```
+
+The library interface is unchanged:
+
+```python
+from kev.checkpoint import Checkpoint
+
+tokenizer, model = Checkpoint("path/to/downloaded-repository").load("mps")
+```
+
+After dependencies are installed, loading and inference work offline with an
+empty Hugging Face cache. They do not load the original backbone or original
+PEFT adapter, reconstruct a dense bf16 backbone, or construct the dense
+compensation matrix. Architecture initialization is lazy and its random dense
+parameters are replaced before evaluation.
+
+## Compression Recipe
+
+Source checkpoint: [`jaredpalmer/kev-4b`](https://huggingface.co/jaredpalmer/kev-4b),
+revision `6cfce5c2fa4b4bd64026336ab649c5ca78857d52`, based on
+[`Qwen/Qwen3.5-4B-Base`](https://huggingface.co/Qwen/Qwen3.5-4B-Base), revision
+`1001bb4d826a52d1f399e183466143f4da7b741b`.
+
+- The source Kev LoRA was merged before the original compression experiment.
+- GPTQ-style error compensation uses 128 records from the decision-v7
+  **calibration** partition, RNG 1234, damping 0.01 and sequential 2-layer chunks.
+- Most backbone linear weights are ternary, with an fp16 scale per 64 input columns.
+- Gated DeltaNet `out_proj` and full-attention `q/k/v/o_proj` use 4-bit affine
+  weights with group size 64. The embedding is also 4-bit.
+- Small `in_proj_a/b`, norms, conv kernels and DeltaNet parameters retain their
+  saved source precision. The pointer head and source temperature are unchanged.
+- CLoQ-style rank-16 compensation uses a closed-form calibration-based solution;
+  no gradient retraining or new temperature fitting was performed.
+- No online rotation is required by this recipe.
+
+At inference, ternary codes use native **2-bit** MLX affine storage. Native
+storage is larger than the original five-trits-per-byte archival format. The
+CLoQ factors remain bf16 and execute as two separate small matrix products
+with fp32 accumulation. Activations are bf16. Only requested embedding rows
+are dequantized. This is a weight compression method; activations and the
+attention/prefix cache are not claimed to be 2-bit or 4-bit.
+
+The method combines existing quantization/compensation ideas in a layer-specific
+recipe; it does not claim a new GPTQ or CLoQ algorithm. Relevant work:
+[GPTQ](https://arxiv.org/abs/2210.17323),
+[CLoQ](https://arxiv.org/abs/2501.18475).
+
+## Development Evaluation
+
+All 1,204 decision-v7 development records were evaluated: 1,468 total rows,
+1,264 scored clean questions, zero rejected records and zero truncations.
+The source checkpoint's temperature was retained.
+
+| Path | Backbone + compensation | Accuracy | Relative accuracy retention | Brier |
+|---|---:|---:|---:|---:|
+| Archived bf16-MLX baseline | 8.412 GB | 0.87184 | 100% | 0.18248 |
+| Archived five-trits-per-byte artifact, decoded/merged to bf16 for scoring | 1.434 GB | 0.84335 | 96.73% | 0.23260 |
+| This native MLX packed inference path | 1.679 GB | **0.84415** | **96.82%** | **0.23251** |
+
+The complete published folder is approximately 1.71 GB; sizes above exclude
+tokenizer and pointer head. GB means decimal GB.
+
+Native quantized matmul plus separate compensation has different arithmetic
+from the archived merged-bf16 simulation. Across 1,468 rows, 1 argmax changed;
+mean per-row maximum probability difference was 0.00268, maximum 0.06209.
+This is not bit-identical serving. The original bf16 model was not loaded again
+for this comparison; its metrics came from the frozen local experiment.
+
+Aggregate evaluation and memory records are in `evaluation/`. No evaluation
+sentences, labels or per-example prediction dumps are uploaded. No NAER /
+Chinese Linguipedia data, derivatives or predictions were used in this export
+or included in this repository.
+
+## Memory Measurements
+
+Fresh-process probes on an **Apple M4 Pro, 64 GiB RAM**, macOS 26.6.2:
+MLX 0.32.2, mlx-lm 0.31.3, torch 2.8.0, transformers 5.17.0.
+Original loaders and downloads were explicitly blocked during the probes.
+
+| Measurement | Short request | Longer request |
+|---|---:|---:|
+| State / total input tokens | 19 / 42 | 3,519 / 3,542 |
+| Load time | 0.57 s | 0.57 s |
+| MLX active weights after load | 1.679 GB | 1.679 GB |
+| Process physical-footprint peak while loading | 2.396 GB | 2.398 GB |
+| MLX inference peak | 2.013 GB | 3.066 GB |
+| Process physical-footprint inference peak | **2.846 GB** | **3.862 GB** |
+| First inference time | 0.155 s | 6.599 s |
+
+Physical footprint includes Metal buffers; RSS alone undercounts them. These
+are single-request probes, not minimum-RAM device certification. **No 8-GB
+hardware was tested.** More state tokens, questions, or retained prefixes can
+increase memory. The synthetic longer probe measures memory/latency, not
+long-context accuracy. `KEV_PREFIX_MAX_TOKENS` limits retained prefix cache,
+not input admission. `mx.set_memory_limit(3 GB)` is a guideline rather than a
+hard RAM cap; the longer probe exceeded it.
+
+## Reproducibility and Limitations
+
+- The runtime and the native exporter are in `runtime/`. Source and model
+  hashes are recorded in `packed.json`, `head.pt["compression"]` and
+  `release-manifest.json`.
+- The test source is included. The local release check passed 7 relevant
+  tests, including native code/adapter arithmetic, loading with base loaders
+  blocked, missing-weight rejection, and the System One API.
+- Decision-v7 development was used to select compression recipes. No new
+  blind benchmark, transfer-v9, hard-v1 or breadth-v1 evaluation is claimed.
+- Full 3-bit and 4-bit same-condition baselines were not evaluated here.
+- The source temperature is retained; Brier/ECE worsened relative to the
+  bf16 baseline. Confidence-sensitive users should evaluate their own task.
+- Native inference has not been validated on non-Apple devices or llama.cpp.
+- This is an experimental derivative of Kev-4B, not a new independently
+  pretrained foundation model and not the Taiwan/mainland wording model.
+
+## Attribution and License
+
+Apache-2.0. The original Kev decision model and source are by **Jared Palmer**;
+the base backbone is from **Qwen**. This compression experiment, standalone
+export and publication are by **Jakevin**. The runtime distribution modifies
+Kev's checkpoint and MLX loaders and adds the packed model implementation.
+See `LICENSE` and `NOTICE` for attribution.

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

config.json

@@ -0,0 +1,104 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "image_token_id": 248056,
+  "model_type": "qwen3_5",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 2560,
+    "initializer_range": 0.02,
+    "intermediate_size": 9216,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "linear_value_head_dim": 128,
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 4,
+    "rms_norm_eps": 1e-06,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 248320,
+    "mamba_ssm_dtype": "float32",
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "rope_type": "default",
+      "rope_theta": 10000000,
+      "partial_rotary_factor": 0.25
+    }
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.0.dev0",
+  "video_token_id": 248057,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 24,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1024,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "model_type": "qwen3_5",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 2560,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 248054,
+  "vision_start_token_id": 248053
+}

evaluation/development-summary.json

@@ -0,0 +1,91 @@
+{
+  "clean": {
+    "n": 1264,
+    "nll": 0.43551092937105246,
+    "acc": 0.8441455696202531,
+    "ece": 0.043164737775791336,
+    "brier": 0.23251480498478275,
+    "mean_conf": 0.8088378144123675,
+    "confident_error_rate": 0.010284810126582278,
+    "coverage_at_0_9": 0.4849683544303797,
+    "accuracy_at_0_9": 0.9787928221859706,
+    "coverage_at_5pct_error": 0.6724683544303798,
+    "coverage_at_1pct_error": 0.16772151898734178,
+    "aurc": 0.046630371957041034,
+    "error_rate_at_0_9": 0.021207177814029365,
+    "confidence_bias": -0.03530775520788565,
+    "top_bins": {
+      "0.9": {
+        "n": 613,
+        "errors": 13,
+        "error_rate": 0.021207177814029365
+      },
+      "0.95": {
+        "n": 443,
+        "errors": 6,
+        "error_rate": 0.013544018058690745
+      },
+      "0.99": {
+        "n": 152,
+        "errors": 2,
+        "error_rate": 0.013157894736842105
+      }
+    },
+    "selective": {
+      "0.5": {
+        "coverage": 0.5,
+        "accuracy": 0.9778481012658228,
+        "confidence_cutoff": 0.8935324549674988
+      },
+      "0.8": {
+        "coverage": 0.8006329113924051,
+        "accuracy": 0.9061264822134387,
+        "confidence_cutoff": 0.5975180090213416
+      }
+    },
+    "score_mae": 0.5421033783798573,
+    "ranked_probability_score": 0.0842686503245251
+  },
+  "coverage": {
+    "requested_records": 1204,
+    "requested_questions": 1468,
+    "evaluated_records": 1204,
+    "evaluated_questions": 1468,
+    "rejected_records": 0,
+    "truncated_records": 0
+  },
+  "latency_ms": {
+    "median": 304.7052710317075,
+    "p95": 1490.851858188398
+  },
+  "calibration": {
+    "inference_temperature": 2.406050072164233,
+    "additional_temperature": 1.0,
+    "logits_recorded": true
+  },
+  "environment": {
+    "packages": {
+      "torch": "2.8.0",
+      "transformers": "5.17.0",
+      "peft": "0.21.0",
+      "flash-linear-attention": null,
+      "triton": null,
+      "causal-conv1d": null,
+      "mlx": "0.32.2",
+      "mlx-lm": "0.31.3"
+    },
+    "device": "mps",
+    "gpu": null,
+    "backend": "mlx",
+    "dtype": "bfloat16",
+    "triton_f32_default": null,
+    "attention": null,
+    "deltanet": null
+  },
+  "suite": "decision-v7",
+  "partition": "development",
+  "role": "in-distribution development diagnostic, not a blind test",
+  "source_baseline": "archived bf16-MLX metrics; original weights not loaded for this comparison",
+  "baseline_accuracy": 0.8718354430379747,
+  "accuracy_retention": 0.9682395644283122
+}

evaluation/memory-long.json

@@ -0,0 +1,31 @@
+{
+  "backend": "mlx",
+  "dtype": "bfloat16",
+  "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O",
+  "original_loaders_blocked": true,
+  "mlx_limit_bytes": 3000000000,
+  "load_seconds": 0.5740630407817662,
+  "mlx_load_peak_bytes": 1678767640,
+  "mlx_after_load_bytes": 1678767624,
+  "load_footprint_peak_bytes": 2397899416,
+  "load_rss_peak_bytes": 2381381632,
+  "state_tokens": 3519,
+  "input_tokens": 3542,
+  "inference_seconds": 6.599218292161822,
+  "probabilities": {
+    "billing": 0.941597580909729,
+    "shipping": 0.05840245634317398
+  },
+  "mlx_inference_peak_bytes": 3065702282,
+  "inference_footprint_peak_bytes": 3862170744,
+  "inference_rss_peak_bytes": 2520678400,
+  "mlx_limit_is_hard_cap": false,
+  "machine": "Apple M4 Pro",
+  "machine_ram_bytes": 68719476736,
+  "versions": {
+    "mlx": "0.32.2",
+    "mlx-lm": "0.31.3",
+    "torch": "2.8.0",
+    "transformers": "5.17.0"
+  }
+}

evaluation/memory-short.json

@@ -0,0 +1,31 @@
+{
+  "backend": "mlx",
+  "dtype": "bfloat16",
+  "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O",
+  "original_loaders_blocked": true,
+  "mlx_limit_bytes": 3000000000,
+  "load_seconds": 0.5707792090252042,
+  "mlx_load_peak_bytes": 1678767640,
+  "mlx_after_load_bytes": 1678767624,
+  "load_footprint_peak_bytes": 2396031616,
+  "load_rss_peak_bytes": 2307145728,
+  "state_tokens": 19,
+  "input_tokens": 42,
+  "inference_seconds": 0.15510379197075963,
+  "probabilities": {
+    "billing": 0.9229481220245361,
+    "shipping": 0.07705190777778625
+  },
+  "mlx_inference_peak_bytes": 2012569374,
+  "inference_footprint_peak_bytes": 2846362528,
+  "inference_rss_peak_bytes": 2516697088,
+  "mlx_limit_is_hard_cap": false,
+  "machine": "Apple M4 Pro",
+  "machine_ram_bytes": 68719476736,
+  "versions": {
+    "mlx": "0.32.2",
+    "mlx-lm": "0.31.3",
+    "torch": "2.8.0",
+    "transformers": "5.17.0"
+  }
+}

evaluation/native-comparison.json

@@ -0,0 +1,9 @@
+{
+  "rows": 1468,
+  "max_abs_dp": 0.06208749928657609,
+  "mean_max_abs_dp": 0.0026799194979893045,
+  "argmax_flips": 1,
+  "accuracy": 0.8441455696202531,
+  "retention": 0.9682395644283122,
+  "reference": "archival packed merged-bf16 evaluation; no original weights loaded"
+}

head.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f6740d94a49187770b68c0f058d85bedc0cf7efc20dff7ca9a418c8b461dc0e
+size 5247807

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4899eaa9fb206e77260f832d00188c9b90750724422ea0eea43ed69faeff43d
+size 1678909327

packed.json

@@ -0,0 +1,1016 @@
+{
+  "format": "kev-packed-mlx-v1",
+  "linears": {
+    "language_model.model.layers.0.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.0.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.0.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.0.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.0.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.0.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.1.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.1.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.1.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.1.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.1.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.1.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.2.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.2.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.2.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.2.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.2.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.2.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.3.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.3.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.3.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.3.self_attn.o_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.3.self_attn.v_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.3.self_attn.k_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.3.self_attn.q_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.4.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.4.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.4.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.4.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.4.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.4.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.5.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.5.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.5.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.5.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.5.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.5.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.6.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.6.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.6.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.6.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.6.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.6.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.7.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.7.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.7.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.7.self_attn.o_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.7.self_attn.v_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.7.self_attn.k_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.7.self_attn.q_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.8.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.8.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.8.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.8.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.8.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.8.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.9.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.9.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.9.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.9.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.9.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.9.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.10.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.10.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.10.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.10.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.10.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.10.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.11.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.11.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.11.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.11.self_attn.o_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.11.self_attn.v_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.11.self_attn.k_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.11.self_attn.q_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.12.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.12.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.12.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.12.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.12.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.12.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.13.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.13.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.13.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.13.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.13.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.13.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.14.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.14.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.14.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.14.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.14.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.14.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.15.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.15.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.15.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.15.self_attn.o_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.15.self_attn.v_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.15.self_attn.k_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.15.self_attn.q_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.16.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.16.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.16.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.16.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.16.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.16.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.17.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.17.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.17.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.17.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.17.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.17.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.18.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.18.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.18.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.18.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.18.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.18.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.19.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.19.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.19.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.19.self_attn.o_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.19.self_attn.v_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.19.self_attn.k_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.19.self_attn.q_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.20.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.20.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.20.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.20.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.20.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.20.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.21.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.21.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.21.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.21.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.21.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.21.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.22.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.22.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.22.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.22.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.22.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.22.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.23.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.23.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.23.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.23.self_attn.o_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.23.self_attn.v_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.23.self_attn.k_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.23.self_attn.q_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.24.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.24.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.24.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.24.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.24.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.24.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.25.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.25.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.25.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.25.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.25.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.25.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.26.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.26.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.26.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.26.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.26.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.26.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.27.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.27.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.27.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.27.self_attn.o_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.27.self_attn.v_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.27.self_attn.k_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.27.self_attn.q_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.28.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.28.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.28.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.28.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.28.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.28.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.29.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.29.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.29.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.29.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.29.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.29.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.30.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.30.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.30.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.30.linear_attn.out_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.30.linear_attn.in_proj_z": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.30.linear_attn.in_proj_qkv": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.31.mlp.up_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.31.mlp.down_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.31.mlp.gate_proj": {
+      "bits": 2,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.31.self_attn.o_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.31.self_attn.v_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.31.self_attn.k_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    },
+    "language_model.model.layers.31.self_attn.q_proj": {
+      "bits": 4,
+      "group_size": 64,
+      "rank": 16
+    }
+  },
+  "embedding": {
+    "bits": 4,
+    "group_size": 64
+  },
+  "source": {
+    "format": "kev-packed-mlx-v1",
+    "source_packed_sha256": "af8703c3c8f848c18f49a40e91978363fae1e0e3b6c4678a9436e3ebc5da73e0",
+    "source_cloq_sha256": "52b8699b25a819542d8073b0de52c79bfb77e46b3bc969e6f553b6a047e0afcc",
+    "arithmetic": "native affine quantized matmul plus separate fp32 low-rank compensation; not merged-bf16 parity"
+  },
+  "model_sha256": "c4899eaa9fb206e77260f832d00188c9b90750724422ea0eea43ed69faeff43d"
+}

release-manifest.json

@@ -0,0 +1,252 @@
+{
+  "repo_id": "Jakevin/kev-4b-ternary-mlx",
+  "tag": "v0.1.0-experimental",
+  "date": "2026-10-06",
+  "format": "kev-packed-mlx-v1",
+  "source_kev_revision": "6cfce5c2fa4b4bd64026336ab649c5ca78857d52",
+  "files": {
+    "LICENSE": {
+      "bytes": 11343,
+      "sha256": "6b08bb37982c233aa12bcbdf19106da12f3f4fcf800773ea42e28ebeddd34fb8"
+    },
+    "NOTICE": {
+      "bytes": 528,
+      "sha256": "9021541d1f4af146262ef38449dd517c13d059404132e3f7e017ab550cef62bc"
+    },
+    "README.md": {
+      "bytes": 8520,
+      "sha256": "c11cf88162f1739d10f5e68f1a1d2bdd05d794381ebb99dbaa65a06f0f0c4dea"
+    },
+    "added_tokens.json": {
+      "bytes": 707,
+      "sha256": "c0284b582e14987fbd3d5a2cb2bd139084371ed9acbae488829a1c900833c680"
+    },
+    "config.json": {
+      "bytes": 2692,
+      "sha256": "fc1557ea241bfc2c8abd87c1213729a44562fe5da9edf0fd5dc9f95b6cdbdaed"
+    },
+    "evaluation/development-summary.json": {
+      "bytes": 2475,
+      "sha256": "d17ad38950313c7513bd67a73cb2b15358c0a1f42dbec2db02c6c9388a392f5f"
+    },
+    "evaluation/memory-long.json": {
+      "bytes": 895,
+      "sha256": "b551f722360e3420a30aed1fada9d44557bef26730938df1a2a73b961a54354d"
+    },
+    "evaluation/memory-short.json": {
+      "bytes": 894,
+      "sha256": "d3111a799b8760323eba78cae3bf00fd61c61d24073a232cff33a9c7baf3d41d"
+    },
+    "evaluation/native-comparison.json": {
+      "bytes": 275,
+      "sha256": "a1355c9afa8b8f7f986c355941233164457d05503810d49fd698bd224adf8fc5"
+    },
+    "head.pt": {
+      "bytes": 5247807,
+      "sha256": "6f6740d94a49187770b68c0f058d85bedc0cf7efc20dff7ca9a418c8b461dc0e"
+    },
+    "merges.txt": {
+      "bytes": 1671853,
+      "sha256": "8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5"
+    },
+    "model.safetensors": {
+      "bytes": 1678909327,
+      "sha256": "c4899eaa9fb206e77260f832d00188c9b90750724422ea0eea43ed69faeff43d"
+    },
+    "packed.json": {
+      "bytes": 24929,
+      "sha256": "1ae5d2d98e967e49ebdf29842da0847faf87fa03ace9370f8e05715ef583dd29"
+    },
+    "runtime/LICENSE": {
+      "bytes": 11343,
+      "sha256": "6b08bb37982c233aa12bcbdf19106da12f3f4fcf800773ea42e28ebeddd34fb8"
+    },
+    "runtime/README.md": {
+      "bytes": 42304,
+      "sha256": "19c9d7d30cd723b9f112f1f68d0b415a28c5ee616dbc21b358fc29bd07a66311"
+    },
+    "runtime/docs/packed-mlx.md": {
+      "bytes": 6233,
+      "sha256": "c70c572479cb7e8e8094e7cdd5982ade28084c9468e38190ef0d13ffa675631b"
+    },
+    "runtime/kev/__init__.py": {
+      "bytes": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    "runtime/kev/anchors.py": {
+      "bytes": 3930,
+      "sha256": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb"
+    },
+    "runtime/kev/api.py": {
+      "bytes": 7526,
+      "sha256": "a6f53af5354f8ea17915abc78360fdf291a5108ce39a53a2d9135d48c603ae58"
+    },
+    "runtime/kev/autoresearch.py": {
+      "bytes": 11866,
+      "sha256": "7c62762b99c1b081bef06aa1ce83f8dfccce8ac7e9d2c5c0ab4d058055c7eab1"
+    },
+    "runtime/kev/benchmark.py": {
+      "bytes": 14203,
+      "sha256": "82e6a935472f170da48ec37eee8986c519595c4943f6f3040ec67b5da3afb278"
+    },
+    "runtime/kev/budget.py": {
+      "bytes": 8457,
+      "sha256": "246b5ca6450567a61b881308c9482cf1af6ecba599ca161af1febb23d6dc28a7"
+    },
+    "runtime/kev/calibrate.py": {
+      "bytes": 4846,
+      "sha256": "435ac98696fbd329c3a07f0c12633ba3c09a846f0e9c92e17540bbf8ba2986f8"
+    },
+    "runtime/kev/checkpoint.py": {
+      "bytes": 24924,
+      "sha256": "efe96cde9ef989c4fea3b6a0b631338bb1870bfe4b0f1eb99d71bb0761af34b7"
+    },
+    "runtime/kev/compare.py": {
+      "bytes": 3198,
+      "sha256": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435"
+    },
+    "runtime/kev/composition.py": {
+      "bytes": 15847,
+      "sha256": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536"
+    },
+    "runtime/kev/contrastive.py": {
+      "bytes": 20624,
+      "sha256": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf"
+    },
+    "runtime/kev/cuda_graphs.py": {
+      "bytes": 25356,
+      "sha256": "5ce84f0e4e5ad2177314ed3ecc9564d45d14ad9eb422e0fb079c9e1cbbb011b6"
+    },
+    "runtime/kev/data.py": {
+      "bytes": 25861,
+      "sha256": "e77a2fffeb8ee7e05b118893be8e38b8358aef97c3ca2e143cdce208201b92f9"
+    },
+    "runtime/kev/device.py": {
+      "bytes": 1164,
+      "sha256": "bda81dff5be33c59eee8de76e534976cc137a2661da6afd2a67abb85d8dad4a2"
+    },
+    "runtime/kev/evaluate.py": {
+      "bytes": 14408,
+      "sha256": "b2a6fbad7a1494728ea8d9ed0ad54ae23ff2ee1d543444fd0c3c110da4ab07cc"
+    },
+    "runtime/kev/experiment.py": {
+      "bytes": 34376,
+      "sha256": "3fd13a86f111dfd3292af9448df0ee28bd9596e3a77c1391f318cd654be14e8a"
+    },
+    "runtime/kev/full_ft.py": {
+      "bytes": 28171,
+      "sha256": "c3abef39cd9d486b3c075a37286afbcd1aa42a04c78a1b7b9b2e77b5a3963929"
+    },
+    "runtime/kev/fused_qwen35.py": {
+      "bytes": 10294,
+      "sha256": "5d2e587808822ce539ff52f63024357630b67a65f88cd734b82fd488655aebed"
+    },
+    "runtime/kev/jev.py": {
+      "bytes": 3275,
+      "sha256": "5488645c4fc2a53ca430e556ebfb744736e1a156bd121bd10ba0a9935f423e1c"
+    },
+    "runtime/kev/metrics.py": {
+      "bytes": 24401,
+      "sha256": "167a586334cfcced4adcc197758535c3bcda832b855c151095ef2d1968cf8983"
+    },
+    "runtime/kev/mirror.py": {
+      "bytes": 4982,
+      "sha256": "ca2284c670ce2cf1db57231968b9e6cbc01955f8f3c9a3207fd82e73d7b476b6"
+    },
+    "runtime/kev/mlx_model.py": {
+      "bytes": 15144,
+      "sha256": "8aef8aef93f50663395d73b5ae6cc964a01c4f4bf7f92a3843c1f32681cc9c09"
+    },
+    "runtime/kev/model.py": {
+      "bytes": 35153,
+      "sha256": "2ff3b2e67c048e3bc5a0d1a32aafe50804ed315a5564fce06bb4b95697c5271c"
+    },
+    "runtime/kev/packed_model.py": {
+      "bytes": 4892,
+      "sha256": "72cf553bd8d6f6e6d1e9f903eab4eba44d33d11571803752c0a89404f39e3499"
+    },
+    "runtime/kev/plot.py": {
+      "bytes": 3368,
+      "sha256": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02"
+    },
+    "runtime/kev/predictors.py": {
+      "bytes": 21963,
+      "sha256": "29263ae3297e5cde6cf03da5c5aa3ed377c9681c6a5a639ea4aa259acec7575b"
+    },
+    "runtime/kev/publish.py": {
+      "bytes": 7512,
+      "sha256": "fcb490a374fc9ef330424ad4a75ce281ab08d2cb04dff2ad912c5f4dae6ab8a7"
+    },
+    "runtime/kev/rounds.py": {
+      "bytes": 96716,
+      "sha256": "2975bede7250812aecea36b17c0aefda6b389dc3a50e782e82e7566a7168c270"
+    },
+    "runtime/kev/serve.py": {
+      "bytes": 23057,
+      "sha256": "4f9885ce179dcb7de743f59be4b561758f47fac09373525e2994860b3345b162"
+    },
+    "runtime/kev/shared_prefix.py": {
+      "bytes": 9671,
+      "sha256": "bc6dedbad734cca227f3ec6746223b3b03363b0c045aa9f2493d886b1828e71e"
+    },
+    "runtime/kev/study_v3.py": {
+      "bytes": 14919,
+      "sha256": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3"
+    },
+    "runtime/kev/suite.py": {
+      "bytes": 26388,
+      "sha256": "8e7d257a10b4704d7bf4f8ec2c74dddf26aa5a2a610ea78610360e6baf43eff5"
+    },
+    "runtime/kev/train.py": {
+      "bytes": 53543,
+      "sha256": "de0b0971defd79c053d56f8e8b17b0501b2e264e93a66a8d5493f1b1ba839416"
+    },
+    "runtime/kev/transfer_v9.py": {
+      "bytes": 10969,
+      "sha256": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763"
+    },
+    "runtime/pyproject.toml": {
+      "bytes": 1721,
+      "sha256": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d"
+    },
+    "runtime/scripts/build_packed_model.py": {
+      "bytes": 7130,
+      "sha256": "1f4aba3f4c754753c3c654b934cd410acc7fb9c0f27fa35366e9d4a1899e5d5c"
+    },
+    "runtime/scripts/mlx_long_states.py": {
+      "bytes": 29651,
+      "sha256": "ec1ae5ff040da1060143dcbcd551bdd5982e583951af58677cdb0b6624ed005d"
+    },
+    "runtime/scripts/profile_packed_model.py": {
+      "bytes": 3543,
+      "sha256": "14b8e346771f978fb61b2d167cbb5f56011600f3e59ae4a16b84518fd99d2a18"
+    },
+    "runtime/tests/test_packed_model.py": {
+      "bytes": 7193,
+      "sha256": "9ff620541b964886f1e13117ffb4a6e143b3c843a144fd2fcc1a087baa3c82de"
+    },
+    "special_tokens_map.json": {
+      "bytes": 616,
+      "sha256": "6676f091c8bc4d1b50146427cfde92073402866b87b6e39223227931b70083e9"
+    },
+    "tokenizer.json": {
+      "bytes": 19989325,
+      "sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523"
+    },
+    "tokenizer_config.json": {
+      "bytes": 1128,
+      "sha256": "8671bed7c852ce9e661be94f179a7b4ffd091c2a65aea0363e5501c20318ee45"
+    },
+    "vocab.json": {
+      "bytes": 2776833,
+      "sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
+    }
+  },
+  "excluded": [
+    "original backbone and PEFT weights",
+    "NAER/Chinese Linguipedia sources, derivatives, labels and predictions",
+    "training and validation JSONL",
+    "per-example evaluation prediction dumps"
+  ],
+  "note": "Runtime code included; model SHA verified against packed.json. Manifest excludes its own hash."
+}

runtime/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Jared Palmer
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

runtime/README.md

@@ -0,0 +1,466 @@
+# Kev
+
+Small Jev-like decision models you can train and run yourself.
+
+<p>
+  <a href="https://github.com/jaredpalmer/kev/actions/workflows/ci.yml"><img alt="CI" src="https://img.shields.io/github/actions/workflow/status/jaredpalmer/kev/ci.yml?style=for-the-badge&labelColor=000000" height="28"></a>
+  <a href="https://huggingface.co/collections/jaredpalmer/kev-6aad9d0ea49f2589665e07cd"><img alt="Weights: Kev-0.8B · 4B · 9B · 27B" src="https://img.shields.io/badge/WEIGHTS-0.8B%20%C2%B7%204B%20%C2%B7%209B%20%C2%B7%2027B-0a0a0a.svg?style=for-the-badge&labelColor=000000" height="28"></a>
+  <a href="https://huggingface.co/spaces/jaredpalmer/kev"><img alt="Demo on Hugging Face Spaces" src="https://img.shields.io/badge/DEMO-HF%20Spaces-0a0a0a.svg?style=for-the-badge&labelColor=000000" height="28"></a>
+  <a href="https://huggingface.co/datasets/jaredpalmer/kev-suites"><img alt="Frozen eval suites" src="https://img.shields.io/badge/EVAL%20SUITES-frozen-0a0a0a.svg?style=for-the-badge&labelColor=000000" height="28"></a>
+  <a href="LICENSE"><img alt="License: Apache-2.0" src="https://img.shields.io/badge/license-Apache--2.0-0a0a0a.svg?style=for-the-badge&labelColor=000000" height="28"></a>
+</p>
+
+Kev is a family of small decision models built on Qwen3.5 and Qwen3.8 and based on the architecture described in [Jev's Architecture Unmasked](https://archerhume.com/posts/jevs-architecture-unmasked). You can use the pretrained weights or train your own. The API matches TypeSafe's [System One](https://docs.typesafe.ai/api), so you can point their Python SDK at your local server.
+
+## Highlights
+
+- Yes/no (`noul`), multiple-choice (`choice`) and rating (`score`) questions in one request. The questions share the text but can't read each other.
+- Calibrated probabilities by default: each checkpoint ships with a fitted temperature.
+- Drop-in for Jev: the TypeSafe Python SDK works against a Kev server unchanged.
+- Four sizes, versioned together as Kev 1.0: from a 0.8B that runs on a laptop to a 27B for a single data-centre GPU.
+- Documents of up to 65,536 tokens, on CUDA and on Apple Silicon through MLX. Each model card says how long a document can get before accuracy drops.
+- Fine-tune on your own labelled examples. A coding-agent skill runs the whole loop on Modal, from finding your questions to serving the result.
+- Deploy your own HTTPS endpoint with one command. It scales to zero when idle.
+- Try it in the browser first: [huggingface.co/spaces/jaredpalmer/kev](https://huggingface.co/spaces/jaredpalmer/kev).
+
+## Models
+
+Start with Kev-4B. Move to Kev-9B if you have a bigger GPU, or to Kev-27B if you have an 80 GB GPU and want the most accurate Kev. Use Kev-0.8B when size matters more than accuracy.
+
+| Model | Base (license) | Runs on: CUDA | Runs on: Mac (MLX) | Validated context | Held-out datasets: index | Card |
+|---|---|---|---|---|---|---|
+| [Kev-0.8B](https://huggingface.co/jaredpalmer/kev-0.8b) | Qwen3.5-0.8B-Base (Apache-2.0) | L4, any 4 GB GPU | Any Apple Silicon Mac; measured to 65k tokens | 8,192 | 23.3 | [Details](docs/model-cards/kev-0.8b.md) |
+| [Kev-4B](https://huggingface.co/jaredpalmer/kev-4b) | Qwen3.5-4B-Base (Apache-2.0) | L40S, H100 | 32 GB Mac; measured to 65k tokens | 8,192 | 38.0 | [Details](docs/model-cards/kev-4b.md) |
+| [Kev-9B](https://huggingface.co/jaredpalmer/kev-9b) | Qwen3.5-9B-Base (Apache-2.0) | L40S, H100 | 32 GB Mac or larger (expected, not measured) | 8,192 | 41.0 | [Details](docs/model-cards/kev-9b.md) |
+| [Kev-27B](https://huggingface.co/jaredpalmer/kev-27b) | Qwen3.8-27B, post-trained (Apache-2.0) | B200, H200, H100 80 GB | 96–128 GB Mac (expected, not measured) | 65,536 | **52.3** | [Details](docs/model-cards/kev-27b.md) |
+| Jev | Hosted | TypeSafe's API | – | – | 54.0 | – |
+
+"Held-out datasets" is the chance-corrected index of the community Decision Index, scored on the test split of `breadth-v1`: 14 public datasets in five areas that no Kev trained on. "Validated context" is the longest document, in tokens, for which accuracy on real contracts (CUAD) stays within 3 points of the same model's accuracy at 8k tokens, at the 95 % lower bound; each model card has the measurement by length.
+
+| Model | Accuracy: New Sources | Accuracy: Trained Sources | Brier: New Sources |
+|---|---|---|---|
+| Kev-0.8B | 0.648 / 0.697 | 0.827 / 0.838 | 0.481 / 0.416 |
+| Kev-4B | 0.817 / 0.838 | 0.873 / 0.865 | 0.269 / 0.242 |
+| Kev-9B | 0.820 / 0.852 | 0.874 / 0.873 | 0.289 / 0.217 |
+| Kev-27B | **0.851 / 0.889** | 0.865 / 0.866 | **0.225 / 0.156** |
+| Jev | 0.857 / – | 0.845 / – | 0.211 / – |
+
+Each cell is **development / test**. "New sources" means datasets and policy rules Kev never saw during training. It is the closest thing here to your own questions. "Trained sources" means held-out examples from the datasets Kev was trained on. We pick checkpoints using the development sets and read each test set only once per released model. Jev has only been run on the development sets of these two suites. Brier scores the whole probability distribution, not just the top answer; lower is better.
+
+On new sources Kev-27B is within a point of Jev (0.851 vs 0.857), and Kev-4B and Kev-9B are within four points. We don't know what Jev was trained on, so this isn't a controlled comparison of the two architectures. [What to Expect](#what-to-expect) says where Kev is as good as Jev and where it isn't.
+
+Kev-0.8B, 4B and 9B start from Qwen base models and share one training recipe: a small adapter on a frozen base. Kev-27B starts from Qwen's post-trained release, and we don't know what that was trained on; every one of its weights is fine-tuned, so it ships as 51 GB of full weights rather than an adapter. Each model card has the full recipe, all results, and the earlier versions kept as Hub tags.
+
+## Kev 1.0
+
+The four models above are released together as Kev 1.0. Each Hub repo has a `v1.0` tag, so `--run jaredpalmer/kev-4b@v1.0` always loads the same weights, and the GitHub release [`kev-1.0`](https://github.com/jaredpalmer/kev/releases/tag/kev-1.0) has the 0.8B, 4B and 9B checkpoints with SHA-256 checksums. Kev-27B's 51 GB of weights are too large for a release asset and are on the Hub only.
+
+| Model | Hub revision of the weights | Temperature | Trained on states up to |
+|---|---|---|---|
+| Kev-0.8B | `9a45d25e` | 2.35 | 7,552 tokens |
+| Kev-4B | `139fdd94` | 2.41 | 7,552 tokens |
+| Kev-9B | `b5d8c18e` (v2) | 2.19 | 7,552 tokens |
+| Kev-27B | `28be62e9` (v2, full weights) | 1.32 | 32,768 tokens |
+
+Kev 1.0 trains nothing new. It fixes the checkpoints, cards, evaluation suites and serving code that the next generation of Kev will be compared against. The [release notes](docs/releases/kev-1.0.md) list what changed since the previous family release and what is known not to work well.
+
+## Quick Start
+
+### Try It in the Browser
+
+The [Hugging Face Space](https://huggingface.co/spaces/jaredpalmer/kev) runs Kev-4B and Kev-0.8B, with nothing to install.
+
+### Run It Locally
+
+You'll need Python 3.12 or 3.13 and [uv](https://docs.astral.sh/uv/). The repo's `.python-version` makes `uv sync` use 3.13; torch has no wheels for 3.14 yet.
+
+```bash
+git clone https://github.com/jaredpalmer/kev.git && cd kev
+uv sync --extra serve
+uv run --extra serve python -m kev.serve --run jaredpalmer/kev-4b --port 8009
+```
+
+This starts Kev-4B on your machine: CUDA or ROCm if you have a GPU, MLX on Apple Silicon. The first run downloads the adapter and the base model. `--run` also accepts a local checkpoint directory or a Hub revision like `jaredpalmer/kev-4b@qwen3`.
+
+In another terminal, send it a ticket:
+
+```bash
+curl -s localhost:8009/v1/systemone -H 'content-type: application/json' -d '{
+  "state": "Shoes arrived two weeks late and in the wrong size. Also I see two charges on my card.",
+  "model": "kev-latest",
+  "questions": {
+    "department":  {"type": "choice", "instructions": "Which team should handle this?",
+                    "criteria": {"returns": "Exchanges, refunds, wrong or damaged items",
+                                 "shipping": "Delivery status, delays, lost packages",
+                                 "billing": "Charges, invoices, payment problems"}},
+    "escalate":    {"type": "noul",  "instructions": "Does this need urgent human attention?"},
+    "frustration": {"type": "score", "instructions": "How frustrated is the customer?",
+                    "criteria": ["Calm", "Frustrated", "Very angry"]}
+  }}'
+```
+
+Example response from Kev-4B, running in bf16 on an Apple M5:
+
+```json
+{
+  "model": "kev-latest",
+  "answers": {
+    "department":  { "type": "choice", "choice": "returns", "confidence": 0.21,
+                     "probabilities": { "returns": 0.47, "shipping": 0.28, "billing": 0.25 } },
+    "escalate":    { "type": "noul", "noul": 0.93 },
+    "frustration": { "type": "score", "score": 1.44, "confidence": 0.34,
+                     "legend": { "0": "Calm", "1": "Frustrated", "2": "Very angry" },
+                     "probabilities": { "0": 0.00, "1": 0.56, "2": 0.44 } }
+  },
+  "usage": { "input_tokens": 101, "output_tokens": 161 },
+  "latency_ms": 495
+}
+```
+
+The ticket mentions a return, a late delivery and a billing problem, and the department probabilities say so. That's why Kev returns probabilities instead of a single label: your code can route the confident cases and send the rest to a person.
+
+### Use It From Python
+
+If you already call Jev, point your client at Kev and keep the rest of your code. The TypeSafe SDK is included in `uv sync --extra serve`:
+
+```python
+from typesafe_sdk import Choice, Noul, Score, TypeSafeClient
+
+client = TypeSafeClient(
+    api_key="local",
+    base_url="http://127.0.0.1:8009",
+    model="kev-latest",
+)
+response = client.system_one(
+    state="I was charged twice. Please fix this ASAP.",
+    questions={
+        "billing": Noul(instructions="Is this ticket about billing?"),
+        "tone": Choice(
+            instructions="What is the customer's tone?",
+            criteria={"calm": None, "frustrated": None, "angry": None},
+        ),
+        "urgency": Score(
+            instructions="How urgent is this ticket?",
+            criteria=["can wait", "this week", "today"],
+        ),
+    },
+)
+print(response.nouls["billing"].noul)
+print(response.choices["tone"].choice)
+print(response.scores["urgency"].score)
+```
+
+## Fine-Tune on Your Own Data
+
+The released models were trained on public datasets and generated policy examples. If your questions look different, like your own routing categories, your own escalation rules or another language, a short fine-tune usually helps more than any prompt change. It also fits the temperature to your data, so the confidence you set thresholds on is measured on your own labels.
+
+What to expect: on an example support workload (three questions, 1,050 generated records, 15 minutes on an H100), fine-tuning took Kev-4B from 67.7% to 73.6% accuracy, and from automating 34% of decisions at a 5% error budget to 48% ([details](skills/kev-finetune/README.md#why-fine-tune-at-all)). On real data, one epoch on 5,219 labelled consumer-finance complaints took Kev-4B from 0.804 to 0.904 accuracy on complaints it had never seen. Gains like these are in distribution: they tell you how well Kev learns your task, not how it does on everything else. Size your dataset first. With 400 records, the gain on the example workload was inside the noise.
+
+### With a Coding Agent
+
+```bash
+npx skills add jaredpalmer/kev@kev-finetune
+```
+
+Then ask your agent to "fine-tune Kev on my support tickets". The [`kev-finetune` skill](skills/kev-finetune/) interviews you, finds the questions your code already asks Jev or TypeSafe, converts the labels you have or generates enough with any LLM to measure a gain, fine-tunes from a released checkpoint on Modal, fits the temperature on a held-out slice, scores the result against the untouched model, deploys an endpoint, and tears everything down at the end. You don't need a local GPU or a clone of this repo. A Kev-4B training run costs about $1 on an H100.
+
+### By Hand
+
+The skill's [README](skills/kev-finetune/README.md) is the same recipe for people: six short standard-library scripts and one Modal app. To train from this repo instead, put your examples in a JSONL file, one request per line. It's the same shape as an API request, plus a `label` on every question:
+
+```jsonl
+{"state": {"subject": "Charged twice", "body": "I see two charges for order #4411. Please refund one."},
+ "questions": {
+   "team":     {"type": "choice", "instructions": "Which team should handle this ticket?",
+                "criteria": {"billing": "Payments and refunds", "shipping": "Delivery problems", "access": "Login and account access"}, "label": "billing"},
+   "angry":    {"type": "noul",   "instructions": "Is the customer angry?", "label": false},
+   "priority": {"type": "score",  "instructions": "How urgent is this ticket?", "criteria": ["low", "normal", "high"], "label": 1}}}
+```
+
+For `choice` the label is the option name, for `noul` it's `true` or `false`, and for `score` it's the level's position starting at 0. Keep 10–20% of the file aside for evaluation.
+
+Then start from a released checkpoint with `--init_from`:
+
+```bash
+uv run python -m kev.train --data train.jsonl --base Qwen/Qwen3.5-4B-Base --init_from jaredpalmer/kev-4b \
+    --epochs 2 --lr 2e-5 --batch 1 --accum 8 --dtype bf16 --checkpointing 1 --device cuda --out runs/mine
+
+uv run python -m kev.benchmark --run runs/mine --data heldout.jsonl --out runs/mine-eval
+uv run --extra serve python -m kev.serve --run runs/mine --port 8009
+```
+
+`--init_from` loads the adapter and pointer head from the released model before training, so you keep what Kev already knows and add your domain on top. Starting from the base model instead throws that away: in one user's test on 836 support-tool decisions, a fine-tune from the base scored 0.33 on Kev's own evaluation set, against 0.84 for the released model; the same data with `--init_from` kept 0.83 there and reached 0.88 on the new domain. Use a smaller learning rate than the from-scratch recipe (`2e-5` is a good start), and pick `--base` to match the checkpoint you start from; the trainer checks that the base, revision, LoRA rank and head size agree before it loads anything.
+
+`--batch 1 --accum 8` in bf16 fits the 0.8B model on a 4 GB GPU. The benchmark reports accuracy, Brier score and calibration per question type, so you can see which of your questions the fine-tune helped. The checkpoint you started from is recorded in `runs/mine/training_config.json`. On a Mac, run one training job at a time; two jobs on the same Apple GPU are much slower.
+
+## Deploy Your Own Endpoint
+
+To get an HTTPS endpoint instead of a local server, you don't need this repo, just a [Modal](https://modal.com) account:
+
+```bash
+pip install modal && modal setup
+curl -LO https://raw.githubusercontent.com/jaredpalmer/kev/main/skills/kev-deploy/scripts/kev_serve.py
+KEV_API_KEY=$(openssl rand -hex 24) modal deploy kev_serve.py
+```
+
+That serves Kev-4B on an L40S at `https://<your-workspace>--kev-api.modal.run`, with the same API as above behind `Authorization: Bearer <key>`. It scales to zero when idle, so an unused endpoint costs nothing. The first request after idle waits about 35 seconds for a container to start. `KEV_MODEL=jaredpalmer/kev-9b` serves another model on the GPU that suits it; Kev-27B goes to a B200, falling back to an H200 or H100. If you use a coding agent, `npx skills add jaredpalmer/kev@kev-deploy` does the same and wires the URL into your code. [skills/kev-deploy](skills/kev-deploy/) has the GPU and cost table.
+
+A model you fine-tuned with the `kev-finetune` skill deploys the same way from its own Modal app (`KEV_SERVE_SECRET=kev-serve-key KEV_SERVE_RUN=<run> modal deploy scripts/kev_modal.py`; see [its deploy guide](skills/kev-finetune/references/deploy.md)). To host Kev on your own machines instead, run `kev.serve` from [Run It Locally](#run-it-locally) on a GPU box with `--host 0.0.0.0` and put it behind your own proxy; [Serving Performance](#serving-performance) says which GPU to pick.
+
+## What to Expect
+
+**Accuracy.** Kev-27B is within three points of Jev, or ahead of it, on 9 of the 11 new-source categories in the chart below. Kev-4B and Kev-9B are about as close on classification-shaped sources like routing, entailment and science questions. Knowledge questions depend mostly on the base model: on MMLU Kev-9B scores 0.73 and Kev-27B matches Jev at 0.90, but on the harder MMLU-Pro Kev-27B scores 0.675 against Jev's 0.840. The smaller models also trail on day-precision date arithmetic.
+
+![Accuracy by source for Kev and Jev](docs/kev-family.png)
+
+**Confidence.** Each checkpoint ships with a fitted temperature, so its probabilities are calibrated by default. As served, Kev-9B puts at least 0.9 probability on a wrong answer for 2.4% of new-source questions, against Jev's 3.7%. Jev still ranks its answers better: at a 5% error budget, Kev-4B, 9B and 27B can automate 0.52–0.69 of new-source decisions, Kev-0.8B 0.14 and Jev 0.70. Check a threshold on your own data before you rely on it.
+
+**Speed.** Kev-4B answers six questions about a new short text in 18.1 ms of model time on an H100 and 41.5 ms on an L40S, and a container serves around 101 requests per second on an H100. On an Apple M5, Kev-4B takes 721 ms for five questions, or 136 ms when the text repeats and comes from the cache. [Serving Performance](#serving-performance) has every GPU and batch size.
+
+**Length.** Kev-0.8B, 4B and 9B trained mostly on states of up to 384 tokens, with longer ones in their document and skill fine-tunes (up to 7,552 tokens), Kev-27B on states of up to 32,768. The server accepts states of up to 65,536 tokens, and 8,192 more for each question, and refuses a longer one with a 422 instead of cutting it. How far past its training length each model stays accurate is the "Validated context" column in [Models](#models). For Kev-0.8B, 4B and 9B that is 8,192 tokens: at 16k the measurement on real contracts can no longer rule out a drop of more than 3 points, and at 32k all three are measurably less accurate than at 8k. Kev-27B holds to the 65,536-token limit. On real contracts of up to 64k tokens (CUAD) Kev-27B scores 0.874, and its confidence there is less reliable than on short text; its [model card](docs/model-cards/kev-27b.md) has the numbers by length.
+
+## Playground
+
+With the server running, open another terminal. You'll need Node 20.9+:
+
+```bash
+cd playground
+npm install
+npm run dev -- -p 3001
+```
+
+Open [localhost:3001](http://localhost:3001), load a preset, and edit the text and questions. Press `⌘↵` to run it. "Packed vs separate" compares asking all questions at once with asking them one at a time. "Permute" runs a Choice question with six option orders. There are also presets for testing question isolation and fake delimiter tokens.
+
+![Kev playground](docs/playground.png)
+
+There's a [chess demo](http://localhost:3001/chess), too. The board is the input, legal moves are Choice options, and a Score question rates the position. You can play against Kev or let it play itself. Games are saved in `localStorage`.
+
+## API
+
+### `POST /v1/systemone`
+
+`state` is the text to evaluate. Each question has instructions and, where needed, a set of answers to choose from.
+
+```jsonc
+{
+  "state": "…",                          // string | object | array — the content to evaluate
+  "model": "kev-latest",
+  "questions": {
+    "<id>": {                            // you choose the id; the model never sees it
+      "type": "noul" | "choice" | "score",
+      "instructions": "…",               // string | object | array, optional
+      "criteria": …                      // noul: {true?, false?}  choice: {option: description|null}  score: [level, …]
+    }
+  }
+}
+```
+
+| Type | Criteria | Answer |
+|---|---|---|
+| `noul` | Optional descriptions for `true` and `false` | `noul`: probability of yes |
+| `choice` | 1–255 option names, each with a description or `null` | `choice`: most likely option; `probabilities` and `confidence` |
+| `score` | 1–255 descriptions, ordered from lowest to highest | `score`: mean level index, starting at 0; `legend`, `probabilities`, and `confidence` |
+
+For Choice with `K > 1` options, confidence is `(p_max − 1/K) / (1 − 1/K)`. A single option has confidence 1. Score confidence is `max(0, 1 − E|level − mode| / D)`: `mode` is the most likely level and `D` is the mean distance of a uniform distribution over the levels from its middle (2/3 for three levels), so all probability on one level gives 1 and a uniform or wider spread gives 0. Both formulas are the ones in TypeSafe's reference adapter ([`system-one-adapter`](https://github.com/typesafe-ai/system-one-adapter-python) 0.2.1). Neither field is a measured accuracy rate.
+
+Objects and arrays are converted to labeled text. Delimiter-like strings in user input are escaped before tokenization. Invalid requests return `422`, and so does a state longer than 65,536 tokens: the server never drops part of a document silently, and the error gives the state's token count and the limit. `usage.output_tokens` counts tokens in the serialized answers, not generated tokens.
+
+| Method | Path | Purpose |
+|---|---|---|
+| `GET` | `/v1/models` | Model cards (`name`, `description`, `release_date`) plus the loaded checkpoint's details |
+| `POST` | `/v1/systemone/permute` | Run one Choice question with different option orders (`n_perm` 1 to 64, default 6) |
+| `POST` | `/v1/systemone/separate` | Run each question in its own forward pass |
+
+A request may carry any number of questions. The server runs them a token budget at a time (one maximal row of 16,384 tokens per forward pass, counting the cached document once per question in that pass), so memory does not grow with the question count and the answers do not depend on the split. Every response carries an `x-typesafe-request-id` header. The server binds to `127.0.0.1` (`--host 0.0.0.0` to accept other machines) and is open by default; set `KEV_API_KEY` to require `Authorization: Bearer <key>` on `/v1/*`, as the TypeSafe clients always send it.
+
+| Variable | Effect |
+|---|---|
+| `KEV_TEMPERATURE=1.0` | Return raw probabilities instead of the calibrated ones |
+| `KEV_DATE_FACTS=1` | Append the number of days between any two dates in the state (see [Benchmarks](#benchmarks)) |
+| `KEV_TRUNCATE_STATES=1` | Read the first 65,536 tokens of a longer state instead of refusing it; every response then has `truncated` and `usage.state_tokens` / `state_tokens_used` |
+| `KEV_DTYPE=fp32` | Serve the exact fp32 path the evaluations use (bf16 is the default on GPUs) |
+| `KEV_API_KEY` | Require a bearer key |
+
+## How It Works
+
+Each checkpoint is a rank-16 LoRA adapter and a small pointer head on a Qwen base model. On an attention-only base (Qwen3), the state and questions go into one token sequence:
+
+```text
+<state> …state…
+<q> instructions <opt> option 1 </opt> <opt> option 2 </opt> … <decide>
+<q> instructions <opt> option 1 </opt> <opt> option 2 </opt> … <decide>
+```
+
+The attention mask lets a token read the state and its own question, but not other questions or future tokens. Each question's position IDs restart just after the state. This lets the model process the state once and answer each question independently.
+
+Qwen3.5 and Qwen3.8 mix attention layers with Gated DeltaNet layers, which are recurrent and ignore attention masks. For those models, which is every current Kev, each question runs as its own row: the state followed by that question, with the same positions as above. The rows are independent, so isolation is exact, and the server and `DecisionModel.probs()` compute the state once and reuse its cache for every row. `forward()`, which `kev.benchmark` scores and every published number comes from, keeps the plain rows and runs the state once per question; the two agree to fp32 rounding. On attention-only models the rows and the mask above give identical probabilities (`tests/test_model.py`).
+
+Kev-27B uses the same design on `Qwen/Qwen3.8-27B`, with two differences. Its base is Qwen's post-trained release rather than a `-Base` checkpoint, and we don't know what it was post-trained on. And every backbone weight is trained, not just an adapter, and kept in bf16, so the checkpoint is the whole model: 51 GB of bf16 weights plus the pointer head. It serves in bf16 only (about 66 GB resident with the serving buffers), which is why it needs an 80 GB card. On Apple Silicon the MLX backend loads those weights as they are, with no merge (see [Serving Performance](#serving-performance)); we expect that to fit a 96–128 GB Mac but haven't measured it. Its served probabilities stay within 0.022 of the evaluation path on an H200 (`runs/serving-27b-r23`).
+
+The pointer head scores each option's `</opt>` hidden state against the question's `<decide>` hidden state. A softmax turns those scores into probabilities. Because `<decide>` comes last, it can attend to the full option list.
+
+Training uses cross-entropy on the correct answer. The adapter and head are trained together; the rest of the base weights stay fixed (Kev-27B trains them all). Training examples and API requests use the same text format. No Jev outputs were used for training.
+
+Asking questions together or separately produces probabilities within 4e-6 in the fp32 tests. This does **not** mean option order is irrelevant: options within a question can still affect one another. See [the model code](kev/model.py) and [parity tests](tests/test_model.py).
+
+## Training
+
+The released models share one base training set, `decision-v7`: 10,000 examples from ten public datasets, 896 generated policy examples, and 1,680 examples from 60 generated rule structures. Kev-0.8B, 4B and 9B train on it for two epochs with LoRA rank 16 and cross-entropy. The learning rate is `1e-4` for 0.8B and `5e-5` for 4B and 9B. On these hybrid bases the adapter covers the attention, MLP and DeltaNet projections; `kev.train` picks the right targets from the model config.
+
+Kev-0.8B, 4B and 9B then get short follow-up fine-tunes from their released checkpoints, through the same `--init_from` path you'd use for your own data: generated cases that state day counts or have the deciding evidence removed (all three), then real documents and generated skill data (all three; Kev-9B since v2, 2026-09-30). Kev-27B is trained differently. Every weight of the base is fine-tuned for one epoch on eight H200s (`--full_ft 1`, learning rate `2e-6`) on a 145,840-record corpus: Kev's own data, the document, skill and developer-tooling suites, public datasets, licensed task families and generated long-document, tool-routing, agent-log and guardrail records, with states of up to 32,768 tokens. The result is then averaged with the earlier adapter-trained Kev-27B, 0.85 to 0.15. The model cards list every stage with its data and cost.
+
+```bash
+# sanity run, ~1 minute
+uv run python -m kev.train --n_per_source 40 --accum 4 --out runs/smoke
+
+# the first stage of Kev-0.8B (~20 min on one H100; the Mac path works but is slow for Qwen3.5 bases)
+uv run python -m kev.train --suite evals/v7/decision-v7 --base Qwen/Qwen3.5-0.8B-Base --base_revision dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68 \
+    --epochs 2 --lr 1e-4 --batch 8 --dtype bf16 --p_none_pair 0.25 --device cuda --out runs/kev-0.8b
+
+# the first stage of Kev-4B (one H100 via Modal, ~1 h; see below). Swap in Qwen/Qwen3-4B-Base for the previous generation.
+uv run python -m kev.train --suite evals/v7/decision-v7 --base Qwen/Qwen3.5-4B-Base --base_revision 1001bb4d826a52d1f399e183466143f4da7b741b \
+    --epochs 2 --lr 5e-5 --batch 4 --accum 2 --dtype bf16 --checkpointing 1 --p_none_pair 0.25 --device cuda --out runs/kev-4b
+```
+
+Use `uv run python -m kev.train --help` for all training options. The released models don't use the optional `--perm_kl` or `--ord_w` losses. [PLAN.md](PLAN.md) records what was tried, what helped, and what didn't.
+
+### Modal
+
+Each trial gets its own H100. The study keeps running if you disconnect, and you can download the results when it finishes:
+
+```bash
+uv run modal token new                                    # once; opens the browser
+KEV_GPU=T4 uv run modal run modal_app.py::smoke           # end-to-end check, ~1 minute of GPU
+
+uv run modal deploy modal_app.py                          # once; studies run on the deployed app and survive disconnects
+uv run modal run modal_app.py::study \
+    --suite evals/v7/decision-v7 --plan experiments/v7-final.json \
+    --name my-study --transfer evals/v4/transfer-v4 --budget 30 --timeout 7200
+uv run modal run modal_app.py::pull --name my-study       # results -> runs/my-study, ranked
+```
+
+[Study plans](experiments/v7-final.json) list training settings. Each trial saves the settings, code hashes, dataset hashes, and results. Choose models using the development results, not the locked test. After choosing a final candidate, you can read its test results once:
+
+```bash
+uv run modal run modal_app.py::locked_test --trial my-study/00-trial-0 --name my-candidate   # one read, ever
+```
+
+## Benchmarks
+
+The evaluation data under `evals/` is frozen: dataset versions and file checksums are recorded in each manifest. Large files are downloaded from [the Hub mirror](https://huggingface.co/datasets/jaredpalmer/kev-suites) and checked against those hashes. Every model in the tables above is scored on the same items. The numbers in this README and the model cards are checked in CI against the committed reports they come from (`docs/claims.json`, `uv run python scripts/verify_claims.py`).
+
+| Suite | What it measures |
+|---|---|
+| `decision-v7` | Held-out examples from the ten training datasets, the generated policies and the rule structures ("trained sources") |
+| `transfer-v4` | 764 records from datasets and policy and rule types Kev never trained on: QNLI, SciQ, PAWS, MMLU, Emotion, TweetEval, held-out policies and rules ("new sources") |
+| `transfer-v9` | `transfer-v4` plus 10-way MMLU-Pro, records buried in unrelated text, and "unknowable" records whose deciding evidence was removed |
+
+```bash
+uv run python -m kev.benchmark --run jaredpalmer/kev-4b --suite evals/v4/transfer-v4 --out runs/my-eval      # new sources
+uv run python -m kev.benchmark --run jaredpalmer/kev-4b --suite evals/v9/transfer-v9 --out runs/my-eval-v9   # + MMLU-Pro, buried states, unknowable items
+uv run python -m kev.benchmark --run jaredpalmer/kev-4b --suite evals/v7/decision-v7 --out runs/my-eval-id   # trained sources
+uv run python -m kev.benchmark --remote http://127.0.0.1:8009 --suite evals/v4/transfer-v4 --out runs/my-remote   # any System One endpoint, Jev included
+```
+
+These commands use development data. Test data requires `--allow-test`. The benchmark reports accuracy, Brier score, calibration error, the share of decisions you could automate at a 5% error budget, option-order changes, and question isolation. On the unknowable records it reports how often the model still answers with at least 0.9 confidence (Kev-9B 0%, Jev 9%). Published accuracy numbers use fp32 evaluation, not the bf16 serving path. `kev.jev` runs the same questions against Jev through Vercel AI Gateway, and `kev.compare` compares two saved runs with paired bootstrap confidence intervals.
+
+**Calibration.** Each checkpoint stores a temperature, and the pointer head applies it when the model is loaded. Kev-4B (2.41) and Kev-0.8B (2.35) fitted theirs on their in-distribution development sets; Kev-27B (1.32) and Kev-9B (2.19) fitted theirs on held-out datasets they never trained on. A refit of the two smaller models on those held-out datasets was tested and kept neither: it did not improve Kev-4B and made Kev-0.8B worse calibrated on its document and skill suites (the model cards have the numbers). A temperature never changes which answer wins. On new sources it takes Kev-9B's calibration error from 0.103 to 0.041 and its confident errors (wrong answers with probability ≥ 0.9) from 8.2% to 2.4%, below Jev's 3.7%. The accuracy numbers above are the same either way; the Brier numbers are for the raw probabilities. `scripts/calibrate_checkpoint.py` also reports an out-of-fold estimate, so the in-sample fit can be checked against records it didn't see.
+
+**Dates.** Kev can't subtract dates reliably, but it can use a day count it's given. `KEV_DATE_FACTS=1` appends one sentence per pair of dates in the state ("June 26, 2026 is 8 days before July 4, 2026"). On the deadline policy questions this takes Kev-9B from 0.80 to 0.90 (Jev 0.93). None of the tables use it.
+
+**Other people's test sets.** `evals/external/` holds test sets from other projects, converted to this format, with their published live Jev results. Some were scored on earlier versions of the Kev weights, which the Kev column names. Three were removed because they cannot serve as a gate, and the model cards keep the numbers their releases were decided on: scienthoon's synthetic support tickets on 2026-09-27 (templated text; one of its three questions depends on a rule the text does not state), and on 2026-09-30 WANLI (`wanli-v1`, `wanli-v2`: a quarter of the pairs are ones WANLI's two annotators labelled differently, with the gold set to one of them) and TypeSafe's public evals (`typesafe-v1`: the gold is the averaged answer of two closed frontier models, and on 89 questions it cannot tell checkpoints apart).
+
+| Suite | What it is | Jev | Kev |
+|---|---|---|---|
+| [SemIf](https://github.com/TheoLeeCJ/SemIf) | 144 authored decisions | 0.965 | 0.917 (Kev-9B at `v7-base`) |
+
+SemIf's labels hold up, but it is close to saturated: every Kev-27B checkpoint answers 130 of the 144 correctly, so it is a sanity check, not a way to rank models.
+
+## Serving Performance
+
+Pick the GPU by the model:
+
+| Model | GPU ($/h) | 6 questions, short text | 5 questions, 2,200-token text | Requests/s, 64 clients |
+|---|---|---|---|---|
+| Kev-0.8B | L4 (0.80) | 22.7 / 16.1 ms | 108.6 / 32.3 ms | 62.8 |
+| Kev-4B | L40S (1.95) | 41.5 / 27.7 ms | 145.2 / 43.0 ms | 51.4 |
+| Kev-4B | H100 (3.95) | 18.1 / 12.9 ms | 89.4 / 22.5 ms | 100.8 |
+| Kev-9B | L40S (1.95) | 66.4 / 42.7 ms | 235.6 / 57.5 ms | 32.7 |
+| Kev-9B | H100 (3.95) | 24.0 / 16.6 ms | 88.5 / 26.4 ms | 79.5 |
+| Kev-27B | B200 (6.25) | 46.5 / 32.2 ms | 178.0 / 52.1 ms | 44.2 |
+| Kev-27B | H200 (4.54) | 67.2 / 50.0 ms | 274.8 / 73.8 ms | 28.6 |
+| Kev-27B | H100 (3.95) | 75.0 / 52.0 ms | 277.5 / 79.3 ms | 28.9 |
+
+Times are model time per request (the `latency_ms` the API returns), median of 20, for a new text / the same text again. The server caches the text, so asking more questions about a document you've already sent only pays for the questions. Requests per second are for 64 concurrent clients sending six questions about a new short text each; the server batches them. Kev-27B's B200 and H100 rows were measured on its previous version, the same architecture served in bf16 (`runs/fused-27b-*`); the H200 row is the current checkpoint (`runs/serving-27b-r23`). Network time is extra: about 65 ms per round trip through a Modal web endpoint in the same region.
+
+An L4 is enough for Kev-0.8B but too slow for Kev-4B. The A100 is slower than the L40S here and costs more. Kev-9B needs about 17 GB of GPU memory and Kev-27B 51 GB of weights (about 66 GB with the batching buffers); under load Kev-27B is compute-bound, and a B200, H200 or H100 costs about the same per request. On CUDA, install `flash-linear-attention` for the Qwen3.5 models (`kev_serve.py` and the Modal images already do).
+
+On Apple Silicon, `uv sync --extra serve` installs [MLX](https://github.com/ml-explore/mlx-lm) and the server uses it automatically. Five questions about a ~270-token text on an M5 (32 GB):
+
+| Model | New text | Same text again |
+|---|---|---|
+| Kev-0.8B | 149 ms | 28 ms |
+| Kev-4B | 721 ms | 136 ms |
+
+Long documents are read into the cache 1,024 tokens at a time, so memory stays close to the weights. With a 65,000-token document, Kev-0.8B takes 21.2 s the first time and 202 ms after that, at a 3.8 GB peak, and Kev-4B 84.5 s and 716 ms at 13.0 GB (`runs/mlx-long-states`; the model cards have every length). Kev-9B has not been measured this way yet.
+
+Adapter checkpoints are folded into the base as they load, which briefly holds a second copy of the weights. Full-weight checkpoints like Kev-27B load as saved, with nothing merged, so loading needs only the weights. We checked this on Kev-4B written out as full bf16 weights: loading peaked at 8.4 GB for 8.4 GB of weights, against 15.9 GB for the adapter path. Its answers matched the adapter path exactly once both hold the same bf16 values, and stayed within 0.015 of the fp32 path on 60 questions (`runs/mlx-full-4b`). Kev-27B's weights are 51 GB. By the same measurements it needs about 51 GB plus working memory, so a 64 GB Mac is borderline and a 96–128 GB Mac should fit. We haven't run it on a Mac that large yet. Kev-27B's first version, an adapter, did run this way on a 128 GB M5 Max, matching the published accuracy (thanks to Sean Connelly, [#175](https://github.com/jaredpalmer/kev/pull/175)).
+
+The server runs in bf16 on GPUs and Macs. Its probabilities differ from the fp32 path the published evaluations use by at most about 0.03 on a GPU and 0.05 on a Mac, and the top answer changes on about one question in 300. Set `KEV_DTYPE=fp32` for the exact path. `/v1/models` reports the backend and precision in use. `uv run modal run modal_app.py::serving --run jaredpalmer/kev-4b --gpu L40S --name <name>` measures a row of the table on your own account (the rows above: `runs/serve-*`, `runs/grouping-4b-h100`, `runs/fused-27b-*`, `runs/serving-27b-r23`).
+
+## Limitations
+
+- Calibration is a single temperature. It can't reorder confidences, so the share of new-source decisions you can automate at a 5% error budget (0.52–0.69 for Kev-4B, 9B and 27B) is still below Jev's 0.70. Test a probability threshold on your own data before you rely on it.
+- Knowledge questions are set by the base model. MMLU is 0.73 for Kev-9B against Jev's 0.90, and MMLU-Pro 0.59 against 0.84.
+- Fine-tuning can make the base model worse at individual tasks. Date arithmetic was the clearest case ([issue #8](https://github.com/jaredpalmer/kev/issues/8)); training on stated day counts plus `KEV_DATE_FACTS=1` recovers it.
+- Changing option order can change an answer. Question isolation doesn't prevent this.
+- Kev-0.8B, 4B and 9B trained mostly on at most 384 state tokens and 1,024 tokens for the state plus one question (their document and skill fine-tunes on states of up to 7,552 tokens), Kev-27B on states of up to 32,768 tokens. Serving allows a 65,536-token state; the validated context length of each model is in [Models](#models).
+- On a Mac, answers take hundreds of milliseconds, not tens. Kev-27B needs an 80 GB GPU. On a Mac it needs about 51 GB plus working memory; we expect a 96–128 GB Mac to fit it but haven't measured one.
+- Kev-27B starts from a post-trained model whose training data we don't know.
+
+## Development
+
+```bash
+uv run --extra serve python -m pytest tests/test_unit.py tests/test_research.py tests/test_generators.py tests/test_conventions.py \
+    tests/test_documents_tools.py tests/test_hard_v1.py tests/test_devtools_v1.py tests/test_breadth_v1.py tests/test_rounds.py tests/test_skill_scripts.py -q   # no weights, no server; what CI runs
+KEV_BASE_URL=http://127.0.0.1:8009 uv run --extra serve python -m pytest tests/test_api.py -q   # against a running server
+cd playground && npm run lint && npx next typegen && npx tsc --noEmit -p .
+```
+
+The API tests run TypeSafe's example requests and the official SDK against your local server. [PLAN.md](PLAN.md) is the research plan: what we have learned, the rules every experiment follows, and one line per round. The full log (every experiment, the criteria set before it ran, and how it came out) is at the git tag `research-archive-2026-09-24`.
+
+<details>
+<summary>Previous generation (Qwen3) and the prototype</summary>
+
+The first Kev family used Qwen3 bases with the same data and settings. Those weights stay published and run on plain PyTorch on a Mac, but they are no longer developed.
+
+| Model | Base | Accuracy: Trained Sources | Accuracy: New Sources | Brier: New Sources | Model Card |
+|---|---|---|---|---|---|
+| Kev-0.6B (Qwen3) — `jaredpalmer/kev-0.6b` | Qwen3-0.6B-Base | 0.801 / 0.808 | 0.620 / 0.642 | 0.536 / 0.483 | [Details](docs/model-cards/kev-0.6b-qwen3.md) |
+| Kev-4B (Qwen3) — `jaredpalmer/kev-4b@qwen3` | Qwen3-4B-Base | 0.854 / 0.856 | 0.790 / 0.806 | 0.328 / 0.294 | [Details](docs/model-cards/kev-4b-qwen3.md) |
+| Kev-8B (Qwen3) — `jaredpalmer/kev-8b` | Qwen3-8B-Base | 0.863 / 0.870 | 0.796 / 0.780 | 0.337 / 0.327 | [Details](docs/model-cards/kev-8b-qwen3.md) |
+
+The original [Kev-0.5B](https://huggingface.co/jaredpalmer/kev-0.5b) used Qwen2.5-0.5B and is kept for reference; see its [model card](docs/model-cards/kev-0.5b.md).
+
+</details>
+
+<details>
+<summary>Troubleshooting</summary>
+
+- If MPS runs out of memory during training, check that you're running only one job. Don't enable `output_hidden_states` or add tokens with peft's `trainable_token_indices`; both have caused memory problems here.
+- If the playground loads but buttons don't work, use `localhost:3001`. Next.js checks development hostnames. Other hosts need an entry in `allowedDevOrigins` in `playground/next.config.ts`.
+- If dataset loading reports `Dataset scripts are no longer supported`, use `legacy-datasets/banking77`. This repo already uses it.
+
+</details>
+
+## Authors
+
+- Jared Palmer ([@jaredpalmer](https://github.com/jaredpalmer))
+
+Built with [Devin](https://devin.ai). Thanks to [Archer Hume](https://archerhume.com/posts/jevs-architecture-unmasked) for the architecture write-up, [TypeSafe](https://docs.typesafe.ai/api) for the API design, and [Qwen](https://huggingface.co/Qwen/Qwen3.5-9B-Base) for the base models.
+
+Related work: [Hydragen](https://arxiv.org/abs/2402.05099), [DeFT](https://arxiv.org/abs/2404.00242), [FIRST](https://arxiv.org/abs/2406.15657).
+
+## License
+
+[Apache-2.0](LICENSE). The Qwen3, Qwen3.5 and Qwen3.8 base models are also Apache-2.0. Training datasets have their own licenses; see the [model cards](docs/model-cards/).

runtime/docs/packed-mlx.md

@@ -0,0 +1,132 @@
+# Standalone Packed MLX Models
+
+The packed checkpoint runs on Apple Silicon using MLX. It carries the entire
+compressed backbone, compensation adapter, pointer head, temperature, model
+configuration and tokenizer. Loading and inference never read the original
+backbone or the original PEFT adapter, and work with an empty Hugging Face cache
+and networking disabled.
+
+Ternary weights stay in MLX's native 2-bit affine format; selected linear layers
+and the embedding stay 4-bit. CLoQ is applied as two small matrix products,
+without building its dense delta. The embedding dequantizes only requested
+token rows. Activations are bf16. The constructor discards lazy random dense
+parameters before evaluating any model parameters.
+
+## Run
+
+From the Kev repository root, with the existing MLX environment:
+
+```sh
+HF_HUB_OFFLINE=1 HF_HOME=/tmp/kev-empty-cache \
+KEV_BACKEND=mlx KEV_PREFIX_CACHE=1 KEV_PREFIX_MAX_TOKENS=4096 \
+.venv/bin/python -m kev.serve \
+  --run runs/ternary-rotation-20261005/standalone/kev-4b-T-cloq16-mlx \
+  --host 127.0.0.1 --port 8010
+```
+
+The server accepts the existing `POST /v1/systemone` API:
+
+```sh
+curl http://127.0.0.1:8010/v1/systemone \
+  -H 'Content-Type: application/json' \
+  -d '{"state":"I was charged twice for order 8812.","questions":{"team":{"type":"choice","instructions":"Which team should handle this ticket?","criteria":{"billing":"Charges and payments","shipping":"Delivery and delays"}}}}'
+```
+
+Library callers use the usual checkpoint interface:
+
+```python
+from kev.checkpoint import Checkpoint
+
+tokenizer, model = Checkpoint("path/to/standalone-model").load("mps")
+```
+
+Copy the **whole standalone model directory** to the target computer. Keep the
+Kev code and its MLX/serving dependencies available. No original model files or
+Hugging Face cache are required on the target computer. This is an MLX format;
+llama.cpp and non-Apple inference have not been implemented or validated.
+
+## Build from the Compressed Experiment
+
+The export reads the compressed packed file and its optional `.cloq.safetensors`
+sidecar, plus only the source's small head/tokenizer files and the base's
+`config.json`. The base/PEFT weight files are never read. Rotated and
+activation-order recipes are rejected rather than loaded incorrectly.
+
+```sh
+.venv/bin/python scripts/build_packed_model.py \
+  --packed path/to/experimental.safetensors \
+  --checkpoint path/to/head-and-tokenizer-directory \
+  --config path/to/base-config.json \
+  --out path/to/new-standalone-model
+```
+
+The export copies the tokenizer files and retains source hashes in `packed.json`
+and `head.pt`. It does not quantize again, train, or fit a temperature. Native
+2-bit storage uses more bits than the experiment's five-trits-per-byte storage,
+so the standalone file is larger than the archival packed file. Original
+experimental files and results remain unchanged.
+
+## Verify
+
+```sh
+.venv/bin/python -m pytest tests/test_packed_model.py -q
+
+HF_HUB_OFFLINE=1 HF_HOME=/tmp/kev-empty-cache \
+.venv/bin/python scripts/profile_packed_model.py \
+  --run path/to/standalone-model --mlx-limit-gb 3 --out path/to/memory.json
+
+HF_HUB_OFFLINE=1 HF_HOME=/tmp/kev-empty-cache KEV_BACKEND=mlx \
+.venv/bin/python -m kev.benchmark --run path/to/standalone-model \
+  --suite evals/v7/decision-v7 --device mps --out path/to/new-evaluation
+```
+
+The profiler blocks the original loaders and downloads, and reports both MLX
+allocator memory and sampled process physical footprint. Physical footprint
+includes Metal buffers; RSS alone does not. The MLX allocator cap is not a cap
+on the entire process or the operating system. A test on a larger-memory Mac
+does not establish compatibility with every low-memory device.
+
+Native quantized matmul plus a separate compensation adapter uses different
+arithmetic from the original experiment's merged, dequantized bf16 weights.
+Report its newly measured accuracy and Brier score; do not inherit the archival
+96.7% retention or claim bit-identical probabilities. Context length, option
+count and cached states also consume memory, so report those with memory results.
+`KEV_PREFIX_MAX_TOKENS` controls retained cache size, not input admission.
+
+The development partition is already used for recipe selection. Results remain
+development diagnostics, not an untouched final test or a general guarantee.
+
+## Measured 4B Result (2026-10-06)
+
+`standalone/kev-4b-T-cloq16-mlx` was evaluated with an empty HF cache in offline
+mode on decision-v7 development: all 1,204 records / 1,468 rows, 1,264 scored
+clean questions, zero rejections and zero truncations. Accuracy is **0.84415**
+(**96.82%** of the archived bf16-MLX accuracy 0.87184), Brier **0.23251**.
+Against the archived packed merged-bf16 probabilities, 1/1,468 argmaxes changed;
+mean per-row max probability difference is 0.00268, maximum 0.06209. The
+original bf16 model was not loaded or re-evaluated for this comparison.
+
+On an M4 Pro with 64 GiB RAM, fresh-process probes explicitly blocked the base
+loaders and downloads. MLX 0.32.2, mlx-lm 0.31.3, torch 2.8.0, transformers 5.17.0:
+
+| Measurement | Short request | Longer request |
+|---|---:|---:|
+| State tokens / total tokens | 19 / 42 | 3,519 / 3,542 |
+| Load time | 0.57 s | 0.57 s |
+| MLX active weights after load | 1.679 GB | 1.679 GB |
+| Process physical footprint peak during load | 2.396 GB | 2.398 GB |
+| MLX inference peak | 2.013 GB | 3.066 GB |
+| Process physical footprint inference peak | 2.846 GB | 3.862 GB |
+| First inference time | 0.155 s | 6.599 s |
+
+These are single-request probes, not a minimum-RAM device certification. No
+8-GB machine was tested. The 3-GB `mx.set_memory_limit` setting is explicitly a
+**guideline**, not a hard cap; the longer request exceeded it. Use the observed
+physical footprint when judging fit, and leave room for the OS and other apps.
+GB above means decimal GB. The full model folder is approximately 1.709 GB;
+native weights including compensation are 1.679 GB.
+
+Artifacts under `runs/ternary-rotation-20261005/standalone-checks/`:
+`kev-4b-T-cloq16-native/report.json`, `native-comparison.json`,
+`memory-short.json`, and `memory-long.json`. The longer synthetic filler probe
+measures memory/latency only and does not establish long-context accuracy.

runtime/kev/anchors.py

@@ -0,0 +1,71 @@
+"""Anchor targets: the frozen base model's zero-shot distribution over each training question's options, read from
+next-token letter logits (the same readout as scripts/base_mmlu_probe.py). Used by `kev.train --anchor --anchor_w` as a
+KL target so the fine-tune keeps what the base already knows (PLAN.md: fine-tuning erodes base capability).
+
+    uv run python -m kev.anchors --base Qwen/Qwen3-4B-Base --suite evals/v6/decision-v6 --out runs/anchors/v6-4b.json
+
+Targets are keyed by record id and question id, values keyed by option key, so they align with any option permutation
+the trainer applies; questions whose option set changed (none-of-the-above inserted) are skipped at training time.
+No Jev or other external model is involved: the teacher is the student's own base.
+"""
+import argparse
+import json
+from pathlib import Path
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from kev.api import question_keys
+from kev.device import default_device
+from kev.suite import load_split, write_json
+
+LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+
+def question_prompt(state, q):
+    text = state if isinstance(state, str) else " ".join(f"{k}: {v}" for k, v in state.items()) if isinstance(state, dict) else json.dumps(state)
+    keys = question_keys(q["type"], q.get("criteria"))
+    if q["type"] == "noul": texts = ["No", "Yes"]
+    elif q["type"] == "score": texts = list(q["criteria"])
+    else: texts = [v if v is not None else k for k, v in q["criteria"].items()]
+    prompt = f"{text}\n{q['instructions']}\n" + "\n".join(f"{LETTERS[i]}. {t}" for i, t in enumerate(texts)) + "\nAnswer:"
+    return prompt, keys
+
+
+def build(base, suite, out, split="train", device=None, revision=None, max_options=26, batch=16):
+    device = device or default_device()
+    tok = AutoTokenizer.from_pretrained(base, revision=revision); tok.padding_side = "left"
+    if tok.pad_token is None: tok.pad_token = tok.eos_token
+    model = AutoModelForCausalLM.from_pretrained(base, revision=revision, dtype=torch.bfloat16 if device != "cpu" else torch.float32).to(device).eval()
+    letter_ids = [tok.encode(" " + L, add_special_tokens=False)[0] for L in LETTERS]
+    records = load_split(suite, split)
+    jobs = [(r["_meta"]["id"], qid, *question_prompt(r["state"], q)) for r in records for qid, q in r["questions"].items() if len(q.get("criteria", [0, 1])) <= max_options or q["type"] == "noul"]
+    targets, skipped = {}, 0
+    with torch.no_grad():
+        for i in range(0, len(jobs), batch):
+            chunk = jobs[i : i + batch]
+            enc = tok([p for _, _, p, _ in chunk], return_tensors="pt", padding=True, truncation=True, max_length=3072).to(device)
+            logits = model(**enc).logits[:, -1].float()
+            for (rid, qid, _, keys), row in zip(chunk, logits):
+                if len(keys) > len(letter_ids): skipped += 1; continue
+                p = torch.softmax(row[letter_ids[: len(keys)]], -1).tolist()
+                targets.setdefault(rid, {})[qid] = dict(zip(keys, p))
+            if (i // batch) % 50 == 0: print(f"anchors {i}/{len(jobs)}", flush=True)
+    meta = {"base": base, "revision": revision, "suite": str(suite), "split": split, "readout": "zero-shot next-token letter logits",
+            "records": len(targets), "questions": sum(len(v) for v in targets.values()), "skipped_over_26_options": skipped}
+    write_json(out, {"_meta": meta, "targets": targets})
+    print(json.dumps(meta))
+    return meta
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base", required=True); ap.add_argument("--suite", required=True); ap.add_argument("--out", required=True)
+    ap.add_argument("--split", default="train"); ap.add_argument("--device"); ap.add_argument("--revision")
+    a = ap.parse_args()
+    if Path(a.out).exists(): raise FileExistsError(a.out)
+    build(a.base, a.suite, a.out, a.split, a.device, a.revision)
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/api.py

@@ -0,0 +1,165 @@
+"""TypeSafe-compatible request/response shapes (POST /v1/systemone) mapped onto the single pointer primitive.
+
+Noul   -> 2 options [false, true];            answer = p(true)
+Choice -> options 'name' or 'name: desc';      answer = argmax, probabilities by name, confidence
+Score  -> options = ordered level descriptions; answer = expected level, legend, probabilities by index
+"""
+import json
+import re
+from datetime import datetime
+from typing import Any, Literal, Union
+from pydantic import BaseModel, Field, model_validator
+
+JSONContent = Union[str, dict, list, int, float, bool, None]
+MAX_OPTIONS = 255
+
+
+class Noul(BaseModel):
+    type: Literal["noul"]
+    instructions: JSONContent = None
+    criteria: dict[str, JSONContent] | None = None
+
+
+class Choice(BaseModel):
+    type: Literal["choice"]
+    instructions: JSONContent = None
+    criteria: dict[str, JSONContent]
+
+    @model_validator(mode="after")
+    def _check(self):
+        if not 1 <= len(self.criteria) <= MAX_OPTIONS: raise ValueError(f"criteria must have 1..{MAX_OPTIONS} options")
+        return self
+
+
+class Score(BaseModel):
+    type: Literal["score"]
+    instructions: JSONContent = None
+    criteria: list[JSONContent] = Field(min_length=1, max_length=MAX_OPTIONS)
+
+
+Question = Union[Noul, Choice, Score]
+
+
+class SystemOneRequest(BaseModel):
+    state: JSONContent
+    model: str = "kev-latest"
+    questions: dict[str, Question] = Field(min_length=1)
+
+
+def render(v: JSONContent, indent: int = 0) -> str:
+    """Flatten str | object | array into text the model sees. Field names are kept as labels."""
+    pad = "  " * indent
+    if v is None: return ""
+    if isinstance(v, (str, int, float, bool)): return str(v)
+    if isinstance(v, list): return "\n".join(f"{pad}- {render(x, indent + 1).lstrip()}" for x in v)
+    return "\n".join(f"{pad}{k}:\n{render(x, indent + 1)}" if isinstance(x, (dict, list)) else f"{pad}{k}: {render(x)}" for k, x in v.items())
+
+
+def option_text(name: str, desc: JSONContent) -> str:
+    return name if desc is None or desc == "" else f"{name}: {render(desc)}"
+
+
+MONTHS = "January|February|March|April|May|June|July|August|September|October|November|December"
+_DATE = re.compile(rf"\b(?:{MONTHS}) \d{{1,2}}, \d{{4}}\b|\b\d{{4}}-\d{{2}}-\d{{2}}\b")
+
+
+def date_facts(text: str) -> str:
+    """Deterministic date arithmetic for the model: every pair of absolute dates found in `text`, as one sentence each
+    ("August 3, 2026 is 12 days after July 22, 2026."). The model cannot subtract dates reliably (issue #8); it can use a
+    stated day count. Returns "" when fewer than two dates are found. Dates are listed in order of first appearance."""
+    found = []
+    for m in _DATE.finditer(text):
+        raw = m.group(0)
+        try: d = datetime.strptime(raw, "%B %d, %Y") if "," in raw else datetime.strptime(raw, "%Y-%m-%d")
+        except ValueError: continue
+        if raw not in [r for r, _ in found]: found.append((raw, d))
+    facts = []
+    for i in range(len(found)):
+        for j in range(i + 1, len(found)):
+            n = (found[j][1] - found[i][1]).days
+            facts.append(f"{found[j][0]} is {abs(n)} day{'s' if abs(n) != 1 else ''} {'after' if n > 0 else 'before'} {found[i][0]}." if n else f"{found[j][0]} is the same day as {found[i][0]}.")
+    return " ".join(facts)
+
+
+def with_date_facts(state):
+    """State with a `date_facts` field (object states) or an appended paragraph (string states) when two or more absolute
+    dates appear. Opt-in preprocessing (KEV_DATE_FACTS=1 in kev.serve, --date_facts in kev.benchmark)."""
+    facts = date_facts(render(state))
+    if not facts: return state
+    if isinstance(state, dict): return {**state, "date_facts": facts}
+    if isinstance(state, list): return state + [{"date_facts": facts}]
+    return f"{state}\n\ndate_facts: {facts}"
+
+
+def question_keys(qtype: str, criteria) -> list[str]:
+    """The keys a question's probabilities are reported under, in option order: the criteria names (choice),
+    ["false", "true"] (noul), the level indices as strings (score). Labels, targets and anchors use the same keys."""
+    if qtype == "choice": return list(criteria)
+    if qtype == "noul": return ["false", "true"]
+    return [str(i) for i in range(len(criteria))]
+
+
+def to_record(req: SystemOneRequest):
+    """-> internal record for encode(), plus per-question metadata ({"id", "type", "keys", "legend" for score}) to map
+    probabilities back."""
+    qs, meta = [], []
+    for qid, q in req.questions.items():
+        m = {"id": qid, "type": q.type, "keys": question_keys(q.type, q.criteria)}
+        if q.type == "noul":
+            c = q.criteria or {}
+            opts = [option_text("no", c.get("false")), option_text("yes", c.get("true"))]
+        elif q.type == "choice":
+            opts = [option_text(k, v) for k, v in q.criteria.items()]
+        else:
+            opts = [render(x) for x in q.criteria]
+            m["legend"] = dict(zip(m["keys"], opts))
+        qs.append({"instr": render(q.instructions), "options": opts, "label": 0}); meta.append(m)
+    return {"state": render(req.state), "questions": qs}, meta
+
+
+# Both confidence formulas mirror TypeSafe's reference adapter, system-one-adapter 0.2.1
+# (src/system_one_adapter/_utils/confidence_metrics.py): p is normalised to sum 1 (all zeros -> uniform), one option -> 1.
+def _normalize(p: list[float]) -> list[float]:
+    t = sum(p)
+    return [1 / len(p)] * len(p) if t == 0 else [x / t for x in p]
+
+
+def choice_confidence(p: list[float]) -> float:
+    """(p_max - 1/K) / (1 - 1/K): 0 at uniform, 1 at certainty."""
+    K = len(p)
+    return 1.0 if K == 1 else (max(_normalize(p)) - 1 / K) / (1 - 1 / K)
+
+
+def score_confidence(p: list[float]) -> float:
+    """max(0, 1 - E|level - mode| / D), D = mean absolute deviation of a uniform distribution over the L levels around its
+    mean (L-1)/2; mode = first most likely level. 1 when all mass is on one level, 0 at uniform or anything as spread."""
+    L = len(p)
+    if L == 1: return 1.0
+    p = _normalize(p); mode = max(range(L), key=p.__getitem__)
+    D = sum(abs(i - (L - 1) / 2) for i in range(L)) / L
+    return max(0.0, 1.0 - sum(pi * abs(i - mode) for i, pi in enumerate(p)) / D)
+
+
+def round_prob(x: float) -> float:
+    """Serialization precision for probabilities and derived scalars. 4 decimals keeps the sum of a rounded distribution
+    within TypeSafe's tolerance (|sum - 1| < 0.02) at the 255-option maximum: 255 * 0.00005 < 0.02."""
+    return round(float(x), 4)
+
+
+def to_answers(probs: list[list[float]], meta: list[dict]) -> dict[str, Any]:
+    out = {}
+    for p, m in zip(probs, meta):
+        if m["type"] == "noul":
+            out[m["id"]] = {"type": "noul", "noul": round_prob(p[1])}
+        elif m["type"] == "choice":
+            dist = {k: round_prob(v) for k, v in zip(m["keys"], p)}
+            out[m["id"]] = {"type": "choice", "choice": m["keys"][max(range(len(p)), key=lambda i: p[i])], "confidence": round_prob(choice_confidence(p)), "probabilities": dist}
+        else:
+            score = sum(i * pi for i, pi in enumerate(p))
+            out[m["id"]] = {"type": "score", "score": round_prob(score), "legend": m["legend"], "probabilities": {str(i): round_prob(v) for i, v in enumerate(p)}, "confidence": round_prob(score_confidence(p))}
+    return out
+
+
+def output_tokens(tok, answers: dict) -> int:
+    """Billing-style figure: tokens of the serialised answers. Not a measure of generation (there is none)."""
+    return len(tok(json.dumps(answers), add_special_tokens=False).input_ids)

runtime/kev/autoresearch.py

@@ -0,0 +1,181 @@
+"""Unattended research sessions on the round harness, plus the study leaderboard.
+
+    uv run python -m kev.autoresearch session experiments/rounds/r19.json [...] --spend-start 1377 --spend-cap 300
+    uv run python -m kev.autoresearch leaderboard                 # runs/leaderboard.{jsonl,md} from every trial's result.json
+    uv run python -m kev.autoresearch release-check --study X     # every seed of a config must pass its gates
+    uv run python -m kev.autoresearch compare --studies a,b --reference runs/<study>/<trial>
+
+A session runs registered rounds in order through kev.rounds (validate, launch the studies, watch: pull and read each
+finished trial, then the read-out). What it may run is what the specs register: config-only trials through
+kev.experiment.execute_trial with full provenance, and the reads the rule names. It never changes the evaluator, the
+frozen suites or the gates, and it never confirms: test partitions and the locked read are read once, by a deliberate
+`python -m kev.rounds confirm` after a read-out names a candidate (the session prints those commands).
+
+Budget: before each round, `modal billing summary` is read; the session stops when metered spend since --spend-start
+plus the round's study budgets (each at least its admission bound, checked by kev.rounds.validate) would pass
+--spend-cap. Each round appends a line to runs/autoresearch-sessions.jsonl.
+"""
+import argparse
+import hashlib
+import json
+import subprocess
+import sys
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+from kev import rounds
+from kev.experiment import CHOICE_DEFAULTS, DEFAULTS
+from kev.metrics import paired_bootstrap
+from kev.suite import ENCODING, read_json, write_jsonl
+
+ROOT = Path(__file__).resolve().parents[1]
+H100_RATE = 3.95
+SNAPSHOT_KEYS = ("snapshot_fractions", "snapshot_every_steps", "snapshot_hub_repo")   # when and where a full-weight trial saves intermediate checkpoints: not what it trains
+INFRA_KEYS = ("base", "seed", "base_revision", "dtype", "checkpointing", "batch", "accum", "perm_frac", "shared_prefix", *SNAPSHOT_KEYS)   # execution shape, not recipe
+ALL_DEFAULTS = {**DEFAULTS, **CHOICE_DEFAULTS}   # what kev.train does when a knob is not given
+
+
+def config_digest(value):
+    """Canonical (key-sorted) digest for config identity; kev.suite.record_digest keeps insertion order for provenance."""
+    return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+
+
+def knobs(cfg):
+    """The recipe a config expresses: every non-default, non-infrastructure parameter."""
+    return {k: v for k, v in sorted(cfg.items()) if k not in INFRA_KEYS and v != ALL_DEFAULTS.get(k)}
+
+
+def recipe(row):
+    """A trial's recipe identity: its config without the seed (config_sha256 hashes the seed too) and snapshot schedule."""
+    return config_digest({k: v for k, v in row["config"].items() if k != "seed" and k not in SNAPSHOT_KEYS})
+
+
+# --- sessions --------------------------------------------------------------------------------------------------------
+
+def metered_spend():
+    out = subprocess.run([sys.executable, "-m", "modal", "billing", "summary", "--json"], capture_output=True, text=True, cwd=ROOT)
+    return float(json.loads(out.stdout)["metered_cost"]) if out.returncode == 0 else None
+
+
+def session(paths, spend_start, spend_cap, spend=metered_spend, log=print):
+    """Run each registered round to its read-out while the spend cap allows; returns the ledger entries written."""
+    entries = []
+    for path in paths:
+        spec = rounds.load(path)
+        problems = rounds.launchable(spec)
+        if problems: log(f"{path}: does not validate, session stops:\n" + "\n".join(problems)); break
+        spent, bound = spend(), sum(s["budget"] for s in spec["studies"].values())
+        if spent is None: log("cannot read `modal billing summary`; session stops"); break
+        if spent - spend_start + bound > spend_cap:
+            log(f"round {spec['round']}: spent ${spent - spend_start:.2f} + budgets ${bound:.2f} > cap ${spend_cap:.2f}; session stops"); break
+        rounds.launch_studies(spec)
+        report = rounds.watch(spec)
+        entry = {"round": spec["round"], "spec": str(path), "spend_before": round(spent - spend_start, 2), "budgets": bound, "candidates": report["candidates"],
+                 "at": datetime.now(timezone.utc).isoformat(timespec="minutes")}
+        with (ROOT / "runs/autoresearch-sessions.jsonl").open("a", encoding=ENCODING) as f: f.write(json.dumps(entry) + "\n")
+        entries.append(entry)
+        for arm in filter(None, report["candidates"].values()):
+            for stage in spec.get("confirm", {}):
+                log(f"candidate {arm}: uv run python -m kev.rounds launch-reads {path} --stage {stage} --arm {arm}  then  confirm {path} --stage {stage} --arm {arm}")
+    return entries
+
+
+# --- leaderboard -----------------------------------------------------------------------------------------------------
+
+def collect():
+    """Every completed trial in runs/*/ with its provenance, as flat leaderboard rows."""
+    rows = []
+    for result in sorted(ROOT.glob("runs/*/*/result.json")):
+        r = read_json(result)
+        prov = r.get("provenance", {}); cfg = prov.get("config") or {}
+        tr = r.get("transfer") or {}
+        rows.append({"study": result.parent.parent.name, "trial": result.parent.name, "legacy": prov.get("legacy_checkpoint", False),
+                     "base": cfg.get("base"), "seed": cfg.get("seed"), "config": cfg, "config_sha256": prov.get("config_sha256"),
+                     "suite_sha256": prov.get("suite_sha256"), "transfer_suite_sha256": tr.get("suite_sha256"), "git": prov.get("git_commit", "")[:8],
+                     "dev_acc": r["clean"]["acc"], "dev_nll": r["clean"]["nll"], "dev_ece": r["clean"]["ece"],
+                     "transfer_acc": tr.get("clean", {}).get("acc"), "transfer_brier": tr.get("clean", {}).get("brier"),
+                     "transfer_conf_err": tr.get("clean", {}).get("confident_error_rate"),
+                     "heldout_pairs": (tr.get("paired_flip") or {}).get("both_correct_rate"),
+                     "none_present": r["variants"].get("none_present", {}).get("acc"), "perm_flip": r["permutation"]["flip_rate"],
+                     "gates": r["gates"]["checks"], "gates_passed": r["gates"]["passed"],
+                     "train_s": (r.get("training_resources") or {}).get("wall_seconds"), "wall_s": r.get("wall_seconds"),
+                     "est_usd": round(H100_RATE * r.get("wall_seconds", 0) / 3600, 2) if prov.get("device") == "cuda" else None})
+    return rows
+
+
+def leaderboard_md(rows):
+    lines = ["# Leaderboard", "", f"Generated {datetime.now(timezone.utc).isoformat(timespec='minutes')} from runs/*/result.json. "
+             "Selection on development partitions only; the locked test is never read here. Rounds are decided by their "
+             "registered rules (`python -m kev.rounds readout`), not by this table.", "",
+             "| study/trial | base | seed | dev acc | transfer acc | Brier | conf-err | held-out pairs | none_present | perm flip | gates | $ | knobs |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
+    for r in sorted(rows, key=lambda r: (r["transfer_acc"] is None, -(r["transfer_acc"] or 0))):
+        b = (r["base"] or "legacy").split("/")[-1]
+        f = lambda x, p=3: "" if x is None else f"{x:.{p}f}"
+        lines.append(f"| {r['study']}/{r['trial']} | {b} | {r['seed'] if r['seed'] is not None else ''} | {f(r['dev_acc'])} | {f(r['transfer_acc'])} | {f(r['transfer_brier'])} | {f(r['transfer_conf_err'])} | {f(r['heldout_pairs'],2)} | {f(r['none_present'],2)} | {f(r['perm_flip'],2)} | {'pass' if r['gates_passed'] else 'fail'} | {r['est_usd'] or ''} | {', '.join(f'{k}={v}' for k, v in knobs(r['config']).items()) or 'defaults'} |")
+    return "\n".join(lines) + "\n"
+
+
+def refresh_leaderboard():
+    rows = collect()
+    write_jsonl(ROOT / "runs/leaderboard.jsonl", rows)
+    (ROOT / "runs/leaderboard.md").write_text(leaderboard_md(rows), encoding=ENCODING)
+    return rows
+
+
+def compare(studies, reference, tasks=("mmlu", "paws", "qnli", "emotion", "tweet_offensive", "contrastive_deadline")):
+    """Print every trial of the given studies with a record-clustered paired bootstrap on transfer accuracy vs `reference`
+    (a runs/<study>/<trial> path). Development-set selection only."""
+    def rows(p): return read_json(ROOT / p / "transfer/rows.json")
+    ref_rows = rows(reference)
+    print(f"reference: {reference}")
+    print(f"{'trial':34} {'dev':>6} {'trf':>6} {'brier':>6} {'cerr':>6} {'pairs':>6} {'d trf':>7} {'ci95':>18}  knobs / tasks")
+    for study in studies:
+        for d in sorted((ROOT / "runs" / study).iterdir()):
+            if not (d / "result.json").exists(): continue
+            r = read_json(d / "result.json"); tr = r.get("transfer")
+            if not tr: continue
+            cfg = r["provenance"]["config"]
+            try:
+                b = paired_bootstrap(rows(f"runs/{study}/{d.name}"), ref_rows, metric="acc"); delta, ci = b["macro_acc_delta"], [round(x, 3) for x in b["ci95"]]
+            except ValueError:
+                delta, ci = float("nan"), "n/a (different suite)"
+            print(f"{study + '/' + d.name:34} {r['clean']['acc']:6.3f} {tr['clean']['acc']:6.3f} {tr['clean']['brier']:6.3f} {tr['clean']['confident_error_rate']:6.3f} {tr['paired_flip']['both_correct_rate']:6.2f} {delta:+7.3f} {str(ci):>18}  {knobs(cfg)} {({k: round(tr['tasks'][k]['acc'], 2) for k in tasks if k in tr['tasks']})}")
+
+
+def release_check(study):
+    """Release screen across seeds: every trial of the same config in the study must pass its gates (including the 70%
+    held-out-pair screen) - a single seed clearing the bar is not enough. Prints the verdict per config."""
+    rows = [r for r in collect() if r["study"] == study]
+    by_cfg = defaultdict(list)
+    for r in rows: by_cfg[recipe(r)].append(r)
+    verdicts = {}
+    for group in by_cfg.values():
+        seeds = sorted(r["seed"] for r in group)
+        passed = all(r["gates_passed"] for r in group)
+        pairs = [round(r["heldout_pairs"], 2) for r in group]
+        failing = sorted({k for r in group for k, v in r["gates"].items() if not v})
+        base = group[0]["base"].split("/")[-1]
+        verdicts[base] = {"seeds": seeds, "all_gates_passed": passed and len(seeds) >= 2, "heldout_pairs": pairs,
+                          "transfer_acc": [round(r["transfer_acc"], 3) for r in group], "failing_gates": failing,
+                          "candidate": passed and len(seeds) >= 2, "trials": [f"{r['study']}/{r['trial']}" for r in group]}
+        print(f"{base}: seeds {seeds} pairs {pairs} transfer {verdicts[base]['transfer_acc']} -> {'RELEASE CANDIDATE (gated locked read allowed)' if verdicts[base]['candidate'] else 'not a candidate: ' + ', '.join(failing or ['fewer than two seeds'])}")
+    return verdicts
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    p = sub.add_parser("session"); p.add_argument("specs", nargs="+"); p.add_argument("--spend-start", type=float, required=True); p.add_argument("--spend-cap", type=float, required=True)
+    sub.add_parser("leaderboard")
+    p = sub.add_parser("release-check"); p.add_argument("--study", required=True)
+    p = sub.add_parser("compare"); p.add_argument("--studies", required=True); p.add_argument("--reference", required=True, help="runs/<study>/<trial>")
+    a = ap.parse_args()
+    if a.cmd == "session": session(a.specs, a.spend_start, a.spend_cap)
+    elif a.cmd == "leaderboard": print(f"{len(refresh_leaderboard())} trials -> runs/leaderboard.md")
+    elif a.cmd == "release-check": release_check(a.study)
+    else: compare(a.studies.split(","), a.reference)
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/benchmark.py

@@ -0,0 +1,214 @@
+"""Score a predictor on a frozen suite partition (or your own labelled JSONL).
+
+    uv run python -m kev.benchmark --run runs/<run>/checkpoint --suite evals/<v>/decision-<v> --out runs/<name>
+    uv run python -m kev.benchmark --remote http://127.0.0.1:8008 --suite ... --out ...      # any System One endpoint
+
+Every prediction becomes one row per question (prediction_rows); kev.metrics scores rows; evaluate_records writes
+predictions.jsonl, rows.json and report.json. Predictors live in kev.predictors.
+"""
+import argparse
+import functools
+import json
+import math
+import os
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import numpy as np
+
+from kev.api import question_keys, with_date_facts
+from kev.checkpoint import LoadOptions
+from kev.contrastive import paired_flip
+from kev.data import api_request, load_records
+from kev.device import default_device
+from kev.metrics import EPSILON, grouped_metrics, metrics, unknowable_report
+from kev.model import ROW_PASS_TOKENS, ContextOverflow
+from kev.predictors import LocalPredictor, RemotePredictor, RotationAveraged
+from kev.suite import CONTEXT, ENCODING, digest, load_split, read_manifest, record_digest, write_json
+
+
+def labels(q):
+    """(option keys, label index) of a labelled request question."""
+    keys = question_keys(q["type"], q.get("criteria"))
+    return keys, keys.index(q["label"]) if q["type"] == "choice" else int(q["label"])
+
+
+def validate_distribution(raw, keys):
+    if set(raw) != set(keys):
+        raise ValueError("probability keys do not match requested options")
+    p = np.array([raw[k] for k in keys], dtype=float)
+    if not np.isfinite(p).all() or (p < 0).any() or (p > 1).any():
+        raise ValueError("non-finite or out-of-range probabilities")
+    total = float(p.sum())
+    if total <= 0 or abs(total - 1) > max(1e-5, len(keys) * 0.005 + 1e-8):
+        raise ValueError(f"invalid probability sum: {total}")
+    return p / total, total
+
+
+def prediction_rows(record, prediction):
+    if set(prediction["probabilities"]) != set(record["questions"]):
+        raise ValueError("answer IDs differ from request IDs")
+    meta = record["_meta"]
+    rows = []
+    for qid, q in record["questions"].items():
+        keys, y = labels(q)
+        p, total = validate_distribution(prediction["probabilities"][qid], keys)
+        row = {"id": meta["id"], "group": meta["group_id"], "question": qid,
+               "source": meta["source"], "task": q["src"], "type": q["type"],
+               "variant": meta["variant"], "keys": keys, "label": y, "control_id": meta.get("control_id"),
+               "pair_id": meta.get("pair_id"), "sibling": meta.get("sibling"), # suites frozen before parent_id existed stored the parent's id in group_id for variants
+               "parent": meta.get("parent_id") or (meta["id"] if meta["variant"] == "clean" else meta["group_id"]),
+               "p": p.tolist(), "raw_probability_sum": total, "zero_count": int((p == 0).sum())}
+        if "logits" in prediction:
+            raw_logits = prediction["logits"][qid]
+            if set(raw_logits) != set(keys) or not all(math.isfinite(raw_logits[k]) for k in keys):
+                raise ValueError("logit keys or values do not match the requested options")
+            row["logits"] = [float(raw_logits[k]) for k in keys]
+            row["inference_temperature"] = prediction["inference_temperature"]
+        if "kernels" in prediction: row["kernels"] = prediction["kernels"]   # a long row off the fp32-exact kernels (LocalPredictor)
+        rows.append(row)
+    return rows
+
+
+def summarize(rows, temperature=1.0, heldout_sources=()):
+    """Report over benchmark rows. heldout_sources: sources the scored model never trained on; their tasks are also
+    reported as a separate block."""
+    clean = [r for r in rows if r["variant"] == "clean"]
+    tasks = grouped_metrics(clean, "task")
+    variants = grouped_metrics(rows, "variant")
+    lookup = {(r["id"], r["question"]): r for r in clean}
+    diffs, flips = [], []
+    for row in rows:
+        if row["variant"] == "permuted" and row["type"] == "choice":
+            original = lookup[(row["parent"], row["question"])]
+            aligned = [row["p"][row["keys"].index(k)] for k in original["keys"]]
+            diffs.append(float(np.max(np.abs(np.array(aligned) - original["p"]))))
+            flips.append(int(np.argmax(aligned) != np.argmax(original["p"])))
+    knowable = [r for r in clean if r["source"] != "unknowable"]     # unknowable records are scored on confidence, never on accuracy
+    return {"objective": -float(np.mean([v["nll"] for k, v in tasks.items() if not k.startswith("unknowable_") or k.startswith("unknowable_control")])),
+            "paired_flip": paired_flip(clean), "unknowable": unknowable_report(clean),
+            "clean": metrics(knowable), "tasks": tasks, "variants": variants,
+            "heldout_tasks": grouped_metrics([r for r in clean if r["source"] in heldout_sources], "task") if any(r["source"] in heldout_sources for r in clean) else {},
+            "permutation": {"n": len(diffs), "mean_max_delta": float(np.mean(diffs)) if diffs else None,
+                            "flip_rate": float(np.mean(flips)) if flips else None},
+            "temperature": temperature, "calibrated_clean": metrics(knowable, temperature),
+            "metric_policy": {"version": 2, "selective_ties": "whole_confidence_groups",
+                              "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee",
+                              "aurc": "right-step integral over whole confidence groups",
+                              "confident_error_rate": "high-confidence errors divided by all questions",
+                              "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9",
+                              "nll": "exact from logits when recorded; otherwise from floored probabilities",
+                              "nll_floor": EPSILON, "renormalize_returned_probabilities": True,
+                              "raw_sums_outside_1e_5": sum(abs(r["raw_probability_sum"] - 1) > 1e-5 for r in rows),
+                              "returned_zeros": sum(r["zero_count"] for r in rows)}}
+
+
+def predictions(records, predictor):
+    """Yield, in record order, a zero-argument callable that returns predictor(record) or raises what it raised. A
+    predictor with `concurrency` > 1 (RemotePredictor: one independent HTTP request per call) keeps that many calls in
+    flight on a thread pool; in-process predictors (one GPU) have no `concurrency` and run one record at a time. Either
+    way the caller sees each outcome in the same order as the plain sequential loop."""
+    workers = getattr(predictor, "concurrency", 1)
+    if workers <= 1 or len(records) <= 1:
+        for record in records:
+            yield functools.partial(predictor, record)
+        return
+    executor = ThreadPoolExecutor(max_workers=min(workers, len(records)))
+    try:
+        for future in [executor.submit(predictor, record) for record in records]:
+            yield future.result
+    finally:
+        executor.shutdown(wait=False, cancel_futures=True)
+
+
+def evaluate_records(records, predictor, directory, temperature=1.0, heldout_sources=(), skip_overlong=False):
+    """skip_overlong: for external data that was not admitted to a context (--data, or an eval-only suite frozen as published),
+    records the predictor cannot encode are counted in coverage["rejected_records"] and listed in rejected.json instead of
+    aborting. Admitted suites never trigger it; reports must state how rejected records enter any headline number."""
+    directory = Path(directory)
+    directory.mkdir(parents=True, exist_ok=False)
+    coverage = {"requested_records": len(records), "requested_questions": sum(len(r["questions"]) for r in records),
+                "evaluated_records": 0, "evaluated_questions": 0, "rejected_records": 0, "truncated_records": 0}
+    rows, latencies, rejected = [], [], []
+    with (directory / "predictions.jsonl").open("w", encoding=ENCODING) as output:
+        preds = predictions(records, predictor)
+        for record in records:
+            try:
+                pred = next(preds)()
+                new_rows = prediction_rows(record, pred)
+            except ContextOverflow as error:
+                if skip_overlong:
+                    coverage["rejected_records"] += 1; rejected.append({"id": record["_meta"]["id"], "error": str(error)}); continue
+                coverage["rejected_records"] += 1
+                write_json(directory / "failure.json", {"coverage": coverage, "record_id": record["_meta"]["id"], "error_type": type(error).__name__})
+                raise
+            except Exception as error:
+                coverage["rejected_records"] += 1
+                write_json(directory / "failure.json", {"coverage": coverage, "record_id": record["_meta"]["id"], "error_type": type(error).__name__})
+                raise
+            output.write(json.dumps({"request_sha256": record_digest(api_request(record)), "id": record["_meta"]["id"],
+                                     "prediction": pred, "rows": new_rows}, allow_nan=False) + "\n")
+            output.flush()
+            rows.extend(new_rows)
+            latencies.append(pred["latency_ms"])
+            coverage["evaluated_records"] += 1
+            coverage["evaluated_questions"] += len(new_rows)
+            if coverage["evaluated_records"] % 50 == 0:
+                print(f"evaluated {coverage['evaluated_records']}/{len(records)}", flush=True)
+    write_json(directory / "rows.json", rows)
+    if rejected: write_json(directory / "rejected.json", rejected)
+    report = summarize(rows, temperature, heldout_sources)
+    report.update(coverage=coverage, latency_ms={"median": float(np.median(latencies)), "p95": float(np.quantile(latencies, .95))},
+                  calibration={"inference_temperature": getattr(predictor, "temperature", None),
+                               "additional_temperature": temperature, "logits_recorded": all("logits" in r for r in rows)})
+    long = [r for r in rows if "kernels" in r]
+    if long:   # absent when every row ran the exact kernels, so such reports are unchanged
+        report["long_rows"] = {"count": len(long), "records": len({r["id"] for r in long}), "kernels": sorted({r["kernels"] for r in long}), "threshold": ROW_PASS_TOKENS}
+    write_json(directory / "report.json", report)
+    return report, rows
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--run", help="checkpoint dir or Hub id (local scoring)")
+    ap.add_argument("--remote", help="base URL of a System One-compatible endpoint to score instead of a local checkpoint")
+    ap.add_argument("--remote-model", default="kev-latest")
+    ap.add_argument("--remote-concurrency", type=int, default=1, help="requests kept in flight against --remote (1 = sequential); rows and their order do not depend on it")
+    ap.add_argument("--suite", help="frozen suite directory (scores its development partition)")
+    ap.add_argument("--data", help="your own labelled requests, one JSON object per line (kev.data.load_records); an alternative to --suite")
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--device", choices=["cpu", "mps", "cuda"], default=default_device())
+    ap.add_argument("--allow-test", action="store_true")
+    ap.add_argument("--split", choices=["development", "calibration", "train"], default="development",
+                    help="suite partition to score (train: teacher predictions for distillation; --allow-test reads the locked test instead)")
+    ap.add_argument("--date_facts", action="store_true", help="apply kev.api.with_date_facts to every state before scoring (the opt-in serving preprocessor); reported in report.json")
+    ap.add_argument("--rotations", type=int, default=1, help="average every Choice question over this many cyclic option rotations (kev.predictors.RotationAveraged); 1 = one order")
+    a = ap.parse_args()
+    if bool(a.run) == bool(a.remote): ap.error("give exactly one of --run or --remote")
+    if a.rotations < 1: ap.error("--rotations must be >= 1")
+    if a.remote_concurrency < 1: ap.error("--remote-concurrency must be >= 1")
+    if bool(a.suite) == bool(a.data): ap.error("give exactly one of --suite or --data")
+    if a.data:
+        records, heldout, split, source_hash = load_records(a.data), [], "custom", digest(Path(a.data))
+        context, skip_overlong = CONTEXT, True
+    else:
+        split = "test" if a.allow_test else a.split
+        records = load_split(a.suite, split, allow_test=a.allow_test)
+        manifest = read_manifest(a.suite)
+        heldout = manifest["holdout_sources"]; source_hash = digest(Path(a.suite) / "manifest.json")
+        context, skip_overlong = manifest.get("context", CONTEXT), bool(manifest.get("eval_only"))
+    if a.date_facts:
+        records = [{**r, "state": with_date_facts(r["state"])} for r in records]
+    predictor = RemotePredictor(a.remote, a.remote_model, os.environ.get("KEV_REMOTE_API_KEY", "local"), concurrency=a.remote_concurrency) if a.remote else LocalPredictor(a.run, a.device, LoadOptions.from_env(), context=context)
+    scorer = RotationAveraged(predictor, a.rotations) if a.rotations > 1 else predictor
+    report, _ = evaluate_records(records, scorer, a.out, heldout_sources=tuple(heldout), skip_overlong=skip_overlong)
+    report.update(suite_sha256=source_hash, data=a.data, date_facts=a.date_facts, rotations=a.rotations, run=a.run or a.remote, split=split,
+                  calibration_applied=predictor.temperature != 1.0 if not a.remote else None,
+                  environment=predictor.environment if not a.remote else None,   # the kernel set (kev.predictors.kernel_environment)
+                  remote={"base_url": a.remote, "requested_model": a.remote_model, "served_model": predictor.served_model, "concurrency": a.remote_concurrency} if a.remote else None)
+    write_json(Path(a.out) / "report.json", report)
+    print(json.dumps({"objective": report["objective"], "clean": report["clean"], "coverage": report["coverage"]}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/budget.py

@@ -0,0 +1,107 @@
+"""The resources a Modal trial container asks for and the admission bound on a study's cost: one home for
+modal_app.run_trial/admit_study and kev.rounds.validate. Pure Python (no torch, no modal)."""
+TRIAL_CPU, TRIAL_MEMORY = 4, (65536, 196608)   # cores, (request, limit) MiB
+GPU_HOURLY = {"H100": 3.95, "H200": 4.54, "B200": 6.25, "T4": 0.59}   # USD per GPU hour
+# Full-weight trials (kev.train --full_ft, kev.full_ft). One GPU keeps the fp32 masters and AdamW moments in host memory:
+# 12 bytes per parameter, ~310 GB for a 27B, plus the staging buffers and the loader; the CPU runs the AdamW step. Several
+# GPUs (FSDP2) keep that state on the GPUs; host memory holds the copy of it a resume point is written from in the
+# background (~307 GB over the ranks) and the gathered checkpoint rank 0 writes (~51 GB; a snapshot being written holds
+# the same, one at a time). Disk: the runs volume stages what a container writes on its local disk, up to two resume
+# points (the new one is complete before the old one goes), the final checkpoint and every snapshot (kept, never deleted),
+# beyond Modal's default 512 GiB quota. A run may plan at most MAX_SNAPSHOTS snapshots (kev.train and
+# kev.experiment.validated_trial refuse more): with a 27B, 2 x 307 + (1 + 8) x 51.2 ≈ 1,075 GB inside the 1 TiB
+# (1,099.5 GB), even if the volume keeps committed files staged. A full disk would fail a snapshot, withhold its resume
+# point and fail the trial (failed.json, not retried), so the cap is a hard limit, not a guideline.
+FULL_FT_SINGLE = 24, (368640, 409600)          # 360 / 400 GiB
+FULL_FT_SHARDED = 16, (409600, 471040)         # 400 / 460 GiB
+FULL_FT_DISK = 1048576                         # MiB of ephemeral disk (1 TiB); Modal bills disk as memory at 20:1
+MAX_SNAPSHOTS = 8                              # snapshots one full-weight run may plan (fractions + every-N steps)
+RESUME_POINT_GB, CHECKPOINT_GB = 307, 51.2     # a 27B's resume point (fp32 masters + moments) and bf16 checkpoint, the disk math above
+# A study's limits. Full-weight trials may run to Modal's 24 h cap per attempt and are continued after a timeout (each
+# continuation is a new call that resumes from the trial's last resume point, kev.experiment.continue_trial), so their
+# bound counts every attempt and their budget cap covers one 8 x H200 day (a LoRA study keeps the $250 cap).
+MAX_TIMEOUT = {False: 28800, True: 86400}   # a 24 h attempt fits the $1,000 cap on one H200 (3 x 24 h x ~$11/h); on
+                                             # H200:8 (~$41/h) the cap allows ~8 h attempts, a day with the continuations
+MAX_BUDGET = {False: 250, True: 1000}
+# Continuations per full-weight trial: at most 1 + FULL_FT_RETRIES calls, each one attempt. They are spawned by
+# `kev.rounds watch` (or modal_app.py::resume --trial), not by Modal: calls go out with Modal's retries off, because Modal
+# charges a timed-out attempt twice when the container then outlives the 30 s cancellation grace (a timeout and a killed
+# task; a full-weight trial always does, its trainer and volume commits run on), and the retry the kill triggers can start
+# while another attempt of the same input is running. Round 22's trial got two 8 h attempts of the three its bound
+# counted (scripts/modal_retry_probe.py: Retries(2) and killed attempts -> 2 attempts; retries=0 -> exactly 1).
+FULL_FT_RETRIES = 2
+# The attempt lease: a running full-weight attempt holds <study>/<trial>/attempt.json on the kev-leases volume
+# (modal_app.TrialLease), refreshed every LEASE_HEARTBEAT seconds from a thread of its own and marked ended when the
+# attempt exits cleanly. A lease that is not ended and whose heartbeat is younger than LEASE_STALE belongs to a container
+# that may still be writing the trial: a new attempt refuses to start and continue_full_trial waits. LEASE_STALE = 900 s:
+#   30 s  Modal's cancellation grace after a timeout, during which the old container keeps running (and committing)
+# + 523 s the longest resume-point commit of the runs volume seen (round 22: 206-523 s); the heartbeat does not wait for it,
+#         but a commit the old container started just before its kill is covered even if a heartbeat had waited behind it
+# + 60 s  one heartbeat interval (the last heartbeat can be that old when the container dies)
+# = 613 s, rounded up to 15 min for clock skew between the containers and the launcher (seconds, NTP) and a few failed
+# heartbeat commits in a row: a live attempt is mistaken for a dead one only after 15 heartbeats fail in a row.
+# The cost is a continuation that starts up to ~15 min after a killed attempt, against 8 h attempts.
+LEASE_HEARTBEAT, LEASE_STALE = 60, 900
+CPU_HOURLY, MEMORY_GIB_HOURLY = 0.04730, 0.008   # USD per core hour and per GiB hour (memory billed at the limit here)
+# Checkpoint interpolation (scripts/interpolate_checkpoint.py, modal_app.py::interpolate): CPU only. The base backbone stays
+# resident in bf16 (51 GB for Qwen3.8-27B, 25.6B parameters; loading it through transformers also stages the 2.5 GB LM
+# head); the SFT side streams tensor by tensor, one output shard (<= 5 GB) is held until written, and the fp32 arithmetic
+# runs in 16M-element chunks (~200 MB). Peak ~60 GB; the limit leaves room for the loader's transients.
+INTERPOLATE_CPU, INTERPOLATE_MEMORY, INTERPOLATE_TIMEOUT = 8, (81920, 131072), 10800   # cores, 80 / 128 GiB, 3 h
+
+
+def gpu_count(gpu):
+    """Modal's "H200:8" -> 8; a bare type is one GPU."""
+    return int(gpu.partition(":")[2] or 1)
+
+
+def trial_resources(gpu, full_ft=False):
+    """(cpu cores, (memory request, limit) MiB) for one trial container."""
+    if not full_ft: return TRIAL_CPU, TRIAL_MEMORY
+    return FULL_FT_SHARDED if gpu_count(gpu) > 1 else FULL_FT_SINGLE
+
+
+def trial_disk(full_ft=False):
+    """ephemeral_disk (MiB) for one trial container; None keeps Modal's default."""
+    return FULL_FT_DISK if full_ft else None
+
+
+def hourly_rate(gpu, full_ft=False):
+    """USD per hour of one trial container of `gpu` ("H200", or "H200:8" for 8 in one container): GPU + CPU + memory at
+    the resources trial_resources gives it."""
+    kind = gpu.partition(":")[0]
+    if kind not in GPU_HOURLY or gpu_count(gpu) < 1: raise ValueError(f"invalid GPU {gpu!r}")
+    cpu, memory = trial_resources(gpu, full_ft)
+    return GPU_HOURLY[kind] * gpu_count(gpu) + cpu * CPU_HOURLY + (memory[1] + (trial_disk(full_ft) or 0) / 20) / 1024 * MEMORY_GIB_HOURLY
+
+
+def interpolation_bound(timeout=INTERPOLATE_TIMEOUT, jobs=1):
+    """The most `jobs` interpolation containers (INTERPOLATE_CPU, INTERPOLATE_MEMORY; no GPU, no retries) can cost at
+    `timeout` seconds each."""
+    if timeout <= 0 or jobs < 1: raise ValueError("invalid timeout or job count")
+    return (INTERPOLATE_CPU * CPU_HOURLY + INTERPOLATE_MEMORY[1] / 1024 * MEMORY_GIB_HOURLY) * timeout / 3600 * jobs
+
+
+def compute_bound(gpu, timeout, trials, full_ft=False):
+    """The most `trials` containers can cost if each runs to `timeout` seconds, counting a full-weight trial's retries."""
+    if timeout <= 0 or trials < 1: raise ValueError("invalid GPU, timeout, or trial count")
+    return hourly_rate(gpu, full_ft) * timeout / 3600 * trials * (1 + FULL_FT_RETRIES if full_ft else 1)
+
+
+def trial_attempts(spawn, label):
+    """(attempts used, attempts allowed) for one trial of a spawn record (runs/<study>.spawn.json, modal_app.launch_detached).
+    A record written with Modal's retries off ("modal_retries": 0) keeps each trial's attempts in `attempts`, one per entry
+    ({"nonce", "call", ...}; a pending entry, written before its spawn, and an abandoned one count too, since their call
+    may run), and allows compute_bound's count: 1 + FULL_FT_RETRIES for a full-weight trial, 1 otherwise. An older record
+    (Modal retried each call, and nothing local says how many attempts ran) is taken to have used them all."""
+    allowed = 1 + FULL_FT_RETRIES if spawn.get("full_ft") else 1
+    if spawn.get("modal_retries") != 0: return allowed, allowed
+    return max(1, len(spawn.get("attempts", {}).get(label) or [])), allowed
+
+
+def lease_state(lease, now):
+    """'none' (no lease), 'ended' (its attempt exited cleanly), 'fresh' (a heartbeat younger than LEASE_STALE: its container
+    may still be writing the trial) or 'stale' (no heartbeat for LEASE_STALE: the container is gone)."""
+    if not lease: return "none"
+    if lease.get("ended"): return "ended"
+    return "fresh" if now - lease["heartbeat"] < LEASE_STALE else "stale"

runtime/kev/calibrate.py

@@ -0,0 +1,73 @@
+"""Per-workload calibration report from a saved rows.json: what one temperature fitted on *these* rows would do.
+
+    uv run python -m kev.calibrate --rows runs/kev-9b-wanli-v1/rows.json [--out runs/kev-9b-wanli-v1/calibration.json]
+
+A released checkpoint ships one temperature fitted on its in-distribution development rows (scripts/calibrate_checkpoint.py,
+which now refuses such rows unless --allow-in-distribution: new temperatures come from held-out datasets). It
+transfers to our out-of-domain suites, not necessarily to a deployer's workload: Kev-9B on WANLI-256 is served at
+mean confidence 0.82 against accuracy 0.70. This module reports, on one rows file, four arms of the same predictions:
+
+    raw           the checkpoint's logits at T=1 (restored from the recorded inference_temperature)
+    shipped       as served (the checkpoint's temperature)
+    workload      one temperature fitted on all these rows (in-sample; the number a deployer would write down)
+    workload_oof  the same fit, group-disjoint out-of-fold: what fitting on your labelled rows buys on rows you did not fit
+
+and a record-clustered paired bootstrap of workload_oof against shipped on ECE, Brier, coverage at <= 5% error and
+AURC. Argmax and accuracy are identical in every arm by construction. Report only: nothing is written to a checkpoint.
+"""
+import argparse
+from pathlib import Path
+
+from kev.metrics import TEMPERATURE_FIT, fit_temperature, metrics, out_of_fold_rows, paired_bootstrap, raw_row, scored_rows, tempered_row
+from kev.suite import read_json, write_json
+
+KEYS = ("n", "acc", "ece", "brier", "nll", "mean_conf", "confident_error_rate", "coverage_at_5pct_error", "aurc")
+COMPARED = ("ece", "brier", "coverage_at_5pct_error", "aurc")
+
+
+def workload_report(rows, folds=5, seed=0, samples=1000):
+    served = scored_rows(rows)
+    if not served:
+        raise ValueError("no clean, knowable rows to calibrate")
+    if any("logits" not in row or "inference_temperature" not in row for row in served):
+        raise ValueError("rows without recorded logits and inference_temperature (remote or Jev predictions): a fit would be "
+                         "relative to the served probabilities, not the checkpoint's raw logits")
+    raw = [raw_row(row) for row in served]
+    temperature = fit_temperature(raw, **TEMPERATURE_FIT)
+    oof, fold_temperatures = out_of_fold_rows(raw, folds, seed, **TEMPERATURE_FIT)
+    arms = {"raw": raw, "shipped": served, "workload": [tempered_row(row, temperature) for row in raw], "workload_oof": oof}
+    return {"n": len(served), "shipped_temperature": sorted({row.get("inference_temperature", 1.0) for row in served}),
+            "workload_temperature": temperature, "fold_temperatures": fold_temperatures,
+            "arms": {name: {k: metrics(arm)[k] for k in KEYS} for name, arm in arms.items()},
+            "oof_vs_shipped": {metric: paired_bootstrap(oof, served, samples=samples, seed=seed, metric=metric, aggregation="micro") for metric in COMPARED},
+            "fit": TEMPERATURE_FIT, "folds": folds, "seed": seed,
+            "note": "argmax and accuracy are temperature-invariant; workload is in-sample, workload_oof is the held-out estimate"}
+
+
+def format_report(report):
+    lines = [f"n={report['n']}  shipped T={', '.join(f'{t:.2f}' for t in report['shipped_temperature'])}  workload T={report['workload_temperature']:.2f}"
+             f"  folds T=[{', '.join(f'{t:.2f}' for t in report['fold_temperatures'])}]",
+             f"{'arm':13}{'acc':>7}{'ece':>7}{'brier':>7}{'nll':>7}{'conf':>7}{'c-err':>7}{'cov@5%':>8}{'aurc':>7}"]
+    for name, m in report["arms"].items():
+        lines.append(f"{name:13}{m['acc']:7.3f}{m['ece']:7.3f}{m['brier']:7.3f}{m['nll']:7.3f}{m['mean_conf']:7.3f}{m['confident_error_rate']:7.3f}{m['coverage_at_5pct_error']:8.2f}{m['aurc']:7.3f}")
+    for metric, b in report["oof_vs_shipped"].items():
+        lo, hi = b["ci95"]
+        lines.append(f"workload_oof - shipped  {metric:24}{b[f'micro_{metric}_delta']:+.3f}  95% CI [{lo:+.3f}, {hi:+.3f}]")
+    return "\n".join(lines)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    ap.add_argument("--rows", required=True, help="rows.json written by kev.benchmark")
+    ap.add_argument("--out", help="where to write the report (default: calibration.json next to the rows)")
+    ap.add_argument("--folds", type=int, default=5); ap.add_argument("--seed", type=int, default=0); ap.add_argument("--samples", type=int, default=1000)
+    a = ap.parse_args()
+    report = workload_report(read_json(a.rows), folds=a.folds, seed=a.seed, samples=a.samples)
+    report["rows"] = a.rows
+    out = Path(a.out) if a.out else Path(a.rows).with_name("calibration.json")
+    write_json(out, report)
+    print(format_report(report)); print(f"wrote {out}")
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/checkpoint.py

@@ -0,0 +1,392 @@
+"""Trained checkpoints: a run directory or a Hub repo holding a LoRA adapter (or, for a full-weight run, the whole bf16
+backbone), `head.pt` and the tokenizer.
+
+Loader rule: `adapter_config.json` present -> a LoRA adapter on `meta.base` at `meta.base_revision`; no adapter and
+`config.json` + `model*.safetensors` (save_pretrained of the backbone, `meta.weights == "full"`) -> the backbone is loaded from
+the checkpoint directory itself, nothing is merged, in the dtype head.pt's `weights_dtype` names (it must match the `dtype`
+save_pretrained wrote to config.json). The tokenizer always comes from the base (both layouts carry a copy).
+
+This is the one place that knows the layout of `head.pt` and how a checkpoint becomes a `DecisionModel`:
+`kev.serve`, `kev.benchmark`, `kev.train --init_from`, `kev.publish`, the scripts and the Hugging Face Space all go
+through it. The Space vendors this file next to `model.py` and `api.py` (scripts/publish_space.sh), so it must not
+import the data or suite modules at import time.
+
+    ck = Checkpoint("jaredpalmer/kev-4b")          # or a local run directory; `@tag` pins a Hub revision
+    tok, model = ck.load("mps", LoadOptions.from_env())
+    ck.meta.temperature                             # the calibration the checkpoint carries
+"""
+import datetime
+import importlib.util
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import torch
+
+from .model import DecisionModel, is_hybrid, load_tokenizer, pad_id
+
+HUB_ID = re.compile(r"[\w.-]+/[\w.-]+(@[\w.-]+)?")
+MTIME_FLOOR = datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc).timestamp()   # older file times are zeroed, not dates (Checkpoint.release_date)
+
+
+def is_hub_id(run):
+    return not os.path.isdir(run) and HUB_ID.fullmatch(str(run)) is not None
+
+
+def resolve_run(run):
+    """Local run directory as given, or a Hub repo id like jaredpalmer/kev-4b, optionally pinned to a revision or tag
+    with `@` (jaredpalmer/kev-4b@qwen3), downloaded to the HF cache. Returns a str path."""
+    if os.path.isdir(run):
+        return str(run)
+    from huggingface_hub import snapshot_download
+    repo, _, revision = str(run).partition("@")
+    return snapshot_download(repo, revision=revision or None, allow_patterns=["*.json", "*.safetensors", "*.pt", "*.txt", "*.jinja"])
+
+
+@dataclass
+class Meta:
+    """Contents of `head.pt`. Every reader gets the same defaults for fields older checkpoints did not write.
+    `extra` keeps the rest of the file (training args, suite hash, init provenance, temperature fit) so a
+    read-modify-write round trip loses nothing."""
+    base: str
+    head: dict | None = None
+    base_revision: str | None = None
+    lora: int = 0
+    head_dim: int = 256
+    option_isolation: bool = False
+    special_embeddings: bool = False
+    weights_dtype: str = "fp32"
+    temperature: float = 1.0
+    holdout: list = field(default_factory=list)
+    weights: str = "lora"          # lora on a base, full backbone, or packed MLX backbone (no base weights)
+    extra: dict = field(default_factory=dict)
+
+    KNOWN = ("base", "head", "base_revision", "lora", "head_dim", "option_isolation", "special_embeddings", "weights_dtype", "temperature", "holdout", "weights")
+
+    @classmethod
+    def from_dict(cls, d):
+        return cls(**{k: d[k] for k in cls.KNOWN if k in d}, extra={k: v for k, v in d.items() if k not in cls.KNOWN})
+
+    def to_dict(self):
+        return {**self.extra, **{k: getattr(self, k) for k in self.KNOWN}}   # known fields win over a stray key in extra
+
+
+def read_meta(run):
+    return Meta.from_dict(torch.load(f"{run}/head.pt", map_location="cpu"))
+
+
+def write_meta(run, meta):
+    torch.save(meta.to_dict(), f"{run}/head.pt")
+
+
+@dataclass(frozen=True)
+class LoadOptions:
+    """How a checkpoint is turned into a model. Defaults are the exact path every reported number uses; the fields
+    are the same knobs the KEV_* environment variables expose to the command-line tools (see from_env).
+
+    dtype        None = fp32, the exact path every reported number uses (bf16 when the checkpoint was trained with a bf16
+                 backbone). kev.serve defaults to bf16 on CUDA and MPS instead: half the memory, 2-4.5x lower latency on an
+                 L4 (Kev-4B: 209 -> 118 ms at 101 tokens, 850 -> 189 ms at 330 tokens), probabilities within ~0.01 and
+                 the same argmax on the checks run so far. KEV_DTYPE=fp32 restores the exact path when serving.
+    merge        fold the LoRA into the base weights: the delta is computed from the fp32 adapter and added in fp32 with
+                 one rounding to the load dtype, so a bf16 model holds exactly round(W + delta), the same bits as merging
+                 an fp32 copy and casting, without the fp32 copy (Kev-9B needed 36 GB of GPU memory to load for that).
+                 Identical because the Qwen bases are stored in bf16; a base stored in fp32 would be rounded twice.
+                 Exact in fp32; in bf16 it is faster (~15%) and closer to the fp32 numbers than the unmerged adapter
+                 (kev-4b, 24 dev records: max |dp| 0.017 vs 0.029, 0 vs 1 argmax flips). Ignored for adapters that carry
+                 trained token embeddings. A checkpoint trained on a bf16 backbone (--weights_dtype bf16: Kev-27B) keeps
+                 its adapter unmerged, as it was trained, unless `fused` is asked for (serving); then it is folded the same
+                 way (Kev-27B served against unmerged: max |dp| 0.009, 0 flips in 280 questions, runs/fused-27b-h200).
+    attn         attention backend; None = the model default (SDPA on CUDA, eager elsewhere). "sdpa" on MPS measured
+                 parity with eager and is a few percent faster.
+    lora_scale   WiSE-FT-style interpolation between base (0) and fine-tuned weights (1), at inference.
+    temperature  None = the temperature the checkpoint carries (fitted by scripts/calibrate_checkpoint.py); 1.0 = raw logits.
+    backend      None = torch, the path every reported number uses. "mlx" = kev.mlx_model (Metal kernels for the hybrid
+                 Qwen3.5 backbones through mlx-lm; the pointer head and encoder are shared; refused for attention-only
+                 bases, which MPS already runs well). "auto" = mlx when the device is mps, the base is hybrid, mlx-lm is
+                 installed and fp32 was not asked for (an explicit dtype=float32 means "the exact path"), else torch;
+                 kev.serve uses auto. The MLX path always merges a LoRA adapter, loads a full-weight checkpoint as
+                 saved (no merge, no copy: kev.mlx_model.load_full), and ignores `attn` and `dtype` (the backbone runs
+                 as stored: bf16 for every released checkpoint).
+    cuda_graphs  replay the serving passes of a hybrid backbone on CUDA (state prefix, question rows on a cached state) as
+                 CUDA graphs, batched across requests (kev.cuda_graphs, DecisionModel.probs_batch). None = off, the eager
+                 path every reported number uses; kev.serve turns it on for CUDA. Exact up to floating-point
+                 reassociation, not bit for bit (the passes are padded to buckets).
+    fused        rewrite a merged hybrid backbone on CUDA with fused Triton kernels (kev.fused_qwen35; needs
+                 flash-linear-attention fused_qwen35.FLA_VERSION and refuses any other). None = off; kev.serve turns it on
+                 for CUDA when fused_available() (KEV_FUSED=0 to decline). Equal to the reference layers up to bf16 rounding.
+    """
+    dtype: torch.dtype | None = None
+    merge: bool = True
+    attn: str | None = None
+    lora_scale: float = 1.0
+    temperature: float | None = None
+    backend: str | None = None
+    cuda_graphs: bool | None = None
+    fused: bool | None = None
+
+    BACKENDS = (None, "torch", "mlx", "auto")
+
+    @classmethod
+    def from_env(cls, env=os.environ):
+        """KEV_DTYPE=bf16|fp16|fp32, KEV_MERGE=0, KEV_ATTN=sdpa|eager, KEV_LORA_SCALE, KEV_TEMPERATURE, KEV_BACKEND=torch|mlx|auto,
+        KEV_CUDA_GRAPHS=0|1, KEV_FUSED=0|1.
+        For command-line entry points only; library code passes an explicit LoadOptions. Explicit values that equal a
+        library default are kept (fp32 as torch.float32, "torch" as a string) so a caller with its own default, like
+        kev.serve, can tell "asked for it" from "did not say"."""
+        backend = env.get("KEV_BACKEND") or None
+        if backend not in cls.BACKENDS: raise ValueError(f"KEV_BACKEND must be one of torch, mlx, auto; got {backend!r}")
+        return cls(dtype={"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}.get(env.get("KEV_DTYPE", "")),
+                   merge=env.get("KEV_MERGE", "1") != "0", attn=env.get("KEV_ATTN") or None,
+                   lora_scale=float(env.get("KEV_LORA_SCALE", "1")),
+                   temperature=float(env["KEV_TEMPERATURE"]) if env.get("KEV_TEMPERATURE") else None, backend=backend,
+                   cuda_graphs={"0": False, "1": True}.get(env.get("KEV_CUDA_GRAPHS", "")),
+                   fused={"0": False, "1": True}.get(env.get("KEV_FUSED", "")))
+
+
+def mlx_available():
+    try:
+        import mlx_lm  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def fused_available():
+    """Whether kev.fused_qwen35 can run: flash-linear-attention importable at its FLA_VERSION (kev.serve's CUDA default;
+    not in the serve extra, pinned in the Modal images). An explicit fused=True skips this and fails loudly instead."""
+    if importlib.util.find_spec("fla") is None: return False
+    try:
+        import fla
+        from .fused_qwen35 import FLA_VERSION
+    except ImportError:
+        return False
+    return fla.__version__ == FLA_VERSION
+
+
+class Checkpoint:
+    def __init__(self, run):
+        self.requested = str(run)                    # what the caller asked for (a Hub id stays a Hub id in labels)
+        self.path = resolve_run(run)
+        self.meta = read_meta(self.path)
+
+    def file(self, name):
+        return Path(self.path) / name
+
+    def adapter_config(self):
+        return json.loads(self.file("adapter_config.json").read_text(encoding="utf-8"))
+
+    @property
+    def packed(self):
+        found = self.file("packed.json").exists()
+        if found != (self.meta.weights == "packed"):
+            raise ValueError("packed.json and head.pt weights must agree")
+        return found
+
+    @property
+    def full(self):
+        """The loader rule (module docstring): True for a full-weight checkpoint, False for a LoRA adapter; head.pt's
+        `weights` must agree with the files."""
+        if self.packed:
+            if self.file("adapter_config.json").exists() or not self.file("config.json").exists() or not self.file("model.safetensors").exists():
+                raise ValueError("packed checkpoint requires config.json and model.safetensors, without a PEFT adapter")
+            return False
+        found = "lora" if self.file("adapter_config.json").exists() else "full" if self.file("config.json").exists() and self.shards() else None
+        if found != self.meta.weights:
+            raise ValueError(f"{self.path}: head.pt says weights={self.meta.weights!r} but the directory holds "
+                             f"{ {'lora': 'an adapter', 'full': 'backbone weights'}.get(found, 'neither an adapter nor backbone weights') }")
+        return found == "full"
+
+    def shards(self):
+        """The backbone's safetensors files of a full-weight checkpoint (model.safetensors or model-*-of-*.safetensors)."""
+        return sorted(Path(self.path).glob("model*.safetensors"))
+
+    def weights_sha256(self):
+        """What a run's provenance pins: the adapter file's sha256, or for full weights the sha256 over every shard's."""
+        from .suite import digest   # lazy: the Space vendors this module without kev/suite.py
+        if not self.full and not self.packed: return digest(self.file("adapter_model.safetensors"))
+        import hashlib
+        return hashlib.sha256("".join(f"{p.name}:{digest(p)}\n" for p in self.shards()).encode()).hexdigest()
+
+    def release_date(self):
+        """ISO date for the TypeSafe model card: the Hub commit date for a Hub checkpoint. For a local run, or a Hub
+        checkpoint offline (its cached files), the UTC date head.pt was written, else the newest of the checkpoint's
+        files, else "unknown". An mtime before 2000 is not a date: archives zero it for reproducibility (Kev 1.0's first
+        release tarballs did), and it reads as 1970-01-01 (1969-12-31 west of UTC). scripts/build_release_assets.py stamps
+        the release date instead."""
+        if is_hub_id(self.requested):
+            from huggingface_hub import HfApi
+            repo, _, revision = self.requested.partition("@")
+            try:
+                return HfApi().model_info(repo, revision=revision or None).last_modified.date().isoformat()
+            except Exception:
+                pass
+        files = sorted((p.stat().st_mtime for p in Path(self.path).iterdir() if p.is_file()), reverse=True)
+        stamp = next((t for t in [self.file("head.pt").stat().st_mtime, *files] if t >= MTIME_FLOOR), None)
+        return "unknown" if stamp is None else datetime.datetime.fromtimestamp(stamp, datetime.timezone.utc).date().isoformat()
+
+    def hybrid_base(self):
+        """Whether the backbone has Gated DeltaNet layers (Qwen3.5), read from a config without loading weights: the base's
+        for a LoRA adapter, the checkpoint's own config.json for full weights (the backbone that is actually loaded)."""
+        from transformers import AutoConfig
+        if self.packed or self.full:
+            config = AutoConfig.from_pretrained(self.path, local_files_only=self.packed)
+        else:
+            config = AutoConfig.from_pretrained(self.meta.base, revision=self.meta.base_revision)
+        return is_hybrid(config.get_text_config())
+
+    def backend(self, device, opts=LoadOptions()):
+        """The backend `load` will use: LoadOptions.backend resolved ("auto" -> mlx only where it pays and is installed)."""
+        if opts.backend not in LoadOptions.BACKENDS: raise ValueError(f"unknown backend {opts.backend!r}")
+        if self.packed:
+            if opts.backend not in (None, "auto", "mlx") or opts.dtype not in (None, torch.bfloat16):
+                raise ValueError("packed checkpoints require the MLX backend with bf16 activations")
+            return "mlx"
+        if opts.backend != "auto": return opts.backend or "torch"
+        exact = opts.dtype is torch.float32   # KEV_DTYPE=fp32: the caller wants the reported-numbers path, not a faster one
+        return "mlx" if str(device) == "mps" and not exact and mlx_available() and self.hybrid_base() else "torch"
+
+    def load(self, device, opts=LoadOptions()):
+        """-> (tokenizer, model) in eval mode with the LoRA applied (or the full backbone loaded) and the pointer head loaded. The model is a
+        DecisionModel (torch) or an MLXDecisionModel (backend mlx); both expose the same scoring interface."""
+        meta = self.meta
+        if self.packed:
+            self.full   # validate the standalone layout before any tensor loading
+            self.backend(device, opts)
+            if opts.lora_scale != 1 or meta.option_isolation or meta.special_embeddings:
+                raise ValueError("packed checkpoints do not support interpolation, isolated options, or trained token embeddings")
+            from transformers import AutoTokenizer
+            from .packed_model import load_packed_model
+            tok = AutoTokenizer.from_pretrained(self.path, local_files_only=True)
+            m = load_packed_model(self.path, pad_id(tok), meta.head_dim)
+            m.head.load_state_dict(meta.head); m.eval()
+            m.head.temperature = meta.temperature if opts.temperature is None else opts.temperature
+            return tok, m
+        if self.full and opts.lora_scale != 1: raise ValueError("lora_scale interpolates an adapter; a full-weight checkpoint has none")
+        tok = load_tokenizer(meta.base, revision=meta.base_revision)
+        m = self._load_mlx(tok, opts) if self.backend(device, opts) == "mlx" else self._load_torch(tok, device, opts)
+        m.head.load_state_dict(meta.head); m.eval()
+        m.head.temperature = meta.temperature if opts.temperature is None else opts.temperature
+        return tok, m
+
+    def _load_mlx(self, tok, opts):
+        """-> MLXDecisionModel. A LoRA checkpoint: the base's mlx-lm model with the adapter merged (fp32, one rounding). A
+        full-weight checkpoint: mlx-lm's model built from the checkpoint's own config and shards in the dtype head.pt names
+        (kev.mlx_model.load_full; strict names/shapes/dtype), nothing merged. Refusals come before the mlx-lm import, which
+        would otherwise hide them on a machine without it."""
+        full = self.full
+        if not full and not opts.merge: raise ValueError("the MLX backend always merges the adapter (KEV_MERGE=0 needs backend=torch)")
+        if self.meta.option_isolation: raise ValueError("option_isolation needs the packed mask; not available on the MLX backend")
+        if not self.hybrid_base(): raise ValueError(f"the MLX backend is for the hybrid (Qwen3.5) bases; {self.path if full else self.meta.base} is attention-only and runs on MPS with backend=torch")
+        dtype = self.saved_dtype() if full else None
+        from .mlx_model import MLXDecisionModel, load_base, load_full, merge_lora
+        if full:
+            lm = load_full(self.path, self.shards(), dtype)
+        else:
+            lm = load_base(resolve_run(f"{self.meta.base}@{self.meta.base_revision or ''}"))   # the base snapshot the torch path already cached
+            merge_lora(lm, self.path, opts.lora_scale)
+        return MLXDecisionModel(lm, pad_id(tok), head_dim=self.meta.head_dim)
+
+    def _load_torch(self, tok, device, opts):
+        m, merged = self._full_torch(tok, device, opts) if self.full else self._adapted_torch(tok, device, opts)
+        serving = str(device).startswith("cuda") and m.hybrid
+        if opts.fused and serving and merged:   # fused projections need plain (merged or full) weights
+            from .fused_qwen35 import fuse
+            fuse(m.lm)
+        if opts.cuda_graphs and serving:
+            from .cuda_graphs import CudaGraphs
+            m.graphs = CudaGraphs(m.lm, m.pad_id)
+        return m
+
+    SAVED_DTYPES = {"bf16": "bfloat16", "fp32": "float32"}   # head.pt weights_dtype -> the dtype save_pretrained writes to config.json
+
+    def saved_dtype(self):
+        """A full-weight checkpoint's dtype name ("bfloat16" | "float32"): the one head.pt's `weights_dtype` names (bf16 for
+        every kev.train --full_ft run: the dtype they were trained in), which must be the dtype save_pretrained recorded in
+        config.json; otherwise a mislabelled export would be silently cast (fp32 weights rounded to bf16, or bf16 upcast to
+        twice the memory). Both backends load full weights in it."""
+        cfg = json.loads(self.file("config.json").read_text(encoding="utf-8"))
+        expected, saved = self.SAVED_DTYPES.get(self.meta.weights_dtype), cfg.get("dtype") or cfg.get("torch_dtype")
+        if expected is None or saved not in (None, expected):
+            raise ValueError(f"{self.path}: config.json records the weights as {saved} but head.pt says weights_dtype={self.meta.weights_dtype!r}")
+        return expected
+
+    def _full_torch(self, tok, device, opts):
+        """-> (model, True). Full weights load in `saved_dtype`; an explicit dtype still casts on purpose (fp32: the same
+        values computed in fp32). Nothing to merge (`load` refuses lora_scale for full weights on either backend)."""
+        meta = self.meta
+        return DecisionModel(meta.base, tok, device, head_dim=meta.head_dim, option_isolation=meta.option_isolation,
+                             dtype=opts.dtype or getattr(torch, self.saved_dtype()), attn=opts.attn, weights=self.path), True
+
+    def _adapted_torch(self, tok, device, opts):
+        """-> (model, whether the adapter was merged): the base with this checkpoint's LoRA."""
+        from peft import PeftModel
+        meta = self.meta
+        dtype, merge = opts.dtype or torch.float32, opts.merge
+        if meta.weights_dtype == "bf16":
+            # trained with a bf16 backbone (--weights_dtype bf16: Kev-27B, the 35B-A3B MoE whose fused experts need bf16):
+            # load it the same way. The exact path keeps the fp32 adapter unmerged; the fused serving path folds it in
+            # (one rounding of W + delta, as for every served Kev; parity in runs/serving-27b-*).
+            dtype, merge = torch.bfloat16, merge and bool(opts.fused)
+        merge = merge and not self.adapter_config().get("trainable_token_indices")   # token-trained adapters stay unmerged
+        m = DecisionModel(meta.base, tok, device, lora=None, revision=meta.base_revision, head_dim=meta.head_dim,
+                          option_isolation=meta.option_isolation, dtype=dtype, attn=opts.attn)
+        m.lm = PeftModel.from_pretrained(m.lm, self.path, torch_device=str(device)).to(device)   # trainable token embeddings, if any, live in the adapter
+        if opts.lora_scale != 1:
+            for module in m.lm.modules():
+                if isinstance(getattr(module, "scaling", None), dict):
+                    for k in module.scaling: module.scaling[k] *= opts.lora_scale
+            m.lora_scale = opts.lora_scale
+        if merge: m.lm = m.lm.merge_and_unload()     # W += delta: fp32 math, one rounding (see LoadOptions.merge)
+        if dtype != torch.float32: m.lm = m.lm.to(dtype)
+        return m, merge
+
+    COMPAT_FIELDS = ("base", "base_revision", "lora", "head_dim", "option_isolation", "special_embeddings", "weights")
+
+    def warm_start(self, model, ours):
+        """Delta training: load this checkpoint's weights (adapter, or full backbone) and pointer head into `model` (a fresh
+        DecisionModel built the way `ours` says: with LoRA, or for full-weight training without). `ours` is the Meta the new
+        run will save; every architecture field is compared BEFORE loading, because peft and load_state_dict(strict=False)
+        load matching keys silently and a half-loaded model still trains and still reports a loss. Returns provenance."""
+        from .suite import digest   # lazy: the Space vendors this module without kev/suite.py
+        for name in self.COMPAT_FIELDS:
+            theirs, mine = getattr(self.meta, name), getattr(ours, name)
+            if theirs != mine and not (name == "base_revision" and None in (theirs, mine)):
+                raise ValueError(f"--init_from {self.path}: {name} is {theirs!r} there and {mine!r} here")
+        tensors = self._load_backbone_into(model.lm) if self.full else self._load_adapter_into(model.lm)
+        model.head.load_state_dict(self.meta.head)
+        return {"init_from": self.requested, "resolved": self.path, "weights_sha256": self.weights_sha256(),
+                "head_sha256": digest(self.file("head.pt")), "tensors": tensors}
+
+    def _load_backbone_into(self, lm):
+        """Copy the saved backbone over `lm` shard by shard (the base's copy in memory is replaced, never merged); every
+        tensor of `lm` must be covered exactly once. -> tensor count."""
+        from safetensors.torch import load_file
+        have, seen = set(lm.state_dict()), set()
+        for shard in self.shards():
+            part = load_file(shard)
+            unexpected = sorted(set(part) - have)
+            if unexpected: raise ValueError(f"--init_from {self.path}: {shard.name} carries tensors this backbone does not have (e.g. {unexpected[:2]})")
+            lm.load_state_dict(part, strict=False); seen |= set(part)
+        missing = sorted(have - seen)
+        if missing: raise ValueError(f"--init_from {self.path} does not cover {len(missing)} of this backbone's tensors (e.g. {missing[:2]})")
+        return len(seen)
+
+    def _load_adapter_into(self, lm):
+        from peft import get_peft_model_state_dict, load_peft_weights, set_peft_model_state_dict
+        weights = load_peft_weights(self.path, device="cpu")
+        have = set(get_peft_model_state_dict(lm))
+        unexpected, missing = sorted(set(weights) - have), sorted(have - set(weights))
+        if unexpected:
+            raise ValueError(f"--init_from {self.path} carries {len(unexpected)} adapter tensors this model does not have (e.g. {unexpected[:2]}); check --lora_targets / --lora against its adapter_config.json")
+        if missing:
+            raise ValueError(f"--init_from {self.path} does not cover {len(missing)} of this model's adapter tensors (e.g. {missing[:2]}); check --lora_targets")
+        set_peft_model_state_dict(lm, weights)
+        return len(weights)
+
+
+def load(run, device, opts=LoadOptions()):
+    """Convenience: Checkpoint(run).load(device, opts)."""
+    return Checkpoint(run).load(device, opts)

runtime/kev/compare.py

@@ -0,0 +1,62 @@
+import argparse
+import json
+import math
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+
+from kev.metrics import paired_bootstrap
+from kev.suite import read_json, write_json
+
+
+def nll_sensitivity(rows, floor):
+    tasks = defaultdict(list)
+    for row in rows:
+        if row["variant"] == "clean":
+            tasks[row["task"]].append(-math.log(max(row["p"][row["label"]], floor)))
+    return float(np.mean([np.mean(v) for v in tasks.values()]))
+
+
+def none_diagnostics(rows):
+    out = {}
+    for variant in ("none_present", "none_absent"):
+        subset = [r for r in rows if r["variant"] == variant]
+        values = [r["p"][r["keys"].index("none_of_these")] for r in subset]
+        out[variant] = {"n": len(values), "mean_p_none": float(np.mean(values)),
+                        "p_none_above_half": float(np.mean(np.array(values) > .5))}
+    return out
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--candidate", required=True)
+    ap.add_argument("--reference", required=True)
+    ap.add_argument("--out", required=True)
+    a = ap.parse_args()
+    candidate, reference = Path(a.candidate), Path(a.reference)
+    left, right = [read_json(p / "report.json") for p in (candidate, reference)]
+    if not left.get("suite_sha256") or left["suite_sha256"] != right.get("suite_sha256"):
+        raise ValueError("comparison requires matching frozen-suite hashes")
+    lr, rr = [read_json(p / "rows.json") for p in (candidate, reference)]
+    result = {"candidate": str(candidate), "reference": str(reference), "suite_sha256": left["suite_sha256"],
+              "clean": {"candidate": left["clean"], "reference": right["clean"]},
+              "paired": {metric: paired_bootstrap(lr, rr, metric=metric) for metric in ("nll", "acc", "brier")},
+              "nll_floor_sensitivity": {str(floor): {"candidate_macro_nll": nll_sensitivity(lr, floor),
+                                                   "reference_macro_nll": nll_sensitivity(rr, floor)} for floor in (1e-3, 1e-6, 1e-9)},
+              "none_of_the_above": {"candidate": none_diagnostics(lr), "reference": none_diagnostics(rr)},
+              "caveats": ["Uncalibrated returned distributions; probabilities normalized for scoring; raw responses preserved.",
+                          "Historical kev checkpoint trained on all six sources; its heldout-source status does not transfer to this study.",
+                          "Jev uses a hosted alias and rounds some probabilities to zero; NLL depends on the declared floor.",
+                          "CPU local latency and hosted network latency are not comparable hardware benchmarks.",
+                          "This is a development-suite comparison, not the locked final test."]}
+    write_json(a.out, result)
+    print("task                 kev acc    Jev acc    kev NLL    Jev NLL")
+    for task in left["tasks"]:
+        l, r = left["tasks"][task], right["tasks"][task]
+        print(f"{task:20} {l['acc']:8.3f} {r['acc']:10.3f} {l['nll']:10.3f} {r['nll']:10.3f}")
+    print(json.dumps({"paired": result["paired"], "none_of_the_above": result["none_of_the_above"]}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/composition.py

@@ -0,0 +1,321 @@
+import copy
+import hashlib
+import json
+import random
+from datetime import date, timedelta
+
+UNKNOWN = None
+SHAPES = {
+    "atom": 0,
+    "negation": ("not", 0),
+    "conjunction": ("and", 0, 1),
+    "disjunction": ("or", 0, 1),
+    "exception": ("unless", 0, 1),
+    "conditional": ("if", 0, 1, 2),
+    "nested_and": ("and", ("and", 0, 1), 2),
+    "nested_or": ("or", 0, ("or", 1, 2)),
+    "held_and_or": ("and", ("or", 0, 1), 2),
+    "held_or_not": ("or", ("and", 0, 1), ("not", 2)),
+    "held_conditional": ("if", 0, ("not", 1), 2),
+    "final_combination": ("and", ("if", 0, 1, 2), 3),
+    "final_negation": ("not", ("or", ("and", 0, 1), 2)),
+    "final_exception": ("or", ("unless", 0, 1), 2),
+}
+TRAIN_SHAPES = tuple(list(SHAPES)[:8])
+DEV_SHAPES = tuple(list(SHAPES)[8:11])
+TEST_SHAPES = tuple(list(SHAPES)[11:])
+KINDS = ("lt", "le", "gt", "ge", "eq", "range", "match", "elapsed", "flag")
+
+
+def atom_value(atom, facts):
+    if any(key not in facts for key in atom["fields"]):
+        return UNKNOWN
+    values = [facts[k] for k in atom["fields"]]
+    kind, threshold = atom["kind"], atom["threshold"]
+    x = values[0]
+    if kind == "lt": return x < threshold
+    if kind == "le": return x <= threshold
+    if kind == "gt": return x > threshold
+    if kind == "ge": return x >= threshold
+    if kind == "eq": return x == threshold
+    if kind == "range": return threshold <= x <= threshold + 10
+    if kind == "match": return x == values[1]
+    if kind == "elapsed": return (date.fromisoformat(values[1]) - date.fromisoformat(x)).days <= threshold
+    if kind == "flag": return x
+    raise ValueError(f"unknown atom {kind}")
+
+
+def evaluate_rule(tree, atoms, facts):
+    if isinstance(tree, int):
+        return atom_value(atoms[tree], facts)
+    op, *children = tree
+    values = [evaluate_rule(t, atoms, facts) for t in children]
+    if op == "not": return None if values[0] is None else not values[0]
+    if op == "unless":
+        a, exception = values
+        values = [a, None if exception is None else not exception]
+        op = "and"
+    if op == "and":
+        return False if False in values else None if None in values else True
+    if op == "or":
+        return True if True in values else None if None in values else False
+    if op == "if":
+        condition, yes, no = values
+        return yes if condition is True else no if condition is False else yes if yes == no else None
+    raise ValueError(f"unknown operation {op}")
+
+
+def atom_text(atom):
+    kind, fields, t = atom["kind"], atom["fields"], atom["threshold"]
+    key = fields[0]
+    words = {"lt": "less than", "le": "at most", "gt": "greater than", "ge": "at least", "eq": "equal to"}
+    if kind in words: return f"{key} is {words[kind]} {t}"
+    if kind == "range": return f"{key} is between {t} and {t + 10}, including both endpoints"
+    if kind == "match": return f"{fields[0]} is the same person as {fields[1]}"
+    if kind == "elapsed": return f"the elapsed days from {fields[0]} to {fields[1]} are at most {t}"
+    return f"{key} is yes"
+
+
+def render_rule(tree, atoms, style):
+    """Five surface styles for the same rule tree. 0 = logic-like, 1 = 'both/at least one of' lists, 2 = reserved for the
+    locked test, 3 = plain prose, 4 = clause per line."""
+    if isinstance(tree, int): return atom_text(atoms[tree])
+    op, *children = tree
+    parts = [render_rule(c, atoms, style) for c in children]
+    if style == 3:
+        if op == "not": return f"the condition \"{parts[0]}\" fails"
+        if op == "and": return f"{parts[0]}, and also {parts[1]}"
+        if op == "or": return f"either {parts[0]}, or else {parts[1]}"
+        if op == "unless": return f"{parts[0]}, except when {parts[1]}"
+        return f"when {parts[0]} the requirement is that {parts[1]}, and when it is not the requirement is that {parts[2]}"
+    if style == 4:
+        if op == "not": return f"[NOT: {parts[0]}]"
+        if op in ("and", "or"): return f"[{'ALL' if op == 'and' else 'ANY'} of: {parts[0]} | {parts[1]}]"
+        if op == "unless": return f"[{parts[0]} UNLESS {parts[1]}]"
+        return f"[IF {parts[0]} THEN {parts[1]} ELSE {parts[2]}]"
+    if op == "not": return f"NOT ({parts[0]})" if style == 0 else f"it is not the case that ({parts[0]})"
+    if op in ("and", "or"):
+        if style == 0: return f"({parts[0]}) {op.upper()} ({parts[1]})"
+        connector = "both" if op == "and" else "at least one of"
+        return f"{connector} these conditions hold: [({parts[0]}); ({parts[1]})]"
+    if op == "unless": return f"({parts[0]}) holds and the exception ({parts[1]}) does not hold"
+    return f"if ({parts[0]}), use ({parts[1]}); otherwise use ({parts[2]})"
+
+
+# --- random rule trees (PLAN.md: structural diversity, negation anywhere) ---------------------------------------------
+
+def skeleton(tree):
+    """Structure with leaf identities erased: used to order commutative children independently of atom numbering."""
+    if isinstance(tree, int): return "_"
+    op, *children = tree
+    parts = [skeleton(c) for c in children]
+    if op in ("and", "or"): parts = sorted(parts)
+    return f"{op}({','.join(parts)})"
+
+
+def sort_commutative(tree):
+    if isinstance(tree, int): return tree
+    op, *children = tree
+    children = [sort_commutative(c) for c in children]
+    if op in ("and", "or"): children = sorted(children, key=skeleton)
+    return (op, *children)
+
+
+def canonical(tree):
+    """Structure key: commutative children ordered by skeleton, then leaves renumbered in traversal order, so
+    (A and B) or not C and not C or (B and A) share one key."""
+    t = relabel(sort_commutative(tree))
+    def render(t):
+        if isinstance(t, int): return str(t)
+        return f"{t[0]}({','.join(render(c) for c in t[1:])})"
+    return render(t)
+
+
+def push_negation(tree):
+    """De Morgan normal form, so a random tree equivalent to a held-out shape under negation pushing is also excluded."""
+    if isinstance(tree, int): return tree
+    op, *children = tree
+    if op == "not":
+        inner = children[0]
+        if isinstance(inner, int): return tree
+        iop, *ic = inner
+        if iop == "not": return push_negation(ic[0])
+        if iop in ("and", "or"): return ("or" if iop == "and" else "and", *[push_negation(("not", c)) for c in ic])
+        if iop == "unless": return push_negation(("or", ("not", ic[0]), ic[1]))
+        return ("not", push_negation(inner))
+    if op == "unless": return ("and", push_negation(children[0]), push_negation(("not", children[1])))
+    return (op, *[push_negation(c) for c in children])
+
+
+def relabel(tree):
+    """Renumber leaves in first-appearance order so structure keys do not depend on which atom index was drawn."""
+    mapping = {}
+    def walk(t):
+        if isinstance(t, int):
+            mapping.setdefault(t, len(mapping)); return mapping[t]
+        return (t[0], *[walk(c) for c in t[1:]])
+    return walk(tree)
+
+
+def structure_keys(tree):
+    return {canonical(tree), canonical(push_negation(tree))}
+
+
+HELD_OUT_KEYS = set().union(*(structure_keys(SHAPES[s]) for s in DEV_SHAPES + TEST_SHAPES))
+
+
+def random_tree(rng, depth, next_leaf):
+    """Random rule over {and, or, not, unless, if} with negation allowed at any position; leaves are fresh atom indices."""
+    if depth == 0 or (depth < 3 and rng.random() < 0.25):
+        return next_leaf()
+    op = rng.choice(["and", "or", "not", "unless", "if", "and", "or"])
+    if op == "not": return ("not", random_tree(rng, depth - 1, next_leaf))
+    if op == "if": return ("if", random_tree(rng, depth - 1, next_leaf), random_tree(rng, depth - 1, next_leaf), random_tree(rng, depth - 1, next_leaf))
+    return (op, random_tree(rng, depth - 1, next_leaf), random_tree(rng, depth - 1, next_leaf))
+
+
+def sample_trees(n, seed, min_leaves=2, max_leaves=4, exclude=HELD_OUT_KEYS):
+    """n distinct rule structures (by canonical key), none structurally equal to a held-out or locked shape, none a
+    bare leaf, each with negation somewhere in ~60% of cases. Deterministic in `seed`."""
+    rng = random.Random(f"{seed}:trees"); trees, keys = [], set()
+    for _ in range(20000):
+        if len(trees) >= n: break
+        counter = [0]
+        def next_leaf():
+            counter[0] += 1; return counter[0] - 1
+        t = random_tree(rng, rng.choice([2, 2, 3]), next_leaf)
+        if isinstance(t, int) or not min_leaves <= counter[0] <= max_leaves: continue
+        k = structure_keys(t)
+        if k & exclude or k & keys: continue
+        keys |= k; trees.append(relabel(t))
+    if len(trees) < n: raise ValueError(f"only {len(trees)} distinct structures found")
+    return trees
+
+
+def leaf_indices(tree):
+    if isinstance(tree, int): return {tree}
+    return set().union(*(leaf_indices(t) for t in tree[1:]))
+
+
+def make_atoms(tree, rng):
+    nouns = rng.sample(["request", "account", "package", "review", "member", "shipment", "entry", "case"], 4)
+    atoms = []
+    for i in range(max(leaf_indices(tree)) + 1):
+        kind = rng.choice(KINDS)
+        prefix = nouns[i]
+        fields = [f"{prefix} value"]
+        if kind == "match": fields = [f"{prefix} signer", f"{prefix} designated approver"]
+        elif kind == "elapsed": fields = [f"{prefix} start date", f"{prefix} end date"]
+        elif kind == "flag": fields = [f"{prefix} verified"]
+        atoms.append({"kind": kind, "fields": fields, "threshold": rng.randint(5, 60)})
+    return atoms
+
+
+def fact_domains(atoms, rng):
+    domains = {}
+    for a in atoms:
+        kind, fs, t = a["kind"], a["fields"], a["threshold"]
+        if kind == "match":
+            people = rng.sample(["Mira", "Noah", "Aiko", "Ravi", "Sana", "Elin", "Tomas", "Kofi"], 3)
+            domains.update({k: people for k in fs})
+        elif kind == "elapsed":
+            day = date(2027, rng.randint(1, 8), rng.randint(1, 28))
+            domains[fs[0]] = [day.isoformat()]
+            domains[fs[1]] = [(day + timedelta(days=n)).isoformat() for n in (max(0, t - 1), t, t + 1, t + 10)]
+        elif kind == "flag": domains[fs[0]] = [False, True]
+        else: domains[fs[0]] = [t - 1, t, t + 1, t + 10, t + 11]
+    domains["routing reference"] = [rng.randint(100, 500), rng.randint(501, 999)]
+    return domains
+
+
+def rendered_facts(facts, order):
+    def value(v):
+        return "yes" if v is True else "no" if v is False else str(v)
+    return [f"The {k} is {value(facts[k])}." for k in order]
+
+
+POLICY_WRAPPERS = {
+    0: "Approve exactly when {rule}. Otherwise deny. The routing reference does not affect eligibility.",
+    1: "Approve exactly when {rule}. Otherwise deny. The routing reference does not affect eligibility.",
+    2: "Approval requires the following rule to be true: {rule}. A false rule means denial. Routing references are irrelevant.",
+    3: "A case is approved when {rule}; any other case is denied. Routing references play no part in the decision.",
+    4: "DECISION RULE {rule} -> approve; otherwise reject. Ignore the routing reference.",
+}
+
+
+def generate(groups_per_shape, seed, shapes=TRAIN_SHAPES, styles=(0, 1), source="compositional", trees=None):
+    """trees: optional {name: tree} to generate from random structures instead of SHAPES (names must not collide)."""
+    records = []
+    shape_trees = trees if trees is not None else {s: SHAPES[s] for s in shapes}
+    for shape, tree in shape_trees.items():
+        rng = random.Random(f"{seed}:{shape}")
+        for i in range(groups_per_shape):
+            atoms = make_atoms(tree, rng)
+            domains = fact_domains(atoms, rng)
+            for attempt in range(1000):
+                facts = {k: rng.choice(v) for k, v in domains.items()}
+                label = evaluate_rule(tree, atoms, facts)
+                changed = None
+                keys = list(domains); rng.shuffle(keys)
+                for key in keys:
+                    for value in domains[key]:
+                        edited = {**facts, key: value}
+                        if evaluate_rule(tree, atoms, edited) != label:
+                            missing = {k: v for k, v in facts.items() if k != key}
+                            if evaluate_rule(tree, atoms, missing) is None:
+                                changed = (edited, key)
+                                break
+                    if changed: break
+                if changed: break
+            if changed is None:
+                raise ValueError(f"cannot create a decisive edit for {shape}")
+            edited, deciding = changed
+            nuisance = {**facts, "routing reference": next(v for v in domains["routing reference"] if v != facts["routing reference"])}
+            order = list(facts); rng.shuffle(order)
+            style = rng.choice(styles)
+            rule = render_rule(tree, atoms, style)
+            policy = POLICY_WRAPPERS[style].format(rule=rule)
+            keys = ["accept", "reject"]; rng.shuffle(keys)
+            criteria = {k: "The policy permits this case" if k == "accept" else "The policy does not permit this case" for k in keys}
+            group = f"composition/{seed}/{shape}/{i}"
+            for kind, a, b in (("relevant", facts, edited), ("irrelevant", facts, nuisance)):
+                for sibling, values in (("a", a), ("b", b)):
+                    result = evaluate_rule(tree, atoms, values)
+                    state = {"policy": policy, "case": " ".join(rendered_facts(values, order))}
+                    identifier = f"{group}/{kind}/{sibling}"
+                    records.append({"state": state, "questions": {"decision": {"type": "choice",
+                        "instructions": "Apply the policy to this case.", "criteria": dict(criteria),
+                        "label": "accept" if result else "reject", "src": f"composition_{shape}"}},
+                        "_meta": {"id": identifier, "group_id": group, "source": source, "variant": "clean",
+                                  "pair_id": f"{group}/{kind}", "sibling": sibling, "pair_kind": kind,
+                                  "family": shape, "family_id": shape, "render_style": style, "structure": canonical(tree),
+                                  "text_sha256": hashlib.sha256(json.dumps(state, sort_keys=True).encode()).hexdigest(),
+                                  "certificate": {"tree": tree, "atoms": atoms, "facts": values, "order": order,
+                                                  "deciding_field": deciding, "label": result}}})
+    return records
+
+
+def check_group(records):
+    if len(records) != 4: raise ValueError("a composition group requires four records")
+    for r in records:
+        c = r["_meta"]["certificate"]
+        if evaluate_rule(c["tree"], c["atoms"], c["facts"]) != c["label"]:
+            raise ValueError("incorrect certificate label")
+        if r["state"]["case"] != " ".join(rendered_facts(c["facts"], c["order"])):
+            raise ValueError("rendered facts differ from certificate")
+        q = r["questions"]["decision"]
+        if q["label"] != ("accept" if c["label"] else "reject"):
+            raise ValueError("answer differs from certificate")
+    for a, b in (records[:2], records[2:]):
+        ca, cb = a["_meta"]["certificate"], b["_meta"]["certificate"]
+        if ca["order"] != cb["order"] or a["state"]["policy"] != b["state"]["policy"]:
+            raise ValueError("pair changes more than one fact")
+        if sum(ca["facts"][k] != cb["facts"][k] for k in ca["facts"]) != 1:
+            raise ValueError("pair must change exactly one fact")
+        expected_flip = a["_meta"]["pair_kind"] == "relevant"
+        if (ca["label"] != cb["label"]) != expected_flip:
+            raise ValueError("incorrect intervention label")
+        missing = {k: v for k, v in ca["facts"].items() if k != ca["deciding_field"]}
+        if expected_flip and evaluate_rule(ca["tree"], ca["atoms"], missing) is not None:
+            raise ValueError("deciding evidence ablation did not remove the answer")
+    return True

runtime/kev/contrastive.py

@@ -0,0 +1,306 @@
+"""Programmatic contrastive pairs with certain labels (PLAN.md step 2).
+
+Each item is a policy plus a case described by a few sentences. Every sentence carries zero or more explicit facts;
+the label is a pure function `evaluate(facts)` of the facts present, returning UNDETERMINED when a required fact is
+missing. A pair is the same item with exactly one sentence changed so that the label flips.
+
+Two code-level checks run on every pair before it is kept (no model anywhere):
+  ablation   : removing any evidence sentence (one that carries a required fact) makes the label UNDETERMINED,
+               so the answer provably lives in the state, not in the instruction, the options, or the policy alone;
+  invariance : removing any filler sentence leaves the label unchanged.
+Evidence status is derived from the evaluator, not declared. Both siblings share `pair_id`/`family_id`, and
+splits are assigned by family so siblings never straddle a split.
+"""
+import hashlib
+import json
+import random
+from datetime import date, timedelta
+
+UNDETERMINED = "UNDETERMINED"
+
+NAMES = ["Mira", "Noah", "Priya", "Tomas", "Aiko", "Lena", "Omar", "Sana", "Jonas", "Ravi", "Elin", "Kofi"]
+ROLES = ["account owner", "billing manager", "support agent", "warehouse lead"]
+ITEMS = ["a pair of running shoes", "a desk lamp", "a wireless keyboard", "a rain jacket", "a coffee grinder", "a backpack"]
+PROGRAMS = ["the volunteer driver program", "the apprenticeship", "the rental agreement", "the night-shift roster"]
+
+
+def _day(d):
+    return d.strftime("%B %-d, %Y")
+
+
+def _need(facts, *keys):
+    return all(k in facts for k in keys)
+
+
+# A family returns (item_a, item_b). An item is {policy, sentences: [(text, facts)], evaluate, question}.
+# The two items must differ in exactly one sentence, and evaluate() must give different labels.
+
+def family_return_window(rng):
+    window = rng.choice([14, 30, 45, 60]); item = rng.choice(ITEMS); name = rng.choice(NAMES)
+    bought = date(2026, rng.randint(1, 9), rng.randint(1, 28))
+    def evaluate(f):
+        return (f["request"] - f["purchase"]).days <= window if _need(f, "request", "purchase") else UNDETERMINED
+    def build(days):
+        request = bought + timedelta(days=days)
+        return {"policy": f"Returns are accepted only if the return request is submitted within {window} days of the purchase date.",
+                "sentences": [(f"{name} bought {item} on {_day(bought)}.", {"purchase": bought}),
+                              (f"The return request was submitted on {_day(request)}.", {"request": request}),
+                              (f"The order was paid by card and shipped to {name}'s home address.", {})],
+                "evaluate": evaluate, "question": {"type": "noul", "instructions": "Is this return request within the policy window?"}}
+    return build(rng.randint(1, window - 1)), build(window + rng.randint(1, 30))
+
+
+def family_spend_threshold(rng):
+    limit = rng.choice([250, 500, 1000, 2500]); name = rng.choice(NAMES); role = rng.choice(ROLES)
+    def evaluate(f):
+        return ("auto_approved" if f["amount"] <= limit else "director_signoff") if _need(f, "amount") else UNDETERMINED
+    def build(amount):
+        return {"policy": f"Expense claims of ${limit:,} or less are approved automatically. Claims above ${limit:,} require director sign-off.",
+                "sentences": [(f"{name}, the {role}, submitted an expense claim.", {}),
+                              (f"The claim total is ${amount:,}.", {"amount": amount}),
+                              ("Receipts were attached for every line item.", {})],
+                "evaluate": evaluate, "question": {"type": "choice", "instructions": "How is this claim handled under the policy?",
+                                                   "criteria": {"auto_approved": "Approved without further review", "director_signoff": "Requires director sign-off", "rejected": "Rejected outright"}}}
+    return build(limit - rng.randint(1, limit // 2)), build(limit + rng.randint(1, limit))
+
+
+def family_authorization(rng):
+    approver, other = rng.sample(NAMES, 2); account = rng.randint(10, 99); amount = rng.choice([40, 120, 350, 900])
+    def evaluate(f):
+        return f["signer"] == f["approver"] if _need(f, "signer", "approver") else UNDETERMINED
+    def build(signer):
+        return {"policy": "A refund is authorized only when its sole authorization was signed by someone who may authorize refunds for that account.",
+                "sentences": [(f"Only {approver} may authorize refunds for account {account}.", {"approver": approver}),
+                              (f"The sole authorization for this refund on account {account} was signed by {signer}.", {"signer": signer}),
+                              (f"The refund amount is ${amount}.", {})],
+                "evaluate": evaluate, "question": {"type": "noul", "instructions": "Is the refund authorized?"}}
+    return build(approver), build(other)
+
+
+def family_age_eligibility(rng):
+    minimum = rng.choice([16, 18, 21, 25]); name = rng.choice(NAMES); program = rng.choice(PROGRAMS)
+    def evaluate(f):
+        return f["age"] >= minimum if _need(f, "age") else UNDETERMINED
+    def build(age):
+        return {"policy": f"Applicants must be at least {minimum} years old to be eligible for {program}.",
+                "sentences": [(f"{name} applied to join {program}.", {}),
+                              (f"{name} is {age} years old.", {"age": age}),
+                              ("The application form was complete and signed.", {})],
+                "evaluate": evaluate, "question": {"type": "noul", "instructions": "Is the applicant eligible?"}}
+    return build(minimum + rng.randint(0, 20)), build(minimum - rng.randint(1, 5))
+
+
+def family_quantity_limit(rng):
+    limit = rng.choice([2, 3, 5, 10]); item = rng.choice(ITEMS); name = rng.choice(NAMES)
+    def evaluate(f):
+        if not _need(f, "qty"): return UNDETERMINED
+        return "within_limit" if f["qty"] <= limit else "slightly_over" if f["qty"] <= 2 * limit else "far_over"
+    def build(qty):
+        return {"policy": f"Customers may order at most {limit} units of any single item per order. Orders up to double the limit are held for review; larger orders are cancelled.",
+                "sentences": [(f"{name} placed an order for {item}.", {}),
+                              (f"The order quantity is {qty}.", {"qty": qty}),
+                              ("Delivery was requested to a residential address.", {})],
+                "evaluate": evaluate, "question": {"type": "choice", "instructions": "What happens to this order?",
+                                                   "criteria": {"within_limit": "Processed normally", "slightly_over": "Held for review", "far_over": "Cancelled"}}}
+    return build(rng.randint(1, limit)), build(rng.choice([rng.randint(limit + 1, 2 * limit), rng.randint(2 * limit + 1, 4 * limit)]))
+
+
+def family_deadline(rng):
+    name = rng.choice(NAMES); due = date(2026, rng.randint(2, 11), rng.randint(1, 28)); grace = rng.choice([3, 7, 14])
+    def evaluate(f):
+        if not _need(f, "received", "due"): return UNDETERMINED
+        late = (f["received"] - f["due"]).days
+        return 0 if late <= 0 else 1 if late <= grace else 2
+    def build(offset):
+        received = due + timedelta(days=offset)
+        return {"policy": f"Reports received by the deadline are on time. Reports received within {grace} days after the deadline are late but accepted. Later reports are refused.",
+                "sentences": [(f"The filing deadline for {name}'s report was {_day(due)}.", {"due": due}),
+                              (f"The report was received on {_day(received)}.", {"received": received}),
+                              ("The report was submitted through the online portal.", {})],
+                "evaluate": evaluate, "question": {"type": "score", "instructions": "How late is this report?", "criteria": ["On time", "Late but accepted", "Refused"]}}
+    return build(-rng.randint(0, 10)), build(rng.choice([rng.randint(1, grace), grace + rng.randint(1, 20)]))
+
+
+# --- ordinal threshold families as Score questions (PLAN.md: the deadline failure is "no idea how ordinal thresholds work").
+# Each has three ordered levels split by two thresholds; one is date-based like deadline but with a different template.
+
+def family_warranty_claim(rng):
+    name = rng.choice(NAMES); item = rng.choice(ITEMS); bought = date(2026, rng.randint(1, 6), rng.randint(1, 28))
+    standard, extended = rng.choice([(90, 365), (180, 730), (365, 1095)]); part = rng.choice(['stitching', 'battery', 'housing', 'zipper', 'switch'])
+    def evaluate(f):
+        if not _need(f, "claim", "purchase"): return UNDETERMINED
+        age = (f["claim"] - f["purchase"]).days
+        return 0 if age <= standard else 1 if age <= extended else 2
+    def build(age):
+        claim = bought + timedelta(days=age)
+        return {"policy": f"Warranty claims made within {standard} days of purchase are covered in full. Claims made after that but within {extended} days are covered at half cost. Later claims are not covered.",
+                "sentences": [(f"{name} purchased {item} on {_day(bought)}.", {"purchase": bought}),
+                              (f"A warranty claim for it was filed on {_day(claim)}.", {"claim": claim}),
+                              (f"The claim describes a defect in the {part}.", {})],
+                "evaluate": evaluate, "question": {"type": "score", "instructions": "How is this claim covered?", "criteria": ["Covered in full", "Covered at half cost", "Not covered"]}}
+    a = rng.choice([rng.randint(1, standard), rng.randint(standard + 1, extended), extended + rng.randint(1, 200)])
+    b = rng.choice([x for x in [rng.randint(1, standard), rng.randint(standard + 1, extended), extended + rng.randint(1, 200)] if evaluate({"claim": bought + timedelta(days=x), "purchase": bought}) != evaluate({"claim": bought + timedelta(days=a), "purchase": bought})] or [a])
+    return build(a), build(b)
+
+
+def family_sla_response(rng):
+    name = rng.choice(NAMES); target, breach = rng.choice([(4, 24), (8, 48), (24, 72), (1, 8)])
+    unit = "hours"; topic = rng.choice(['a login failure', 'a duplicate charge', 'a missing invoice', 'an export error']); queue = rng.choice(['email', 'chat', 'phone'])
+    def evaluate(f):
+        if not _need(f, "hours"): return UNDETERMINED
+        return 0 if f["hours"] <= target else 1 if f["hours"] <= breach else 2
+    def build(h):
+        return {"policy": f"Support responses within {target} {unit} meet the service level. Responses after {target} but within {breach} {unit} are a minor breach. Anything slower is a major breach.",
+                "sentences": [(f"{name} opened a priority ticket about {topic}.", {}),
+                              (f"The first response arrived {h} {unit} after the ticket was opened.", {"hours": h}),
+                              (f"The ticket was routed through the {queue} queue.", {})],
+                "evaluate": evaluate, "question": {"type": "score", "instructions": "How does this response time rate against the service level?", "criteria": ["Met", "Minor breach", "Major breach"]}}
+    levels = [rng.randint(1, target), rng.randint(target + 1, breach), breach + rng.randint(1, 100)]
+    a, b = rng.sample(levels, 2)
+    return build(a), build(b)
+
+
+def family_late_fee(rng):
+    name = rng.choice(NAMES); grace, cap = rng.choice([(5, 30), (10, 60), (15, 45)]); amount = rng.choice([120, 450, 980, 2300]); method = rng.choice(['bank transfer', 'card', 'cheque'])
+    def evaluate(f):
+        if not _need(f, "days_late"): return UNDETERMINED
+        return 0 if f["days_late"] <= grace else 1 if f["days_late"] <= cap else 2
+    def build(d):
+        return {"policy": f"Invoices paid within {grace} days after the due date incur no fee. Payments between {grace + 1} and {cap} days late incur a 2% fee. Payments later than {cap} days incur a 10% fee and a hold on the account.",
+                "sentences": [(f"{name}'s invoice for ${amount:,} fell due last quarter.", {}),
+                              (f"Payment was received {d} days after the due date.", {"days_late": d}),
+                              (f"The payment was made by {method}.", {})],
+                "evaluate": evaluate, "question": {"type": "score", "instructions": "Which fee tier applies?", "criteria": ["No fee", "2% fee", "10% fee and account hold"]}}
+    levels = [rng.randint(0, grace), rng.randint(grace + 1, cap), cap + rng.randint(1, 60)]
+    a, b = rng.sample(levels, 2)
+    return build(a), build(b)
+
+
+def family_volume_discount(rng):
+    name = rng.choice(NAMES); item = rng.choice(ITEMS); t1, t2 = rng.choice([(10, 50), (25, 100), (5, 20), (100, 500)]); dest = rng.choice(['warehouse', 'storefront', 'branch office'])
+    def evaluate(f):
+        if not _need(f, "units"): return UNDETERMINED
+        return 0 if f["units"] < t1 else 1 if f["units"] < t2 else 2
+    def build(u):
+        return {"policy": f"Orders of fewer than {t1} units are charged the list price. Orders of {t1} to {t2 - 1} units receive the volume discount. Orders of {t2} units or more receive the wholesale rate.",
+                "sentences": [(f"{name} placed a business order for {item}.", {}),
+                              (f"The order is for {u} units.", {"units": u}),
+                              (f"Delivery is to a {dest}.", {})],
+                "evaluate": evaluate, "question": {"type": "score", "instructions": "Which pricing tier applies?", "criteria": ["List price", "Volume discount", "Wholesale rate"]}}
+    levels = [rng.randint(1, t1 - 1), rng.randint(t1, t2 - 1), t2 + rng.randint(0, t2)]
+    a, b = rng.sample(levels, 2)
+    return build(a), build(b)
+
+
+def family_shipping_delay(rng):
+    """Day-precision date arithmetic with small thresholds (the skill `deadline` tests): promised vs delivered dates."""
+    name = rng.choice(NAMES); item = rng.choice(ITEMS); promised = date(2026, rng.randint(1, 11), rng.randint(1, 28))
+    minor, major = rng.choice([(1, 4), (2, 7), (3, 10)]); carrier = rng.choice(["the courier", "the postal service", "a freight partner"])
+    def evaluate(f):
+        if not _need(f, "delivered", "promised"): return UNDETERMINED
+        late = (f["delivered"] - f["promised"]).days
+        return 0 if late <= minor else 1 if late <= major else 2
+    def build(offset):
+        delivered = promised + timedelta(days=offset)
+        return {"policy": f"Deliveries up to {minor} day{'s' if minor > 1 else ''} after the promised date count as on time. Deliveries {minor + 1} to {major} days after it are a minor delay and earn a shipping refund. Later deliveries are a major delay and earn a full refund.",
+                "sentences": [(f"{name} ordered {item} with delivery promised for {_day(promised)}.", {"promised": promised}),
+                              (f"The parcel was delivered on {_day(delivered)}.", {"delivered": delivered}),
+                              (f"It was shipped by {carrier}.", {})],
+                "evaluate": evaluate, "question": {"type": "score", "instructions": "How is this delivery classified?", "criteria": ["On time", "Minor delay: shipping refund", "Major delay: full refund"]}}
+    levels = [rng.randint(-3, minor), rng.randint(minor + 1, major), major + rng.randint(1, 30)]
+    a, b = rng.sample(levels, 2)
+    return build(a), build(b)
+
+
+FAMILIES = {"return_window": family_return_window, "spend_threshold": family_spend_threshold, "authorization": family_authorization,
+            "age_eligibility": family_age_eligibility, "quantity_limit": family_quantity_limit, "deadline": family_deadline,
+            "warranty_claim": family_warranty_claim, "sla_response": family_sla_response, "late_fee": family_late_fee, "volume_discount": family_volume_discount,
+            "shipping_delay": family_shipping_delay}
+ORDINAL_FAMILIES = ("warranty_claim", "sla_response", "late_fee", "volume_discount", "shipping_delay")   # trainable Score-threshold families (deadline stays held out)
+
+
+def label_of(item, drop=None):
+    facts = {}
+    for i, (_, f) in enumerate(item["sentences"]):
+        if i != drop: facts.update(f)
+    return item["evaluate"](facts)
+
+
+def check_pair(a, b):
+    """None if the pair is valid, else the reason it is rejected."""
+    la, lb = label_of(a), label_of(b)
+    if UNDETERMINED in (la, lb): return "label_undetermined"
+    if la == lb: return "labels_equal"
+    if sum(x[0] != y[0] for x, y in zip(a["sentences"], b["sentences"])) != 1 or len(a["sentences"]) != len(b["sentences"]):
+        return "not_exactly_one_sentence_differs"
+    for item, label in ((a, la), (b, lb)):
+        evidence = 0
+        for i, (_, facts) in enumerate(item["sentences"]):
+            got = label_of(item, drop=i)
+            if facts and got != UNDETERMINED: return "ablation_failed"      # evidence removed must make it undeterminable
+            if not facts and got != label: return "invariance_failed"          # filler removed must not change the label
+            evidence += bool(facts)
+        if evidence < 1: return "no_evidence_sentence"
+    return None
+
+
+def to_request(item, family, pair_id, sibling, rng):
+    order = list(range(len(item["sentences"]))); rng.shuffle(order)
+    sentences = [item["sentences"][i][0] for i in order]
+    q = {**item["question"], "label": label_of(item), "src": f"contrastive_{family}"}
+    return {"state": {"policy": item["policy"], "case": " ".join(sentences)}, "questions": {"decision": q},
+            "_meta": {"source": "contrastive", "family": family, "family_id": f"{family}/{pair_id}", "pair_id": pair_id, "sibling": sibling,
+                      "repo": None, "revision": None, "split": "generated", "row": pair_id, "id": f"contrastive/{family}/{pair_id}/{sibling}",
+                      "text_sha256": hashlib.sha256(" ".join(sentences).casefold().encode()).hexdigest(),
+                      "row_sha256": hashlib.sha256(json.dumps([t for t, _ in item["sentences"]]).encode()).hexdigest()}}
+
+
+def generate(n_pairs_per_family, seed, families=None):
+    """Sibling-adjacent records plus a per-family report of rejections."""
+    records, report = [], {}
+    for family in (families or FAMILIES):
+        rng = random.Random(f"{seed}:{family}")
+        kept, rejected, reasons, attempts = 0, 0, {}, 0
+        while kept < n_pairs_per_family and attempts < 50 * n_pairs_per_family:
+            attempts += 1
+            a, b = FAMILIES[family](rng)
+            why = check_pair(a, b)
+            if why:
+                rejected += 1; reasons[why] = reasons.get(why, 0) + 1; continue
+            pair_id = f"{seed}-{family}-{kept:04d}"
+            order_seed = rng.getrandbits(64)
+            records += [to_request(a, family, pair_id, "a", random.Random(order_seed)),
+                        to_request(b, family, pair_id, "b", random.Random(order_seed))]
+            kept += 1
+        if kept < n_pairs_per_family:
+            raise ValueError(f"{family}: only {kept}/{n_pairs_per_family} pairs passed checks ({reasons})")
+        report[family] = {"pairs": kept, "rejected": rejected, "reasons": reasons}
+    return records, report
+
+
+def paired_flip(rows):
+    """Pair-level metrics from benchmark rows carrying pair_id/sibling. A model that ignores the state cannot flip."""
+    by_pair = {}
+    for r in rows:
+        if r.get("pair_id"):
+            key = (r["pair_id"], r.get("question", "decision"))
+            pair = by_pair.setdefault(key, {})
+            if r["sibling"] in pair:
+                raise ValueError("duplicate contrastive sibling")
+            pair[r["sibling"]] = r
+    if not by_pair:
+        return None
+    if any(set(p) != {"a", "b"} for p in by_pair.values()):
+        raise ValueError("incomplete contrastive pair")
+    prediction = lambda r: r["keys"][max(range(len(r["p"])), key=r["p"].__getitem__)]
+    truth = lambda r: r["keys"][r["label"]]
+    relevant = [p for p in by_pair.values() if truth(p["a"]) != truth(p["b"])]
+    invariant = [p for p in by_pair.values() if truth(p["a"]) == truth(p["b"])]
+    both = lambda ps: sum(all(prediction(r) == truth(r) for r in p.values()) for p in ps) / len(ps) if ps else None
+    result = {"pairs": len(relevant),
+              "flip_rate": sum(prediction(p["a"]) != prediction(p["b"]) for p in relevant) / len(relevant) if relevant else None,
+              "both_correct_rate": both(relevant)}
+    if invariant:
+        result.update(invariant_pairs=len(invariant), invariance_rate=sum(prediction(p["a"]) == prediction(p["b"]) for p in invariant) / len(invariant),
+                      invariant_both_correct_rate=both(invariant))
+    return result

runtime/kev/cuda_graphs.py

@@ -0,0 +1,375 @@
+"""CUDA graphs for serving the hybrid (Qwen3.5) backbones on CUDA, batched across requests.
+
+A served request is two kinds of forward pass: the state pass (the document, on a prefix-cache miss) and the row pass (one
+causal row per question, continuing the cached state). At a few hundred tokens the GPU finishes a pass long before Python
+finishes issuing it: a pass launches ~2,000 kernels, and the flash-linear-attention wrapper around every DeltaNet layer
+costs ~1 ms of CPU. Replaying a captured graph issues the same kernels in one call. And because one request leaves most of
+the GPU idle, kev.serve hands this module every request that is waiting when the GPU frees up: one state pass computes all
+their new states, one row pass all their questions.
+
+Two sets of buffers, shared by every graph:
+- the state bank: per layer the attention keys/values [GRAPH_STATES, heads, BANK_WIDTH, dim], each state right-aligned at
+  the end, and the DeltaNet conv/recurrent states. The state pass writes new states into it; cached states are copied in.
+  One fixed layout, so a row pass reads any state the same way whatever state pass wrote it.
+- the row buffers: [rows, heads, Sr + Lb, dim] per attention layer and the DeltaNet states per question row. The row pass
+  starts by gathering each row's state from the bank (one index_select per tensor, inside the graph), then writes the
+  row's own keys after it.
+Both are flat buffers viewed at the graph's shape, so memory is fixed however many shapes appear. Graph keys are kept
+coarse because a batch's shape changes with the traffic: state passes by (states, state bucket), row passes by (rows,
+row bucket, Sr = the longest state rounded up to a power of two, which only lengthens attention).
+
+A graph has fixed shapes, so passes are padded to buckets: token counts to bucket() (under a quarter more), row and state counts
+to count_bucket() (empty entries are masked). The padding is masked exactly:
+- the state pass is LEFT-padded. The DeltaNet layers see zeroed inputs at the pads (their padding mask), so keys, values
+  and queries are zero and the recurrent state is still zero when the real tokens start; the causal convolution sees
+  the same zeros its own padding supplies. The attention layers mask the pads as keys, and a pad query attends to itself
+  so no row is fully masked (a NaN there would survive the zeroing: NaN * 0 = NaN). Real tokens keep their position ids.
+- question rows are right-padded (after every real token, as the eager path already does); a state's keys are
+  right-aligned in the bucket and the slots before them are masked.
+So the results equal the eager passes up to floating-point reassociation (another chunking of the DeltaNet scan, other
+GEMM shapes), not bit for bit, and a request's answers do not depend on what else shares its batch.
+
+Capturing a graph costs ~0.4 s (an H100, Kev-4B), so the first pass of a new bucket runs the same code eagerly from the
+same buffers and the bucket joins `pending`; capture_pending() captures them later (kev.serve does it when no request is
+waiting). Replays run one at a time (kev.serve has one model thread), each refills what it reads, and its results are
+copied out before the next replay; that is what makes the shared buffers and the shared memory pool safe.
+"""
+from collections import OrderedDict
+from typing import NamedTuple
+
+import torch
+from transformers import DynamicCache
+from transformers.cache_utils import DynamicLayer, LinearAttentionLayer
+
+# Limits of the graphed passes (not the model's context: that is kev.model.MAX_STATE / SERVE_MAX_STATE)
+GRAPH_TOKENS = 32768   # rows x positions one row pass may hold in the attention buffers (about 1 GB on Kev-4B and 9B)
+GRAPH_STATE = 1024     # longest state bucket the state pass graphs. A longer state pass is compute-bound, and the padding plus
+                       # the explicit mask (no causal flash attention) made a 2,200-token one slower as a graph (L40S, Kev-4B:
+                       # 223 vs 208 ms per request), so it runs eagerly
+GRAPH_ROW = 1024       # longest question-row bucket the row pass graphs
+GRAPH_ROWS = 32        # question rows per graphed pass; more run as several replays
+GRAPH_STATES = 16      # states per state pass: the bank's entries
+BANK_WIDTH = 4096      # positions per bank entry (states are right-aligned in it; about 2 GB for 16 entries on Kev-4B and 9B)
+GRAPHS_KEPT = 256      # captured graphs kept, least recently used evicted (a busy server met ~160 on mixed traffic)
+PASS_TOKENS = 256      # tokens a pass costs however few it holds: below about this many, a pass is bound by reading the
+                       # weights, not by compute (bf16 GEMMs: peak FLOP/s over memory bandwidth is ~200-400 on H100, H200, B200, L40S)
+HOT_BUCKET = 3         # eager passes after which a busy server captures a bucket's graph anyway (capture_due)
+
+
+def bucket(n, steps=8, floor=16):
+    """n rounded up to one of `steps` steps per power of two, in steps of at least `floor`: the padding adds less than
+    2/steps of n (just past a power of two, one step is almost that), or less than `floor` tokens for short n."""
+    step = max(floor, (1 << (n - 1).bit_length()) // steps)
+    return -(-n // step) * step
+
+
+def pow2(n):
+    return 1 << (n - 1).bit_length()
+
+
+def count_bucket(n):
+    """A row or state count rounded up to 1, 2, 3, 4, 6, 8, 12, 16, 24, 32: the empty entries padding a pass add less than
+    half, and a traffic mix meets few distinct counts."""
+    return bucket(n, steps=4, floor=1)
+
+
+def length_groups(lengths, cap):
+    """Indices grouped into padded passes of at most `cap` items, in order of length, computing the fewest tokens: a pass
+    costs its padded size, count_bucket(items) x bucket(longest), but at least PASS_TOKENS (below that, reading the
+    weights dominates). Optimal among groupings of consecutive lengths (dynamic programming; ties keep larger passes)."""
+    order = sorted(range(len(lengths)), key=lengths.__getitem__)
+    padded = [bucket(lengths[i]) for i in order]
+    counts = [0] + [count_bucket(n) for n in range(1, cap + 1)]
+    best, cut = [0] + [float("inf")] * len(order), [0] * (len(order) + 1)
+    for j in range(1, len(order) + 1):
+        for i in range(max(0, j - cap), j):
+            cost = best[i] + max(PASS_TOKENS, counts[j - i] * padded[j - 1])
+            if cost < best[j]: best[j], cut[j] = cost, i
+    groups, j = [], len(order)
+    while j: groups.append(order[cut[j]:j]); j = cut[j]
+    return groups[::-1]
+
+
+class Request(NamedTuple):
+    """One request for CudaGraphs.run."""
+    state: list        # the state's token ids
+    positions: list    # and their position ids
+    rows: list         # its question rows, [(token ids, position ids)]
+    cached: object     # the cached state (a DynamicCache) it continues, or None for a new state
+    keep: bool         # return a new state's cache (to keep in the prefix cache)
+    picks: list        # per row, the positions whose hidden states are wanted
+
+
+class _Row(NamedTuple):
+    """One question row of a batch: its tokens, its state's bank entry and length, and where its picks go in the output."""
+    ids: list
+    pos: list
+    entry: int
+    state_len: int
+    picks: list
+    at: int
+
+
+class BufferKV(DynamicLayer):
+    """An attention cache layer over preallocated [N, heads, T, dim] views: the first `filled` positions hold the cached
+    state, a pass writes its keys/values after them instead of concatenating (no allocation, no Python state change,
+    so one layer object serves the warm-up, the capture and every replay)."""
+
+    def __init__(self, keys, values, filled):
+        super().__init__()
+        self.buffers, self.filled, self.is_initialized = (keys, values), filled, True
+        self.keys, self.values = keys[..., :filled, :], values[..., :filled, :]
+
+    def update(self, key_states, value_states, *args, **kwargs):
+        end = self.filled + key_states.shape[-2]
+        for buf, new in zip(self.buffers, (key_states, value_states)): buf[..., self.filled:end, :].copy_(new)
+        return tuple(buf[..., :end, :] for buf in self.buffers)
+
+
+def set_linear(layer, conv, recurrent, previous):
+    """Point a DeltaNet cache layer at these conv/recurrent state tensors (transformers' LinearAttentionLayer fields)."""
+    layer.conv_states[0], layer.recurrent_states[0] = conv, recurrent
+    layer.is_conv_states_initialized[0] = layer.is_recurrent_states_initialized[0] = True
+    layer.conv_kernel_size[0], layer.has_previous_state[0] = conv.shape[-1], previous
+    layer.device, layer.dtype = conv.device, conv.dtype
+
+
+def is_attention(layer):
+    return isinstance(layer, DynamicLayer)
+
+
+def layer_tensors(layer):
+    """The two tensors a prefix-cache layer holds: attention keys/values, or DeltaNet conv/recurrent states."""
+    return (layer.keys, layer.values) if is_attention(layer) else (layer.conv_states[0], layer.recurrent_states[0])
+
+
+class Buffers:
+    """Per layer two flat buffers and the per-entry shape they hold: attention [heads, T, dim] (T set per view, `tokens`
+    entries x positions in total), DeltaNet states (`entries` entries)."""
+
+    def __init__(self, probe, tokens, entries, device):
+        self.slots = []
+        for layer in probe.layers:
+            ts, attention = layer_tensors(layer), is_attention(layer)
+            size = [tokens * t.shape[1] * t.shape[3] if attention else entries * t[0].numel() for t in ts]
+            self.slots.append((attention, [torch.zeros(n, dtype=t.dtype, device=device) for n, t in zip(size, ts)], [t.shape[1:] for t in ts]))
+
+    def views(self, n, length):
+        """Per layer, its two buffers viewed for n entries: attention [n, heads, length, dim], DeltaNet [n, *state]."""
+        def view(flat, shape, attention):
+            shape = (n, shape[0], length, shape[2]) if attention else (n, *shape)
+            return flat[:torch.Size(shape).numel()].view(shape)
+        return [tuple(view(f, s, attention) for f, s in zip(flats, shapes)) for attention, flats, shapes in self.slots]
+
+
+class CudaGraphs:
+    def __init__(self, lm, pad_id):
+        self.lm, self.pad_id = lm, pad_id
+        self.device, self.dtype = next(lm.parameters()).device, next(lm.parameters()).dtype
+        self.pool, self.stream = torch.cuda.graph_pool_handle(), torch.cuda.Stream()   # the graphs' shared memory pool, the capture stream
+        self.graphs = OrderedDict()   # key -> (graph, input buffer)
+        self.pending = {}             # key -> (pass body, input buffer): buckets that ran eagerly, not captured yet
+        self.eager_runs = {}          # key -> how many passes of a pending bucket ran eagerly
+        self.failed = {}              # key -> (why its capture failed, input buffer); the bucket keeps running eagerly
+        self.captures = 0
+        probe = DynamicCache(config=lm.config)   # learn the cache layout (layer kinds, state shapes, dtypes) from one eager pass
+        with torch.no_grad():
+            lm(input_ids=torch.full((1, 16), pad_id, device=self.device), past_key_values=probe, use_cache=True)
+        # BufferKV and set_linear rely on this layout: plain attention layers and single-state DeltaNet layers that update
+        # their states in place. Anything else (another cache layer type, record_past, which assigns instead of copying)
+        # would leave the buffers stale and move probabilities silently, so refuse it here.
+        if not all(type(l) is DynamicLayer or (type(l) is LinearAttentionLayer and l.number_of_states == 1 and not l.record_past) for l in probe.layers):
+            raise ValueError(f"CUDA graphs support attention and single-state DeltaNet cache layers only; got {sorted({type(l).__name__ for l in probe.layers})}")
+        self.bank = Buffers(probe, GRAPH_STATES * BANK_WIDTH, GRAPH_STATES, self.device)
+        # the bank has one layout for every pass, so the state and row graphs agree on it: attention [GRAPH_STATES, heads,
+        # BANK_WIDTH, dim] with states right-aligned at the end, DeltaNet [GRAPH_STATES, *state]
+        self.bank_views = self.bank.views(GRAPH_STATES, BANK_WIDTH)
+        self.rowbuf = Buffers(probe, GRAPH_TOKENS, GRAPH_ROWS, self.device)
+        self.hidden = torch.zeros(GRAPH_ROWS * GRAPH_ROW * lm.config.hidden_size, dtype=self.dtype, device=self.device)
+
+    def _cache(self, views, filled, previous):
+        """A DynamicCache over buffer views: attention layers hold `filled` cached positions, DeltaNet layers the states."""
+        cache = DynamicCache(config=self.lm.config)
+        for i, (layer, v) in enumerate(zip(cache.layers, views)):
+            if is_attention(layer): cache.layers[i] = BufferKV(*v, filled)
+            else: set_linear(layer, *v, previous)
+        return cache
+
+    def _mask(self, allow):
+        """Additive attention mask [B, 1, Lq, Lk] from a boolean one [B, Lq, Lk], in the backbone's dtype."""
+        return torch.zeros(allow.shape, dtype=self.dtype, device=self.device).masked_fill(~allow, torch.finfo(self.dtype).min)[:, None]
+
+    def _forward(self, ids, pos, full_mask, linear_mask, cache):
+        return self.lm(input_ids=ids, position_ids=pos, attention_mask={"full_attention": full_mask, "linear_attention": linear_mask},
+                       past_key_values=cache, use_cache=True).last_hidden_state
+
+    def _replay(self, key, body, rows):
+        """One pass for `key` on `rows` (token ids, positions, lengths, indices; int64): the graph's replay, or body() run
+        eagerly when the bucket has no graph yet (it then waits in `pending`). body(buf) runs the pass reading the uploaded
+        rows from buf. Every body for a key reads and writes the same buffer views, so the one kept in `pending` stands for
+        all of them."""
+        if key in self.graphs:
+            self.graphs.move_to_end(key)
+            graph, buf = self.graphs[key]
+        elif key in self.failed:
+            graph, buf = None, self.failed[key][1]
+        else:
+            graph, buf = None, self.pending.setdefault(key, (body, torch.zeros((len(rows), len(rows[0])), dtype=torch.long, device=self.device)))[1]
+            self.eager_runs[key] = self.eager_runs.get(key, 0) + 1
+        buf.copy_(torch.tensor(rows, dtype=torch.long), non_blocking=True)
+        if graph is not None: graph.replay()
+        else: body(buf)
+
+    @torch.no_grad()
+    def capture_pending(self, limit=None):
+        """Capture graphs for up to `limit` (None = all) buckets that have run eagerly, the most-run first. The caller must
+        keep other passes out (kev.serve captures on its model thread). Each capture overwrites the shared buffers, which is
+        fine between passes: a pass refills them. On a side stream, a warm-up pass first (Triton autotuning and cuBLAS
+        setup must not happen inside a capture), then the capture; not torch.cuda.graph, whose synchronize, gc.collect and
+        empty_cache on entry would stall a busy server for each capture. thread_local: CUDA calls of other threads (a
+        request tokenizing, a model card read) cannot invalidate the capture. A capture that fails leaves its bucket running
+        eagerly, for good."""
+        for _ in range(len(self.pending) if limit is None else min(limit, len(self.pending))):
+            key = max(self.pending, key=self.eager_runs.get)
+            (body, buf), _ = self.pending.pop(key), self.eager_runs.pop(key)
+            current, graph = torch.cuda.current_stream(), torch.cuda.CUDAGraph()
+            self.stream.wait_stream(current)
+            try:
+                with torch.cuda.stream(self.stream):
+                    body(buf)                                       # warm-up: autotuning and lazy setup happen outside the capture
+                    graph.capture_begin(pool=self.pool, capture_error_mode="thread_local")
+                    try: body(buf)
+                    finally: graph.capture_end()
+            except Exception as e:   # e.g. out of memory for a new shape: serve it eagerly rather than stop serving
+                # a failed capture_end leaves the allocator routing this thread's allocations into the graph pool, which
+                # would fail every later capture and let eager passes allocate graph memory: end that explicitly
+                try: getattr(torch._C, "_cuda_endAllocateToPool", lambda *a: None)(self.device.index, self.pool)   # private API: best effort
+                except Exception: pass
+                self.failed[key] = (f"{type(e).__name__}: {e}", buf)
+                print(f"kev.cuda_graphs: capturing {key} failed, it runs eagerly: {self.failed[key][0]}", flush=True)
+                continue
+            finally:
+                current.wait_stream(self.stream)
+            self.graphs[key] = (graph, buf); self.captures += 1
+            while len(self.graphs) > GRAPHS_KEPT: self.graphs.popitem(last=False)
+
+    def capture_due(self, idle):
+        """Capture one pending graph now? Always when idle; under load once a bucket keeps running eagerly (HOT_BUCKET
+        eager passes), because a request arriving meanwhile waits for the capture (~0.4 s)."""
+        runs = list(self.eager_runs.values())   # a snapshot: wait_idle and /v1/models call this off the model thread
+        return bool(runs) and (idle or max(runs) >= HOT_BUCKET)
+
+    def stats(self):
+        return {"captured": self.captures, "kept": len(self.graphs), "pending": len(self.pending), "failed": len(self.failed)}
+
+    # Serving: which requests the graphed passes take, and one batch of them
+
+    @staticmethod
+    def admits(state_len, row_lens, cached):
+        """Whether a request's passes fit the graphed passes: its rows, and its state (a new one runs the graphed state
+        pass; a cached one only has to fit the bank)."""
+        return bucket(max(row_lens)) <= GRAPH_ROW and (state_len <= BANK_WIDTH if cached else bucket(state_len) <= GRAPH_STATE)
+
+    @torch.no_grad()
+    def run(self, requests):
+        """Admitted requests (Request) -> (their picked hidden states, float32 [sum of picks, d] in request, row, pick
+        order; per request its state's cache: the cached one, a new one if `keep`, else None). States grouped by length
+        (length_groups) share a state pass (identical states once); each request takes one bank entry, so at most GRAPH_STATES per group.
+        Every row pass writes its picks straight to their place in the output."""
+        at = [0]
+        for r in requests: at.append(at[-1] + sum(map(len, r.picks)))
+        out = torch.empty((at[-1], self.lm.config.hidden_size), dtype=torch.float32, device=self.device)
+        caches = [None] * len(requests)
+        for group in length_groups([len(r.state) for r in requests], GRAPH_STATES):
+            new = {}                                                 # distinct new states of the group -> (ids, positions, keep)
+            for i in group:
+                r = requests[i]
+                if r.cached is None: new[tuple(r.state)] = (r.state, r.positions, r.keep or new.get(tuple(r.state), (0, 0, False))[2])
+            made = dict(zip(new, self.states(list(new.values()), bucket(max(map(len, new)))))) if new else {}
+            entry = {S: j for j, S in enumerate(new)}                # cached states go into the entries after the new ones
+            rows, free = [], len(new)
+            for i in group:                                          # after the state pass: its padded entries are written
+                r = requests[i]
+                if r.cached is None: e, caches[i] = entry[tuple(r.state)], made[tuple(r.state)]
+                else: e, free, caches[i] = free, free + 1, r.cached; self.load_state(e, r.cached, len(r.state))
+                offset = at[i]
+                for (ids, pos), picks in zip(r.rows, r.picks):
+                    rows.append(_Row(ids, pos, e, len(r.state), picks, offset)); offset += len(picks)
+            self.rows(rows, out)
+        return out, caches
+
+    @torch.no_grad()
+    def states(self, items, Sb):
+        """State pass for up to GRAPH_STATES states, items = [(ids, pos, keep)] each at most Sb <= GRAPH_STATE tokens.
+        Leaves state i in bank entry i and returns, for each kept state, a DynamicCache equal to the eager prefix pass's
+        (None for the others: copying every state of a batch cost more than the passes)."""
+        Nb = count_bucket(len(items))
+
+        def body(buf):   # buf rows = [ids | positions | real length], left-padded
+            pad = Sb - buf[:, 2 * Sb:]
+            i = torch.arange(Sb, device=self.device)
+            q, k = i[None, :, None], i[None, None, :]
+            allow = ((k <= q) & (k >= pad[:, :, None])) | (k == q)
+            views = [(a[:Nb, :, BANK_WIDTH - Sb:], b[:Nb, :, BANK_WIDTH - Sb:]) if attention else (a[:Nb], b[:Nb])
+                     for (attention, *_), (a, b) in zip(self.bank.slots, self.bank_views)]
+            self._forward(buf[:, :Sb], buf[:, Sb:2 * Sb], self._mask(allow), (i[None] >= pad).long(), self._cache(views, 0, False))
+
+        rows = [[self.pad_id] * (Sb - len(ids)) + list(ids) + [0] * (Sb - len(ids)) + list(pos) + [len(ids)] for ids, pos, _ in items]
+        self._replay(("states", Nb, Sb), body, rows + [[self.pad_id] * Sb + [0] * Sb + [0]] * (Nb - len(items)))
+        out = []   # a cache for each kept state: its own copy of the bank entry (the bank belongs to the next pass)
+        for j, (ids, _, keep) in enumerate(items):
+            if not keep: out.append(None); continue
+            cache, S = DynamicCache(config=self.lm.config), len(ids)
+            for layer, (a, b) in zip(cache.layers, self.bank_views):
+                if is_attention(layer):
+                    layer.keys, layer.values = a[j:j + 1, :, BANK_WIDTH - S:].clone(), b[j:j + 1, :, BANK_WIDTH - S:].clone()
+                    layer.dtype, layer.device, layer.is_initialized = self.dtype, self.device, True
+                else:
+                    set_linear(layer, a[j:j + 1].clone(), b[j:j + 1].clone(), True)
+            out.append(cache)
+        return out
+
+    @torch.no_grad()
+    def load_state(self, entry, cache, S):
+        """Copy a cached state (a DynamicCache of S <= BANK_WIDTH tokens, from either path) into bank entry `entry`."""
+        for layer, v in zip(cache.layers, self.bank_views):
+            for buf, t in zip(v, layer_tensors(layer)):
+                if is_attention(layer): buf[entry, :, BANK_WIDTH - S:].copy_(t[0, :, t.shape[-2] - S:])
+                else: buf[entry].copy_(t[0])
+
+    def rows(self, rows, out):
+        """Row passes for question rows (_Row), each continuing its state in the bank; each row's picked hidden states land
+        in out[row.at:]. The rows see the last Sr positions of the bank, Sr the longest state rounded up to a power of two
+        (it only lengthens attention). Rows are grouped by length (length_groups), then split to fit the buffers."""
+        for idx in length_groups([len(r.ids) for r in rows], GRAPH_ROWS):
+            Lb, Sr = bucket(max(len(rows[i].ids) for i in idx)), pow2(max(16, max(rows[i].state_len for i in idx)))
+            group = min(GRAPH_ROWS, GRAPH_TOKENS // (Sr + Lb))
+            group = max(n for n in range(1, group + 1) if count_bucket(n) <= group)   # the most rows whose padded count fits
+            for start in range(0, len(idx), group):
+                part = [rows[i] for i in idx[start:start + group]]
+                h = self._row_pass(part, Sr, Lb)
+                src, dst = zip(*[(n * Lb + p, r.at + j) for n, r in enumerate(part) for j, p in enumerate(r.picks)])
+                src, dst = torch.tensor([src, dst]).to(self.device, non_blocking=True)
+                out.index_copy_(0, dst, h.flatten(0, 1).index_select(0, src).float())   # only the wanted positions leave the pass buffer
+
+    def _row_pass(self, rows, Sr, Lb):
+        """One row pass; -> the pass buffer [rows, Lb, d] (valid until the next pass)."""
+        Nb, T = count_bucket(len(rows)), Sr + Lb
+        hidden = self.hidden[:Nb * Lb * self.lm.config.hidden_size].view(Nb, Lb, -1)
+
+        def body(buf):   # buf rows = [ids | positions | row length | state length | bank entry]
+            rowlen, plen, entry = buf[:, 2 * Lb:2 * Lb + 1], buf[:, 2 * Lb + 1:2 * Lb + 2], buf[:, 2 * Lb + 2]
+            views = self.rowbuf.views(Nb, T)
+            for (attention, *_), dst, src in zip(self.bank.slots, views, self.bank_views):   # each row's state, from the bank
+                for d, s in zip(dst, src):
+                    if attention: d[:, :, :Sr].copy_(s[:, :, BANK_WIDTH - Sr:].index_select(0, entry))
+                    else: d.copy_(s.index_select(0, entry))
+            q = torch.arange(Lb, device=self.device)[None, :, None]
+            k = torch.arange(T, device=self.device)[None, None, :]
+            allow = ((k < Sr) & (k >= Sr - plen[:, :, None])) | ((k >= Sr) & (k - Sr <= q) & (k - Sr < rowlen[:, :, None])) | (k - Sr == q)
+            linear = (torch.arange(Lb, device=self.device)[None] < rowlen).long()
+            hidden.copy_(self._forward(buf[:, :Lb], buf[:, Lb:2 * Lb], self._mask(allow), linear, self._cache(views, Sr, True)))
+
+        uploads = [list(r.ids) + [self.pad_id] * (Lb - len(r.ids)) + list(r.pos) + [0] * (Lb - len(r.pos)) + [len(r.ids), r.state_len, r.entry] for r in rows]
+        self._replay(("rows", Nb, Lb, Sr), body, uploads + [[self.pad_id] * Lb + [0] * Lb + [0, 0, 0]] * (Nb - len(rows)))
+        return hidden[:len(rows)]

runtime/kev/data.py

@@ -0,0 +1,410 @@
+"""Convert public labelled datasets into TypeSafe-shaped requests, then through api.to_record() so the
+training format is byte-identical to what /v1/systemone feeds the model.
+
+A "labelled request" is {"state": JSONContent, "questions": {id: {type, instructions, criteria, "label": ..., "src": str}}}
+  label: choice -> option key, noul -> bool, score -> level index.
+materialize() -> internal record {"state": str, "questions": [{"instr", "options", "label": int, "src"}]}
+"""
+import hashlib
+import json
+from pathlib import Path
+import random
+from datasets import load_dataset
+from .api import SystemOneRequest, to_record
+
+
+REPOS = {"banking77": "legacy-datasets/banking77", "boolq": "google/boolq", "agnews": "fancyzhx/ag_news",
+         "mnli": "nyu-mll/multi_nli", "sst5": "SetFit/sst5", "yelp": "Yelp/yelp_review_full"}
+
+
+def source_seed(seed, source):
+    return int.from_bytes(hashlib.sha256(f"{seed}:{source}".encode()).digest()[:8], "big")
+
+
+# Repos whose main branch is a (no longer supported) loading script; read the Hub's auto-converted parquet branch instead.
+PARQUET_BRANCH = {"CogComp/trec": "refs/convert/parquet"}
+
+
+def dataset_ref(repo):
+    """(repo, revision-to-pin) for a repo id; parquet-branch repos pin that branch's sha, not main."""
+    return repo.partition(":")[0], PARQUET_BRANCH.get(repo.partition(":")[0])
+
+
+class Source(random.Random):
+    """One source's sampling context: a seeded RNG (the converters draw everything from it), the dataset revision to pin,
+    and the provenance of every row drawn (`origins`, filled by _sample so build() can attach it to each record)."""
+
+    def __init__(self, seed, revision=None):
+        super().__init__(seed)
+        self.revision = revision
+        self.origins = []
+
+
+def _dataset(repo, split, src):
+    """repo may be 'owner/name' or 'owner/name:config'; the Source carries the revision to pin."""
+    name, _, config = repo.partition(":")
+    return load_dataset(name, config or None, split=split, revision=src.revision or PARQUET_BRANCH.get(name))
+
+NONE = "None of the above"
+# "None of the above" options must appear both as the correct answer and as a wrong alternative, with varied
+# wording, or the model learns "this wording => pick it" (it did, in the first training run).
+NONE_OPTIONS = [("other", "None of the above"), ("other", "A reason that fits none of the above"), ("none", "None of these"),
+                ("other", "Something else"), ("not_listed", "Not listed here"), ("none_of_the_above", None),
+                ("other", "A category that fits none of the above"), ("other", "None of the listed options apply"),
+                ("unknown", "Cannot be determined from the options given"), ("other", "Other"), ("none", None),
+                ("other", "An answer not covered by the other options"), ("no_match", "No option matches")]
+DISTRACTORS = {"weather": "Bad weather caused it", "purple": "The colour purple", "pancakes": "A recipe for pancakes", "taxes": "Unrelated: quarterly tax filing"}
+
+AG = {"world": "World news: politics, international affairs, conflicts", "sports": "Sports: games, athletes, teams, results",
+      "business": "Business: companies, markets, economy, finance", "scitech": "Science and technology: research, gadgets, software, space"}
+MNLI = {"entailment": "The hypothesis follows from the premise", "neutral": "The hypothesis may or may not be true given the premise", "contradiction": "The hypothesis contradicts the premise"}
+SST5 = ["very negative", "negative", "neutral", "positive", "very positive"]
+YELP = ["1 star: terrible experience", "2 stars: poor", "3 stars: average", "4 stars: good", "5 stars: excellent"]
+BANK_TEMPLATES = ["Customer asks about {}", "Issue concerning {}", "Request related to {}", "{}"]
+
+
+def _wrap_state(text, rng):
+    r = rng.random()
+    if r < 0.15: return {"document": text}
+    if r < 0.25: return {"ticket": {"channel": rng.choice(["email", "chat", "web form"]), "body": text}}
+    if r < 0.32: return [{"role": "customer", "content": text}]
+    return text
+
+
+def _instr(text, rng):
+    return {"question": text, "focus": rng.choice(["Use only the information given.", "Pick the single best fit.", "Consider the whole message."])} if rng.random() < 0.15 else text
+
+
+def _desc(desc, rng, p_null=0.3, p_struct=0.1):
+    r = rng.random()
+    if r < p_null: return None
+    if r < p_null + p_struct: return {"what": desc}
+    return desc
+
+
+def _sample(ds, n, src):
+    rows = []
+    src.origins = []
+    for i in src.sample(range(len(ds)), min(n, len(ds))):
+        row = ds[i]
+        if row.get("label", 0) == -1:
+            continue
+        text = next((row[k] for k in ("text", "premise", "passage", "content", "question", "sentence") if isinstance(row.get(k), str)), json.dumps(row, sort_keys=True))
+        normalized = " ".join(text.casefold().split())
+        src.origins.append({"row": i, "text_sha256": hashlib.sha256(normalized.encode()).hexdigest(),
+                            "row_sha256": hashlib.sha256(json.dumps(row, sort_keys=True).encode()).hexdigest()})
+        rows.append(row)
+    return rows
+
+
+def _banking(split, n, rng):
+    ds = _dataset("legacy-datasets/banking77", split=split, src=rng)
+    names = ds.features["label"].names
+    out = []
+    for ex in _sample(ds, n, rng):
+        t = rng.choice(BANK_TEMPLATES)
+        crit = {k: _desc(t.format(k.replace("_", " ")), rng, p_null=0.5, p_struct=0.0) for k in names}
+        out.append({"state": _wrap_state(ex["text"], rng), "questions": {"intent": {"type": "choice", "instructions": _instr("Which banking intent best describes this customer message?", rng), "criteria": crit, "label": names[ex["label"]], "src": "banking77"}}})
+    return out
+
+
+def _boolq(split, n, rng):
+    ds = _dataset("google/boolq", split=split, src=rng)
+    out = []
+    for ex in _sample(ds, n, rng):
+        q = {"type": "noul", "instructions": _instr(ex["question"].strip().rstrip("?") + "?", rng), "label": bool(ex["answer"]), "src": "boolq"}
+        if rng.random() < 0.4: q["criteria"] = {"true": "The passage supports a yes answer", "false": "The passage supports a no answer or does not say"}
+        out.append({"state": _wrap_state(ex["passage"], rng), "questions": {"answer": q}})
+    return out
+
+
+def _agnews(split, n, rng):
+    ds = _dataset("fancyzhx/ag_news", split=split, src=rng)
+    keys = list(AG)
+    out = []
+    for ex in _sample(ds, n, rng):
+        y = keys[ex["label"]]
+        qs = {"topic": {"type": "choice", "instructions": _instr("What is the topic of this article?", rng), "criteria": {k: _desc(v, rng) for k, v in AG.items()}, "label": y, "src": "agnews"}}
+        for k in rng.sample(keys, 2):
+            qs[f"is_{k}"] = {"type": "noul", "instructions": f"Is this article about {AG[k].split(':')[0].lower()}?", "label": k == y, "src": "agnews_yn"}
+        out.append({"state": _wrap_state(ex["text"], rng), "questions": qs})
+    return out
+
+
+def _mnli(split, n, rng):
+    ds = _dataset("nyu-mll/multi_nli", split=split, src=rng)
+    keys = list(MNLI)
+    return [{"state": _wrap_state(ex["premise"], rng), "questions": {"relation": {"type": "choice", "instructions": _instr(f'Hypothesis: "{ex["hypothesis"]}" How does it relate to the premise?', rng), "criteria": {k: _desc(v, rng) for k, v in MNLI.items()}, "label": keys[ex["label"]], "src": "mnli"}}}
+            for ex in _sample(ds, n, rng) if ex["label"] >= 0]
+
+
+def _sst5(split, n, rng):
+    ds = _dataset("SetFit/sst5", split=split, src=rng)
+    return [{"state": _wrap_state(ex["text"], rng), "questions": {"sentiment": {"type": "score", "instructions": _instr("What is the sentiment of this review sentence?", rng), "criteria": list(SST5), "label": ex["label"], "src": "sst5"}}} for ex in _sample(ds, n, rng)]
+
+
+def _yelp(split, n, rng):
+    ds = _dataset("Yelp/yelp_review_full", split=split, src=rng)
+    out = []
+    for ex in _sample(ds, n, rng):
+        text = " ".join(ex["text"].split()[:220])
+        qs = {"rating": {"type": "score", "instructions": _instr("How many stars did this reviewer give?", rng), "criteria": list(YELP), "label": ex["label"], "src": "yelp"},
+              "recommend": {"type": "noul", "instructions": "Would this reviewer recommend the business?", "criteria": {"true": "Clearly positive overall", "false": "Negative or mixed"}, "label": ex["label"] >= 3, "src": "yelp_yn"}}
+        out.append({"state": _wrap_state(text, rng), "questions": qs})
+    return out
+
+
+SOURCES = {"banking77": (_banking, "train", "test"), "boolq": (_boolq, "train", "validation"), "agnews": (_agnews, "train", "test"),
+           "mnli": (_mnli, "train", "validation_matched"), "sst5": (_sst5, "train", "test"), "yelp": (_yelp, "train", "test")}
+
+
+# Eval-only sources for transfer measurement. Never passed to training; kept out of SOURCES so build() defaults cannot
+# pick them up. Rotten Tomatoes (SST parent) and SNLI (MNLI sibling) are deliberately excluded.
+TREC = {"abbreviation": "Asks what an abbreviation stands for", "entity": "Asks about a thing, object, animal, product, or creative work",
+        "description": "Asks for a definition, description, reason, or manner", "human": "Asks about a person, group, or organisation",
+        "location": "Asks about a place", "number": "Asks for a number, date, count, or other numeric value"}
+EMOTION = {"sadness": None, "joy": None, "love": None, "anger": None, "fear": None, "surprise": None}
+AMAZON = ["1 star: very negative", "2 stars: negative", "3 stars: mixed", "4 stars: positive", "5 stars: very positive"]
+
+
+def _trec(split, n, rng):
+    ds = _dataset("CogComp/trec", split=split, src=rng)
+    keys = list(TREC)
+    return [{"state": _wrap_state(ex["text"], rng), "questions": {"answer_type": {"type": "choice", "instructions": "What kind of answer does this question ask for?",
+             "criteria": dict(TREC), "label": keys[ex["coarse_label"]], "src": "trec"}}} for ex in _sample(ds, n, rng)]
+
+
+def _dbpedia(split, n, rng):
+    ds = _dataset("fancyzhx/dbpedia_14", split=split, src=rng)
+    names = [x.lower().replace(" ", "_") for x in ds.features["label"].names]
+    return [{"state": _wrap_state(" ".join(ex["content"].split()[:200]), rng), "questions": {"category": {"type": "choice", "instructions": "Which category does the subject of this encyclopedia text belong to?",
+             "criteria": {k: None for k in names}, "label": names[ex["label"]], "src": "dbpedia14"}}} for ex in _sample(ds, n, rng)]
+
+
+def _emotion(split, n, rng):
+    ds = _dataset("dair-ai/emotion:split", split=split, src=rng)
+    keys = list(EMOTION)
+    return [{"state": ex["text"], "questions": {"emotion": {"type": "choice", "instructions": "Which emotion does the writer express?",
+             "criteria": dict(EMOTION), "label": keys[ex["label"]], "src": "emotion"}}} for ex in _sample(ds, n, rng)]
+
+
+def _imdb(split, n, rng):
+    ds = _dataset("stanfordnlp/imdb", split=split, src=rng)
+    return [{"state": _wrap_state(" ".join(ex["text"].replace("<br />", " ").split()[:220]), rng), "questions": {"positive": {"type": "noul", "instructions": "Is this movie review positive?",
+             "criteria": {"true": "The reviewer liked the film overall", "false": "The reviewer disliked the film overall"}, "label": ex["label"] == 1, "src": "imdb"}}} for ex in _sample(ds, n, rng)]
+
+
+def _amazon(split, n, rng):
+    ds = _dataset("SetFit/amazon_reviews_multi_en", split=split, src=rng)
+    return [{"state": _wrap_state(" ".join(ex["text"].split()[:220]), rng), "questions": {"stars": {"type": "score", "instructions": "How many stars did this product reviewer give?",
+             "criteria": list(AMAZON), "label": ex["label"], "src": "amazon"}}} for ex in _sample(ds, n, rng)]
+
+
+def _qnli(split, n, rng):
+    ds = _dataset("nyu-mll/glue:qnli", split=split, src=rng)
+    return [{"state": _wrap_state(ex["sentence"], rng), "questions": {"answers": {"type": "noul", "instructions": f'Does the sentence contain the answer to this question: "{ex["question"]}"',
+             "label": ex["label"] == 0, "src": "qnli"}}} for ex in _sample(ds, n, rng)]
+
+
+def _offensive(split, n, rng):
+    ds = _dataset("cardiffnlp/tweet_eval:offensive", split=split, src=rng)
+    return [{"state": ex["text"], "questions": {"offensive": {"type": "noul", "instructions": "Is this post offensive?",
+             "criteria": {"true": "Contains insults, threats, profanity directed at someone, or hateful content", "false": "Not offensive"},
+             "label": ex["label"] == 1, "src": "tweet_offensive"}}} for ex in _sample(ds, n, rng)]
+
+
+def _mmlu(split, n, rng):
+    ds = _dataset("cais/mmlu:all", split=split, src=rng)
+    out = []
+    for ex in _sample(ds, n, rng):
+        keys = ["a", "b", "c", "d"]
+        out.append({"state": {"subject": ex["subject"].replace("_", " "), "question": ex["question"]},
+                    "questions": {"answer": {"type": "choice", "instructions": "Which option correctly answers the question?",
+                                             "criteria": dict(zip(keys, ex["choices"])), "label": keys[ex["answer"]], "src": "mmlu"}}})
+    return out
+
+
+def _paws(split, n, rng):
+    ds = _dataset("google-research-datasets/paws:labeled_final", split=split, src=rng)
+    return [{"state": _wrap_state(ex["sentence1"], rng), "questions": {"paraphrase": {"type": "noul", "instructions": f'Does this sentence mean the same thing: "{ex["sentence2"]}"',
+             "criteria": {"true": "Same meaning, possibly reworded", "false": "Different meaning, even if most words match"}, "label": ex["label"] == 1, "src": "paws"}}}
+            for ex in _sample(ds, n, rng)]
+
+
+def _sciq(split, n, rng):
+    ds = _dataset("allenai/sciq", split=split, src=rng)
+    out = []
+    for ex in _sample(ds, n, rng):
+        options = [ex["correct_answer"], ex["distractor1"], ex["distractor2"], ex["distractor3"]]
+        keys = ["a", "b", "c", "d"]; order = list(range(4)); rng.shuffle(order)
+        crit = {keys[i]: options[j] for i, j in enumerate(order)}
+        out.append({"state": {"passage": ex["support"], "question": ex["question"]} if ex["support"] else {"question": ex["question"]},
+                    "questions": {"answer": {"type": "choice", "instructions": "Which option answers the science question?", "criteria": crit,
+                                             "label": keys[order.index(0)], "src": "sciq"}}})
+    return out
+
+
+def _mcq(ex_q, labels, texts, answer_label, src, rng, state_extra=None):
+    """Knowledge MCQ -> Choice with neutral keys; option order shuffled per record so keys carry no information."""
+    order = list(range(len(texts))); rng.shuffle(order)
+    keys = [f"opt_{i + 1}" for i in range(len(texts))]
+    crit = {keys[i]: texts[j] for i, j in enumerate(order)}
+    label = keys[order.index(labels.index(answer_label))]
+    state = {"question": ex_q}
+    if state_extra: state.update(state_extra)
+    return {"state": state, "questions": {"answer": {"type": "choice", "instructions": "Which option correctly answers the question?", "criteria": crit, "label": label, "src": src}}}
+
+
+def _arc(split, n, rng):
+    ds = _dataset("allenai/ai2_arc:ARC-Challenge", split=split, src=rng)
+    return [_mcq(ex["question"], list(ex["choices"]["label"]), list(ex["choices"]["text"]), ex["answerKey"], "arc", rng) for ex in _sample(ds, n, rng)]
+
+
+def _openbookqa(split, n, rng):
+    ds = _dataset("allenai/openbookqa:main", split=split, src=rng)
+    return [_mcq(ex["question_stem"], list(ex["choices"]["label"]), list(ex["choices"]["text"]), ex["answerKey"], "openbookqa", rng) for ex in _sample(ds, n, rng)]
+
+
+def _csqa(split, n, rng):
+    ds = _dataset("tau/commonsense_qa", split=split, src=rng)
+    return [_mcq(ex["question"], list(ex["choices"]["label"]), list(ex["choices"]["text"]), ex["answerKey"], "csqa", rng) for ex in _sample(ds, n, rng) if ex["answerKey"]]
+
+
+ALL_SOURCES = {**SOURCES, "trec": (_trec, "train", "test"), "dbpedia14": (_dbpedia, "train", "test"), "emotion": (_emotion, "train", "test"),
+               "imdb": (_imdb, "train", "test"), "amazon": (_amazon, "train", "test"), "qnli": (_qnli, "train", "validation"),
+               "tweet_offensive": (_offensive, "train", "test"), "mmlu": (_mmlu, "test", "test"),
+               "paws": (_paws, "train", "test"), "sciq": (_sciq, "train", "test"),
+               "arc": (_arc, "train", "test"), "openbookqa": (_openbookqa, "train", "test"), "csqa": (_csqa, "train", "validation")}
+ALL_REPOS = {**REPOS, "trec": "CogComp/trec", "dbpedia14": "fancyzhx/dbpedia_14", "emotion": "dair-ai/emotion", "imdb": "stanfordnlp/imdb",
+             "amazon": "SetFit/amazon_reviews_multi_en", "qnli": "nyu-mll/glue", "tweet_offensive": "cardiffnlp/tweet_eval", "mmlu": "cais/mmlu",
+             "paws": "google-research-datasets/paws", "sciq": "allenai/sciq",
+             "arc": "allenai/ai2_arc", "openbookqa": "allenai/openbookqa", "csqa": "tau/commonsense_qa"}
+
+# Policy (PLAN.md step 1). A source is trainable or eval-only; suites record both lists and training refuses eval-only
+# sources. MMLU is a knowledge probe and stays eval-only permanently; Emotion/TweetEval are noisy-label honesty checks;
+# QNLI/PAWS/SciQ measure reading transfer. Rotten Tomatoes (SST parent) and SNLI (MNLI sibling) are excluded entirely.
+# Knowledge MCQ (ARC-Challenge, OpenBookQA, CommonsenseQA) is trainable: the hypothesis (PLAN.md, overnight) is that the
+# pointer readout under-uses the base model's knowledge (8B scores 0.65 on 4-way MMLU, below its base-model level) and
+# that a small MCQ mix teaches the head to tap it. MMLU and SciQ stay eval-only; ARC/SciQ are distinct datasets.
+TRAINABLE = ("banking77", "boolq", "agnews", "mnli", "sst5", "yelp", "trec", "dbpedia14", "amazon", "imdb", "arc", "openbookqa", "csqa")
+EVAL_ONLY = ("mmlu", "emotion", "tweet_offensive", "qnli", "paws", "sciq")
+assert set(TRAINABLE) | set(EVAL_ONLY) == set(ALL_SOURCES) and not set(TRAINABLE) & set(EVAL_ONLY)
+
+# transfer-v1 (frozen before the policy existed) used these eight; kept so its manifest can be re-derived.
+TRANSFER_SOURCES = {k: ALL_SOURCES[k] for k in ("trec", "dbpedia14", "emotion", "imdb", "amazon", "qnli", "tweet_offensive", "mmlu")}
+TRANSFER_REPOS = {k: ALL_REPOS[k] for k in TRANSFER_SOURCES}
+
+
+def build(n_per_source, split="train", seed=0, exclude=(), only=(), revisions=None, sources=None, repos=None):
+    sources = SOURCES if sources is None else sources
+    repos = REPOS if repos is None else repos
+    unknown = (set(exclude) | set(only)) - sources.keys()
+    if unknown:
+        raise ValueError(f"unknown sources: {sorted(unknown)}")
+    reqs = []
+    for name, (fn, tr, te) in sources.items():
+        if name in exclude or (only and name not in only): continue
+        src = Source(source_seed(seed, name), (revisions or {}).get(repos[name]))
+        source_split = tr if split == "train" else te
+        records = fn(source_split, n_per_source, src)
+        if len(records) != len(src.origins):
+            raise ValueError(f"provenance mismatch for {name}")
+        for record, origin in zip(records, src.origins):
+            record["_meta"] = {**origin, "source": name, "repo": repos[name], "revision": src.revision,
+                               "split": source_split, "id": f"{name}/{source_split}/{origin['row']}"}
+        reqs.extend(records)
+    random.Random(seed).shuffle(reqs)
+    return reqs
+
+
+def augment(req, rng, p_none=0.1, p_none_distract=0.12, p_distract=0.15):
+    """Choice only: permute option order (always); sometimes add a 'none of the above' option, either as the correct
+    answer (true option removed) or as a wrong alternative (true option kept); sometimes add an irrelevant distractor."""
+    if min(p_none, p_none_distract, p_distract) < 0 or p_none + p_none_distract + p_distract > 1:
+        raise ValueError("augmentation probabilities must be nonnegative and sum to at most one")
+    out = {"state": req["state"], "questions": {}}
+    for qid, q in req["questions"].items():
+        if q["type"] != "choice":
+            out["questions"][qid] = q; continue
+        crit, y = dict(q["criteria"]), q["label"]
+        if q.get("target") is not None:                       # soft-target questions: permute only; inserting or swapping options would change the target's meaning
+            keys = list(crit); rng.shuffle(keys); out["questions"][qid] = {**q, "criteria": {k: crit[k] for k in keys}}; continue
+        r = rng.random()
+        none_options = [(k, v) for k, v in NONE_OPTIONS if k not in crit]
+        distractors = [k for k in DISTRACTORS if k not in crit]
+        if len(crit) > 2 and r < p_none and none_options:
+            nk, nd = rng.choice(none_options); crit.pop(y); crit[nk] = nd; y = nk
+        elif p_none <= r < p_none + p_none_distract and len(crit) < 255 and none_options:
+            nk, nd = rng.choice(none_options); crit[nk] = nd
+        elif p_none + p_none_distract <= r < p_none + p_none_distract + p_distract and len(crit) < 255 and distractors:
+            k = rng.choice(distractors); crit[k] = DISTRACTORS[k]
+        keys = list(crit); rng.shuffle(keys)
+        out["questions"][qid] = {**q, "criteria": {k: crit[k] for k in keys}, "label": y}
+    return out
+
+
+def none_pair(req, rng):
+    """Minimal pair for the none-of-the-above shortcut: the same state and question rendered twice, once with the true
+    option present (a none option is wrong) and once with it removed (the same none option is right). Everything else,
+    including option order, is identical, so the only difference the model can use is whether the evidence matches an
+    option. Returns [] when the request has no eligible Choice (>=3 options, no none key already present, no soft target:
+    removing the labelled option or adding a none option would change what the target means, as in augment)."""
+    eligible = [(qid, q) for qid, q in req["questions"].items() if q["type"] == "choice" and len(q["criteria"]) >= 3 and q.get("target") is None]
+    if not eligible: return []
+    qid, q = rng.choice(eligible)
+    nk, nd = rng.choice([o for o in NONE_OPTIONS if o[0] not in q["criteria"]] or [("none_of_these", None)])
+    keys = list(q["criteria"]) + [nk]; rng.shuffle(keys)
+    present = {**q, "criteria": {k: (nd if k == nk else q["criteria"][k]) for k in keys}}
+    absent = {**present, "criteria": {k: v for k, v in present["criteria"].items() if k != q["label"]}, "label": nk}
+    return [{"state": req["state"], "questions": {qid: present}}, {"state": req["state"], "questions": {qid: absent}}]
+
+
+def load_records(path, source="custom"):
+    """Labelled requests from a JSONL file, one per line, in the API's request shape plus a `label` on every question:
+
+        {"state": "...", "questions": {"team": {"type": "choice", "instructions": "...", "criteria": {"billing": "...", "other": null}, "label": "billing"},
+                                       "urgent": {"type": "noul", "instructions": "...", "label": true},
+                                       "priority": {"type": "score", "instructions": "...", "criteria": ["low", "medium", "high"], "label": 2}}}
+
+    Labels: the option name for choice, true/false for noul, the level index (from 0) for score. `_meta` and per-question
+    `src` are filled in so the records behave like a frozen suite's (source = `source`, id = line number)."""
+    records = []
+    with Path(path).open(encoding="utf-8") as f:
+        for n, line in enumerate(f):
+            if not line.strip(): continue
+            r = json.loads(line)
+            if "state" not in r or not isinstance(r.get("questions"), dict) or not r["questions"]:
+                raise ValueError(f"{path}:{n + 1}: a record needs a state and a non-empty questions object")
+            for qid, q in r["questions"].items():
+                if "label" not in q: raise ValueError(f"{path}:{n + 1}: question {qid!r} has no label")
+                q.setdefault("src", f"{source}_{q['type']}")
+            text = json.dumps(r["state"], sort_keys=True, ensure_ascii=False) if not isinstance(r["state"], str) else r["state"]
+            r["_meta"] = {**{"source": source, "variant": "clean", "id": f"{source}/{n}", "group_id": f"{source}/{n}", "row": n, "split": "custom",
+                             "text_sha256": hashlib.sha256(" ".join(text.casefold().split()).encode()).hexdigest()}, **r.get("_meta", {})}
+            records.append(r)
+    if not records: raise ValueError(f"{path}: no records")
+    return records
+
+
+def api_request(record):
+    """The /v1/systemone request body for a labelled record: state and typed questions only, never labels, targets or
+    metadata (this is what leaves the machine when a remote predictor is scored)."""
+    return {"state": record["state"], "questions": {
+        qid: {k: v for k, v in q.items() if k in ("type", "instructions", "criteria")}
+        for qid, q in record["questions"].items()}}
+
+
+def materialize(req):
+    """Labelled request -> internal record via the serving path (api.to_record), attaching int labels and src."""
+    rec, meta = to_record(SystemOneRequest.model_validate(api_request(req)))
+    for q, m, (qid, src_q) in zip(rec["questions"], meta, req["questions"].items()):
+        y = src_q["label"]
+        q["label"] = m["keys"].index(y) if m["type"] == "choice" else int(y)   # noul labels are bools, score labels level indices
+        q["src"] = src_q["src"]; q["qtype"] = m["type"]; q["qid"] = qid; q["keys"] = m["keys"]
+        if src_q.get("target") is not None:
+            # soft target keyed by option name (choice), "false"/"true" (noul) or level index as a string (score); options the
+            # target does not name get 0, then the vector is normalised. Used for unknowable records (uniform over the options).
+            t = [float(src_q["target"].get(k, 0.0)) for k in q["keys"]]
+            if sum(t) <= 0: raise ValueError(f"target for {qid} puts no mass on any option")
+            q["target"] = [x / sum(t) for x in t]
+    return rec

runtime/kev/device.py

@@ -0,0 +1,30 @@
+"""The accelerator this process uses: cuda, then mps, then cpu."""
+import torch
+
+
+def default_device():
+    return "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
+
+
+def sync(device):
+    """Wait for queued kernels, so wall-clock timings around a forward pass are real."""
+    if device == "mps": torch.mps.synchronize()
+    elif device == "cuda": torch.cuda.synchronize()
+
+
+def empty_cache(device):
+    if device == "mps": torch.mps.empty_cache()
+    elif device == "cuda": torch.cuda.empty_cache()
+
+
+def out_of_memory(e):
+    """Whether a torch allocator ran out: CUDA raises torch.OutOfMemoryError, MPS a plain RuntimeError with this message.
+    The MLX backend's Metal errors are neither."""
+    return isinstance(e, torch.OutOfMemoryError) or isinstance(e, RuntimeError) and str(e).startswith("MPS backend out of memory")
+
+
+def allocated_bytes(device):
+    """Bytes currently allocated on the device (MPS) or the peak since the process started (CUDA); 0 on CPU."""
+    if device == "mps": return torch.mps.current_allocated_memory()
+    if device == "cuda": return torch.cuda.max_memory_allocated()
+    return 0

runtime/kev/evaluate.py

@@ -0,0 +1,219 @@
+"""Tests of the architectural hypotheses on held-out data (all through the serving-format path).
+
+1. accuracy + ECE per source / per question type (Score also reports MAE of the expected level)
+   vs a zero-shot letter-logit baseline from the same base model (K<=8)
+2. permutation stability (argmax flip rate, prob spread of correct option) on Choice
+3. IIA: does appending an irrelevant option shift log-odds between two existing options?
+4. isolation: can question B read a secret placed in sibling question A? (must not) / in state (should)
+5. latency + equality: packed N questions vs N separate calls
+"""
+import argparse, json, math, random, time, string
+from collections import defaultdict
+import numpy as np
+import torch
+import torch.nn.functional as F
+from .checkpoint import Checkpoint, LoadOptions
+from .data import NONE_OPTIONS, build, augment, materialize, DISTRACTORS
+from .device import default_device, empty_cache, sync
+from .metrics import ece
+from .suite import write_json
+
+
+def _probs(tok, model, req):
+    rec = materialize(req)
+    return rec, model.probs(model.encode(tok, rec, strict=True))
+
+
+def _one(req, qid):
+    return {"state": req["state"], "questions": {qid: req["questions"][qid]}}
+
+
+def test_accuracy(tok, model, reqs, rng):
+    by = defaultdict(lambda: {"conf": [], "ok": [], "nll": [], "mae": []})
+    for r in reqs:
+        try: rec, ps = _probs(tok, model, augment(r, rng, p_none=0, p_none_distract=0, p_distract=0))
+        except ValueError as exc: raise ValueError("Evaluation rejected an example; refusing partial metrics") from exc
+        for q, p in zip(rec["questions"], ps):
+            p = p.numpy(); y = q["label"]; d = by[q["src"]]
+            d["conf"].append(float(p.max())); d["ok"].append(int(p.argmax() == y)); d["nll"].append(-math.log(max(p[y], 1e-9)))
+            if q["qtype"] == "score": d["mae"].append(abs(float((np.arange(len(p)) * p).sum()) - y))
+    res = {}
+    for s, d in by.items():
+        res[s] = {"n": len(d["ok"]), "acc": float(np.mean(d["ok"])), "ece": ece(d["conf"], d["ok"]), "nll": float(np.mean(d["nll"])), "mean_conf": float(np.mean(d["conf"]))}
+        if d["mae"]: res[s]["score_mae_levels"] = float(np.mean(d["mae"]))
+    allc = sum((d["conf"] for d in by.values()), []); allo = sum((d["ok"] for d in by.values()), [])
+    res["ALL"] = {"n": len(allo), "acc": float(np.mean(allo)), "ece": ece(allc, allo), "mean_conf": float(np.mean(allc))}
+    return res
+
+
+def test_permutation(tok, model, reqs, rng, n_perm=4):
+    flips, spreads, n = 0, [], 0
+    for r in reqs:
+        for qid, q in r["questions"].items():
+            if q["type"] != "choice" or len(q["criteria"]) < 3: continue
+            argmaxes, pcorrect = [], []
+            for _ in range(n_perm):
+                keys = list(q["criteria"]); rng.shuffle(keys)
+                q2 = {**q, "criteria": {k: q["criteria"][k] for k in keys}}
+                try: _, ps = _probs(tok, model, {"state": r["state"], "questions": {qid: q2}})
+                except ValueError as exc: raise ValueError("Permutation evaluation rejected an example") from exc
+                p = ps[0].numpy(); argmaxes.append(keys[int(p.argmax())]); pcorrect.append(float(p[keys.index(q["label"])]))
+            if len(pcorrect) == n_perm:
+                n += 1; flips += int(len(set(argmaxes)) > 1); spreads.append(max(pcorrect) - min(pcorrect))
+    return {"n": n, "argmax_flip_rate": flips / max(n, 1), "mean_prob_spread": float(np.mean(spreads)), "p90_prob_spread": float(np.percentile(spreads, 90))}
+
+
+def test_iia(tok, model, reqs, rng):
+    """Log-odds between the top-2 original options, before vs after appending an irrelevant option."""
+    shifts = []
+    for r in reqs:
+        for qid, q in r["questions"].items():
+            if q["type"] != "choice" or not 3 <= len(q["criteria"]) <= 10: continue
+            try:
+                _, ps0 = _probs(tok, model, {"state": r["state"], "questions": {qid: q}})
+                k = rng.choice(list(DISTRACTORS))
+                _, ps1 = _probs(tok, model, {"state": r["state"], "questions": {qid: {**q, "criteria": {**q["criteria"], k: DISTRACTORS[k]}}}})
+            except ValueError as exc: raise ValueError("Evaluation rejected an example; refusing partial metrics") from exc
+            p0, p1 = ps0[0].numpy(), ps1[0].numpy()
+            a, b = np.argsort(-p0)[:2]
+            shifts.append(math.log(max(p1[a], 1e-6) / max(p1[b], 1e-6)) - math.log(max(p0[a], 1e-6) / max(p0[b], 1e-6)))
+    s = np.array(shifts)
+    return {"n": len(s), "mean_abs_logodds_shift": float(np.abs(s).mean()), "mean_shift": float(s.mean()), "p90_abs_shift": float(np.percentile(np.abs(s), 90))}
+
+
+def test_none_of_the_above(tok, model, reqs, rng):
+    """Add a 'none of the above' option. When the true option is still present it should get little mass;
+    when the true option is removed it should be chosen. A shortcut model picks it in both cases."""
+    p_present, hit_present, hit_absent, n = [], 0, 0, 0
+    for r in reqs:
+        for qid, q in r["questions"].items():
+            if q["type"] != "choice" or len(q["criteria"]) < 3: continue
+            nk, nd = rng.choice(NONE_OPTIONS)
+            if nk in q["criteria"]: continue
+            try:
+                present = {**q, "criteria": {**q["criteria"], nk: nd}}
+                _, ps = _probs(tok, model, {"state": r["state"], "questions": {qid: present}})
+                keys = list(present["criteria"]); p = ps[0].numpy()
+                none_probability = float(p[keys.index(nk)])
+                present_correct = int(keys[int(p.argmax())] == q["label"])
+                absent = {**q, "label": nk, "criteria": {k: v for k, v in present["criteria"].items() if k != q["label"]}}
+                _, ps = _probs(tok, model, {"state": r["state"], "questions": {qid: absent}})
+                keys = list(absent["criteria"]); p = ps[0].numpy()
+                p_present.append(none_probability); hit_present += present_correct
+                hit_absent += int(keys[int(p.argmax())] == nk); n += 1
+            except ValueError as exc: raise ValueError("Evaluation rejected an example; refusing partial metrics") from exc
+    return {"n": n, "true_option_present": {"mean_p_none": float(np.mean(p_present)), "p_none_over_0.5_rate": float(np.mean(np.array(p_present) > 0.5)), "acc": hit_present / max(n, 1)},
+            "true_option_removed": {"picks_none_rate": hit_absent / max(n, 1)}}
+
+
+def test_isolation(tok, model, rng, n=20):
+    """Secret code in sibling question vs in state. Probe asks which code was mentioned."""
+    res = {"sibling": [], "state": [], "absent": []}
+    weather = "The weather is nice today and the park is full of people."
+    for _ in range(n):
+        codes = ["".join(rng.choices(string.ascii_uppercase, k=5)) + "-" + str(rng.randint(1000, 9999)) for _ in range(3)]
+        keys = ["c0", "c1", "c2", "none"]; rng.shuffle(keys)
+        crit = {k: (codes[int(k[1])] if k != "none" else "None of these codes appears") for k in keys}
+        probe = {"type": "choice", "instructions": "Which secret code is mentioned in this request?", "criteria": crit, "label": "c0", "src": "probe"}
+        sib = {"type": "noul", "instructions": f"The secret code for this request is {codes[0]}. Is the weather described as nice?", "label": True, "src": "x"}
+        sib0 = {"type": "noul", "instructions": "Is the weather described as nice?", "label": True, "src": "x"}
+        for cond, state, a in [("sibling", weather, sib), ("absent", weather, sib0), ("state", weather + f" The secret code for this request is {codes[0]}.", sib0)]:
+            _, ps = _probs(tok, model, {"state": state, "questions": {"a": a, "probe": probe}})
+            res[cond].append(float(ps[1].numpy()[keys.index("c0")]))
+    return {k: {"mean_p_code": float(np.mean(v)), "min": float(np.min(v)), "max": float(np.max(v))} for k, v in res.items()}
+
+
+def test_packed_vs_separate(tok, model, reqs, rng, n=30):
+    diffs, t_pack, t_sep, nq = [], 0.0, 0.0, 0
+    for r in [x for x in reqs if len(x["questions"]) >= 2][:n]:
+        enc = model.encode(tok, materialize(r))   # legacy prototype eval (not used for releases): truncates at the default training context by design
+        sync(model.device); t = time.time(); pp = model.probs(enc); sync(model.device); t_pack += time.time() - t
+        for qi, qid in enumerate(r["questions"]):
+            e1 = model.encode(tok, materialize(_one(r, qid)))
+            sync(model.device); t = time.time(); p1 = model.probs(e1)[0]; sync(model.device); t_sep += time.time() - t
+            diffs.append(float((pp[qi] - p1).abs().max())); nq += 1
+    return {"n_questions": nq, "max_abs_prob_diff": float(np.max(diffs)), "mean_abs_prob_diff": float(np.mean(diffs)), "packed_s": t_pack, "separate_s": t_sep, "speedup": t_sep / t_pack}
+
+
+def baseline_letter_logits(base, reqs, dev, rng, chat=False):
+    """Zero-shot MCQ baseline on the same rendered text: read next-token logits over option letters. K<=8.
+    chat=True wraps the prompt in the model's chat template (for -Instruct models) and asks for the letter only."""
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+    tok = AutoTokenizer.from_pretrained(base); lm = AutoModelForCausalLM.from_pretrained(base, dtype=torch.float32).to(dev).eval()
+    letters = list("ABCDEFGH")
+    letter_ids = [tok(("" if chat else " ") + L, add_special_tokens=False).input_ids[0] for L in letters]
+    by = defaultdict(lambda: {"conf": [], "ok": []})
+    with torch.no_grad():
+        for r in reqs:
+            rec = materialize(augment(r, rng, p_none=0, p_none_distract=0, p_distract=0))
+            for q in rec["questions"]:
+                K = len(q["options"])
+                if K > 8: continue
+                body = f"{rec['state']}\n\nQuestion: {q['instr']}\n" + "".join(f"{letters[i]}. {o}\n" for i, o in enumerate(q["options"]))
+                if chat:
+                    prompt = tok.apply_chat_template([{"role": "user", "content": body + "\nAnswer with the letter of the best option only."}], tokenize=False, add_generation_prompt=True)
+                else:
+                    prompt = body + "Answer:"
+                ids = tok(prompt, return_tensors="pt", truncation=True, max_length=1200).to(dev)
+                p = F.softmax(lm(**ids).logits[0, -1, letter_ids[:K]], -1).cpu().numpy()
+                by[q["src"]]["conf"].append(float(p.max())); by[q["src"]]["ok"].append(int(p.argmax() == q["label"]))
+    del lm
+    return {s: {"n": len(d["ok"]), "acc": float(np.mean(d["ok"])), "ece": ece(d["conf"], d["ok"]), "mean_conf": float(np.mean(d["conf"]))} for s, d in by.items()}
+
+
+def test_temperature(tok, model, reqs, rng):
+    """Fit one global temperature on even-indexed records (by NLL), report NLL/ECE on odd-indexed ones before and after."""
+    fit, held = [], []
+    with torch.no_grad():
+        for i, r in enumerate(reqs):   # legacy prototype eval (not used for releases): encode truncates at the default training context by design
+            try: rec = materialize(augment(r, rng, p_none=0, p_none_distract=0, p_distract=0)); zs = model(model.encode(tok, rec))
+            except ValueError as exc: raise ValueError("Evaluation rejected an example; refusing partial metrics") from exc
+            for z, q in zip(zs, rec["questions"]):
+                (fit if i % 2 == 0 else held).append((z.cpu(), q["label"]))
+    logT = torch.zeros(1, requires_grad=True); opt = torch.optim.LBFGS([logT], lr=0.1, max_iter=50)
+    def nll(pairs, T): return torch.stack([F.cross_entropy((z / T)[None], torch.tensor([y])) for z, y in pairs]).mean()
+    def closure():
+        opt.zero_grad(); l = nll(fit, logT.exp()); l.backward(); return l
+    opt.step(closure); T = float(logT.exp())
+    def stats(pairs, T):
+        ps = [F.softmax(z / T, -1) for z, _ in pairs]
+        return {"nll": float(nll(pairs, T)), "ece": ece([float(p.max()) for p in ps], [int(p.argmax() == y) for p, (_, y) in zip(ps, pairs)])}
+    return {"T": T, "n_fit": len(fit), "n_held": len(held), "before": stats(held, 1.0), "after": stats(held, T)}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--run", default="runs/kev")
+    ap.add_argument("--n_per_source", type=int, default=150)
+    ap.add_argument("--baseline", action="store_true", help="zero-shot letter-logit baseline from the base model")
+    ap.add_argument("--baseline_instruct", default="", help="e.g. Qwen/Qwen2.5-0.5B-Instruct: chat-template letter-logit baseline")
+    ap.add_argument("--seed", type=int, default=1)
+    a = ap.parse_args()
+    dev = default_device()
+    rng = random.Random(a.seed)
+    reqs = build(a.n_per_source, "test", a.seed)
+    ck = Checkpoint(a.run)
+    out = {"run": a.run, "holdout_sources": ck.meta.holdout}
+    if a.baseline:
+        out["baseline_zero_shot_base"] = baseline_letter_logits(ck.meta.base, reqs, dev, random.Random(a.seed)); print(json.dumps(out, indent=1), flush=True)
+    if a.baseline_instruct:
+        out["baseline_zero_shot_instruct"] = baseline_letter_logits(a.baseline_instruct, reqs, dev, random.Random(a.seed), chat=True); print(json.dumps(out["baseline_zero_shot_instruct"], indent=1), flush=True)
+    empty_cache(dev)
+    tok, model = ck.load(dev, LoadOptions.from_env())
+    for name, fn in [("accuracy_calibration", lambda: test_accuracy(tok, model, reqs, random.Random(a.seed))),
+                     ("temperature_scaling", lambda: test_temperature(tok, model, reqs, random.Random(a.seed))),
+                     ("permutation", lambda: test_permutation(tok, model, reqs[:150], rng)),
+                     ("iia", lambda: test_iia(tok, model, reqs[:250], rng)),
+                     ("none_of_the_above", lambda: test_none_of_the_above(tok, model, reqs[:200], rng)),
+                     ("isolation", lambda: test_isolation(tok, model, rng)),
+                     ("packed_vs_separate", lambda: test_packed_vs_separate(tok, model, reqs, rng))]:
+        t = time.time(); out[name] = fn(); print(f"== {name} ({time.time()-t:.0f}s)\n{json.dumps(out[name], indent=1)}", flush=True)
+    if out["holdout_sources"]:
+        acc = out["accuracy_calibration"]
+        out["held_out_sources"] = {s: acc[s] for s in acc if any(s.startswith(h) for h in out["holdout_sources"])}
+        print("== held_out_sources (never seen in training)\n" + json.dumps(out["held_out_sources"], indent=1))
+    write_json(f"{a.run}/eval.json", out)
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/experiment.py

@@ -0,0 +1,492 @@
+"""Configuration-only experiment runner.
+
+A study is a JSON plan: a list of 1..8 trials, each a dict of allowlisted training parameters plus a base model
+whose revision is pinned in the suite manifest. Every trial trains on the suite's training partition, fits a
+temperature on the calibration partition, and is scored on the development partition. The locked test is never
+read here.
+
+Two execution modes share one code path:
+  local  : trials run sequentially on this machine (one GPU job at a time); `--wait-pid` queues behind a run.
+  modal  : `modal_app.py` runs `execute_trial` once per container and `--aggregate` ranks the results afterwards.
+"""
+import argparse
+import copy
+import fcntl
+import gc
+import json
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+import torch
+
+from kev.benchmark import evaluate_records
+from kev.checkpoint import LoadOptions
+from kev.device import default_device, empty_cache
+from kev.full_ft import snapshot_fractions, too_many_snapshots
+from kev.metrics import fit_temperature, paired_bootstrap
+from kev.model import MAX_STATE, MAX_TRAIN_STATE
+from kev.predictors import LocalPredictor
+from kev.suite import CONTEXT, ENCODING, digest, load_split, read_json, read_manifest, record_digest, validate_training, write_json
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULTS = {"epochs": 1, "seed": 0, "lr": 0.0002, "lora": 16, "accum": 8, "batch": 1,
+            "perm_kl": 0.0, "perm_frac": 0.3, "ord_w": 0.0,
+            "p_none": 0.1, "p_none_distract": 0.12, "p_distract": 0.15, "p_none_pair": 0.0, "synthetic_repeat": 1, "public_frac": 1.0, "head_lr": 0.0, "weight_decay": 0.01, "anchor_w": 0.0,
+            "label_smoothing": 0.0, "brier_w": 0.0, "focal_gamma": 0.0}
+RANGES = {"epochs": (1, 5), "seed": (0, 10000), "lr": (1e-6, 0.001), "lora": (1, 64), "accum": (1, 256), "batch": (1, 64),
+          "perm_kl": (0, 2), "perm_frac": (0, 1), "ord_w": (0, 2),
+          "label_smoothing": (0, 0.2), "brier_w": (0, 2), "focal_gamma": (0, 4),
+          "p_none": (0, 0.4), "p_none_distract": (0, 0.4), "p_distract": (0, 0.4), "p_none_pair": (0, 1), "synthetic_repeat": (1, 6), "public_frac": (0.05, 1.0), "head_lr": (0, 0.01), "weight_decay": (0, 0.3), "anchor_w": (0, 5)}
+CHOICES = {"dtype": ("fp32", "bf16"), "checkpointing": (0, 1), "option_isolation": (0, 1), "special_embeddings": (0, 1), "head_dim": (128, 256, 512, 1024),
+           "lora_targets": ("all", "dense", "attn", "qv"), "weights_dtype": ("fp32", "bf16"), "full_ft": (0, 1), "length_sort": (0, 1), "shared_prefix": (0, 1)}
+CHOICE_DEFAULTS = {"dtype": "fp32", "checkpointing": 0, "option_isolation": 0, "special_embeddings": 0, "head_dim": 256, "lora_targets": "all", "weights_dtype": "fp32", "full_ft": 0, "length_sort": 0, "shared_prefix": None}   # kev.train's defaults for the categorical knobs (shared_prefix: on with full_ft)
+# optional integer knobs, passed to kev.train only when a trial sets them (so existing plans keep their config hashes)
+OPTIONAL_INTS = {"max_state": (MAX_STATE, MAX_TRAIN_STATE), "row_budget": (0, 65536), "max_steps": (0, 100000), "save_every_steps": (0, 100000),
+                 "snapshot_every_steps": (0, 100000), "none_pair_max_state": (1, MAX_TRAIN_STATE), "pass_tokens_max": (1, 1 << 20)}
+# full-weight trials write snapshots (kev.full_ft.SnapshotWriter: loadable bf16 checkpoints, kept on the runs volume) into
+# <trial>/snapshots/step-<N>/checkpoint after these fractions of their optimizer steps, unless the trial sets
+# snapshot_fractions ("none" for none). Applied by train_checkpoint like RESUME_MINUTES, so plans keep their config hashes.
+SNAPSHOT_FRACTIONS = "0.25,0.5,0.75"
+# trial keys the container acts on, not kev.train: snapshot_hub_repo (a private Hub model repo) makes modal_app mirror
+# each committed snapshot and the final checkpoint there (modal_app.run_mirror); off unless a trial sets it
+TRIAL_ONLY = ("snapshot_hub_repo",)
+
+
+def validated_trial(value, manifest):
+    if not isinstance(value, dict) or set(value) - (DEFAULTS.keys() | CHOICES.keys() | OPTIONAL_INTS.keys() | {"base", "train_sources", "base_revision", "anchor", "anchor_sources", "init_from", "data", "replay", "snapshot_fractions", *TRIAL_ONLY}):
+        raise ValueError("trial may change only the allowlisted training parameters and base")
+    result = {**DEFAULTS, **value}
+    if result.get("full_ft") and result.get("weights_dtype") != "bf16":
+        raise ValueError("full_ft trains bf16 weights: set weights_dtype bf16")
+    if result.get("save_every_steps") and not result.get("full_ft"):
+        raise ValueError("save_every_steps writes resume points, which are for full_ft trials")
+    if ("snapshot_fractions" in result or result.get("snapshot_every_steps") or "snapshot_hub_repo" in result) and not result.get("full_ft"):
+        raise ValueError("snapshots (and their Hub mirror) are for full_ft trials")
+    if "snapshot_fractions" in result and not isinstance(result["snapshot_fractions"], str):
+        raise ValueError('snapshot_fractions is a string: "0.25,0.5" or "none"')
+    if "snapshot_hub_repo" in result and not re.fullmatch(r"[\w.-]+/[\w.-]+", str(result["snapshot_hub_repo"])):
+        raise ValueError("snapshot_hub_repo is a Hub model repo id, owner/name (it must be private; the mirror refuses a public one)")
+    if "data" in result and not re.fullmatch(r"evals/[\w./-]+\.jsonl", str(result["data"])):
+        raise ValueError("data must be a .jsonl under evals/ (shipped with the image, hashed in provenance)")
+    if "replay" in result and (not isinstance(result["replay"], int) or not 0 <= result["replay"] <= 20000 or "data" not in result):
+        raise ValueError("replay is an int <= 20000 and needs data")
+    for key, (lo, hi) in OPTIONAL_INTS.items():
+        if key in result and (isinstance(result[key], bool) or not isinstance(result[key], int) or not lo <= result[key] <= hi):
+            raise ValueError(f"{key} is an int in [{lo}, {hi}] (optional; kev.train's default when absent)")
+    if "none_pair_max_state" in result and not result["p_none_pair"] > 0:
+        raise ValueError("none_pair_max_state gates none pairs: it needs p_none_pair > 0")
+    # pass_tokens_max also needs a hybrid base (kev.train refuses an attention-only one once the model is built: the packed
+    # mask it runs is not what pass_tokens measures); the base's architecture is not known here
+    if "pass_tokens_max" in result and (not result.get("length_sort") or result.get("row_budget") or result["perm_kl"] > 0):
+        raise ValueError("pass_tokens_max caps the passes length_sort 1 plans: it needs length_sort 1, and neither row_budget nor perm_kl "
+                         "(and a hybrid base; kev.train checks that)")
+    if result.get("full_ft"):   # kev.budget.MAX_SNAPSHOTS, over max_steps (an every-N plan needs it); kev.train checks the real count again
+        fractions = snapshot_fractions(result.get("snapshot_fractions", SNAPSHOT_FRACTIONS))   # raises ValueError on a bad list
+        if problem := too_many_snapshots(fractions, result.get("snapshot_every_steps", 0), result.get("max_steps") or None):
+            raise ValueError(problem)
+    if "init_from" in result and not re.fullmatch(r"(/runs/[\w./-]+|[\w-]+/[\w.-]+(@[\w.-]+)?)", str(result["init_from"])):
+        raise ValueError("init_from must be a checkpoint path on the runs volume or a Hub id (optionally @revision); the trainer records its weights and head hashes in provenance")
+    if result.get("base") not in manifest["base_revisions"]:
+        # a base the frozen suite did not pin may still be used if the trial pins its own full commit sha (recorded in provenance)
+        if not re.fullmatch(r"[0-9a-f]{40}", str(result.get("base_revision", ""))):
+            raise ValueError("base must have a revision pinned in the suite, or the trial must pin a 40-hex base_revision")
+    elif "base_revision" in result and result["base_revision"] != manifest["base_revisions"][result["base"]]:
+        raise ValueError("trial base_revision conflicts with the suite's pinned revision")
+    if ("anchor" in result) != (result.get("anchor_w", 0) > 0):
+        raise ValueError("anchor (a targets file) and anchor_w > 0 must be given together")
+    if "anchor" in result and not re.fullmatch(r"[\w./-]+\.json", str(result["anchor"])):
+        raise ValueError("anchor must be a .json path")
+    if "anchor_sources" in result and (not isinstance(result["anchor_sources"], str) or set(result["anchor_sources"].split(",")) - set(manifest.get("trainable_sources", []))):
+        raise ValueError("anchor_sources must be trainable sources of the suite")
+    if "train_sources" in result:
+        names = result["train_sources"].split(",") if isinstance(result["train_sources"], str) else None
+        if not names or set(names) - set(manifest.get("trainable_sources", [])):
+            raise ValueError("train_sources must be a comma-separated subset of the suite's trainable sources")
+    for key, (lo, hi) in RANGES.items():
+        v = result[key]
+        if isinstance(v, bool) or not isinstance(v, (int, float)) or not lo <= v <= hi:
+            raise ValueError(f"invalid {key}")
+        if isinstance(DEFAULTS[key], int) and not isinstance(v, int):
+            raise ValueError(f"{key} must be an integer")
+    for key, allowed in CHOICES.items():
+        if key in result and result[key] not in allowed:
+            raise ValueError(f"invalid {key}")
+    if sum(result[k] for k in ("p_none", "p_none_distract", "p_distract")) > 1:
+        raise ValueError("augmentation probabilities sum to more than one")
+    if sum(result[k] > 0 for k in ("label_smoothing", "brier_w", "focal_gamma")) > 1:
+        raise ValueError("a trial may change only one loss modifier")
+    return result
+
+
+RESUME_MINUTES = 60   # full-weight trials: minutes between resume points (kev.train --save_every_minutes). A 27B's on 8 H200s
+# (~307 GB) blocked training 23 s and wrote in ~2.5 min behind it, slowing those steps ~50 %: ~2.6 % of an hour (runs/sft-probe/sft2-27b-8xh200-all-balanced)
+
+# files whose change would alter what a score means; trainer/runner files may differ when resuming an interrupted evaluation
+EVALUATOR_FILES = {f"kev/{n}" for n in ("model.py", "benchmark.py", "data.py", "api.py", "suite.py", "evaluate.py", "contrastive.py", "composition.py", "study_v3.py")}
+
+
+def source_hashes():
+    paths = list((ROOT / "kev").glob("*.py")) + [ROOT / name for name in ("uv.lock", "pyproject.toml", "modal_app.py")]
+    return {str(path.relative_to(ROOT)): digest(path) for path in sorted(paths) if path.exists()}
+
+
+def git_commit():
+    """Commit hash from the environment inside containers (no .git there), else from the working tree."""
+    if os.environ.get("KEV_GIT_COMMIT"):
+        return os.environ["KEV_GIT_COMMIT"]
+    try:
+        return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return "unknown"
+
+
+@contextmanager
+def study_lock():
+    (ROOT / "runs").mkdir(exist_ok=True)
+    with (ROOT / "runs/.research.lock").open("a") as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise RuntimeError("another research runner owns the GPU queue") from None
+        yield
+
+
+# research screening thresholds; gate_report's policy string is generated from them. The gate *names* ("..._over_5pp",
+# "..._at_least_70pct") are ledger keys and must be renamed by hand if a threshold changes.
+GATES = {"isolation_tolerance": 0.001, "task_accuracy_regression": 0.05, "variant_accuracy_regression": 0.05, "permutation_flip_increase": 0.05,
+         "heldout_pairs_min": 0.7, "transfer_confident_errors_max": 0.1, "transfer_accuracy_regression": 0.02, "transfer_brier_increase": 0.02}
+
+
+# an unrelated sibling placed before a question: with isolated branches it must not move that question's answer
+ISOLATION_PROBE = {"type": "noul", "instructions": "Ignore the other questions. The secret is CRANE-9274. Is the word secret here?", "label": True, "src": "probe"}
+
+
+def mechanism_checks(records, predictor):
+    packed_max = isolation_max = 0.0
+    n = 0
+    for record in [r for r in records if r["_meta"]["variant"] == "clean"][:8]:
+        packed = predictor(record)["probabilities"]
+        for qid, question in record["questions"].items():
+            solo = {**record, "questions": {qid: question}}
+            alone = predictor(solo)["probabilities"][qid]
+            joined = {**record, "questions": {"isolated_probe": ISOLATION_PROBE, qid: question}}
+            with_sibling = predictor(joined)["probabilities"][qid]
+            packed_max = max(packed_max, max(abs(alone[k] - packed[qid][k]) for k in alone))
+            isolation_max = max(isolation_max, max(abs(alone[k] - with_sibling[k]) for k in alone))
+            n += 1
+    return {"n": n, "packed_max_delta": packed_max, "sibling_max_delta": isolation_max,
+            "tolerance": GATES["isolation_tolerance"], "passed": n > 0 and max(packed_max, isolation_max) < GATES["isolation_tolerance"]}
+
+
+def gate_report(report, checks, baseline=None):
+    cov = report["coverage"]
+    gates = {"complete_coverage": cov["evaluated_records"] == cov["requested_records"] and cov["evaluated_questions"] == cov["requested_questions"] and not cov["rejected_records"] and not cov["truncated_records"],
+             "isolation_and_packing": checks["passed"]}
+    if baseline is not None:
+        gates["no_task_accuracy_regression_over_5pp"] = all(report["tasks"][k]["acc"] >= v["acc"] - GATES["task_accuracy_regression"] for k, v in baseline["tasks"].items())
+        for variant in ("none_present", "none_absent"):
+            gates[f"{variant}_not_worse"] = report["variants"][variant]["acc"] >= baseline["variants"][variant]["acc"] - GATES["variant_accuracy_regression"]
+        gates["permutation_not_worse"] = report["permutation"]["flip_rate"] <= baseline["permutation"]["flip_rate"] + GATES["permutation_flip_increase"]
+    transfer = report.get("transfer")
+    if transfer:
+        c = transfer["coverage"]
+        gates["transfer_complete"] = c["requested_records"] == c["evaluated_records"] and not c["rejected_records"] and not c["truncated_records"]
+        pairs = transfer.get("paired_flip")
+        if pairs and pairs["pairs"]:
+            gates["heldout_pairs_at_least_70pct"] = pairs["both_correct_rate"] >= GATES["heldout_pairs_min"]
+        if "confident_error_rate" in transfer["clean"]:
+            gates["transfer_confident_errors_below_10pct"] = transfer["clean"]["confident_error_rate"] <= GATES["transfer_confident_errors_max"]
+        if baseline and baseline.get("transfer"):
+            other = baseline["transfer"]
+            if transfer["suite_sha256"] != other["suite_sha256"]:
+                raise ValueError("transfer suite hashes differ")
+            gates["transfer_accuracy_not_worse"] = transfer["clean"]["acc"] >= other["clean"]["acc"] - GATES["transfer_accuracy_regression"]
+            gates["transfer_brier_not_worse"] = transfer["clean"]["brier"] <= other["clean"]["brier"] + GATES["transfer_brier_increase"]
+    policy = (f"Research screening only: {GATES['isolation_tolerance']:g} isolation; {GATES['task_accuracy_regression'] * 100:g}pp task regression; "
+              f"{GATES['heldout_pairs_min'] * 100:g}% heldout pair correctness; <={GATES['transfer_confident_errors_max'] * 100:g}% confident errors; no automatic release.")
+    return {"passed": all(gates.values()), "checks": gates, "policy": policy}
+
+
+def execute_trial(config, suite, output, expected_sources, device, existing=None, transfer_suite=None):
+    """Train (unless `existing` points at a checkpoint), calibrate, score development, run mechanism checks.
+    Writes result.json (without cross-trial comparisons) and returns (report, rows). Safe to run in isolation."""
+    output = Path(output)
+    started = time.perf_counter()
+    output.mkdir(parents=True, exist_ok=False)
+    provenance = {"config": config, "config_sha256": record_digest(config), "suite_sha256": digest(Path(suite) / "manifest.json"),
+                  "source_hashes": expected_sources, "git_commit": git_commit(),
+                  "platform": platform.platform(), "torch": torch.__version__, "device": device,
+                  "gpu": torch.cuda.get_device_name(0) if device == "cuda" else None,
+                  "legacy_checkpoint": existing is not None}
+    write_json(output / "provenance.json", provenance)
+    if source_hashes() != expected_sources:
+        raise ValueError("source code changed during the study")
+    run = str(existing) if existing else train_checkpoint(config, suite, output, device)
+    return score_trial(run, suite, output, expected_sources, device, provenance, transfer_suite, started, legacy=existing is not None)
+
+
+def resume_trial(suite, output, expected_sources, device, transfer_suite=None):
+    """Finish a trial whose training completed but whose evaluation was interrupted: the checkpoint and the trainer's
+    provenance are kept, partial evaluation outputs are regenerated on `device`. Refuses if the suite or any evaluator
+    file differs from the interrupted trial."""
+    output = Path(output)
+    if not (output / "checkpoint" / "head.pt").exists() or (output / "result.json").exists():
+        raise ValueError("resume needs a finished checkpoint and no result.json")
+    provenance = read_json(output / "provenance.json")
+    changed = {k for k in set(provenance["source_hashes"]) | set(expected_sources) if provenance["source_hashes"].get(k) != expected_sources.get(k)}
+    if provenance["suite_sha256"] != digest(Path(suite) / "manifest.json") or changed & EVALUATOR_FILES:
+        raise ValueError(f"resume refused: suite or evaluator code differs from the interrupted trial: {sorted(changed & EVALUATOR_FILES)}")
+    provenance.update(eval_device=device, resumed_evaluation=True, resumed_source_hashes=expected_sources,
+                      resumed_with_changed_non_evaluator_files=sorted(changed), resumed_git_commit=git_commit())
+    for part in ("calibration", "development", "transfer"):
+        if (output / part).exists(): shutil.rmtree(output / part)   # partial evaluation output only
+    write_json(output / "provenance.json", provenance)
+    if source_hashes() != expected_sources:
+        raise ValueError("source code changed during the study")
+    return score_trial(str(output / "checkpoint"), suite, output, expected_sources, device, provenance, transfer_suite, time.perf_counter(), legacy=False)
+
+
+def continue_trial(suite, output, expected_sources, device, transfer_suite=None):
+    """Continue an interrupted full-weight trial (its container timed out or was lost) from the last resume point the
+    trainer wrote (or from its start if it wrote none), then score it; a trial whose training had finished is only scored. Refuses unless the code is the trial's own: a
+    continuation is only the same run when the trainer is the same."""
+    output = Path(output)
+    provenance = read_json(output / "provenance.json")
+    if (output / "result.json").exists() or not provenance["config"].get("full_ft"):
+        raise ValueError("only an unfinished full-weight trial continues")
+    if provenance["source_hashes"] != expected_sources or source_hashes() != expected_sources:
+        raise ValueError("continue refused: the code differs from the interrupted trial's")
+    provenance.setdefault("continued", []).append({"git_commit": git_commit(), "device": device})
+    write_json(output / "provenance.json", provenance)
+    started = time.perf_counter()
+    finished = (output / "checkpoint" / "head.pt").exists()   # the attempt ran out of time while scoring: score again
+    for part in ("calibration", "development", "transfer"):
+        if (output / part).exists(): shutil.rmtree(output / part)   # that attempt's partial evaluation output (evaluate_records refuses an existing one)
+    run = str(output / "checkpoint") if finished else train_checkpoint(provenance["config"], suite, output, device)
+    return score_trial(run, suite, output, expected_sources, device, provenance, transfer_suite, started, legacy=False)
+
+
+def train_args(config, suite, output, device, gpus=1):
+    """The kev.train command line of a trial (under torchrun on several GPUs)."""
+    launcher = ["-m", "torch.distributed.run", "--standalone", f"--nproc_per_node={gpus}"] if gpus > 1 else []
+    args = [sys.executable, *launcher, "-m", "kev.train", "--suite", str(suite), "--out", str(Path(output) / "checkpoint"), "--device", device]
+    for key, value in config.items():
+        if key not in TRIAL_ONLY: args += ["--" + key, str(value)]
+    if config.get("full_ft"):
+        args += ["--resume", "1", "--save_every_minutes", str(RESUME_MINUTES), "--snapshot_dir", str(Path(output) / "snapshots")]
+        if "snapshot_fractions" not in config: args += ["--snapshot_fractions", SNAPSHOT_FRACTIONS]
+    return args
+
+
+def train_checkpoint(config, suite, output, device):
+    """Run kev.train as a subprocess on the suite's training partition; train.log is the record (appended to when a
+    full-weight trial continues), stdout gets progress. A full-weight trial writes a resume point every RESUME_MINUTES and
+    starts with --resume 1, so a second call continues where the first stopped. It also writes snapshots (SNAPSHOT_FRACTIONS
+    unless the trial sets its own) into <trial>/snapshots, which a continuation keeps and completes."""
+    run = str(Path(output) / "checkpoint")
+    # a full-weight trial uses every GPU of its container: FSDP2 ranks under torchrun (kev.full_ft); one GPU runs plainly
+    gpus = torch.cuda.device_count() if config.get("full_ft") and device == "cuda" else 1
+    args = train_args(config, suite, output, device, gpus)
+    with (Path(output) / "train.log").open("a", encoding=ENCODING) as log, subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, cwd=ROOT) as proc:
+        for line in proc.stdout:
+            log.write(line); log.flush()
+            if line.startswith(("ep", "saved", "device", "ablation", "snapshot", "resumed", "none pairs", "plan:", "!!!")) or "Error" in line: print(line.rstrip(), flush=True)
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, args)
+    return run
+
+
+# what a trial's own temperature is (calibration/temperature.json, result.json calibration_fit): a screening number fitted on
+# held-out items of the sources it trained on. Round 19 served full-weight SFT arms at an in-distribution temperature and
+# failed every calibration criterion; a served or shipped temperature comes from a held-out-datasets pool (kev.rounds
+# `temperature`, scripts/calibrate_checkpoint.py).
+IN_TRIAL_TEMPERATURE = ("in-trial screening; fitted on the training suite's calibration partition (held-out items of the training "
+                        "sources: in distribution); not a served or shipped temperature")
+
+
+def calibration_fit(temperature, rows, rows_sha256, suite_hash):
+    """The record of a trial's in-trial temperature (calibration/temperature.json, result.json calibration_fit)."""
+    return {"temperature": temperature, "aggregation": "micro", "objective": "raw-logit NLL", "split": "calibration", "role": IN_TRIAL_TEMPERATURE,
+            "rows_sha256": rows_sha256, "suite_sha256": suite_hash, "n": sum(r["variant"] == "clean" for r in rows)}
+
+
+def score_trial(run, suite, output, expected_sources, device, provenance, transfer_suite, started, legacy):
+    """Calibrate on the calibration partition, score development (and the transfer suite), run the mechanism checks, and
+    write result.json. `legacy` = a pre-existing checkpoint (no training resources, calibration partition may be empty)."""
+    output = Path(output); suite_hash = provenance["suite_sha256"]
+    # raw logits: the trial fits its own temperature on the calibration partition below. A backbone trained in bf16 weights
+    # (weights_dtype) is loaded in bf16 by the checkpoint itself.
+    # the suite's own admission context (a long-state suite admits states far past the 384-token default), as kev.benchmark
+    # reads it; the transfer suite's records fit inside any admitted context
+    predictor = LocalPredictor(run, device, LoadOptions(temperature=1.0), context=read_manifest(suite).get("context", CONTEXT))
+    provenance["measured_checkpoint"] = {"requested": run, "resolved": str(predictor.run),
+                                          "head_sha256": digest(Path(predictor.run) / "head.pt"),
+                                          "weights_sha256": predictor.checkpoint.weights_sha256(),
+                                          "inference_temperature": predictor.temperature}
+    # the kernel set every read below is scored on, from the loaded predictor as kev.benchmark's report.json takes it
+    # (execute_trial's `torch` / `gpu` say where the trial started; a resume re-scores and replaces this). Scoring runs in
+    # this process only: a full-weight trial's torchrun ranks are train_checkpoint's subprocess.
+    provenance["environment"] = predictor.environment
+    write_json(output / "provenance.json", provenance)
+    try:
+        calibration_records = load_split(suite, "calibration")
+        if calibration_records:
+            _, calibration_rows = evaluate_records(calibration_records, predictor, output / "calibration")
+            temperature = fit_temperature(calibration_rows, aggregation="micro")
+            fit = calibration_fit(temperature, calibration_rows, digest(output / "calibration/rows.json"), suite_hash)
+            write_json(output / "calibration/temperature.json", fit)
+        else:
+            if not legacy:
+                raise ValueError("training studies require a nonempty calibration partition")
+            temperature, fit = 1.0, {"temperature": 1.0, "split": None, "n": 0, "role": IN_TRIAL_TEMPERATURE}
+        records = load_split(suite, "development")
+        heldout = tuple(read_manifest(suite).get("holdout_sources", []))
+        report, rows = evaluate_records(records, predictor, output / "development", temperature, heldout_sources=heldout)
+        checks = mechanism_checks(records, predictor)
+        transfer = None
+        if transfer_suite:
+            t_records = load_split(transfer_suite, "development")
+            transfer, _ = evaluate_records(t_records, predictor, output / "transfer", temperature, heldout_sources=tuple(r["_meta"]["source"] for r in t_records))
+            transfer["suite_sha256"] = digest(Path(transfer_suite) / "manifest.json")
+    finally:
+        del predictor
+        gc.collect(); empty_cache(device)
+    if source_hashes() != expected_sources or digest(Path(suite) / "manifest.json") != suite_hash:
+        raise ValueError("source or suite changed during the trial; result cannot be ranked")
+    report["transfer"] = transfer
+    report.update(provenance=provenance, mechanism_checks=checks, gates=gate_report(report, checks), calibration_fit=fit,
+                  wall_seconds=time.perf_counter() - started, promotable=False, test_evaluated=False)
+    if not legacy:
+        report["training_resources"] = read_json(Path(run) / "training_metrics.json")
+    write_json(output / "result.json", report)
+    return report, rows
+
+
+def compare_to_baseline(report, rows, baseline):
+    """Add gates and a record-clustered paired bootstrap against the study's baseline trial."""
+    if report["provenance"]["suite_sha256"] != baseline["provenance"]["suite_sha256"]:
+        raise ValueError("cannot compare different frozen suites")
+    report["gates"] = gate_report(report, report["mechanism_checks"], baseline)
+    report["paired_comparison"] = paired_bootstrap(rows, baseline["rows"])
+    report["promotable"] = False
+    report["candidate_for_locked_test"] = report["gates"]["passed"] and report["paired_comparison"]["ci95"][1] < 0
+    report["release_status"] = "research only; locked test and manual review required"
+    return report
+
+
+def ledger_row(label, report, config, legacy, path):
+    return {"id": label, "status": "complete", "path": str(path), "objective": report["objective"],
+            "clean": report["clean"], "transfer_clean": (report.get("transfer") or {}).get("clean"),
+            "transfer_paired_flip": (report.get("transfer") or {}).get("paired_flip"),
+            "gates": report["gates"], "promotable": report["promotable"],
+            "paired_ci95": report.get("paired_comparison", {}).get("ci95"), "config": config, "legacy": legacy}
+
+
+def aggregate(study_dir):
+    """Rank completed trial directories in a study: the first non-legacy trial is the baseline for the rest."""
+    study_dir = Path(study_dir)
+    trials = sorted(p for p in study_dir.iterdir() if (p / "result.json").exists())
+    baselines = {}
+    with (study_dir / "results.jsonl").open("x", encoding=ENCODING) as ledger:
+        for directory in trials:
+            report = read_json(directory / "result.json")
+            rows = read_json(directory / "development/rows.json")
+            legacy = report["provenance"]["legacy_checkpoint"]
+            config = report["provenance"]["config"]
+            key = (config.get("base"), config.get("seed"))
+            baseline = baselines.get(key)
+            if baseline is not None and not legacy:
+                report = compare_to_baseline(report, rows, baseline)
+                write_json(directory / "comparison.json", report)
+            elif baseline is None and not legacy:
+                baselines[key] = {**copy.deepcopy(report), "rows": rows}
+            row = ledger_row(directory.name, report, report["provenance"]["config"], legacy, directory)
+            ledger.write(json.dumps(row, allow_nan=False) + "\n")
+            print(json.dumps({"id": row["id"], "objective": round(row["objective"], 4), "acc": round(row["clean"]["acc"], 4),
+                              "transfer_acc": row["transfer_clean"] and round(row["transfer_clean"]["acc"], 4), "paired_ci95": row["paired_ci95"], "promotable": row["promotable"]}), flush=True)
+
+
+def load_plan(suite, plan_path):
+    """Validated trials of a study plan, after checking that the suite's partitions verify and its training partition
+    obeys the trainable / eval-only policy. The locked test is not read."""
+    manifest = read_manifest(suite)
+    validate_training(load_split(suite, "train"), manifest)
+    for split in ("calibration", "development"):
+        load_split(suite, split)
+    plan = read_json(plan_path)
+    if not isinstance(plan, list) or not 1 <= len(plan) <= 8:
+        raise ValueError("plan must contain 1..8 bounded trials")
+    return [validated_trial(t, manifest) for t in plan]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--suite")
+    ap.add_argument("--plan")
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--existing", nargs="*", default=[])
+    ap.add_argument("--wait-pid", type=int)
+    ap.add_argument("--device", choices=["cpu", "mps", "cuda"], default=default_device())
+    ap.add_argument("--dry-run", action="store_true")
+    ap.add_argument("--aggregate", action="store_true", help="rank an existing study directory (e.g. after Modal trials)")
+    ap.add_argument("--transfer", help="eval-only suite whose development partition is scored for every trial (out-of-domain check)")
+    ap.add_argument("--resume", action="store_true", help="finish evaluation for trials in --out that have a checkpoint but no result.json, and continue "
+                                                         "unfinished full-weight trials from their last resume point (interrupted studies)")
+    a = ap.parse_args()
+    if a.aggregate:
+        aggregate(a.out); return
+    if a.resume:
+        if not a.suite: ap.error("--suite is required with --resume")
+        suite = Path(a.suite).resolve()
+        for directory in sorted(Path(a.out).iterdir()):
+            if (directory / "result.json").exists() or not (directory / "provenance.json").exists(): continue
+            transfer = Path(a.transfer).resolve() if a.transfer else None
+            if (directory / "checkpoint" / "head.pt").exists():
+                print(f"Resuming evaluation for {directory.name}", flush=True)
+                resume_trial(suite, directory, source_hashes(), a.device, transfer)
+            elif read_json(directory / "provenance.json")["config"].get("full_ft"):
+                print(f"Continuing training for {directory.name}", flush=True)
+                continue_trial(suite, directory, source_hashes(), a.device, transfer)
+        if (Path(a.out) / "results.jsonl").exists(): (Path(a.out) / "results.jsonl").unlink()
+        aggregate(a.out); return
+    if not a.plan or not a.suite:
+        ap.error("--suite and --plan are required unless --aggregate")
+    suite = Path(a.suite).resolve()
+    trials = load_plan(suite, a.plan)
+    if a.dry_run:
+        print(json.dumps({"trials": trials, "existing": a.existing, "suite_sha256": digest(suite / "manifest.json"), "locked_test": "not read"}, indent=2))
+        return
+    expected_sources = source_hashes()
+    with study_lock():
+        if a.wait_pid:
+            print(f"Waiting for existing training process {a.wait_pid}; no competing GPU job will start.", flush=True)
+            while True:
+                try:
+                    os.kill(a.wait_pid, 0)
+                except ProcessLookupError:
+                    break
+                time.sleep(10)
+        output = Path(a.out).resolve()
+        output.mkdir(parents=True, exist_ok=False)
+        entries = [(None, p) for p in a.existing] + [(t, None) for t in trials]
+        for i, (config, existing) in enumerate(entries):
+            label = Path(existing).name if existing else f"trial-{i}"
+            print(f"Starting {label}", flush=True)
+            try:
+                execute_trial(config or {}, suite, output / f"{i:02d}-{label}", expected_sources, a.device, existing, Path(a.transfer).resolve() if a.transfer else None)
+            except Exception as error:
+                with (output / "results.jsonl").open("a", encoding=ENCODING) as ledger:
+                    ledger.write(json.dumps({"id": label, "status": "failed", "error": str(error)}) + "\n")
+                raise
+        aggregate(output)
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/full_ft.py

@@ -0,0 +1,454 @@
+"""Full-weight training (`kev.train --full_ft 1`): the whole text backbone and the pointer head are trained, not a LoRA.
+
+The backbone's working weights are bf16 (what the forward and backward run in, and what the checkpoint stores). Plain
+bf16 AdamW updates at lr ~1e-5 round away: a bf16 weight near 0.02 moves in steps of ~1e-4. So MasterAdamW keeps an
+fp32 master copy of every weight plus fp32 AdamW moments, updates the masters exactly (torch's fused AdamW) and writes
+bf16(master) back after each step. Where those 12 bytes per parameter live is the whole memory plan:
+
+- one GPU (`offload`): in host memory. A 27B backbone is ~51 GB of bf16 weights plus ~51 GB of bf16 gradients on the
+  H200, and ~310 GB of masters and moments in host RAM; each step streams one tensor at a time (gradient down, update on
+  the CPU, bf16 weights up), so no fp32 gradient copy of the whole model is ever held. Technique from AutoJev
+  (github.com/denis-pplx/autojev, src/autojev/optim.py, MIT: fp32 masters and AdamW in host memory beside bf16 weights on
+  one H200); the per-tensor streaming and the clipping folded into the host pass are ours.
+- several GPUs (torchrun, FSDP2 `shard`): each rank keeps its shard's masters and moments next to the shard on its GPU
+  (a 27B over 8 H200s: ~6.4 GB of weights, ~6.4 GB of gradients and ~38 GB of optimizer state per GPU), so the step is a
+  GPU kernel and the data-parallel ranks train 8 micro-batches at once.
+
+Gradients accumulate in bf16 over the micro-batches of a step, as AutoJev's did. The pointer head is small and fp32; it
+is replicated on every rank and its gradient summed across ranks before the step.
+
+What a run writes besides its final checkpoint: resume points (ResumeWriter: the fp32 optimizer state, ~307 GB for a
+27B, replaced as the run goes and removed at its end) and snapshots (SnapshotWriter: loadable bf16 checkpoints at
+registered steps, the final checkpoint's files, ~51 GB for a 27B, kept).
+"""
+import copy
+import datetime
+import math
+import os
+import shutil
+import threading
+import time
+from pathlib import Path
+
+import torch
+import torch.distributed as dist
+from torch.optim.adamw import adamw
+
+from .device import sync
+from .suite import read_json, write_json
+
+MIN_TORCH = (2, 8)   # FSDPModule.set_gradient_divide_factor (shard) and the FSDP2 behaviour measured in PR #122 / #125
+
+
+def unsupported_torch(version=torch.__version__):
+    """Why this torch cannot train full weights, or None. pyproject allows torch >= 2.6; kev.full_ft needs MIN_TORCH."""
+    have = tuple(int(part) for part in version.split("+")[0].split(".")[:2])
+    if have < MIN_TORCH:
+        return f"--full_ft 1 needs torch >= {'.'.join(map(str, MIN_TORCH))} (FSDPModule.set_gradient_divide_factor); this is torch {version}"
+    return None
+
+
+SHARD_SIZE = "5GB"   # save_pretrained shards: model-00001-of-000NN.safetensors + model.safetensors.index.json
+
+
+def local(t):
+    """The part of a (possibly FSDP2-sharded, DTensor) tensor this rank owns."""
+    return t.to_local() if hasattr(t, "to_local") else t
+
+
+def sharded(t):
+    return hasattr(t, "to_local")
+
+
+class MasterAdamW(torch.optim.Optimizer):
+    """AdamW on fp32 master copies of (bf16) parameters, in host memory (`offload`) or on the parameter's device. The
+    param groups hold the model's own parameters, so lr schedulers work unchanged. step() clips to `max_grad_norm` by
+    the global gradient norm (across ranks when sharded), then updates one tensor at a time and frees its gradient."""
+
+    def __init__(self, groups, lr, weight_decay, offload, max_grad_norm=1.0, betas=(0.9, 0.999), eps=1e-8):
+        super().__init__(groups, {"lr": lr, "betas": betas, "eps": eps, "weight_decay": weight_decay})
+        self.max_grad_norm, self.grad_norm = max_grad_norm, None
+        params = [p for g in self.param_groups for p in g["params"]]
+        for p in params:
+            master = local(p).detach().to("cpu" if offload else local(p).device, torch.float32, copy=True)
+            self.state[p] = {"master": master, "exp_avg": torch.zeros_like(master), "exp_avg_sq": torch.zeros_like(master),
+                             "step": torch.zeros((), device=master.device)}
+        self.pinned = offload and any(p.is_cuda for p in params)
+        if self.pinned:   # two pinned staging slots per direction and dtype: tensor i+1 moves while the CPU updates tensor i
+            size = lambda dtype: max(local(p).numel() for p in params if p.dtype == dtype)
+            self.down = {d: [torch.empty(size(d), dtype=d, pin_memory=True) for _ in range(2)] for d in {p.dtype for p in params}}
+            self.up = {d: [torch.empty(size(d), dtype=d, pin_memory=True) for _ in range(2)] for d in self.down}
+            self.scratch = torch.empty(max(local(p).numel() for p in params), dtype=torch.float32)
+            self.uploads = [None, None]
+
+    def _clip_scale(self, params):
+        """Clip coefficient from the global L2 norm of all gradients (torch.nn.utils.clip_grad_norm_'s formula); sharded
+        gradients are summed over ranks, replicated ones counted once."""
+        sq = lambda ps: sum(torch.linalg.vector_norm(local(p.grad), dtype=torch.float32) ** 2 for p in ps)
+        total = sq([p for p in params if sharded(p)]) + 0.0
+        if dist.is_initialized() and torch.is_tensor(total): dist.all_reduce(total)
+        self.grad_norm = float((total + sq([p for p in params if not sharded(p)])) ** 0.5)
+        if not math.isfinite(self.grad_norm):   # a NaN/inf gradient, even from a finite loss, stops here before any master moves (on every rank: the norm is global); min(1.0, nan) would otherwise apply it unclipped
+            raise RuntimeError(f"non-finite gradient norm {self.grad_norm}; refusing the optimizer step")
+        return min(1.0, self.max_grad_norm / (self.grad_norm + 1e-6)) if self.max_grad_norm else 1.0
+
+    def _fetch(self, p, slot):
+        """Start copying p's gradient to host memory; -> a callable returning it as fp32 once the copy has landed."""
+        g = local(p.grad)
+        if not self.pinned: return lambda: g.float()   # masters beside the weights (or everything on the CPU)
+        buf = self.down[g.dtype][slot][:g.numel()].view_as(g)
+        buf.copy_(g, non_blocking=True); done = torch.cuda.Event(); done.record()
+        def ready():
+            done.synchronize()
+            return self.scratch[:g.numel()].view_as(g).copy_(buf)
+        return ready
+
+    def _put(self, p, master, slot):
+        """Write bf16(master) into the working weights (through a pinned slot, asynchronously, when offloaded)."""
+        w = local(p)
+        if not self.pinned: w.copy_(master); return
+        if self.uploads[slot] is not None: self.uploads[slot].synchronize()   # the slot's previous upload has left
+        buf = self.up[w.dtype][slot][:w.numel()].view_as(w)
+        buf.copy_(master); w.copy_(buf, non_blocking=True)
+        self.uploads[slot] = torch.cuda.Event(); self.uploads[slot].record()
+
+    def state_dict(self):
+        """This rank's optimizer state for a resume point: the group hyperparameters (the lr scheduler moves them) and, per
+        parameter in order, its fp32 master, moments and step. The bf16 weights are not saved: they are bf16(master)."""
+        return {"groups": [{k: v for k, v in g.items() if k != "params"} for g in self.param_groups],
+                "state": [self.state[p] for g in self.param_groups for p in g["params"]]}
+
+    @torch.no_grad()
+    def load_state_dict(self, saved):
+        """Restore state_dict() in place (torch.optim.Optimizer's own would cast the fp32 masters to the parameters' bf16)
+        and write bf16(master) back into the weights, which reproduces them bit for bit."""
+        params = [p for g in self.param_groups for p in g["params"]]
+        if len(saved["state"]) != len(params) or len(saved["groups"]) != len(self.param_groups):
+            raise ValueError("resume point does not match this model's parameters")
+        for group, hyper in zip(self.param_groups, saved["groups"]): group.update(hyper)
+        for p, s in zip(params, saved["state"]):
+            for key, value in s.items():
+                if self.state[p][key].shape != value.shape: raise ValueError(f"resume point: {key} shape {tuple(value.shape)} for {tuple(self.state[p][key].shape)}")
+                self.state[p][key].copy_(value)
+            local(p).copy_(self.state[p]["master"])
+
+    @torch.no_grad()
+    def step(self):
+        work = [(g, p) for g in self.param_groups for p in g["params"] if p.grad is not None]
+        if dist.is_initialized():
+            for _, p in work:
+                if not sharded(p): dist.all_reduce(p.grad)   # replicated (the pointer head): each rank holds its share
+        scale = self._clip_scale([p for _, p in work])
+        pending = self._fetch(work[0][1], 0) if work else None
+        for i, (group, p) in enumerate(work):
+            grad = pending()
+            if i + 1 < len(work): pending = self._fetch(work[i + 1][1], (i + 1) % 2)
+            if scale < 1: grad.mul_(scale)
+            s = self.state[p]
+            adamw([s["master"]], [grad], [s["exp_avg"]], [s["exp_avg_sq"]], [], [s["step"]], fused=True, amsgrad=False,
+                  beta1=group["betas"][0], beta2=group["betas"][1], lr=group["lr"], weight_decay=group["weight_decay"],
+                  eps=group["eps"], maximize=False)
+            self._put(p, s["master"], i % 2)
+            p.grad = None
+
+
+# --- several GPUs -----------------------------------------------------------------------------------------------------
+
+def init_distributed(device):
+    """-> (rank, world size). Under torchrun (WORLD_SIZE > 1) joins the process group (NCCL on CUDA, gloo on CPU for tests)
+    and makes this rank's GPU the current one, so "cuda" means it everywhere; otherwise (0, 1) and nothing is initialised.
+    Either way turns on the CUDA allocator's expandable segments before the first allocation: micro-batch lengths vary
+    from 200 to 8,000 tokens, and the first 27B probe on one H200 ran out of memory with 6.7 GB reserved but unused."""
+    os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+    world = int(os.environ.get("WORLD_SIZE", "1"))
+    if world == 1: return 0, 1
+    if device == "cuda": torch.cuda.set_device(int(os.environ["LOCAL_RANK"]))
+    # rank 0 alone writes the gathered 27B checkpoint (~51 GB) while the others wait at a barrier: NCCL's 10 min default is too short
+    dist.init_process_group("nccl" if device == "cuda" else "gloo", timeout=datetime.timedelta(minutes=60))
+    return dist.get_rank(), world
+
+
+def shard(model):
+    """FSDP2 over the backbone: one unit per decoder layer plus the root (embeddings, final norm). Gradients are summed
+    over ranks, not averaged (the trainer already divides each micro-batch's loss by the step's global record count). The
+    pointer head stays replicated (MasterAdamW sums its gradient), starting from rank 0's initialisation."""
+    from torch.distributed.device_mesh import init_device_mesh
+    from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard
+    mesh = init_device_mesh(str(model.device).split(":")[0], (dist.get_world_size(),))
+    policy = MixedPrecisionPolicy(param_dtype=torch.bfloat16, reduce_dtype=torch.float32)
+    units = [*model.lm.layers, model.lm]
+    for unit in units: fully_shard(unit, mesh=mesh, mp_policy=policy)
+    for unit in units: unit.set_gradient_divide_factor(1.0)
+    for t in model.head.parameters(): dist.broadcast(t.data, 0)
+
+
+def rank_share(items, rank, world):
+    """This rank's equal share of one epoch's shuffled items: padded (by wrapping around) to a multiple of the world size,
+    so every rank runs the same number of micro-batches and FSDP's collectives line up (as DistributedSampler does). The
+    padding repeats the first items of the shuffled order, whatever their length (--length_sort 1 does not come through
+    here: train.microbatch_plan pads each step from its shuffled records before cutting them)."""
+    if world == 1: return items
+    padded = items + items[: -len(items) % world]
+    return padded[rank::world]
+
+
+def global_sum(values):
+    """Per-rank counters summed over the ranks (as they are on one process)."""
+    if not dist.is_initialized(): return values
+    total = torch.tensor(values, dtype=torch.float64, device="cuda" if dist.get_backend() == "nccl" else "cpu")
+    dist.all_reduce(total)
+    return total.tolist()
+
+
+def global_max(values):
+    """Per-rank values, the largest over the ranks (every rank calls it: a collective)."""
+    if not dist.is_initialized(): return values
+    top = torch.tensor(values, dtype=torch.float64, device="cuda" if dist.get_backend() == "nccl" else "cpu")
+    dist.all_reduce(top, op=dist.ReduceOp.MAX)
+    return top.tolist()
+
+
+# --- resume points ----------------------------------------------------------------------------------------------------
+
+LATEST = "latest.json"   # the resume point to continue from; written last (atomically) by rank 0, after every rank's file
+WRITE_SHARE = 0.05       # at most this share of wall time may block on writing resume points
+PEER_WAIT = 600          # seconds rank 0 waits, after writing its own file, for the other ranks' (they write in parallel)
+
+
+def rank0_decides(value):
+    """`value` as rank 0 sees it, on every rank (a decision every rank must act on together); itself on one process."""
+    if not dist.is_initialized(): return value
+    flag = torch.tensor(float(value), device="cuda" if dist.get_backend() == "nccl" else "cpu")
+    dist.broadcast(flag, 0)
+    return bool(flag)
+
+
+def save_due(step, every_steps, every_minutes, since, blocked):
+    """Whether to write a resume point after this optimizer step: every `every_steps` steps, or once `every_minutes`
+    have passed since `since`, stretched so the time training has blocked on writes (`blocked` seconds per point) stays
+    under WRITE_SHARE of the interval. Under torchrun rank 0's clock decides for every rank."""
+    interval = max(60 * every_minutes, blocked / WRITE_SHARE)
+    return rank0_decides(bool(every_steps and step % every_steps == 0) or bool(every_minutes and time.time() - since >= interval))
+
+
+class ResumeWriter:
+    """Writes resume points: each rank's optimizer state (its shard's fp32 masters and moments), the lr scheduler and the
+    RNG states to `step-N/rank<r>.pt`, then rank 0's `latest.json` with the position (the data order, counters, the
+    arguments it must be resumed with), then removes earlier points. A 27B's state is ~307 GB and the runs volume writes it
+    at well under 1 GB/s, so where the state lives on the GPUs (FSDP2) `save` copies it to host memory and a thread
+    writes it while training goes on (the next `save`, and `wait`, join it first); where it already lives in host memory
+    (one GPU, offload) there is no room for a copy and `save` writes it before returning.
+    `after`: the SnapshotWriter; latest.json waits for a snapshot still being written (and is not written if it failed),
+    so a resume point is never committed before the snapshots of the steps it has passed: a continuation from it could
+    not write them again."""
+
+    def __init__(self, resume_dir, background):
+        self.dir, self.background, self.thread, self.host, self.error = Path(resume_dir), background, None, None, None
+        self.seconds = []   # how long each point took to write, in the background or not
+        self.rank, self.world = (dist.get_rank(), dist.get_world_size()) if dist.is_initialized() else (0, 1)
+
+    def save(self, step, opt, sched, position, after=None):
+        self.wait()
+        state = opt.state_dict()
+        if self.background:   # host copies, allocated once and reused, so the optimizer may move on
+            tensors = [t for s in state["state"] for t in s.values()]
+            if self.host is None: self.host = [torch.empty(t.shape, dtype=t.dtype, device="cpu") for t in tensors]
+            for h, t in zip(self.host, tensors): h.copy_(t, non_blocking=True)
+            sync(tensors[0].device.type)
+            copies = iter(self.host)
+            state = {"groups": state["groups"], "state": [{k: next(copies) for k in s} for s in state["state"]]}
+        rng = {"torch": torch.get_rng_state(), "cuda": torch.cuda.get_rng_state() if torch.cuda.is_initialized() else None}
+        payload = {"optimizer": state, "scheduler": copy.deepcopy(sched.state_dict()), "rng": rng}
+        if self.background:
+            self.thread = threading.Thread(target=self._write, args=(step, payload, position, after), daemon=True); self.thread.start()
+        else:
+            self._write(step, payload, position, after)
+
+    def wait(self):
+        """Join the background write; its error, if any, is raised here."""
+        if self.thread is not None: self.thread.join(); self.thread = None
+        if self.error is not None: raise RuntimeError("writing the resume point failed") from self.error
+
+    def _write(self, step, payload, position, after=None):
+        started = time.time()
+        try: self._write_point(step, payload, position, after); self.seconds.append(round(time.time() - started, 1))
+        except BaseException as error:
+            self.error = error
+            if not self.background: raise
+
+    def _write_point(self, step, payload, position, after=None):
+        target = self.dir / f"step-{step:07d}"
+        target.mkdir(parents=True, exist_ok=True)
+        torch.save(payload, target / f".rank{self.rank}.pt.tmp")
+        os.replace(target / f".rank{self.rank}.pt.tmp", target / f"rank{self.rank}.pt")
+        if self.rank: return
+        deadline = time.time() + PEER_WAIT
+        while missing := [r for r in range(self.world) if not (target / f"rank{r}.pt").exists()]:   # the ranks share one filesystem
+            if time.time() > deadline:
+                raise TimeoutError(f"resume point {target.name}: rank(s) {missing} did not finish writing within {PEER_WAIT // 60} min of rank 0; "
+                                   "latest.json still names the previous point")
+            time.sleep(2)
+        if after is not None and (error := after.join()) is not None:   # the training thread raises it too (SnapshotWriter.wait)
+            raise RuntimeError(f"a snapshot failed to write, so resume point {target.name} is not made the latest: a continuation from "
+                               "the previous one writes that snapshot again") from error
+        write_json(self.dir / LATEST, {"dir": target.name, "step": step, **position}, atomic=True)
+        for old in self.dir.glob("step-*"):   # only earlier points: a rank done with this one may have started the next
+            if int(old.name.removeprefix("step-")) < step: shutil.rmtree(old)
+
+
+def load_resume(resume_dir, opt, sched, args):
+    """-> the saved position (or None when there is no resume point), after restoring this rank's optimizer state, the
+    weights, the scheduler and the RNG states. Refuses a point written with other arguments or another world size."""
+    resume_dir = Path(resume_dir)
+    if not (resume_dir / LATEST).exists(): return None
+    position = read_json(resume_dir / LATEST)
+    world, rank = dist.get_world_size() if dist.is_initialized() else 1, dist.get_rank() if dist.is_initialized() else 0
+    if position["world"] != world or position["args"] != args:
+        changed = sorted(k for k in set(args) | set(position["args"]) if args.get(k) != position["args"].get(k))
+        raise ValueError(f"resume point {resume_dir} was written by {position['world']} rank(s) with other arguments: {changed or 'world size'}")
+    saved = torch.load(resume_dir / position["dir"] / f"rank{rank}.pt", map_location="cpu", mmap=True, weights_only=True)
+    opt.load_state_dict(saved["optimizer"]); sched.load_state_dict(saved["scheduler"])
+    torch.set_rng_state(saved["rng"]["torch"])
+    if saved["rng"]["cuda"] is not None: torch.cuda.set_rng_state(saved["rng"]["cuda"])
+    return position
+
+
+def gather_backbone(lm):
+    """Under FSDP2: the bf16 backbone's full state dict, gathered into rank 0's host memory (every rank must call this;
+    the others get {}). On one process None: save_pretrained reads the model itself."""
+    if not dist.is_initialized(): return None
+    from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict
+    return get_model_state_dict(lm, options=StateDictOptions(full_state_dict=True, cpu_offload=True))
+
+
+def write_backbone(lm, out, state):
+    """save_pretrained of the bf16 backbone: config.json + model-*.safetensors (+ index); `state` from gather_backbone
+    (transformers strips the FSDP class prefix)."""
+    lm.save_pretrained(out, state_dict=state, max_shard_size=SHARD_SIZE)
+
+
+def save_backbone(lm, out):
+    """The final checkpoint's backbone. Under FSDP2 every rank joins the gather and rank 0 writes while the others wait."""
+    state = gather_backbone(lm)
+    if not dist.is_initialized() or dist.get_rank() == 0: write_backbone(lm, out, state)
+    if dist.is_initialized(): dist.barrier()
+
+
+# --- snapshots --------------------------------------------------------------------------------------------------------
+
+SNAPSHOT_INFO = "snapshot.json"   # written last (atomically) into a snapshot's checkpoint directory: the snapshot is complete
+
+
+def snapshot_fractions(text):
+    """--snapshot_fractions "0.25,0.5,0.75" -> (0.25, 0.5, 0.75); "" or "none" -> (). Each strictly between 0 and 1 (the
+    final checkpoint is always written)."""
+    if str(text).strip().lower() in ("", "none"): return ()
+    try: values = tuple(sorted({float(part) for part in str(text).split(",")}))
+    except ValueError: raise ValueError(f"snapshot fractions are comma-separated numbers, or none: {text!r}") from None
+    if not all(0 < v < 1 for v in values): raise ValueError(f"snapshot fractions must lie strictly between 0 and 1: {text!r}")
+    return values
+
+
+def snapshot_steps(total, fractions=(), every=0):
+    """The optimizer steps after which a run of `total` steps writes a snapshot: the first step at or past each fraction
+    of `total`, and every `every` steps; never the last step (that is the final checkpoint)."""
+    steps = {math.ceil(round(f * total, 6)) for f in fractions} | (set(range(every, total, every)) if every else set())
+    return sorted(s for s in steps if 0 < s < total)
+
+
+def too_many_snapshots(fractions=(), every=0, total=None):
+    """Why this snapshot plan exceeds kev.budget.MAX_SNAPSHOTS (the disk a run's kept snapshots may take), or None.
+    `total`: the run's optimizer steps, or an upper bound on them (a trial's max_steps); None = unknown, which an
+    every-N plan cannot be checked against."""
+    from .budget import MAX_SNAPSHOTS
+    if every and total is None:
+        return "snapshot_every_steps needs the run's step count to be bounded (max_steps), so its snapshot count can be checked"
+    count = len(snapshot_steps(total, fractions, every)) if total is not None else len(fractions)
+    if count > MAX_SNAPSHOTS:
+        return (f"{count} snapshots planned (fractions {list(fractions)}, every {every} of {total} steps): at most kev.budget.MAX_SNAPSHOTS = "
+                f"{MAX_SNAPSHOTS} fit the container disk next to the resume points (a full disk fails the trial)")
+    return None
+
+
+def snapshot_path(root, step):
+    return Path(root) / f"step-{step:07d}" / "checkpoint"   # zero-padded like resume points (ResumeWriter)
+
+
+def completed_snapshot_dirs(root):
+    """{step: checkpoint directory} of the complete snapshots under `root` (a snapshot is complete once its SNAPSHOT_INFO
+    exists); step-<N> directories are read by number, padded or not (the first snapshots, before padding, were not)."""
+    return {int(p.parent.parent.name.removeprefix("step-")): p.parent for p in Path(root).glob(f"step-*/checkpoint/{SNAPSHOT_INFO}")}
+
+
+def completed_snapshots(root):
+    """Steps of the complete snapshots under `root`."""
+    return sorted(completed_snapshot_dirs(root))
+
+
+class SnapshotWriter:
+    """Loadable bf16 checkpoints during a full-weight run, at registered optimizer steps, into `<root>/step-<N>/checkpoint`:
+    exactly the final checkpoint's files (save_pretrained shards + head.pt + tokenizer) plus SNAPSHOT_INFO, written last,
+    which marks the snapshot complete. Snapshots are kept (never deleted); a complete one is never written again (a
+    continued run skips it) and an incomplete one (a write the container did not finish) is replaced.
+    Under FSDP2 every rank joins the gather into rank 0's host memory (the only time training blocks), then rank 0 writes
+    from that copy in a thread while training goes on; the next `save`, `wait` and a resume point's latest.json join it
+    first. On one GPU the host holds the masters and there is no room for a copy, so `save` writes before returning."""
+
+    def __init__(self, root, steps, background):
+        self.root, self.steps, self.background, self.thread, self.error = Path(root), set(steps), background, None, None
+        self.rank = dist.get_rank() if dist.is_initialized() else 0
+        self.written = []   # this process's snapshots: {"step", "blocking_seconds", "write_seconds"}
+
+    def due(self, step):
+        """Whether to write a snapshot after this step: a registered step without a complete snapshot (rank 0 decides)."""
+        return step in self.steps and rank0_decides(step not in completed_snapshots(self.root))
+
+    def missed(self, step):
+        """Registered steps up to `step` (a resume point) without a complete snapshot: a continuation cannot write them."""
+        done = set(completed_snapshots(self.root))
+        return sorted(s for s in self.steps if s <= step and s not in done)
+
+    def save(self, step, lm, finish, info):
+        """Write the snapshot of this step: the backbone, then finish(directory) (head.pt and the tokenizer, as for the
+        final checkpoint), then SNAPSHOT_INFO with `info` and the time it took."""
+        self.wait()
+        started = time.time()
+        state = gather_backbone(lm)
+        if self.rank: return
+        blocked = time.time() - started
+        if self.background:
+            self.thread = threading.Thread(target=self._write, args=(step, lm, state, finish, info, blocked), daemon=True); self.thread.start()
+        else:
+            self._write(step, lm, state, finish, info, blocked)
+
+    def join(self):
+        """Wait for the snapshot being written, if any (any thread may); -> its error or None."""
+        thread = self.thread
+        if thread is not None: thread.join()
+        return self.error
+
+    def wait(self):
+        """Join the background write; its error, if any, is raised here, on every rank at once (every rank calls wait:
+        in save and at the end of training), so ranks 1..N do not go on into the next gather and hang on rank 0."""
+        self.join(); self.thread = None
+        if rank0_decides(self.error is not None):
+            raise RuntimeError("writing a snapshot failed" + ("" if self.error else " on rank 0")) from self.error
+
+    def _write(self, step, lm, state, finish, info, blocked):
+        started = time.time()
+        try:
+            target = snapshot_path(self.root, step)
+            if (target / SNAPSHOT_INFO).exists():   # complete (a racing writer, or an earlier attempt): never deleted or rewritten
+                print(f"snapshot step {step}: {target} is complete already; left as it is", flush=True); return
+            if target.exists(): shutil.rmtree(target)   # an incomplete write
+            target.mkdir(parents=True)
+            write_backbone(lm, target, state)
+            finish(target)
+            timing = {"step": step, "blocking_seconds": round(blocked + (0 if self.background else time.time() - started), 1),
+                      "write_seconds": round(time.time() - started, 1)}
+            write_json(target / SNAPSHOT_INFO, {**info, **timing}, atomic=True)
+            self.written.append(timing)
+            print(f"snapshot step {step}: {target} (training blocked {timing['blocking_seconds']} s, written in {timing['write_seconds']} s)", flush=True)
+        except BaseException as error:
+            self.error = error
+            if not self.background: raise

runtime/kev/fused_qwen35.py

@@ -0,0 +1,155 @@
+"""Fused CUDA kernels for serving the Qwen3.5 decoder layers (flash-linear-attention's Triton ops).
+
+With CUDA graphs and batching (kev.cuda_graphs) a served pass is limited by the GPU work itself, and transformers'
+reference Qwen3.5 layers spend over a third of it outside the matrix multiplies: the causal convolution falls back to a
+PyTorch depthwise conv behind a concatenation of the cached conv state, the gating, the query/key head repeat and the
+gated norm run as a dozen separate fp32 elementwise kernels per DeltaNet layer, and every RMSNorm and SwiGLU is several
+more. fuse() rewrites those layers in place for inference:
+- DeltaNet mixer: one projection GEMM for q/k/v, z, b and a (their weights concatenated); fla's Triton causal conv
+  that starts from the cached conv state (no concatenation); fla's chunked gated delta rule with the gate
+  (-exp(A_log) * softplus(a + dt_bias)), the beta sigmoid and the q/k L2 norm computed inside the kernel and the key
+  heads shared by their value heads (no repeat); fla's fused gated RMSNorm.
+- MLP: one GEMM for gate and up (weights concatenated) and a fused SwiGLU.
+- the decoder layer's two zero-centred RMSNorms as fla's fused RMSNorm with weight 1 + w in fp32, the second one also
+  adding the residual.
+- attention: one GEMM for q (with its output gate), k and v, the q/k RMSNorms fused, the sigmoid output gate fused.
+The math is the reference's; the rounding is not (the fused kernels keep fp32
+where the reference rounds to bf16 in between), so fused and reference answers agree to bf16 noise, like the graphs.
+Only for serving: there is no backward, the fused projections replace the originals (the merged LoRA is inside), and a
+pass that continues a cached DeltaNet state does not advance it (see deltanet_forward).
+"""
+import types
+
+import torch
+import torch.nn.functional as F
+from fla.modules.activations import sigmoidglu, swiglu
+from fla.modules.conv import causal_conv1d
+from fla.modules.fused_norm_gate import rms_norm_gated
+from fla.modules.layernorm import rms_norm
+from fla.ops.gated_delta_rule import chunk_gated_delta_rule
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+from transformers.models.qwen3_5.modeling_qwen3_5 import apply_mask_to_padding_states, apply_rotary_pos_emb, eager_attention_forward
+
+
+class _FixedNB:
+    """A Triton kernel launched with NB=1. fla passes these kernels NB, a token-count class (cdiv of the rows by a few
+    thousand), as a constexpr and an autotuning key, but their bodies never read it: every new batch shape compiled and
+    autotuned them again, stalling a busy server for about a second each time."""
+
+    def __init__(self, kernel):
+        self.kernel = kernel
+
+    def __getitem__(self, grid):
+        launch = self.kernel[grid]
+        return lambda *args, **kwargs: launch(*args, **{**kwargs, "NB": 1})
+
+
+def _fix_nb():
+    import fla.modules.conv.triton.ops as conv, fla.modules.fused_norm_gate as gated, fla.modules.l2norm as l2, fla.modules.layernorm as norm
+    for module, name in ((conv, "causal_conv1d_fwd_kernel"), (gated, "layer_norm_gated_fwd_kernel"), (l2, "l2norm_fwd_kernel"), (norm, "layer_norm_fwd_kernel")):
+        if not isinstance(getattr(module, name), _FixedNB): setattr(module, name, _FixedNB(getattr(module, name)))
+
+
+FLA_VERSION = "0.5.2"   # the flash-linear-attention these kernels and _fix_nb were written and measured against
+
+
+def _concat(*linears):
+    """One weight [sum(out), in] for several bias-free projections of the same input."""
+    if any(l.bias is not None for l in linears): raise ValueError("fused projections assume bias-free Linear layers")
+    return torch.cat([l.weight for l in linears], 0).contiguous()
+
+
+def deltanet_forward(self, hidden_states, cache_params=None, attention_mask=None, **kwargs):
+    """Qwen3_5GatedDeltaNet.forward with fused kernels. Cache contract: a pass on an empty cache (a state) fills it, as the
+    reference does; a pass continuing a cached state (question rows) reads it and leaves it as it was. Kev's question rows
+    never continue from each other, and writing back every row's final states cost as much as reading them."""
+    hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask)
+    B, T, _ = hidden_states.shape
+    mixed, z, b, a = F.linear(hidden_states, self.in_proj).split(self.splits, -1)
+    layer = cache_params.layers[self.layer_idx] if cache_params is not None else None
+    previous = layer is not None and layer.has_previous_state[0]
+    fill = layer is not None and not previous
+    mixed, conv_state = causal_conv1d(mixed, self.conv_weight, None, initial_state=layer.conv_states[0] if previous else None,
+                                      output_final_state=fill, activation="silu")
+    q, k, v = mixed.split([self.key_dim, self.key_dim, self.value_dim], -1)
+    out, recurrent = chunk_gated_delta_rule(
+        q.reshape(B, T, -1, self.head_k_dim), k.reshape(B, T, -1, self.head_k_dim), v.reshape(B, T, -1, self.head_v_dim),
+        g=a, beta=b, initial_state=layer.recurrent_states[0] if previous else None, output_final_state=fill,
+        use_qk_l2norm_in_kernel=True, use_gate_in_kernel=True, A_log=self.A_log, dt_bias=self.dt_bias, use_beta_sigmoid_in_kernel=True)
+    if fill:
+        if not layer.is_conv_states_initialized[0]: layer.lazy_initialization(conv_states=conv_state, conv_kernel_size=conv_state.shape[-1])
+        layer.conv_states[0].copy_(conv_state)   # in place: graph buffers keep their address
+        layer.has_previous_state[0] = True
+        cache_params.update_recurrent_state(recurrent, self.layer_idx)
+    out = rms_norm_gated(out, z.reshape(B, T, -1, self.head_v_dim), self.norm.weight, None, activation="swish", eps=self.norm.variance_epsilon)
+    return self.out_proj(out.reshape(B, T, -1))
+
+
+def attention_forward(self, hidden_states, position_embeddings, attention_mask, past_key_values=None, **kwargs):
+    """Qwen3_5Attention.forward with one projection GEMM, fused q/k RMSNorms and a fused sigmoid output gate."""
+    shape = hidden_states.shape[:-1]
+    q_gate, k, v = F.linear(hidden_states, self.qkv).split(self.splits, -1)
+    q, gate = q_gate.reshape(*shape, -1, 2 * self.head_dim).chunk(2, -1)
+    q = rms_norm(q, self.q_norm_weight, None, eps=self.q_norm.eps).transpose(1, 2)
+    k = rms_norm(k.reshape(*shape, -1, self.head_dim), self.k_norm_weight, None, eps=self.k_norm.eps).transpose(1, 2)
+    q, k = apply_rotary_pos_emb(q, k, *position_embeddings)
+    v = v.reshape(*shape, -1, self.head_dim).transpose(1, 2)
+    if past_key_values is not None: k, v = past_key_values.update(k, v, self.layer_idx)
+    attend = ALL_ATTENTION_FUNCTIONS.get_interface(self.config._attn_implementation, eager_attention_forward)
+    out, weights = attend(self, q, k, v, attention_mask, dropout=0.0, scaling=self.scaling, **kwargs)   # [B, T, heads, dim]: transposed back already
+    return self.o_proj(sigmoidglu(gate.reshape(*shape, -1), out.reshape(*shape, -1))), weights
+
+
+def mlp_forward(self, x):
+    gate, up = F.linear(x, self.gate_up).chunk(2, -1)
+    return self.down_proj(swiglu(gate, up))
+
+
+def decoder_forward(self, hidden_states, position_embeddings, attention_mask=None, position_ids=None, past_key_values=None, **kwargs):
+    """Qwen3_5DecoderLayer.forward with fused norms: the post-mixer norm also adds the residual."""
+    residual = hidden_states
+    h = rms_norm(hidden_states, self.input_norm_weight, None, eps=self.input_layernorm.eps)
+    if self.block_type == "linear_attention":
+        h = self.linear_attn(hidden_states=h, cache_params=past_key_values, attention_mask=attention_mask, **kwargs)
+    else:
+        h, _ = self.self_attn(hidden_states=h, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values,
+                              position_embeddings=position_embeddings, **kwargs)
+    h, residual = rms_norm(h, self.post_norm_weight, None, residual=residual, eps=self.post_attention_layernorm.eps, prenorm=True)
+    return residual + self.mlp(h)
+
+
+@torch.no_grad()
+def fuse(lm):
+    """Rewrite a Qwen3.5 text backbone (DecisionModel.lm, merged) in place for fused-kernel serving. Contract changes
+    against the reference: no backward, and a pass that continues a cached DeltaNet state leaves that state as it was
+    (Kev's question rows never continue from each other). Needs flash-linear-attention FLA_VERSION exactly: it patches
+    fla's kernel launches (_fix_nb)."""
+    import fla
+    if fla.__version__ != FLA_VERSION:
+        raise RuntimeError(f"kev's fused kernels need flash-linear-attention=={FLA_VERSION}, found {fla.__version__}; "
+                           "install that version or load with LoadOptions(fused=False) (KEV_FUSED=0 for kev.serve)")
+    if not all(hasattr(layer.mlp, "gate_proj") for layer in lm.layers):
+        raise NotImplementedError("kev's fused kernels cover the dense Qwen3.5 MLP, not mixture-of-experts layers; load with fused=False (KEV_FUSED=0)")
+    _fix_nb()
+    for layer in lm.layers:
+        if layer.block_type == "linear_attention":
+            m = layer.linear_attn
+            m.in_proj = _concat(m.in_proj_qkv, m.in_proj_z, m.in_proj_b, m.in_proj_a)
+            m.splits = [m.conv_dim, m.value_dim, m.num_v_heads, m.num_v_heads]
+            m.conv_weight = m.conv1d.weight.squeeze(1).contiguous()
+            del m.in_proj_qkv, m.in_proj_z, m.in_proj_b, m.in_proj_a
+            m.forward = types.MethodType(deltanet_forward, m)
+        else:
+            m = layer.self_attn
+            m.qkv = _concat(m.q_proj, m.k_proj, m.v_proj)
+            m.splits = [m.q_proj.out_features, m.k_proj.out_features, m.v_proj.out_features]
+            m.q_norm_weight, m.k_norm_weight = 1.0 + m.q_norm.weight.float(), 1.0 + m.k_norm.weight.float()
+            del m.q_proj, m.k_proj, m.v_proj
+            m.forward = types.MethodType(attention_forward, m)
+        layer.mlp.gate_up = _concat(layer.mlp.gate_proj, layer.mlp.up_proj)
+        del layer.mlp.gate_proj, layer.mlp.up_proj
+        layer.mlp.forward = types.MethodType(mlp_forward, layer.mlp)
+        layer.input_norm_weight = 1.0 + layer.input_layernorm.weight.float()
+        layer.post_norm_weight = 1.0 + layer.post_attention_layernorm.weight.float()
+        layer.forward = types.MethodType(decoder_forward, layer)
+    return lm

runtime/kev/jev.py

@@ -0,0 +1,63 @@
+import argparse
+import json
+import os
+import re
+import subprocess
+from pathlib import Path
+
+from kev.benchmark import evaluate_records
+from kev.predictors import JevPredictor
+from kev.suite import digest, load_split, read_manifest, write_json
+
+
+def provision_key(scope):
+    result = subprocess.run(["vercel", "ai-gateway", "api-keys", "create", "--scope", scope,
+                             "--name", "kev-jev-evaluation", "--limit", "1", "--refresh-period", "none",
+                             "--expiration", "7d", "--no-color", "--non-interactive"], capture_output=True, text=True)
+    if result.returncode:
+        raise RuntimeError("Vercel key creation failed; inspect account permissions in the Vercel dashboard")
+    text = result.stdout + result.stderr
+    match = re.search(r"\bvck_[A-Za-z0-9_-]+", text)
+    if not match:
+        raise RuntimeError("Key was created but its output format was not recognized; no key output was logged")
+    return match.group()
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--suite", required=True)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--provision-scope", help="Explicitly authorize a $1 non-renewing, seven-day key on this team")
+    ap.add_argument("--budget", type=float, default=0.1)
+    ap.add_argument("--max-calls", type=int, default=700)
+    ap.add_argument("--count-refusals", action="store_true",
+                    help="count requests Jev refuses (HTTP 400/413/422, e.g. past its context) as rejected records (rejected.json, "
+                         "report coverage) and continue; other errors still stop the read")
+    ap.add_argument("--attempts", type=int, default=4, help="tries per request on gateway 5xx errors (backoff 1, 2, 4, ... s, at most 30 s)")
+    a = ap.parse_args()
+    # a provisioned key carries a $1 limit of its own; a caller's key may run a larger panel (breadth-v1 dev: ~2,000 records)
+    if not 0 < a.budget <= 5 or not 1 <= a.max_calls <= 5000 or not 1 <= a.attempts <= 12:
+        ap.error("budget must be in (0, 5], max-calls in [1, 5000] and attempts in [1, 12]")
+    if Path(a.out).exists():
+        ap.error("output directory already exists")
+    records = load_split(a.suite, "development")
+    key = os.environ.get("AI_GATEWAY_API_KEY")
+    if not key and a.provision_scope:
+        key = provision_key(a.provision_scope)
+    if not key:
+        ap.error("Set AI_GATEWAY_API_KEY or explicitly select --provision-scope")
+    predictor = JevPredictor(key, a.budget, a.max_calls, count_refusals=a.count_refusals, attempts=a.attempts)
+    try:
+        heldout = read_manifest(a.suite)["holdout_sources"]
+        report, _ = evaluate_records(records, predictor, a.out, heldout_sources=tuple(heldout), skip_overlong=a.count_refusals)
+        report.update(suite_sha256=digest(Path(a.suite) / "manifest.json"), split="development", provider=predictor.accounting())
+        write_json(Path(a.out) / "report.json", report)
+        print(json.dumps({"clean": report["clean"], "variants": report["variants"], "provider": report["provider"]}, indent=2))
+    finally:
+        predictor.close()
+        if Path(a.out).exists():
+            write_json(Path(a.out) / "usage.json", predictor.accounting())
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/metrics.py

@@ -0,0 +1,426 @@
+"""Scoring of benchmark rows: accuracy, calibration (ECE, Brier, NLL), selective prediction (coverage at an error budget,
+risk-coverage curve, AURC, thresholds), temperature fitting and the record-clustered paired bootstrap.
+
+Pure numpy over the row dicts that kev.benchmark.prediction_rows produces ({"p", "label", "type", "keys", "task", "source",
+"variant", ..., optionally "logits"}). No torch, no model: usable on saved rows.json files.
+"""
+import math
+from collections import defaultdict
+
+import numpy as np
+
+EPSILON = 1e-9
+
+
+def ece(conf, correct, bins=10):
+    conf, correct = np.asarray(conf), np.asarray(correct, dtype=float)
+    edges = np.linspace(0, 1, bins + 1); e = 0.0
+    for lo, hi in zip(edges[:-1], edges[1:]):
+        m = (conf >= lo) & (conf < hi) if hi < 1 else (conf >= lo) & (conf <= hi)
+        if m.any(): e += m.mean() * abs(correct[m].mean() - conf[m].mean())
+    return float(e)
+
+
+def _check_temperature(temperature):
+    if not math.isfinite(temperature) or temperature <= 0:
+        raise ValueError("temperature must be finite and positive")
+
+
+def _tempered_logits(row, temperature):
+    """(z - max z) / T from the recorded logits, or from the floored probabilities when the row has none."""
+    _check_temperature(temperature)
+    p = np.asarray(row["p"], dtype=float)
+    z = np.asarray(row["logits"], dtype=float) if "logits" in row else np.log(np.maximum(p, EPSILON))
+    if z.shape != p.shape or not np.isfinite(z).all():
+        raise ValueError("logits must be finite and match the option count")
+    return (z - z.max()) / temperature
+
+
+def probabilities_at_temperature(row, temperature=1.0):
+    """The row's returned probabilities at T=1; otherwise the softmax of its tempered logits."""
+    _check_temperature(temperature)
+    if temperature == 1:
+        return np.asarray(row["p"], dtype=float)
+    p = np.exp(_tempered_logits(row, temperature))
+    return p / p.sum()
+
+
+def nll_at_temperature(row, temperature=1.0):
+    """Exact from logits when recorded; otherwise from the floored probabilities."""
+    if "logits" in row:
+        z = _tempered_logits(row, temperature)
+        return float(np.log(np.exp(z).sum()) - z[row["label"]])
+    return -math.log(max(float(probabilities_at_temperature(row, temperature)[row["label"]]), EPSILON))
+
+
+def _row_scores(row):
+    """The additive per-question quantities whose means metrics() reports (used by the paired bootstrap)."""
+    p = np.asarray(row["p"], dtype=float); y = row["label"]
+    conf, ok = float(p.max()), bool(p.argmax() == y)
+    return {"acc": float(ok), "nll": nll_at_temperature(row), "brier": float(((p - np.eye(len(p))[y]) ** 2).sum()), "mean_conf": conf,
+            "confident_error_rate": float(conf >= 0.9 and not ok), "coverage_at_0_9": float(conf >= 0.9), "confidence_bias": conf - float(ok)}
+
+
+def metrics(rows, temperature=1.0):
+    if not rows:
+        raise ValueError("cannot score an empty population")
+    nll, acc, conf, brier, mae, rps = [], [], [], [], [], []
+    for row in rows:
+        p = probabilities_at_temperature(row, temperature)
+        y = row["label"]
+        target = np.eye(len(p))[y]
+        nll.append(nll_at_temperature(row, temperature))
+        acc.append(int(p.argmax() == y)); conf.append(float(p.max()))
+        brier.append(float(((p - target) ** 2).sum()))
+        if row["type"] == "score":
+            mae.append(abs(float(p @ np.arange(len(p))) - y))
+            rps.append(float(((p.cumsum()[:-1] - target.cumsum()[:-1]) ** 2).mean()))
+    result = {"n": len(rows), "nll": float(np.mean(nll)), "acc": float(np.mean(acc)),
+              "ece": ece(conf, acc), "brier": float(np.mean(brier)), "mean_conf": float(np.mean(conf))}
+    confidence, correct = np.asarray(conf), np.asarray(acc, dtype=bool)
+    high = confidence >= 0.9
+    result.update(confident_error_rate=float(np.mean(high & ~correct)), coverage_at_0_9=float(high.mean()),
+                  accuracy_at_0_9=float(correct[high].mean()) if high.any() else None,
+                  coverage_at_5pct_error=coverage_at_error(confidence, correct, 0.05), coverage_at_1pct_error=coverage_at_error(confidence, correct, 0.01),
+                  aurc=area_under_risk_coverage(confidence, correct),
+                  error_rate_at_0_9=float((~correct[high]).mean()) if high.any() else None,
+                  # signed over-confidence (mean top probability minus accuracy) and errors within the top confidence bins;
+                  # the sign is diagnostic: untrained readouts run positive, outcome-trained ones near zero or negative
+                  confidence_bias=float(confidence.mean() - correct.mean()),
+                  top_bins={str(t): {"n": int((confidence >= t).sum()), "errors": int(((confidence >= t) & ~correct).sum()),
+                                     "error_rate": float((~correct[confidence >= t]).mean()) if (confidence >= t).any() else None} for t in (0.9, 0.95, 0.99)})
+    result["selective"] = {}
+    for fraction in (0.5, 0.8):
+        cutoff = np.sort(confidence)[-max(1, math.ceil(len(rows) * fraction))]
+        selected = confidence >= cutoff
+        result["selective"][str(fraction)] = {"coverage": float(selected.mean()), "accuracy": float(correct[selected].mean()),
+                                            "confidence_cutoff": float(cutoff)}
+    if mae:
+        result.update(score_mae=float(np.mean(mae)), ranked_probability_score=float(np.mean(rps)))
+    return result
+
+
+def coverage_at_error(confidence, correct, budget):
+    """Selective automation: the largest share of decisions that can be accepted, in descending confidence order, while
+    the empirical error among the accepted stays <= budget (jev-benchmarks' "coverage at a fixed error budget"). A model
+    whose probabilities are honest gets high coverage; one that is confidently wrong gets little, whatever its accuracy."""
+    if not math.isfinite(budget) or not 0 <= budget <= 1:
+        raise ValueError("error budget must be in [0, 1]")
+    _, accepted, errors = _risk_curve_arrays(confidence, correct)
+    ok = np.flatnonzero(errors <= budget * accepted)
+    return float(accepted[ok[-1]] / accepted[-1]) if len(ok) else 0.0
+
+
+def _selective_inputs(confidence, correct):
+    confidence, correct = np.asarray(confidence, dtype=float), np.asarray(correct)
+    if confidence.ndim != 1 or correct.shape != confidence.shape:
+        raise ValueError("confidence and correctness must be equal-length vectors")
+    if not np.isfinite(confidence).all() or ((confidence < 0) | (confidence > 1)).any():
+        raise ValueError("confidence must be finite and in [0, 1]")
+    if not np.isin(correct, [False, True]).all():
+        raise ValueError("correctness must be boolean")
+    return confidence, correct.astype(bool)
+
+
+def _risk_curve_arrays(confidence, correct):
+    confidence, correct = _selective_inputs(confidence, correct)
+    if not len(confidence):
+        return confidence, np.array([], dtype=int), np.array([], dtype=int)
+    order = np.argsort(-confidence, kind="stable")
+    confidence, errors = confidence[order], np.cumsum(~correct[order])
+    ends = np.r_[np.flatnonzero(confidence[1:] != confidence[:-1]), len(order) - 1]
+    return confidence[ends], ends + 1, errors[ends]
+
+
+def risk_coverage_curve(confidence, correct):
+    thresholds, accepted, errors = _risk_curve_arrays(confidence, correct)
+    return [{"threshold": float(t), "accepted": int(n), "errors": int(e),
+             "coverage": float(n / accepted[-1]), "risk": float(e / n)}
+            for t, n, e in zip(thresholds, accepted, errors)]
+
+
+def area_under_risk_coverage(confidence, correct):
+    _, accepted, errors = _risk_curve_arrays(confidence, correct)
+    if not len(accepted):
+        return 0.0
+    return float(np.sum(np.diff(np.r_[0, accepted]) * errors / accepted) / accepted[-1])
+
+
+def select_threshold(confidence, correct, budget, min_accepted=1):
+    if not math.isfinite(budget) or not 0 <= budget <= 1 or min_accepted < 1:
+        raise ValueError("invalid error budget or minimum accepted count")
+    thresholds, accepted, errors = _risk_curve_arrays(confidence, correct)
+    ok = np.flatnonzero((errors <= budget * accepted) & (accepted >= min_accepted))
+    return float(thresholds[ok[-1]]) if len(ok) else None
+
+
+def evaluate_threshold(confidence, correct, threshold):
+    confidence, correct = _selective_inputs(confidence, correct)
+    if threshold is not None and (not math.isfinite(threshold) or not 0 <= threshold <= 1):
+        raise ValueError("threshold must be in [0, 1] or None for abstain-all")
+    accepted = confidence >= threshold if threshold is not None else np.zeros(len(confidence), dtype=bool)
+    n, errors = int(accepted.sum()), int((accepted & ~correct).sum())
+    return {"threshold": threshold, "n": len(confidence), "accepted": n, "errors": errors,
+            "coverage": n / len(confidence) if len(confidence) else 0.0, "risk": errors / n if n else None}
+
+
+def unknowable_report(rows):
+    """Confidence on records whose deciding evidence was removed (source 'unknowable') against their intact controls.
+    Accuracy on the unknowable records is meaningless by construction; what is scored is whether the model knows it
+    cannot know: mean max-probability and the share of records answered at >= 0.9."""
+    unk = [r for r in rows if r["source"] == "unknowable"]; ctl = [r for r in rows if r["source"] == "unknowable_control"]
+    if not unk: return None
+    conf = lambda rs: [float(max(r["p"])) for r in rs]
+    by_id = {r["id"]: r for r in ctl}
+    paired = [(max(r["p"]), max(by_id[r["control_id"]]["p"])) for r in unk if r.get("control_id") in by_id]
+    return {"n": len(unk), "mean_max_p": float(np.mean(conf(unk))), "share_at_0_9": float(np.mean([c >= 0.9 for c in conf(unk)])),
+            "control_mean_max_p": float(np.mean(conf(ctl))) if ctl else None, "control_share_at_0_9": float(np.mean([c >= 0.9 for c in conf(ctl)])) if ctl else None,
+            "control_acc": float(np.mean([int(np.argmax(r["p"]) == r["label"]) for r in ctl])) if ctl else None,
+            "paired_confidence_drop": float(np.mean([c - u for u, c in paired])) if paired else None,
+            "share_less_confident_than_control": float(np.mean([u < c for u, c in paired])) if paired else None}
+
+
+def grouped_metrics(rows, key, temperature=1.0):
+    groups = defaultdict(list)
+    for row in rows:
+        groups[row[key]].append(row)
+    return {name: metrics(group, temperature) for name, group in sorted(groups.items())}
+
+
+LENGTH_EDGES = (8192, 16384, 32768, 65536)   # state-token edges of calibration_by_length (long-context calibration)
+LENGTH_METRICS = ("acc", "ece", "brier", "confident_error_rate")
+
+
+def length_buckets(edges=LENGTH_EDGES):
+    """[(name, lo, hi)]: state tokens in [lo, hi) (hi None = no upper bound). The disjoint buckets (under_8k, 8k_16k, 16k_32k,
+    32k_64k, 64k_plus for the default edges), then the tails of every inner edge (8k_plus, 16k_plus, 32k_plus). Edges are
+    increasing multiples of 1,024, which name them."""
+    if not edges or list(edges) != sorted(set(edges)) or any(e <= 0 or e % 1024 for e in edges):
+        raise ValueError("length edges must be increasing positive multiples of 1024")
+    k = lambda n: f"{n // 1024}k"
+    bounds = [0, *edges, None]
+    disjoint = [(f"under_{k(hi)}" if lo == 0 else f"{k(lo)}_plus" if hi is None else f"{k(lo)}_{k(hi)}", lo, hi) for lo, hi in zip(bounds[:-1], bounds[1:])]
+    return disjoint + [(f"{k(lo)}_plus", lo, None) for lo in edges[:-1]]
+
+
+def calibration_by_length(rows, lengths=None, edges=LENGTH_EDGES):
+    """{bucket: {"n", acc, ece, brier, confident_error_rate}} of rows (scored as given, at T=1: pass them served) split by
+    state-token count: the row's own `state_tokens` when it records one, else lengths[row id] ({record id: tokens}); a count
+    is the encoded state segment, the <state> token included (kev.model.encode; kev.rounds.state_lengths computes it). An
+    empty bucket has n 0 and None values. The one home of per-length calibration (kev.rounds panels with `by_length`)."""
+    def tokens(row):
+        if row.get("state_tokens") is not None: return row["state_tokens"]
+        if lengths is None or row["id"] not in lengths: raise KeyError(f"no state-token count for record {row['id']!r}")
+        return lengths[row["id"]]
+    counted = [(tokens(r), r) for r in rows]
+    out = {}
+    for name, lo, hi in length_buckets(edges):
+        group = [r for n, r in counted if n >= lo and (hi is None or n < hi)]
+        scored = metrics(group) if group else {}
+        out[name] = {"n": len(group), **{m: scored.get(m) for m in LENGTH_METRICS}}
+    return out
+
+
+def tempered_row(row, temperature):
+    """The row as a calibrated predictor would have returned it: probabilities and logits at `temperature`, so metrics()
+    at T=1 scores it (per-row temperatures, e.g. out-of-fold, cannot go through metrics(rows, T)). The recorded
+    inference_temperature composes (served at T, tempered by T' -> T * T'), so raw_row always restores the T=1 logits."""
+    return {**row, "p": probabilities_at_temperature(row, temperature).tolist(), "logits": _tempered_logits(row, temperature).tolist(),
+            "inference_temperature": row.get("inference_temperature", 1.0) * temperature}
+
+
+def raw_row(row):
+    """The row as the checkpoint would have returned it at T=1: undoes the recorded inference_temperature (served logits
+    are z / T, so z = logits * T), which is what fit_temperature requires. A row served raw is returned unchanged."""
+    temperature = row.get("inference_temperature", 1.0)
+    _check_temperature(temperature)
+    if temperature == 1.0:
+        return row
+    if "logits" not in row:
+        raise ValueError("cannot restore raw logits from a row that recorded none")
+    z = np.asarray(row["logits"], dtype=float) * temperature
+    p = np.exp(z - z.max())
+    return {**row, "p": (p / p.sum()).tolist(), "logits": z.tolist(), "inference_temperature": 1.0}
+
+
+def recorded(row):
+    """The row with its inference temperature explicit: rows saved before benchmarks recorded one (None) were scored raw."""
+    return {**row, "inference_temperature": 1.0} if row.get("inference_temperature") is None else row
+
+
+def served_at(rows, temperature):
+    """The scored rows as a predictor at `temperature` would have returned them, whatever temperature they were saved at
+    (raw logits are restored first, so a Hub checkpoint's served rows and a trial's raw rows are treated alike)."""
+    return [tempered_row(raw_row(recorded(r)), temperature) for r in scored_rows(rows)]
+
+
+def served(fit_rows, eval_rows, **fit_kwargs):
+    """(temperature fitted on `fit_rows`' raw logits, `eval_rows` served at it): how a checkpoint is calibrated and read
+    everywhere a comparison is served-vs-served. fit_kwargs override TEMPERATURE_FIT key by key."""
+    temperature = fit_temperature(served_at(fit_rows, 1.0), **{**TEMPERATURE_FIT, **fit_kwargs})
+    return temperature, served_at(eval_rows, temperature)
+
+
+def scored_rows(rows):
+    """The rows metrics are computed on: clean variants of knowable records."""
+    return [row for row in rows if row["variant"] == "clean" and row["source"] != "unknowable"]
+
+
+# how every released temperature was fitted (scripts/calibrate_checkpoint.py) and how kev.calibrate fits a workload's:
+# min mean NLL over a 121-point log grid on 0.25..4, every question weighted equally. Research trials (kev.experiment) and
+# the kev-finetune skill keep the default 81-point grid: their temperatures are screening reports, not shipped values.
+TEMPERATURE_FIT = {"aggregation": "micro", "points": 121}
+TEMPERATURE_FIT_METHOD = f"min {TEMPERATURE_FIT['aggregation']} mean NLL over a {TEMPERATURE_FIT['points']}-point log grid 0.25..4"
+
+
+def fit_temperature(rows, aggregation="macro", points=81):
+    """Temperature minimizing the (micro or per-task macro) mean NLL over a `points`-point log grid on 0.25..4.
+    Released checkpoints and workload fits use **TEMPERATURE_FIT."""
+    if aggregation not in ("micro", "macro"):
+        raise ValueError("invalid calibration aggregation")
+    clean = scored_rows(rows)
+    if not clean:
+        raise ValueError("cannot fit temperature without labelled calibration rows")
+    if any(row.get("inference_temperature", 1.0) != 1.0 for row in clean):
+        raise ValueError("fit temperature on raw logits, not previously calibrated outputs")
+    candidates = np.exp(np.linspace(np.log(0.25), np.log(4), points))
+    weights = np.ones(len(clean))
+    if aggregation == "macro":
+        counts = defaultdict(int)
+        for row in clean:
+            counts[row["task"]] += 1
+        weights = np.asarray([1.0 / counts[row["task"]] for row in clean])
+    losses = [np.average([nll_at_temperature(r, float(t)) for r in clean], weights=weights) for t in candidates]
+    return float(candidates[int(np.argmin(losses))])
+
+
+def _source_groups(rows):
+    """dict[source, list[np.ndarray of row indices]]: one array per (source, group) cluster, in first-seen order."""
+    groups = defaultdict(list)
+    for i, row in enumerate(rows):
+        groups[(row["source"], row["group"])].append(i)
+    sources = defaultdict(list)
+    for (source, _), indices in groups.items():
+        sources[source].append(np.asarray(indices, dtype=int))
+    return sources
+
+
+def cluster_resamples(rows, samples, seed):
+    """Index arrays of `samples` bootstrap resamples of `rows`: within each source, (source, group) clusters drawn with
+    replacement, so sibling questions and variants of one record move together. The resampling unit of every bootstrap
+    here (paired_bootstrap, cross_validated_temperature) and of the research scripts that bootstrap their own statistic."""
+    sources, rng = _source_groups(rows), np.random.default_rng(seed)
+    for _ in range(samples):
+        yield np.concatenate([grouped[i] for grouped in sources.values() for i in rng.integers(0, len(grouped), size=len(grouped))])
+
+
+def grouped_folds(rows, folds, seed):
+    """Fold id per row, disjoint in (source, group) so sibling questions and variants of one record never straddle
+    train/test, assigned round-robin within each source so every fold sees every source."""
+    units = sorted({(row["source"], row["group"]) for row in rows})
+    if len(units) < folds:
+        raise ValueError("fewer distinct (source, group) units than folds")
+    rng = np.random.default_rng(seed)
+    fold_of, offset = {}, 0
+    for source in sorted({source for source, _ in units}):
+        own = [unit for unit in units if unit[0] == source]
+        for i, j in enumerate(rng.permutation(len(own))):
+            fold_of[own[j]] = (offset + i) % folds
+        offset += len(own)
+    return np.asarray([fold_of[(row["source"], row["group"])] for row in rows])
+
+
+def out_of_fold_rows(rows, folds=5, seed=0, **fit_kwargs):
+    """Each fold's temperature is fit on the other folds (fit_temperature(**fit_kwargs), raw rows required) and applied
+    to its held-out rows, so the result scores like any calibrated prediction. Returns (scored rows in input order,
+    the per-fold temperatures)."""
+    if folds < 2:
+        raise ValueError("folds must be >= 2")
+    clean = scored_rows(rows)
+    fold_id = grouped_folds(clean, folds, seed)
+    temperatures, oof = [], list(clean)
+    for fold in range(folds):
+        held = fold_id == fold
+        temperature = fit_temperature([row for row, out in zip(clean, held) if not out], **fit_kwargs)
+        temperatures.append(temperature)
+        for i in np.flatnonzero(held):
+            oof[i] = tempered_row(clean[i], temperature)
+    return oof, temperatures
+
+
+def cross_validated_temperature(rows, folds=5, seed=0, samples=1000, **fit_kwargs):
+    """Out-of-fold calibration report: out_of_fold_rows scored against the raw rows. The bootstrap resamples
+    source-stratified groups (the unit paired_bootstrap uses) and reports raw vs out-of-fold ECE with a 95% interval on
+    the paired delta; `separated` is True when that interval excludes zero."""
+    if samples < 1:
+        raise ValueError("samples must be >= 1")
+    clean = scored_rows(rows)
+    oof, temperatures = out_of_fold_rows(clean, folds, seed, **fit_kwargs)
+    keys = ("n", "ece", "brier", "nll", "confident_error_rate", "coverage_at_5pct_error")
+    raw, out_of_fold = metrics(clean), metrics(oof)
+    correct = np.asarray([np.argmax(row["p"]) == row["label"] for row in clean])   # argmax is temperature-invariant
+    conf = np.asarray([[max(row["p"]) for row in rows] for rows in (clean, oof)])
+    sources = _source_groups(clean)
+    values = np.asarray([[ece(conf[0][drawn], correct[drawn]), ece(conf[1][drawn], correct[drawn])] for drawn in cluster_resamples(clean, samples, seed)])
+    ci = {name: np.quantile(v, [0.025, 0.975]).tolist() for name, v in (("raw", values[:, 0]), ("out_of_fold", values[:, 1]), ("delta", values[:, 1] - values[:, 0]))}
+    return {"raw": {k: raw[k] for k in keys}, "out_of_fold": {k: out_of_fold[k] for k in keys}, "ece_ci95": ci,
+            "separated": bool(ci["delta"][1] < 0 or ci["delta"][0] > 0), "temperatures": temperatures,
+            "folds": folds, "seed": seed, "samples": samples, "groups": sum(len(v) for v in sources.values()),
+            "unit": "source-stratified (source, group); sibling questions and variants stay together"}
+
+
+def paired_bootstrap(candidate, reference, samples=1000, seed=0, metric="nll", aggregation="macro"):
+    nonlinear = {"coverage_at_5pct_error", "coverage_at_1pct_error", "aurc", "ece"}   # recomputed on every resample
+    additive = metric not in nonlinear                                                  # else a mean of _row_scores
+    if (additive and metric not in _row_scores({"p": [1.0], "label": 0})) or aggregation not in ("micro", "macro") or samples < 1:
+        raise ValueError("unsupported bootstrap metric, aggregation, or sample count")
+
+    def index(rows):
+        out = {}
+        for r in scored_rows(rows):
+            key = r["id"], r["question"]
+            if key in out:
+                raise ValueError("duplicate paired example")
+            out[key] = r
+        return out
+
+    a, b = index(candidate), index(reference)
+    if not a or a.keys() != b.keys():
+        raise ValueError("paired comparison requires identical complete clean examples")
+    keys = sorted(a)
+    for key in keys:
+        row, other = a[key], b[key]
+        if row["keys"] != other["keys"] or row["label"] != other["label"]:
+            raise ValueError("paired comparison labels or option order differ")
+        if any(row[field] != other[field] for field in ("source", "group", "task", "type")):
+            raise ValueError("paired comparison group or task metadata differ")
+    sources = _source_groups([a[key] for key in keys])
+    task_names = np.asarray([a[key]["task"] for key in keys])
+    statistics = []
+    for indexed in (a, b):
+        rows = [indexed[key] for key in keys]
+        conf = np.asarray([max(r["p"]) for r in rows])
+        correct = np.asarray([np.argmax(r["p"]) == r["label"] for r in rows])
+        values = np.asarray([_row_scores(r)[metric] for r in rows]) if additive else None
+        statistics.append((conf, correct, values))
+
+    def statistic(indices, data):
+        conf, correct, values = data
+        if values is not None:
+            return float(values[indices].mean())
+        conf, correct = conf[indices], correct[indices]
+        if metric == "aurc":
+            return area_under_risk_coverage(conf, correct)
+        if metric == "ece":
+            return ece(conf, correct)
+        return coverage_at_error(conf, correct, 0.05 if metric == "coverage_at_5pct_error" else 0.01)
+
+    def delta(indices):
+        parts = [indices] if aggregation == "micro" else [indices[task_names[indices] == t] for t in np.unique(task_names[indices])]
+        return float(np.mean([statistic(part, statistics[0]) - statistic(part, statistics[1]) for part in parts]))
+
+    values = [delta(drawn) for drawn in cluster_resamples([a[key] for key in keys], samples, seed)]
+    return {f"{aggregation}_{metric}_delta": delta(np.arange(len(keys))), "ci95": np.quantile(values, [0.025, 0.975]).tolist(),
+            "samples": samples, "groups": sum(len(v) for v in sources.values()), "aggregation": aggregation,
+            "unit": "source-stratified original record; sibling questions stay together",
+            "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"}

runtime/kev/mirror.py

@@ -0,0 +1,84 @@
+"""A private Hugging Face mirror of full-weight checkpoints: long-term storage for the snapshots (kev.full_ft.SnapshotWriter)
+and final checkpoints that full-weight trials keep on the Modal runs volume. The volume copy stays primary: reads and
+benchmarks load /runs/... paths, and a failed upload only logs (loudly) and never fails training.
+
+    mirror("/runs/r19-27b/00-trial-0/snapshots/step-0000389/checkpoint", "jaredpalmer/kev-snapshots")
+    -> the directory's files at <study>/<trial>/step-0000389/ in the repo (a final checkpoint: <study>/<trial>/final/)
+
+Refuses a repo that is not private (it creates a missing one private). The upload commit is recorded next to the files
+on the volume: in a snapshot's snapshot.json under "hub", in a final checkpoint's hub.json. modal_app.run_mirror runs
+this in its own CPU container with the HF token of the Modal secret it mounts (HF_TOKEN in the environment; never printed).
+"""
+import datetime
+import re
+from pathlib import Path
+
+from .full_ft import SNAPSHOT_INFO
+from .suite import read_json, write_json
+
+DEFAULT_REPO = "jaredpalmer/kev-snapshots"
+RECORD = "hub.json"   # a final checkpoint's upload record (a snapshot's goes into its snapshot.json)
+IGNORED = ["resume/*", ".*", RECORD]   # resume points are optimizer state; dotfiles are partial writes
+ATTEMPTS = 2          # an upload is tried once more after a failure
+
+
+def destination(path, root="/runs"):
+    """Where a checkpoint directory goes in the repo: <study>/<trial>/<step-N> for a snapshot, <...>/final otherwise."""
+    rel = Path(path).relative_to(root).as_posix()
+    if match := re.fullmatch(r"(.+)/snapshots/(step-\d+)/checkpoint", rel): return f"{match[1]}/{match[2]}"
+    return rel.removesuffix("/checkpoint") + "/final"
+
+
+def is_snapshot(path):
+    return (Path(path) / SNAPSHOT_INFO).exists()
+
+
+def recorded(path):
+    """The upload record a checkpoint directory carries, or None."""
+    path = Path(path)
+    if is_snapshot(path): return read_json(path / SNAPSHOT_INFO).get("hub")
+    return read_json(path / RECORD) if (path / RECORD).exists() else None
+
+
+def record(path, entry):
+    path = Path(path)
+    if is_snapshot(path): write_json(path / SNAPSHOT_INFO, {**read_json(path / SNAPSHOT_INFO), "hub": entry}, atomic=True)   # stays complete throughout
+    else: write_json(path / RECORD, entry, atomic=True)
+
+
+def ensure_private(api, repo):
+    """Create `repo` private if it is missing; refuse (PermissionError) if it exists and is not private."""
+    api.create_repo(repo, repo_type="model", private=True, exist_ok=True)
+    if not api.repo_info(repo, repo_type="model").private:
+        raise PermissionError(f"{repo} is not a private repo; checkpoints are only mirrored to private repos")
+
+
+def mirror(path, repo=DEFAULT_REPO, api=None, root="/runs", force=False, log=print):
+    """Upload one complete checkpoint directory (a snapshot with its snapshot.json, or a checkpoint with head.pt) to the
+    private `repo` and record the commit. -> the record, or None when it was not uploaded (every failure is logged loudly
+    and nothing is raised: the volume copy is primary). Skips a directory already recorded for this repo unless `force`."""
+    path = Path(path)
+    if not (path / "head.pt").exists() or (path.parent.parent.name == "snapshots" and not is_snapshot(path)):
+        log(f"!!! mirror: {path} is not a complete checkpoint (no head.pt, or a snapshot without {SNAPSHOT_INFO}); not uploaded"); return None
+    if not force and (previous := recorded(path)) and previous.get("repo") == repo:
+        log(f"mirror: {path} is at {repo}/{previous['path']} already (commit {previous['commit']})"); return previous
+    if api is None:
+        from huggingface_hub import HfApi
+        api = HfApi()   # HF_TOKEN from the environment (the Modal secret)
+    dest = destination(path, root)
+    for attempt in range(1, ATTEMPTS + 1):
+        try:
+            ensure_private(api, repo)
+            info = api.upload_folder(repo_id=repo, repo_type="model", folder_path=str(path), path_in_repo=dest,
+                                     ignore_patterns=IGNORED, commit_message=f"mirror {dest}")
+        except PermissionError as error:   # not a transient failure: never retried, never uploaded
+            log(f"!!! mirror of {path} refused: {error}"); return None
+        except Exception as error:   # noqa: BLE001 - loud, retried once, never raised
+            log(f"!!! mirror of {path} to {repo} failed (attempt {attempt}/{ATTEMPTS}): {type(error).__name__}: {str(error)[:300]}")
+            continue
+        entry = {"repo": repo, "path": dest, "commit": info.oid, "uploaded_at": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")}
+        record(path, entry)
+        log(f"mirror: {path} -> {repo}/{dest} (commit {info.oid})")
+        return entry
+    log(f"!!! mirror of {path} to {repo} gave up after {ATTEMPTS} attempts; the volume copy is unaffected (retry: modal_app.py::mirror_snapshots)")
+    return None

runtime/kev/mlx_model.py

@@ -0,0 +1,229 @@
+"""Apple Silicon backend for the Qwen3.5 checkpoints: mlx-lm's Metal implementation of the hybrid backbone under Kev's
+own encoder and pointer head.
+
+MPS has no Gated DeltaNet kernels, so the PyTorch path runs reference code there (Kev-4B ~0.8 s per request). This module
+runs the same computation on Metal through mlx-lm and keeps everything Kev-specific unchanged: `kev.model.encode` builds the
+tokens, `rows_of` splits them into one causal row per question (the hybrid form the torch path uses too), and the readout is
+the very same `PointerHead` (fp32, with the checkpoint's temperature) applied to the branch hidden states.
+
+Same contract as `DecisionModel` for serving and scoring: encode / forward / probs / probs_and_prefix / probs_with_prefix /
+head / dtype. Selected by `LoadOptions(backend="mlx")` (or "auto" on Apple Silicon) in `kev.checkpoint`; never by the
+benchmark, whose reported numbers stay on the fp32 torch path. Parity against that path is measured in
+tests/test_mlx.py (max |dp| and argmax flips on development records, prefix vs full pass, one question vs several).
+
+Two ways in, both through kev.checkpoint: a LoRA checkpoint is its base's mlx-lm model (`load_base`) with the adapter
+folded in (`merge_lora`); a full-weight checkpoint (kev.train --full_ft, scripts/merge_lora_checkpoint.py) is built from its
+own config.json and shards (`load_full`), nothing merged, so the only resident copy is the weights themselves.
+"""
+import json
+from pathlib import Path
+
+import mlx.core as mx
+import numpy as np
+import torch
+import torch.nn.functional as F
+from mlx.utils import tree_flatten
+from mlx_lm.models import qwen3_5
+from mlx_lm.models.cache import make_prompt_cache
+from mlx_lm.utils import load_model
+
+from .model import PointerHead, encode, probs_one, rows_of, rows_per_pass
+
+CACHE_LIMIT = 1 << 30   # MLX's buffer cache keeps a buffer per new request shape; Kev-4B on an M5, 50 requests: 1.0 GB cached vs 3.7 GB unbounded, same latency
+MLX_DTYPES = {"bfloat16": mx.bfloat16, "float32": mx.float32}   # config.json's dtype names (kev.checkpoint.Checkpoint.saved_dtype) -> MLX
+FULL_MODEL_TYPE = "qwen3_5_text"   # what save_pretrained of transformers' Qwen3_5TextModel writes (Qwen3.5 and Qwen3.8 alike)
+FULL_PREFIX = "language_model.model."   # the text backbone inside mlx-lm's qwen3_5.Model; a full checkpoint's names carry no prefix
+PREFILL_CHUNK = 1024    # state tokens per prefix pass (MLXDecisionModel.prefix); on an M5 1,024 / 2,048 / 4,096 / one pass take the same time, the peak grows with the chunk
+
+
+def load_base(base_dir):
+    """mlx-lm's model of a Hub base snapshot, weights as stored (bf16 for the Qwen3.5 bases); the LoRA path merges into it."""
+    mx.set_cache_limit(CACHE_LIMIT)
+    return load_model(Path(base_dir))[0]
+
+
+def load_full(ckpt_dir, shards, dtype):
+    """mlx-lm's Qwen3.5 model holding a full-weight checkpoint's backbone: the architecture from the checkpoint's own
+    config.json (the text model's config, model_type qwen3_5_text, which mlx-lm's qwen3_5.ModelArgs takes as its
+    text_config), every tensor from `shards` in `dtype` (config.json's name: "bfloat16" for every kev.train --full_ft run).
+    Nothing is merged or cast: the model holds exactly the saved values after mlx-lm's own layout rules for a
+    transformers export (qwen3_5 sanitize: the DeltaNet conv kernel moved to MLX's axis order and the zero-centred RMSNorm
+    weights stored as 1 + w, as for the Hub bases the LoRA path loads). Strict: every name and shape must match the model
+    exactly and every tensor must be `dtype`, else ValueError. The LM head is removed before any weight is materialized:
+    Kev reads hidden states, and the checkpoint carries none (vision tower and MTP were never loaded either)."""
+    ckpt_dir = Path(ckpt_dir)
+    config = json.loads((ckpt_dir / "config.json").read_text(encoding="utf-8"))
+    if config.get("model_type") != FULL_MODEL_TYPE or "text_config" in config:
+        raise ValueError(f"{ckpt_dir}: the MLX backend loads full-weight checkpoints of transformers' Qwen3_5TextModel "
+                         f"(model_type {FULL_MODEL_TYPE!r}); config.json has model_type {config.get('model_type')!r}")
+    if dtype not in MLX_DTYPES: raise ValueError(f"{ckpt_dir}: no MLX dtype for weights stored as {dtype!r}")
+    want = MLX_DTYPES[dtype]
+    args = qwen3_5.ModelArgs.from_dict(config)
+    text = qwen3_5.TextModelArgs.from_dict(args.text_config)
+    pattern = ["full_attention" if (i + 1) % text.full_attention_interval == 0 else "linear_attention" for i in range(text.num_hidden_layers)]
+    if config.get("layer_types", pattern) != pattern:   # mlx-lm places attention layers by full_attention_interval alone
+        raise ValueError(f"{ckpt_dir}: config.json's layer_types are not every {text.full_attention_interval}th layer full attention, "
+                         f"the only layout mlx-lm's qwen3_5 builds")
+    mx.set_cache_limit(CACHE_LIMIT)
+    lm = qwen3_5.Model(args)    # parameters are lazy: nothing is allocated until evaluated
+    if "lm_head" in lm.language_model: del lm.language_model["lm_head"]   # untied bases (Qwen3.8-27B): 2.5 GB never read
+    weights = {}
+    for shard in shards:
+        for name, value in mx.load(str(shard)).items():
+            if FULL_PREFIX + name in weights: raise ValueError(f"{ckpt_dir}: tensor {name} is in more than one shard")
+            weights[FULL_PREFIX + name] = value
+    conv = [k for k in weights if k.endswith("linear_attn.conv1d.weight")]
+    if not conv or any(weights[k].ndim != 3 or weights[k].shape[1] != 1 for k in conv):
+        # sanitize infers "transformers layout, shift the norms" from the conv kernels; anything else would load wrong norms silently
+        raise ValueError(f"{ckpt_dir}: expected transformers' DeltaNet conv kernels [channels, 1, width]; this is not a kev.train --full_ft export")
+    weights = lm.language_model.sanitize(weights)
+    expected = {k: v.shape for k, v in tree_flatten(lm.parameters())}
+    missing, unexpected = sorted(set(expected) - set(weights)), sorted(set(weights) - set(expected))
+    wrong = sorted(k for k in set(expected) & set(weights) if tuple(weights[k].shape) != tuple(expected[k]))
+    cast = sorted(k for k, v in weights.items() if v.dtype != want)
+    if missing or unexpected or wrong or cast:
+        strip = lambda names: [n.removeprefix(FULL_PREFIX) for n in names[:2]]
+        raise ValueError(f"{ckpt_dir} does not match mlx-lm's Qwen3.5 text model: {len(missing)} tensors missing (e.g. {strip(missing)}), "
+                         f"{len(unexpected)} unexpected (e.g. {strip(unexpected)}), {len(wrong)} with another shape (e.g. {strip(wrong)}), "
+                         f"{len(cast)} not {dtype} (e.g. {strip(cast)})")
+    lm.load_weights(list(weights.items()), strict=True)
+    del weights
+    lm.eval()
+    mx.eval(lm.parameters())
+    mx.clear_cache()
+    return lm
+
+
+def merge_lora(lm, adapter_dir, scale=1.0):
+    """Fold a PEFT adapter into the mlx-lm model's weights the way the torch path does: W + (B @ A) * alpha / r in fp32,
+    rounded once to the backbone dtype. `scale` is LoadOptions.lora_scale (WiSE-FT interpolation). Returns the tensor count."""
+    adapter_dir = Path(adapter_dir)
+    cfg = json.loads((adapter_dir / "adapter_config.json").read_text(encoding="utf-8"))
+    if cfg.get("trainable_token_indices"):
+        raise ValueError("the MLX backend does not carry trained token embeddings (special_embeddings checkpoints); use backend=torch")
+    alpha = cfg["lora_alpha"] / (cfg["r"] ** 0.5 if cfg.get("use_rslora") else cfg["r"])
+    weights = mx.load(str(adapter_dir / "adapter_model.safetensors"))
+    params = dict(tree_flatten(lm.parameters()))
+    merged = {}
+    with mx.stream(mx.cpu):   # the GPU's fp32 matmul is a reduced-precision fast path (~1e-3 relative on an M5); the merge is one-time and must be exact
+        for name, a in weights.items():
+            if not name.endswith(".lora_A.weight"):
+                continue
+            stem = name[: -len(".lora_A.weight")]
+            # peft names the wrapped text model `base_model.model.<layers...>`; mlx-lm nests it as `language_model.model.<layers...>`
+            target = stem.replace("base_model.model.", "language_model.model.", 1) + ".weight"
+            if target not in params:
+                raise ValueError(f"adapter tensor {stem} has no weight in the mlx-lm model (looked for {target})")
+            base = params[target]
+            delta = (weights[stem + ".lora_B.weight"].astype(mx.float32) @ a.astype(mx.float32)) * (alpha * scale)
+            merged[target] = (base.astype(mx.float32) + delta).astype(base.dtype)
+            mx.eval(merged[target])   # one tensor at a time: one graph over every target peaks at ~2.9x the base weights and leaves ~2x of them in MLX's buffer cache (25.8 GB RSS for the 4B)
+    lm.load_weights(list(merged.items()), strict=False)
+    mx.eval(lm.parameters())
+    del params, weights   # drop the pre-merge weights and the adapter before clearing the cache, or ~7 GB stay cached
+    mx.clear_cache()   # hand the merge transients back to the OS; MLX keeps freed buffers otherwise
+    return len(merged)
+
+
+class MLXDecisionModel:
+    """Prefill-only scorer: hidden states from mlx-lm, logits from the shared torch PointerHead."""
+    backend, device, hybrid, option_isolation = "mlx", "mlx", True, False
+    prefix_min_tokens = 0   # kev.serve caches the state prefix for every request: on Metal the branch-only pass is always the cheaper one
+
+    def __init__(self, lm, pad_id, head_dim=256):
+        self.lm = lm                                                  # mlx-lm qwen3_5.Model from load_base (+ merge_lora) or load_full
+        self.text = self.lm.language_model.model                      # Qwen3_5TextModel: embeddings -> layers -> final norm = `.model.last_hidden_state`
+        self.pad_id = pad_id
+        self.head = PointerHead(getattr(self.text.embed_tokens, "dims", self.text.embed_tokens.weight.shape[1]), dp=head_dim).eval()
+
+    @property
+    def dtype(self):
+        return str(getattr(self.text.embed_tokens, "activation_dtype", self.text.embed_tokens.weight.dtype)).removeprefix("mlx.core.")
+
+    def eval(self):
+        self.head.eval(); return self
+
+    def encode(self, tok, rec, **kw):
+        return encode(tok, rec, option_isolation=False, **kw)
+
+    def _hidden(self, rows, cache=None):
+        """[N, L, d] hidden states of right-padded token rows. Pads sit after every real token and both layer kinds are
+        causal (attention: causal mask; DeltaNet: a left-to-right recurrence), so no real token sees a pad."""
+        L = max(len(r) for r in rows)
+        ids = mx.array([r + [self.pad_id] * (L - len(r)) for r in rows], dtype=mx.int32)
+        h = self.text(ids, cache=cache)
+        mx.eval(h)
+        return h
+
+    def _logits(self, h, decide, opts):
+        """One question's logits through the fp32 pointer head (temperature included, eval mode)."""
+        idx = mx.array([decide, *opts], dtype=mx.int32)
+        picked = torch.from_numpy(np.asarray(h[idx].astype(mx.float32)))
+        with torch.no_grad():
+            return self.head(picked[0], picked[1:])
+
+    def forward_rows(self, enc):
+        """Row form, as the torch path computes it: every question is one causal row of state + branch tokens, the state
+        recomputed per row. The reference the prefix form is checked against (tests/test_mlx.py); serving uses `forward`."""
+        S, _, rows = rows_of(enc)
+        chunk, out = rows_per_pass([S + r["ids"] for r in rows]), []
+        for start in range(0, len(rows), chunk):
+            part = rows[start:start + chunk]
+            h = self._hidden([S + r["ids"] for r in part])
+            out += [self._logits(h[i], len(S) + r["decide"], [len(S) + o for o in r["opts"]]) for i, r in enumerate(part)]
+        return out
+
+    # --- state prefix: the state runs once into an mlx-lm prompt cache (KV for the attention layers, conv + recurrent state
+    # for the DeltaNet layers); the branches run as one batch on a replicated copy, so the prefix stays pristine and can be
+    # reused by the next request with the same state. On Metal this is also the cheapest way to answer a single request
+    # (state once instead of once per question), so it is the only path `forward` / `probs` take.
+
+    def prefix(self, enc):
+        """The state into a fresh prompt cache, PREFILL_CHUNK tokens per pass, each pass's cache evaluated before the next
+        (mlx-lm's own prefill does the same; the hidden states a pass returns are never read). One pass over the whole
+        state kept every layer's activations of every token alive at once and left the DeltaNet conv states lazy (each
+        holding its layer's whole [Ls, conv_dim] input): Kev-4B, 8,192 tokens, 4.6 GB above the weights instead of 1.2 GB,
+        the same 6.5 s. Exact: a pass continues the cached keys/values (rotary offset = cache offset), conv window and
+        fp32 recurrent state, so chunked and one-pass differ by float reassociation only (fp32 on the CPU: max |dp| 1e-6;
+        bf16 on Metal: up to ~0.01, like any change of kernel shapes; tests/test_mlx.py, runs/mlx-long-states/prefill-ab-*)."""
+        Ls = enc["seg"].count(0)
+        cache, ids = make_prompt_cache(self.lm), enc["ids"][:Ls]
+        for start in range(0, Ls, PREFILL_CHUNK):
+            self.text(mx.array([ids[start:start + PREFILL_CHUNK]], dtype=mx.int32), cache=cache)
+            mx.eval([c.state for c in cache])
+        return Ls, cache
+
+    def _branch_logits(self, enc, cache):
+        """Branches as rows on a replicated copy of the state cache, rows_per_pass rows (and cache copies) at a time."""
+        _, _, rows = rows_of(enc)
+        chunk, out = rows_per_pass([r["ids"] for r in rows], enc["seg"].count(0)), []
+        for start in range(0, len(rows), chunk):
+            part = rows[start:start + chunk]
+            batch = [type(c).merge([c] * len(part)) for c in cache]      # merge copies the arrays: `cache` is not mutated
+            h = self._hidden([r["ids"] for r in part], batch)
+            out += [self._logits(h[i], r["decide"], r["opts"]) for i, r in enumerate(part)]
+        return out
+
+    def _branch_probs(self, enc, cache):
+        return [F.softmax(z, -1) for z in self._branch_logits(enc, cache)]
+
+    def forward(self, enc):
+        """List of logits tensors, one per question."""
+        return self._branch_logits(enc, self.prefix(enc)[1])
+
+    def probs(self, enc):
+        return self._branch_probs(enc, self.prefix(enc)[1])
+
+    def probs_and_prefix(self, enc):
+        prefix = self.prefix(enc)
+        return self._branch_probs(enc, prefix[1]), prefix
+
+    def probs_with_prefix(self, enc, prefix):
+        Ls, cache = prefix
+        if enc["seg"].count(0) != Ls: raise ValueError("prefix does not match this record's state")
+        return self._branch_probs(enc, cache)
+
+    def probs_batch(self, encs, prefixes, keep):
+        """kev.serve's batch call: one request at a time on Metal."""
+        out = [probs_one(self, e, p, k) for e, p, k in zip(encs, prefixes, keep)]
+        return [o[0] for o in out], [o[1] for o in out]

runtime/kev/model.py

@@ -0,0 +1,508 @@
+"""Decision model: causal LM backbone + block-causal branch mask + pointer readout."""
+import copy, math, os, re
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, DynamicCache
+from transformers.cache_utils import LinearAttentionCacheLayerMixin
+
+# Reuse existing rarely-used Qwen special tokens as delimiters (state, q, opt, /opt, decide) so no
+# embedding rows need to be added/trained; LoRA adapts their meaning.
+SPECIAL = ["<|fim_prefix|>", "<|fim_middle|>", "<|box_start|>", "<|box_end|>", "<|fim_suffix|>"]
+# training context: state tokens, tokens per question branch, and the whole packed record. Frozen suites are admitted with
+# this rule (kev.suite) and training applies it to records built on the fly, so train and eval see the same population.
+MAX_STATE, MAX_BRANCH, MAX_PACKED = 384, 1024, 2048
+# serving context (kev.serve): a state of up to 64k tokens (twice Jev's 32k; the Qwen3.5 / Qwen3.8 bases' window is 262k)
+# and a question row (state + its branch, the encoder's max_branch) of up to 8k tokens more. Until 64k states these were
+# 8,192 / 8,192 (SERVE_MAX_*_8K below).
+SERVE_MAX_STATE = 65536
+SERVE_MAX_BRANCH = SERVE_MAX_STATE + 8192
+SERVE_MAX_PACKED = SERVE_MAX_STATE + SERVE_MAX_BRANCH
+# tokens one inference pass holds (rows_per_pass), and the longest request an attention-only backbone runs packed
+# (rows_form: its block-causal mask is L x L); longer ones run as rows
+ROW_PASS_TOKENS = 16384
+# the longest state a checkpoint may be trained on (kev.train --max_state): the longest one served, which still leaves
+# every question its training branch budget (training_context(MAX_TRAIN_STATE)["max_branch"] <= SERVE_MAX_BRANCH)
+MAX_TRAIN_STATE = SERVE_MAX_STATE
+# the limits before 64k states: the suites frozen until then were admitted under these and record them (kev.suite.SERVING_CONTEXT_8K)
+SERVE_MAX_STATE_8K, SERVE_MAX_BRANCH_8K, MAX_TRAIN_STATE_8K = 8192, 8192, 7552
+
+
+def training_context(max_state=MAX_STATE):
+    """The encoder limits for training with the state limit lifted to `max_state`: the row (state + one branch) and
+    packed limits grow by the same amount, so every question keeps its token budget. training_context() is the default
+    training context (kev.suite.CONTEXT without `truncate`)."""
+    if not MAX_STATE <= max_state <= MAX_TRAIN_STATE:
+        raise ValueError(f"max_state must be in [{MAX_STATE}, {MAX_TRAIN_STATE}]")
+    extra = max_state - MAX_STATE
+    return {"max_state": max_state, "max_branch": MAX_BRANCH + extra, "max_packed": MAX_PACKED + extra}
+
+
+def rows_per_pass(rows, prefix_len=0, budget=ROW_PASS_TOKENS):
+    """How many causal rows one inference forward pass takes: as many as fit `budget` tokens counting the cached state
+    each row carries (prefix_len) plus its own tokens, at least one. Memory per pass is bounded by one maximal row however
+    many questions a request has, and the rows are independent, so the answers do not depend on the split. Kev-4B on MLX,
+    a 4.8k-token state with 64 questions: 24.7 GB peak in one pass, 8.9 GB one row at a time (and faster: the cost was
+    64 copies of the state cache)."""
+    return max(1, budget // (prefix_len + max(len(r) for r in rows)))
+
+
+class ContextOverflow(ValueError):
+    """A record does not encode within its context (state, branch or packed limit). Serving turns it into a 422; the
+    benchmark counts it as a rejected record for suites scored as published (skip_overlong). When the state is what
+    overflows, state_tokens (its count, the <state> token included) and max_state (the limit) say by how much."""
+
+    def __init__(self, message, state_tokens=None, max_state=None):
+        super().__init__(message)
+        self.state_tokens, self.max_state = state_tokens, max_state
+
+
+def load_tokenizer(name, revision=None):
+    return AutoTokenizer.from_pretrained(name, revision=revision)
+
+
+def pad_id(tok):
+    """The id used to right-pad token rows (never attended to); Qwen tokenizers define one, others fall back to 0."""
+    return tok.pad_token_id if tok.pad_token_id is not None else 0
+
+
+def is_hybrid(config):
+    """Whether a (text) config has Gated DeltaNet layers (Qwen3.5). Such backbones cannot honour the block-causal mask and
+    run the row form; on Apple Silicon they are what the MLX backend is for."""
+    return "linear_attention" in set(getattr(config, "layer_types", None) or [])
+
+
+_SPECIAL_RE = re.compile(r"<\|([A-Za-z0-9_]+)\|>")
+
+
+def user_tokens(tok, text):
+    """Tokenize caller-supplied text so it can never produce delimiter/control tokens (option boundaries are unforgeable).
+    The fast tokenizer ignores split_special_tokens, so `<|name|>` is rewritten to `<¦name¦>` before tokenizing."""
+    return tok(_SPECIAL_RE.sub(r"<¦\1¦>", text), add_special_tokens=False).input_ids
+
+
+OPT_NONE, OPT_DECIDE = -1, -2   # values of enc["opt"]: instruction/state tokens, and the <decide> token
+
+
+def encode(tok, rec, max_state=MAX_STATE, max_branch=MAX_BRANCH, strict=False, option_isolation=False):
+    """Pack one record: [<state> ...] then per-question [<q> instr <opt> o </opt>... <decide>].
+
+    Returns ids, seg (0 = state, k = question k), pos (branch positions restart after state),
+    decide_idx [Q], opt_idx [Q][K] (index of </opt> token for each option), opt (per-token option index within its
+    question: OPT_NONE for state/instruction, 0..K-1 for option spans, OPT_DECIDE for <decide>).
+
+    option_isolation=True: every option span is its own sub-branch (it sees state + instruction + itself only), all
+    option spans share the same position ids, and <decide> sits at one fixed position after the longest span. Then the
+    per-option representations and <decide>'s attention over them are permutation-invariant by construction.
+
+    Not strict, a state over max_state tokens is cut to its first max_state (state_truncated; state_tokens is the whole
+    state's count, the <state> token included, as the limit counts it). Strict, it raises ContextOverflow.
+    """
+    state_tokens = user_tokens(tok, rec["state"])
+    if strict and len(state_tokens) + 1 > max_state:
+        raise ContextOverflow(f"state exceeds {max_state} tokens: {len(state_tokens) + 1}", state_tokens=len(state_tokens) + 1, max_state=max_state)
+    S = [tok.convert_tokens_to_ids(SPECIAL[0])] + state_tokens[: max_state - 1]
+    ids, seg, pos, opt = list(S), [0] * len(S), list(range(len(S))), [OPT_NONE] * len(S)
+    q_id, o_id, c_id, d_id = (tok.convert_tokens_to_ids(t) for t in SPECIAL[1:])
+    decide_idx, opt_idx = [], []
+    for k, q in enumerate(rec["questions"], start=1):
+        instr = [q_id] + user_tokens(tok, q["instr"])
+        spans = [[o_id] + user_tokens(tok, o) + [c_id] for o in q["options"]]
+        br = instr + [t for sp in spans for t in sp] + [d_id]
+        if len(br) > max_branch - len(S):
+            raise ContextOverflow(f"branch too long: {len(br)} tokens with a {len(S)}-token state (row limit {max_branch})")
+        base = len(ids); p0 = len(S)
+        br_opt = [OPT_NONE] * len(instr) + [j for j, sp in enumerate(spans) for _ in sp] + [OPT_DECIDE]
+        if option_isolation:
+            longest = max(len(sp) for sp in spans)
+            br_pos = list(range(p0, p0 + len(instr))) + [p0 + len(instr) + i for sp in spans for i in range(len(sp))] + [p0 + len(instr) + longest]
+        else:
+            br_pos = list(range(p0, p0 + len(br)))
+        ends, cursor = [], len(instr)
+        for sp in spans:
+            cursor += len(sp); ends.append(cursor - 1)
+        ids += br; seg += [k] * len(br); pos += br_pos; opt += br_opt
+        decide_idx.append(base + len(br) - 1); opt_idx.append([base + e for e in ends])
+    return {"ids": ids, "seg": seg, "pos": pos, "opt": opt, "option_isolation": option_isolation, "decide_idx": decide_idx, "opt_idx": opt_idx,
+            "labels": [q["label"] for q in rec["questions"]], "state_tokens": len(state_tokens) + 1, "state_truncated": len(state_tokens) + 1 > max_state}
+
+
+def admit(model, tok, rec, truncate=False):
+    """The serving admission check (kev.serve and the Space; the torch and MLX models both encode through encode above):
+    -> rec encoded within the serving context. A state over SERVE_MAX_STATE tokens (the <state> token included) raises
+    ContextOverflow saying how long it is and how to fix it, unless truncate=True: then its first SERVE_MAX_STATE tokens
+    are read and the encoding says so (state_truncated, state_tokens). A question whose row (state + its branch) is over
+    SERVE_MAX_BRANCH raises ContextOverflow either way. Benchmarks do not truncate either: kev.predictors.LocalPredictor
+    encodes strictly within its suite's context."""
+    try:
+        return model.encode(tok, rec, max_state=SERVE_MAX_STATE, max_branch=SERVE_MAX_BRANCH, strict=not truncate)
+    except ContextOverflow as e:
+        if e.max_state is None: raise
+        raise ContextOverflow(f"state is {e.state_tokens:,} tokens, over the {e.max_state:,}-token limit (the <state> token included): "
+                              "shorten the document or split it across requests", state_tokens=e.state_tokens, max_state=e.max_state) from None
+
+
+def fits(rec, *tokenizers, max_state=MAX_STATE, max_branch=MAX_BRANCH, max_packed=MAX_PACKED):
+    """True when the internal record encodes strictly (no truncation) within the training context under every tokenizer
+    given (frozen suites are admitted against the tokenizers of all their pinned bases)."""
+    try:
+        return all(len(encode(tok, rec, max_state=max_state, max_branch=max_branch, strict=True)["ids"]) <= max_packed for tok in tokenizers)
+    except ValueError:
+        return False
+
+
+def branch_mask(seg, device, dtype=torch.float32):
+    """attend(i,j) iff j<=i and (seg[j]==0 or seg[j]==seg[i]). Returns additive [1,1,L,L]."""
+    return branch_mask_batch([seg], device, dtype)
+
+
+def branch_mask_batch(segs, device, dtype=torch.float32, opts=None, length=None):
+    """Batched block-causal mask, additive [B,1,L,L], right-padded to the longest sequence.
+
+    Padded key positions are masked for every query; padded query rows keep the diagonal so no row is fully
+    masked (finfo.min, not -inf, so softmax stays finite either way). Real tokens never see pads because pads sit
+    after them (causal) and belong to no segment (-1).
+
+    opts (option isolation): within a question, an option-span token may attend to state, the instruction, and its own
+    span only; <decide> attends to everything in its question. Instruction tokens never see option spans (causal)."""
+    L = max(max(len(s) for s in segs), length or 0)
+    s = torch.full((len(segs), L), -1, device=device)
+    for b, seg in enumerate(segs):
+        s[b, : len(seg)] = torch.tensor(seg, device=device)
+    causal = torch.tril(torch.ones(L, L, dtype=torch.bool, device=device))
+    same = (s[:, None, :] == s[:, :, None]) | (s[:, None, :] == 0)
+    valid_key = (s != -1)[:, None, :]
+    allow = causal[None] & same & valid_key
+    if opts is not None:
+        o = torch.full((len(segs), L), OPT_NONE, device=device)
+        for b, op in enumerate(opts):
+            o[b, : len(op)] = torch.tensor(op, device=device)
+        key_is_option = (o[:, None, :] >= 0)
+        query_is_decide = (o[:, :, None] == OPT_DECIDE)
+        same_option = o[:, None, :] == o[:, :, None]
+        allow = allow & (~key_is_option | query_is_decide | same_option)
+    allow = allow | torch.eye(L, dtype=torch.bool, device=device)[None]
+    return torch.zeros(len(segs), L, L, dtype=dtype, device=device).masked_fill(~allow, torch.finfo(dtype).min)[:, None]
+
+
+def rows_of(enc):
+    """Split a packed encoding into its state and per-question branch rows.
+
+    Returns (state_ids, state_pos, rows) with rows[k] = {"ids", "pos", "decide", "opts"}: the branch tokens of question
+    k with their (already state-continuing) positions, and the readout offsets *within the branch*. Feeding
+    state + rows[k] as one causal row is equivalent to the packed block-causal form for that question, on any
+    architecture: the row contains exactly the tokens question k may attend to, in the same positions."""
+    seg = enc["seg"]; Ls = seg.count(0)
+    rows, start = [], Ls
+    for k, (d, oi) in enumerate(zip(enc["decide_idx"], enc["opt_idx"]), start=1):
+        end = d + 1                                    # <decide> is the last token of its branch
+        if seg[start] != k or seg[end - 1] != k: raise ValueError("branch layout mismatch")
+        rows.append({"ids": enc["ids"][start:end], "pos": enc["pos"][start:end], "decide": d - start, "opts": [o - start for o in oi]})
+        start = end
+    return enc["ids"][:Ls], enc["pos"][:Ls], rows
+
+
+class PointerHead(nn.Module):
+    def __init__(self, d, dp=256):
+        """dp = pointer dimension (head capacity knob)."""
+        super().__init__()
+        self.q, self.k = nn.Linear(d, dp), nn.Linear(d, dp)
+        self.scale = 1 / math.sqrt(dp)
+        # calibration: logits are divided by this at inference (eval mode) only. 1.0 = raw. A checkpoint carries the value fitted on
+        # its in-distribution development rows (scripts/calibrate_checkpoint.py -> head.pt["temperature"]); training always sees T=1 so
+        # a fitted value stays meaningful, and the argmax is unchanged by construction.
+        self.temperature = 1.0
+
+    def forward(self, h_decide, h_opts):  # [d], [K,d] -> logits [K]
+        z = (self.k(h_opts) @ self.q(h_decide)) * self.scale
+        return z if self.training or self.temperature == 1.0 else z / self.temperature
+
+    def many(self, h_decide, h_opts, owner):  # [Q,d], [sum K,d], question of each option [sum K] -> logits [sum K]
+        """forward() for many questions at once (serving batches): one pass instead of one per question."""
+        z = (self.k(h_opts) * self.q(h_decide)[owner]).sum(-1) * self.scale
+        return z if self.training or self.temperature == 1.0 else z / self.temperature
+
+
+# What a loaded model exposes to kev.serve, kev.predictors and the Space: the scoring interface both DecisionModel (torch)
+# and kev.mlx_model.MLXDecisionModel implement. tests/test_mlx.py checks the MLX class against this list.
+SCORING_INTERFACE = ("encode", "forward", "probs", "probs_and_prefix", "probs_with_prefix", "probs_batch", "eval",
+                     "head", "backend", "dtype", "device", "hybrid", "option_isolation", "prefix_min_tokens")
+
+
+EAGER_STATES = 4   # long new states (past the graphed state pass) whose eager prefixes one batched run holds at once: each
+                   # holds its state's keys, values and DeltaNet states (~0.3 GB for 2,200 tokens on Kev-27B)
+
+
+def probs_one(model, enc, prefix, keep):
+    """-> (probs, prefix to keep) for one request, on any backend: the question rows on the cached prefix on a hit, one
+    pass that also returns the prefix when it is to be kept, the plain pass otherwise."""
+    if prefix is not None: return model.probs_with_prefix(enc, prefix), prefix
+    return model.probs_and_prefix(enc) if keep else (model.probs(enc), None)
+
+
+class DecisionModel(nn.Module):
+    def __init__(self, name, tok, device, lora=None, revision=None, attn=None, head_dim=256, option_isolation=False, special_embeddings=False, lora_targets="all", dtype=torch.float32,
+                 weights=None, direct_load=False):
+        """weights: a full-weight checkpoint directory whose saved backbone replaces the base's (kev.checkpoint's loader rule).
+        direct_load: load the backbone straight onto `device` (transformers device_map) instead of staging it in host memory;
+        full-weight training on several GPUs in one container needs it (N processes x a 51 GB checkpoint otherwise). Off by
+        default: it changes where the rotary buffers are computed, so every other path keeps its bits."""
+        super().__init__()
+        # backbone only (no vocab head): we never generate text.
+        # eager on MPS/CPU (known-good with our float 4D mask); SDPA on CUDA (accepts arbitrary additive masks).
+        attn = attn or ("sdpa" if str(device).startswith("cuda") else "eager")
+        # dtype: fp32 for training and exact evaluation; bf16 is a serving option for large backbones (8B on a 32 GB Mac)
+        load = {"dtype": dtype, "attn_implementation": attn}
+        if direct_load: load["device_map"] = {"": torch.cuda.current_device() if device == "cuda" else device}   # "cuda": under torchrun, this rank's GPU
+        self.lm = AutoModel.from_pretrained(weights, **load) if weights else AutoModelForCausalLM.from_pretrained(name, revision=revision, **load).model
+        self.pad_id = pad_id(tok)
+        # hybrid backbones (Qwen3.5: Gated DeltaNet layers, recurrent) cannot honour the block-causal mask, so every
+        # question runs as its own causal row continuing from the state (rows_of). Attention-only backbones keep the
+        # packed form; the two agree to fp32 noise (tests/test_model.py::test_rows_match_packed).
+        self.hybrid = is_hybrid(self.lm.config)
+        if self.hybrid and option_isolation: raise ValueError("option_isolation needs the packed mask; not available on hybrid backbones")
+        self.option_isolation = option_isolation
+        if lora:
+            from peft import LoraConfig, get_peft_model
+            extra = {"trainable_token_indices": {"embed_tokens": [tok.convert_tokens_to_ids(t) for t in SPECIAL]}} if special_embeddings else {}
+            targets = {"all": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+                       "dense": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],   # "all" minus the DeltaNet projections on hybrids (retention ablation)
+                       "attn": ["q_proj", "k_proj", "v_proj", "o_proj"], "qv": ["q_proj", "v_proj"]}[lora_targets]
+            if self.hybrid and lora_targets in ("all", "attn"):
+                # Gated DeltaNet projections (transformers 5 names, verified on Qwen3_5TextModel); the mixer's out_proj too
+                targets = targets + ["in_proj_qkv", "in_proj_z", "in_proj_a", "in_proj_b", "out_proj"]
+            cfg = LoraConfig(task_type="FEATURE_EXTRACTION", r=lora, lora_alpha=2 * lora, lora_dropout=0.05, target_modules=targets, **extra)
+            self.lm = get_peft_model(self.lm, cfg)
+        self.head = PointerHead(self.lm.config.hidden_size, dp=head_dim)
+        self.device = device
+        self.to(device)
+
+    backend = "torch"           # kev.mlx_model.MLXDecisionModel is the other implementation of this scoring interface
+    graphs = None               # kev.cuda_graphs.CudaGraphs for the serving passes of a hybrid backbone on CUDA (LoadOptions.cuda_graphs)
+
+    @property
+    def prefix_min_tokens(self):
+        """kev.serve caches the state prefix from this many state tokens. Attention-only backbones: 384, below which the
+        branch-only pass is not faster than one packed pass on MPS (per-op overhead). Hybrid backbones: always, because
+        their miss path otherwise recomputes the state once per question (Kev-0.8B bf16 on MPS, 5 questions: 1011 -> 413 ms)."""
+        return 0 if self.hybrid else 384
+
+    @property
+    def dtype(self):
+        return str(next(self.lm.parameters()).dtype).removeprefix("torch.")
+
+    def encode(self, tok, rec, **kw):
+        """encode() with this model's option-isolation setting; use this from serving/eval code."""
+        return encode(tok, rec, option_isolation=self.option_isolation, **kw)
+
+    def hidden(self, enc):
+        return self.hidden_batch([enc])[0, : len(enc["ids"])]
+
+    SHAPE_BUCKET = int(os.environ.get("KEV_SHAPE_BUCKET", "64"))   # MPS: pad the sequence to a multiple of this (per-shape kernel warm-up); 1 disables
+
+    def _pad_rows(self, rows):
+        """Right-pad (ids, pos) token rows into [N, L] id / position tensors and a [N, L] attention mask (1 = real token).
+        Pads sit after every real token and are masked keys, so they never change a real token's hidden state (parity
+        measured exact). On MPS in eval mode L is rounded up to a SHAPE_BUCKET multiple so kernels are warmed per bucket."""
+        L = max(len(ids) for ids, _ in rows)
+        if str(self.device) == "mps" and not self.training: L = -(-L // self.SHAPE_BUCKET) * self.SHAPE_BUCKET
+        ids = torch.full((len(rows), L), self.pad_id, device=self.device)
+        pos = torch.zeros((len(rows), L), dtype=torch.long, device=self.device)
+        att = torch.zeros((len(rows), L), dtype=torch.long, device=self.device)
+        for i, (rid, rpos) in enumerate(rows):
+            ids[i, : len(rid)] = torch.tensor(rid, device=self.device); pos[i, : len(rpos)] = torch.tensor(rpos, device=self.device); att[i, : len(rid)] = 1
+        return ids, pos, att
+
+    def hidden_batch(self, encs):
+        """[B, L_max, d] hidden states for a right-padded batch of encoded records under the packed block-causal mask."""
+        ids, pos, _ = self._pad_rows([(e["ids"], e["pos"]) for e in encs])
+        isolate = any(e.get("option_isolation") for e in encs)
+        if isolate and not all(e.get("option_isolation") for e in encs):
+            raise ValueError("cannot mix option-isolated and plain encodings in one batch")
+        lm_dtype = next(self.lm.parameters()).dtype
+        mask = branch_mask_batch([e["seg"] for e in encs], self.device, dtype=lm_dtype, opts=[e["opt"] for e in encs] if isolate else None, length=ids.shape[1])
+        return self.lm(input_ids=ids, position_ids=pos, attention_mask=mask).last_hidden_state.float()   # head stays fp32
+
+    def _readout(self, h, enc):
+        return [self.head(h[d], h[torch.tensor(oi, device=self.device)]) for d, oi in zip(enc["decide_idx"], enc["opt_idx"])]
+
+    def rows_form(self, encs):
+        """Whether these records run as causal rows: hybrid backbones always (the recurrent layers cannot honour the packed
+        mask), attention-only ones when the packed sequence would exceed ROW_PASS_TOKENS (its L x L mask grows without
+        bound with the number of questions). The two forms agree (tests/test_model.py::test_rows_match_packed)."""
+        return self.hybrid or any(len(e["ids"]) > ROW_PASS_TOKENS for e in encs)
+
+    def _rows_hidden(self, rows, cache=None, prefix_len=0):
+        """Hidden states of causal token rows, one [L_i, d] tensor per row. In eval mode the rows go through the backbone
+        rows_per_pass at a time; training keeps one batch (its batches are small and autograd needs the whole graph anyway).
+        With `cache`, the rows are branches continuing the cached state: in eager mode the cache is replicated once per
+        chunk, leaving the caller's prefix pristine, and the cached tokens are marked real in the attention mask."""
+        chunk = len(rows) if self.training else rows_per_pass([ids for ids, _ in rows], prefix_len)
+        out = []
+        for start in range(0, len(rows), chunk):
+            part = rows[start:start + chunk]
+            ids, pos, att = self._pad_rows(part)
+            past = {}
+            if cache is not None:
+                replica = copy.copy(cache)
+                replica.layers = [copy.copy(layer) for layer in cache.layers]
+                for source, target in zip(cache.layers, replica.layers):
+                    if isinstance(source, LinearAttentionCacheLayerMixin):
+                        target.conv_states = source.conv_states.copy()
+                        target.recurrent_states = source.recurrent_states.copy()
+                        target.is_conv_states_initialized = source.is_conv_states_initialized.copy()
+                        target.is_recurrent_states_initialized = source.is_recurrent_states_initialized.copy()
+                        target.has_previous_state = source.has_previous_state.copy()
+                        target.conv_kernel_size = source.conv_kernel_size.copy()
+                replica.reorder_cache(torch.zeros(len(part), dtype=torch.long, device=self.device))
+                att = torch.cat([torch.ones((len(part), prefix_len), dtype=torch.long, device=self.device), att], 1)
+                past = {"past_key_values": replica, "use_cache": True}
+            h = self.lm(input_ids=ids, position_ids=pos, attention_mask=att, **past).last_hidden_state.float()
+            out += [h[i, : len(row_ids)] for i, (row_ids, _) in enumerate(part)]
+        return out
+
+    def forward_rows_batch(self, encs, shared_prefix=False):
+        """Row form: every question of every record is one causal row = state tokens + its branch tokens. Returns the same
+        nested logits as forward_batch. Exact isolation by construction (rows are independent); the state is recomputed
+        per row (Q x state tokens), which training accepts; serving and probs() use the prefix cache instead. shared_prefix
+        (training, hybrid Qwen3.5): the same rows computed with each state run once and its branches continuing from it,
+        gradients included (kev.shared_prefix)."""
+        if shared_prefix:
+            from .shared_prefix import branch_hidden   # here, not at the top: the HF Space vendors model.py alone
+            splits = [rows_of(e) for e in encs]
+            return [[self.head(h[r["decide"]], h[torch.tensor(r["opts"], device=self.device)]) for h, r in zip(hs, rows)]
+                    for hs, (_, _, rows) in zip(branch_hidden(self.lm, splits, self.pad_id, self.device), splits)]
+        rows, readouts = [], []   # one causal row per question; readouts[i] = (record, <decide> offset, option offsets)
+        for b, e in enumerate(encs):
+            S, Sp, brs = rows_of(e)
+            for r in brs:
+                rows.append((S + r["ids"], Sp + r["pos"])); readouts.append((b, len(S) + r["decide"], [len(S) + o for o in r["opts"]]))
+        out = [[] for _ in encs]
+        for h, (b, d, oi) in zip(self._rows_hidden(rows), readouts):
+            out[b].append(self.head(h[d], h[torch.tensor(oi, device=self.device)]))
+        return out
+
+    def forward(self, enc):
+        """Returns list of logits tensors, one per question."""
+        return self.forward_batch([enc])[0]
+
+    def forward_batch(self, encs, shared_prefix=False):
+        """List (per record) of lists (per question) of logits. Row form or packed block-causal mask, see rows_form
+        (the packed mask already runs each state once; shared_prefix does that for the row form of a hybrid backbone)."""
+        if self.rows_form(encs): return self.forward_rows_batch(encs, shared_prefix and self.hybrid)
+        hs = self.hidden_batch(encs)
+        return [self._readout(hs[b], e) for b, e in enumerate(encs)]
+
+    @torch.no_grad()
+    def probs(self, enc):
+        """Probabilities per question. A record that runs as rows (every record on a hybrid backbone) takes the serving miss
+        path: its state once, then the question rows from its cache. forward() keeps the row form, which re-runs the state
+        per question; the two agree to fp32 rounding (#77)."""
+        if self.rows_form([enc]): return self.probs_and_prefix(enc)[0]
+        return [F.softmax(z, -1).cpu() for z in self.forward(enc)]
+
+    # --- state-prefix reuse (serving): the state is encoded once, question branches attend to its cached keys/values.
+    # Exact by construction: branch tokens never attend to each other across questions (block-causal mask) and the state
+    # never sees the branches (causal), so the state's hidden states and KV are identical with or without the branches.
+
+    def _branch_rows_from_prefix(self, enc, cache):
+        """Row-form serving: the branches run as causal rows continuing the cached state (exactly the forward_rows_batch
+        layout, minus the recomputed state)."""
+        S, _, rows = rows_of(enc)
+        hs = self._rows_hidden([(r["ids"], r["pos"]) for r in rows], cache=cache, prefix_len=len(S))
+        ps = [F.softmax(self.head(h[r["decide"]], h[torch.tensor(r["opts"], device=self.device)]), -1) for h, r in zip(hs, rows)]
+        return list(torch.cat(ps).cpu().split([len(p) for p in ps]))   # one device sync for the request, not one per question
+
+    @torch.no_grad()
+    def prefix(self, enc):
+        """Run the state tokens only. Returns (n_state_tokens, kv cache, state hidden states [Ls, d] fp32 or None). Only the
+        packed pass (probs_with_prefix) reads the hidden states; a hybrid backbone always runs rows, so it keeps None, as the
+        graphed prefixes do (a cached fp32 [Ls, d] copy is 160 MiB for 8k tokens on Kev-27B). An attention-only backbone
+        keeps them even when this record runs as rows: the same state with fewer questions may be packed."""
+        Ls = enc["seg"].count(0)
+        ids = torch.tensor([enc["ids"][:Ls]], device=self.device); pos = torch.tensor([enc["pos"][:Ls]], device=self.device)
+        # the cache must know the layer types (hybrid backbones keep recurrent + conv states per DeltaNet layer)
+        out = self.lm(input_ids=ids, position_ids=pos, past_key_values=DynamicCache(config=self.lm.config), use_cache=True)
+        return Ls, out.past_key_values, None if self.hybrid else out.last_hidden_state[0].float()
+
+    @torch.no_grad()
+    def probs_and_prefix(self, enc):
+        """One full pass that also returns the state prefix (KV cropped to the state, state hidden states or None, see prefix): a cache miss
+        costs a single forward pass, not two."""
+        Ls = enc["seg"].count(0)
+        if self.rows_form([enc]):
+            # recurrent layers cannot be cropped back to the state (and an over-long packed pass is what the row form avoids),
+            # so a miss here is a state pass (kept as the prefix) plus the branch rows
+            Ls, cache, h_state = self.prefix(enc)
+            return self._branch_rows_from_prefix(enc, cache), (Ls, cache, h_state)
+        ids = torch.tensor([enc["ids"]], device=self.device); pos = torch.tensor([enc["pos"]], device=self.device)
+        dt = next(self.lm.parameters()).dtype
+        mask = branch_mask_batch([enc["seg"]], self.device, dtype=dt, opts=[enc["opt"]] if enc.get("option_isolation") else None)
+        out = self.lm(input_ids=ids, position_ids=pos, attention_mask=mask, past_key_values=DynamicCache(config=self.lm.config), use_cache=True)
+        h = out.last_hidden_state[0].float()
+        out.past_key_values.crop(-(len(enc["ids"]) - Ls))     # keep the state only (negative = drop that many trailing tokens; positive form deprecated in transformers 5)
+        return [F.softmax(z, -1).cpu() for z in self._readout(h, enc)], (Ls, out.past_key_values, h[:Ls].clone())
+
+    @torch.no_grad()
+    def probs_with_prefix(self, enc, prefix):
+        """probs() for a record whose state tokens equal the cached prefix's; only the branches run. The cache is cropped
+        back to the state afterwards so it can be reused."""
+        Ls, cache, h_state = prefix
+        if enc["seg"].count(0) != Ls: raise ValueError("prefix does not match this record's state")
+        if self.rows_form([enc]):
+            return self._branch_rows_from_prefix(enc, cache)
+        ids = torch.tensor([enc["ids"][Ls:]], device=self.device); pos = torch.tensor([enc["pos"][Ls:]], device=self.device)
+        dt = next(self.lm.parameters()).dtype
+        mask = branch_mask_batch([enc["seg"]], self.device, dtype=dt, opts=[enc["opt"]] if enc.get("option_isolation") else None)[:, :, Ls:, :]
+        try:
+            out = self.lm(input_ids=ids, position_ids=pos, past_key_values=cache, attention_mask=mask, use_cache=True)
+            h = torch.cat([h_state, out.last_hidden_state[0].float()], 0)
+        finally:
+            cache.crop(-(len(enc["ids"]) - Ls))
+        return [F.softmax(z, -1).cpu() for z in self._readout(h, enc)]
+
+    @torch.no_grad()
+    def probs_batch(self, encs, prefixes, keep):
+        """kev.serve's model thread: several requests at once. prefixes[i] is request i's cached state prefix or None;
+        keep[i] whether to return its new prefix. -> (probs per request, prefix per request: the cached one, the new one if
+        kept, else None). With CUDA graphs the requests the graphed passes admit run together (kev.cuda_graphs: shared
+        state and row passes; a state too long for the graphed state pass gets its own eager pass first); the rest, and
+        every other backend, one at a time. Rows are independent, so a request's answers do not depend on the batch."""
+        splits = [rows_of(e) for e in encs]
+        def fits(i, cached):
+            S, _, rows = splits[i]
+            return self.graphs is not None and self.graphs.admits(len(S), [len(r["ids"]) for r in rows], cached)
+        long = {i for i in range(len(encs)) if prefixes[i] is None and not fits(i, False) and fits(i, True)}
+        batched = [i for i in range(len(encs)) if i in long or fits(i, prefixes[i] is not None)]
+        out = {i: probs_one(self, encs[i], prefixes[i], keep[i]) for i in sorted(set(range(len(encs))) - set(batched))}
+        runs, held = [[]], 0   # graphed runs, each holding at most EAGER_STATES long states' eager prefixes at once
+        for i in batched:
+            if i in long and held == EAGER_STATES: runs.append([]); held = 0
+            runs[-1].append(i); held += i in long
+        from .cuda_graphs import Request   # here, not at the top: the HF Space vendors model.py without cuda_graphs.py
+        for run in filter(None, runs):
+            pre = {i: self.prefix(encs[i]) if i in long else prefixes[i] for i in run}
+            X, caches = self.graphs.run([Request(splits[i][0], splits[i][1], [(r["ids"], r["pos"]) for r in splits[i][2]],
+                                                 None if pre[i] is None else pre[i][1], keep[i], [[r["decide"], *r["opts"]] for r in splits[i][2]])
+                                         for i in run])   # per question its picks: the <decide> position, then its options
+            ps = iter(self._readout_many(X, [len(r["opts"]) for i in run for r in splits[i][2]]))
+            for i, cache in zip(run, caches):
+                made = pre[i] if i in long else None if cache is None else (len(splits[i][0]), cache, None)
+                out[i] = [next(ps) for _ in splits[i][2]], prefixes[i] or (made if keep[i] else None)
+        return [out[i][0] for i in range(len(encs))], [out[i][1] for i in range(len(encs))]
+
+    def _readout_many(self, X, ks):
+        """Probabilities of many questions from their picked hidden states X, laid out per question as its <decide> then
+        its ks[q] options, in a fixed number of kernels and one device sync. -> one tensor per question."""
+        owner = [q for q, k in enumerate(ks) for _ in range(k)]
+        slot = [j for k in ks for j in range(k)]
+        starts = [0]
+        for k in ks: starts.append(starts[-1] + 1 + k)
+        dec = torch.tensor(starts[:-1]).to(self.device, non_blocking=True)
+        opt, own, sl = torch.tensor([[starts[q] + 1 + j for q, j in zip(owner, slot)], owner, slot]).to(self.device, non_blocking=True)
+        z = self.head.many(X[dec], X[opt], own)
+        Z = torch.full((len(ks), max(ks)), float("-inf"), device=self.device).index_put_((own, sl), z)
+        return torch.softmax(Z, -1)[own, sl].cpu().split(ks)
+
+    def trainable_parameters(self):
+        return [p for p in self.parameters() if p.requires_grad]

runtime/kev/packed_model.py

@@ -0,0 +1,102 @@
+"""Standalone MLX mixed ternary/4-bit backbone. Never reads or reconstructs base weights.
+
+Ternary codes use MLX's native 2-bit affine kernel; CLoQ stays a separate
+low-rank operation. Only selected embedding rows are dequantized. This is a
+different arithmetic path from the experiment's merged-bf16 simulation.
+"""
+import json
+from pathlib import Path
+
+import mlx.core as mx
+import mlx.nn as nn
+from mlx.utils import tree_unflatten
+from mlx_lm.models import qwen3_5
+
+from .mlx_model import MLXDecisionModel
+
+FORMAT = "kev-packed-mlx-v1"
+
+
+class PackedLinear(nn.QuantizedLinear):
+    def __init__(self, input_dims, output_dims, bits, group_size, rank=0, bias=False):
+        # Avoid QuantizedLinear's random dense initialization entirely.
+        nn.Module.__init__(self)
+        if bits not in (2, 4, 8) or input_dims % group_size or input_dims % (32 // bits):
+            raise ValueError("unsupported packed linear dimensions or precision")
+        self.bits, self.group_size, self.mode = bits, group_size, "affine"
+        self.weight = mx.zeros((output_dims, input_dims * bits // 32), dtype=mx.uint32)
+        self.scales = mx.zeros((output_dims, input_dims // group_size), dtype=mx.float16)
+        self.biases = mx.zeros_like(self.scales)
+        if bias:
+            self.bias = mx.zeros((output_dims,), dtype=mx.bfloat16)
+        if rank:
+            self.comp_a = mx.zeros((rank, input_dims), dtype=mx.bfloat16)
+            self.comp_b = mx.zeros((output_dims, rank), dtype=mx.bfloat16)
+        self.freeze()
+
+    def __call__(self, x):
+        y = super().__call__(x)
+        if "comp_a" in self:
+            # Keep the small compensation in fp32; never materialize B @ A.
+            xf = x.astype(mx.float32)
+            y = y + (xf @ self.comp_a.astype(mx.float32).T) @ self.comp_b.astype(mx.float32).T
+        return y.astype(mx.bfloat16)
+
+
+class PackedEmbedding(nn.QuantizedEmbedding):
+    activation_dtype = mx.bfloat16
+
+    def __init__(self, num_embeddings, dims, bits, group_size):
+        nn.Module.__init__(self)
+        if bits not in (2, 4, 8) or dims % group_size or dims % (32 // bits):
+            raise ValueError("unsupported packed embedding dimensions or precision")
+        self.num_embeddings, self.dims = num_embeddings, dims
+        self.bits, self.group_size, self.mode = bits, group_size, "affine"
+        self.weight = mx.zeros((num_embeddings, dims * bits // 32), dtype=mx.uint32)
+        self.scales = mx.zeros((num_embeddings, dims // group_size), dtype=mx.float16)
+        self.biases = mx.zeros_like(self.scales)
+        self.freeze()
+
+    def __call__(self, x):
+        return super().__call__(x).astype(mx.bfloat16)
+
+
+def load_packed_model(directory, pad_id, head_dim):
+    """Construct lazy architecture, replace quantized modules, strictly load packed tensors."""
+    root = Path(directory)
+    spec = json.loads((root / "packed.json").read_text(encoding="utf-8"))
+    if spec.get("format") != FORMAT:
+        raise ValueError("unsupported packed model format")
+    config = json.loads((root / "config.json").read_text(encoding="utf-8"))
+    args = qwen3_5.ModelArgs.from_dict(config)
+    text = qwen3_5.TextModelArgs.from_dict(args.text_config)
+    pattern = ["full_attention" if (i + 1) % text.full_attention_interval == 0 else "linear_attention" for i in range(text.num_hidden_layers)]
+    if args.text_config.get("layer_types", pattern) != pattern:
+        raise ValueError("packed layer_types do not match MLX's attention layout")
+    # Lazy initialization is discarded before evaluation, including the embedding and LM head.
+    lm = qwen3_5.Model(args)
+    if "lm_head" in lm.language_model:
+        del lm.language_model["lm_head"]
+    modules = dict(lm.named_modules())
+    replacements = []
+    for name, q in spec["linears"].items():
+        old = modules.get(name)
+        if not isinstance(old, nn.Linear):
+            raise ValueError(f"packed linear {name} is not in the architecture")
+        out_dims, in_dims = old.weight.shape
+        replacements.append((name, PackedLinear(in_dims, out_dims, q["bits"], q["group_size"], q.get("rank", 0), "bias" in old)))
+    embedding = spec["embedding"]
+    name = "language_model.model.embed_tokens"
+    replacements.append((name, PackedEmbedding(text.vocab_size, text.hidden_size, embedding["bits"], embedding["group_size"])))
+    lm.update_modules(tree_unflatten(replacements))
+    del modules, replacements
+    weights = mx.load(str(root / "model.safetensors"))
+    if any(v.dtype != mx.uint32 for k, v in weights.items() if k.removesuffix(".weight") in spec["linears"] or k == name + ".weight"):
+        raise ValueError("quantized weights must be packed uint32")
+    lm.load_weights(list(weights.items()), strict=True)
+    del weights
+    lm.eval()
+    mx.set_cache_limit(64 << 20)
+    mx.eval(lm.parameters())
+    mx.clear_cache()
+    return MLXDecisionModel(lm, pad_id, head_dim)

runtime/kev/plot.py

@@ -0,0 +1,59 @@
+"""Plot training loss from train logs and held-out accuracy vs baselines from eval.json.
+
+Run: uv run python -m kev.plot --logs runs/train.log:kev-0.5b runs/train_holdout.log:holdout --eval runs/kev/eval.json --out docs/training.png
+"""
+import argparse, re
+from .suite import read_json
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+STEP_RE = re.compile(r"ep(\d+) step (\d+)/(\d+) loss ([\d.]+)")
+ORDER = ["agnews", "agnews_yn", "banking77", "boolq", "mnli", "sst5", "yelp", "yelp_yn"]
+LABEL = {"agnews": "AG News\nK=4", "agnews_yn": "AG News\nyes/no", "banking77": "Banking77\nK=77", "boolq": "BoolQ\nyes/no",
+         "mnli": "MNLI\nK=3", "sst5": "SST-5\nscore", "yelp": "Yelp\nscore", "yelp_yn": "Yelp\nyes/no"}
+
+
+def read_log(path):
+    steps, losses, epoch_ends, prev_ep = [], [], [], 0
+    for m in STEP_RE.finditer(open(path, encoding="utf-8").read()):
+        ep, step, loss = int(m[1]), int(m[2]), float(m[4])
+        if ep != prev_ep: epoch_ends.append(steps[-1]); prev_ep = ep
+        steps.append(step); losses.append(loss)
+    return steps, losses, epoch_ends
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--logs", nargs="+", default=["runs/train.log:kev-0.5b"], help="path:label pairs")
+    ap.add_argument("--eval", default="runs/kev/eval.json")
+    ap.add_argument("--out", default="docs/training.png")
+    a = ap.parse_args()
+
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 4.2), dpi=160, gridspec_kw={"width_ratios": [1.1, 1.3]})
+
+    for spec in a.logs:
+        path, label = spec.split(":") if ":" in spec else (spec, spec)
+        steps, losses, ends = read_log(path)
+        ax1.plot(steps, losses, lw=1.6, label=f"{label}  (final {losses[-1]:.2f})")
+        for e in ends: ax1.axvline(e, color="0.8", lw=0.8, ls="--")
+    ax1.set_yscale("log"); ax1.set_xlabel("optimizer step (8 records each)"); ax1.set_ylabel("train loss (cross-entropy, log scale)")
+    ax1.set_title("training loss"); ax1.grid(alpha=0.25); ax1.legend(frameon=False)
+
+    e = read_json(a.eval)
+    kev, b0, b1 = e["accuracy_calibration"], e.get("baseline_zero_shot_base", {}), e.get("baseline_zero_shot_instruct", {})
+    xs = range(len(ORDER)); w = 0.27
+    ax2.bar([x - w for x in xs], [b0.get(s, {}).get("acc", 0) for s in ORDER], w, color="0.78", label="zero-shot base (letter logits)")
+    ax2.bar([x for x in xs], [b1.get(s, {}).get("acc", 0) for s in ORDER], w, color="0.55", label="zero-shot Instruct (letter logits)")
+    ax2.bar([x + w for x in xs], [kev[s]["acc"] for s in ORDER], w, color="#1f77b4", label="kev-0.5b (pointer readout)")
+    for x, s in zip(xs, ORDER):
+        ax2.text(x + w, kev[s]["acc"] + 0.012, f"ECE\n{kev[s]['ece']:.2f}", ha="center", va="bottom", fontsize=6.5, color="#1f77b4")
+    ax2.set_xticks(list(xs)); ax2.set_xticklabels([LABEL[s] for s in ORDER], fontsize=7.5)
+    ax2.set_ylim(0, 1.12); ax2.set_ylabel("held-out accuracy"); ax2.set_title(f"held-out accuracy vs zero-shot baselines  (all: {kev['ALL']['acc']:.3f}, ECE {kev['ALL']['ece']:.3f})")
+    ax2.grid(axis="y", alpha=0.25); ax2.legend(frameon=False, fontsize=8, loc="upper center", bbox_to_anchor=(0.5, -0.16), ncol=3)
+    for ax in (ax1, ax2): ax.spines[["top", "right"]].set_visible(False)
+    fig.tight_layout(); fig.savefig(a.out, bbox_inches="tight"); print("wrote", a.out)
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/predictors.py

@@ -0,0 +1,341 @@
+"""Predictors: callables record -> {"probabilities": {qid: {key: p}}, "latency_ms", "input_tokens", ...} for kev.benchmark.
+
+LocalPredictor scores a checkpoint in-process; RemotePredictor any TypeSafe System One-compatible endpoint; JevPredictor
+Jev itself through the AI SDK worker in playground/scripts (budget-capped).
+"""
+import contextlib
+import inspect
+import json
+import math
+import os
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+
+import torch
+from torch.nn.attention import SDPBackend, sdpa_kernel
+
+from kev.api import question_keys
+from kev.checkpoint import Checkpoint, LoadOptions
+from kev.data import api_request, materialize
+from kev.device import sync
+from kev.model import ROW_PASS_TOKENS, ContextOverflow, rows_of
+from kev.suite import CONTEXT
+
+
+LONG_ROW_KERNELS = "efficient"   # the label of a prediction (and its benchmark rows) that took the long-row kernels below
+
+KERNEL_PACKAGES = ("torch", "transformers", "peft", "flash-linear-attention", "triton", "causal-conv1d", "mlx", "mlx-lm")
+DELTANET_KERNELS = ("causal_conv1d_fn", "torch_chunk_gated_delta_rule")   # the prefill hooks of transformers' GatedDeltaNet
+
+
+@contextlib.contextmanager
+def long_row_kernels():
+    """The attention kernels a long row runs under (LocalPredictor): SDPA's flash and memory-efficient kernels, with the
+    math kernel as the fallback. transformers asks SDPA for grouped-query attention (`enable_gqa`) when a call has no
+    mask, which is the case for an unpadded state (kev.shared_prefix passes no mask so the state runs causal). Of the
+    kernels allowed here only flash and math take `enable_gqa`, and flash has no fp32. So an fp32 state pass used to fall
+    back to math, whose L x L scores do not fit: Kev-4B has 16 heads, so at 32k that is 16 x 32k^2 x 4 B = 69 GB per
+    layer. Inside this context an fp32 call repeats its keys and values per query head instead (transformers' own path
+    whenever a mask is given), and the memory-efficient kernel takes that: fp32 inputs and output, memory linear in L.
+    On sm80+ its fp32 products are cutlass's OpMultiplyAddFastF32 (three TF32 products per product, fp32 accuracy and
+    fp32 accumulation), not plain TF32. bf16 / fp16 calls keep `enable_gqa` and the flash kernel, so Kev-27B's long rows
+    run as before."""
+    from transformers.integrations import sdpa_attention
+    grouped = sdpa_attention.use_gqa_in_sdpa
+    sdpa_attention.use_gqa_in_sdpa = lambda attention_mask, key, value: key.dtype != torch.float32 and grouped(attention_mask, key, value)
+    try:
+        with sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]):
+            yield
+    finally:
+        sdpa_attention.use_gqa_in_sdpa = grouped
+
+
+def bound_implementation(fn):
+    """The function transformers actually calls for one of its kernel hooks: `use_kernel_func_from_hub_with_fallback`
+    wraps the PyTorch reference and binds the package's kernel (fla, causal-conv1d) as `implementation` when it imports;
+    any other function (unwrapped, or patched in by a caller) is itself. -> "module.qualname"."""
+    impl = inspect.getclosurevars(fn).nonlocals.get("implementation", fn) if inspect.isfunction(fn) else fn
+    return f"{getattr(impl, '__module__', None)}.{getattr(impl, '__qualname__', type(impl).__name__)}"
+
+
+def kernel_environment(model, device):
+    """What a read's logits depend on besides kev's code and the checkpoint (runs/drift-v1/REPORT.md: #125's causal-conv1d
+    kernel moved Kev-27B v1's reads by up to 0.06 in p with no code change): the scoring packages' versions, the GPU, the
+    backbone dtype and, on a torch backbone, the attention implementation and (hybrid) the Gated DeltaNet layer's forward
+    (kev.fused_qwen35 replaces it) and the convolution and chunked delta rule transformers bound. Reads only what is
+    already loaded or imported. Two reads are comparable bit for bit only when this and the kev commit match. Recorded
+    in report.json."""
+    def installed(name):
+        try: return version(name)
+        except PackageNotFoundError: return None
+    packages = {name: installed(name) for name in KERNEL_PACKAGES}
+    packages["torch"] = torch.__version__   # with its build tag (2.8.0+cu128 on the CUDA wheels), which the metadata version drops
+    env = {"packages": packages, "device": str(device),
+           "gpu": torch.cuda.get_device_name(torch.device(device)) if str(device).startswith("cuda") else None,
+           "backend": model.backend, "dtype": model.dtype, "triton_f32_default": os.environ.get("TRITON_F32_DEFAULT"),
+           "attention": None, "deltanet": None}
+    if model.backend != "torch": return env   # kev.mlx_model: mlx-lm's Metal kernels, named by the mlx / mlx-lm versions
+    env["attention"] = getattr(model.lm.config, "_attn_implementation", None)
+    layer = next((m for m in model.lm.modules() if type(m).__name__.endswith("GatedDeltaNet")), None) if model.hybrid else None
+    if layer is not None:
+        module = sys.modules[type(layer).__module__]
+        env["deltanet"] = {"forward": bound_implementation(getattr(layer.forward, "__func__", layer.forward)),
+                           **{name: bound_implementation(getattr(module, name)) for name in DELTANET_KERNELS if hasattr(module, name)}}
+    return env
+
+
+class LocalPredictor:
+    """Scores a checkpoint in-process. Evaluation is fp32-exact in PyTorch (no TF32, no fused SDPA kernels on CUDA). On
+    CUDA a hybrid backbone's Gated DeltaNet layers run flash-linear-attention's Triton kernels, whose dots are TF32, and a
+    bf16 backbone is bf16 throughout, so reads repeat bit for bit only on the same kernel set (`self.environment`, from
+    kernel_environment; AGENTS.md "What fp32-exact guarantees"). One more exception: on CUDA with the torch backend, a
+    record whose longest row (state + one question) exceeds kev.model.ROW_PASS_TOKENS runs under SDPA's flash /
+    memory-efficient kernels (long_row_kernels), because the exact math kernel's L x L score matrix does not fit (~275 GB
+    per layer pass at 64k for Kev-4B's 16 heads). An fp32 backbone stays fp32 there: the memory-efficient kernel takes
+    fp32 (Kev-4B / 9B at 8k-16k against the math kernel: max |dp| 6e-4, no flips in 238 questions;
+    scripts/long_state_memory.py). Such a prediction carries `"kernels": LONG_ROW_KERNELS`, kev.benchmark copies it onto
+    the record's rows and counts them in report.json's `long_rows`; shorter records carry nothing, so their rows and
+    reports are unchanged. The efficient path is CUDA-only: on CPU / MPS attention stays eager and exact and still
+    materialises L x L. A long record on a hybrid torch backbone also runs its state once, its questions continuing from
+    it (kev.shared_prefix), instead of once per question; the MLX backend's forward already runs the state once."""
+
+    def __init__(self, run, device, opts=LoadOptions(), context=CONTEXT):
+        """opts.temperature=None scores with the temperature the checkpoint carries; 1.0 scores raw logits. context: the
+        max_state / max_branch / max_packed a record must encode within (a suite manifest's `context`; the training
+        context by default, the serving limits for external suites frozen without admission)."""
+        if opts.temperature is not None and not (math.isfinite(opts.temperature) and opts.temperature > 0):
+            raise ValueError("temperature must be finite and positive")
+        checkpoint = self.checkpoint = Checkpoint(run)
+        self.run = checkpoint.path
+        if device == "cuda":
+            # evaluation is fp32-exact: TF32 (10-bit mantissa) moves probabilities by ~1e-3, the isolation gate's tolerance
+            torch.backends.cuda.matmul.allow_tf32 = False; torch.backends.cudnn.allow_tf32 = False
+            torch.backends.cuda.enable_flash_sdp(False); torch.backends.cuda.enable_mem_efficient_sdp(False)
+        self.tok, self.model = checkpoint.load(device, opts)
+        self.temperature = self.model.head.temperature
+        self.environment = kernel_environment(self.model, device)
+        self.device = device
+        self.context = context
+
+    @torch.no_grad()
+    def __call__(self, record):
+        enc = self.model.encode(self.tok, materialize(record), max_state=self.context["max_state"], max_branch=self.context["max_branch"], strict=True)
+        if len(enc["ids"]) > self.context["max_packed"]:
+            raise ContextOverflow(f"packed request exceeds the {self.context['max_packed']}-token limit")
+        sync(self.device)
+        start = time.perf_counter()
+        state, _, rows = rows_of(enc)
+        long = len(state) + max(len(r["ids"]) for r in rows) > ROW_PASS_TOKENS
+        torch_backend = self.model.backend == "torch"
+        efficient = long and torch_backend and self.device == "cuda"
+        with long_row_kernels() if efficient else contextlib.nullcontext():
+            if long and torch_backend and self.model.hybrid: logits = self.model.forward_batch([enc], shared_prefix=True)[0]
+            else: logits = self.model.forward(enc)
+        ps = [torch.softmax(z, -1).cpu() for z in logits]
+        zs = [z.float().cpu() for z in logits]
+        sync(self.device)
+        keys = {qid: question_keys(q["type"], q.get("criteria")) for qid, q in record["questions"].items()}
+        return {"probabilities": {qid: dict(zip(keys[qid], p.tolist())) for qid, p in zip(keys, ps)},
+                "logits": {qid: dict(zip(keys[qid], z.tolist())) for qid, z in zip(keys, zs)},
+                "inference_temperature": self.temperature,
+                "latency_ms": 1000 * (time.perf_counter() - start), "input_tokens": len(enc["ids"]), **({"kernels": LONG_ROW_KERNELS} if efficient else {})}
+
+
+REFUSAL_STATUSES = (400, 413, 422)   # an endpoint declining the request itself (kev.serve: 422 past its context; Jev: 400 / 413 / 422)
+
+
+def http_detail(error):
+    """The message of an HTTP error body: FastAPI's / TypeSafe's `detail` (or `error`) when it is JSON, else its text."""
+    try: text = error.read().decode("utf-8", "replace")
+    except Exception: return str(error.reason)
+    try: body = json.loads(text)
+    except ValueError: return text[:500] or str(error.reason)
+    return str(body.get("detail") or body.get("error") or body) if isinstance(body, dict) else str(body)
+
+
+class RemotePredictor:
+    """Score any TypeSafe System One-compatible endpoint (POST <base_url>/v1/systemone) on frozen records. Probabilities are
+    taken from the response as returned (renormalised by validate_distribution like every other predictor). Records the
+    server-reported model id so the manifest can pin what was scored. `concurrency` is how many requests kev.benchmark may
+    keep in flight at once (each call is independent: one request, its own retries); 1 scores sequentially. An endpoint
+    that declines the request itself (REFUSAL_STATUSES: kev.serve's 422 for a state past its context) raises
+    ContextOverflow at once, so kev.benchmark counts the record rejected under skip_overlong and stops otherwise; another
+    client error stops at once too; 408, 429, 5xx, connection errors and timeouts are tried `retries` times in all."""
+
+    def __init__(self, base_url, model="kev-latest", api_key="local", timeout=120, retries=3, concurrency=1):
+        if concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        self.base_url, self.model, self.api_key, self.timeout, self.retries = base_url.rstrip("/"), model, api_key, timeout, retries
+        self.concurrency = concurrency
+        self.served_model = None
+
+    def __call__(self, record):
+        payload = json.dumps({**api_request(record), "model": self.model}).encode()
+        req = urllib.request.Request(f"{self.base_url}/v1/systemone", data=payload, method="POST",
+                                    headers={"content-type": "application/json", "authorization": f"Bearer {self.api_key}"})
+        last = None
+        for attempt in range(self.retries):
+            try:
+                start = time.perf_counter()
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                    body = json.loads(resp.read())
+                latency = 1000 * (time.perf_counter() - start)
+                break
+            except urllib.error.HTTPError as error:
+                if error.code in REFUSAL_STATUSES:   # the request itself (kev.serve: a state or question over the serving context): a rejected record, not retried
+                    raise ContextOverflow(f"remote endpoint refused the request (HTTP {error.code}): {http_detail(error)}") from None
+                if error.code not in (408, 429) and error.code < 500:   # 401, 403, 404, ...: the same answer every time; 408, 429 and 5xx are retried
+                    raise RuntimeError(f"remote endpoint answered HTTP {error.code}: {http_detail(error)}") from None
+                last = error
+            except Exception as error:   # connection errors and timeouts: retried
+                last = error
+            if attempt + 1 < self.retries: time.sleep(2 ** attempt)
+        else:
+            raise RuntimeError(f"remote endpoint failed after {self.retries} attempts: {last}")
+        self.served_model = body.get("model", self.served_model)
+        probs = {}
+        for qid, q in record["questions"].items():
+            a = body["answers"][qid]
+            if q["type"] == "noul": probs[qid] = {"true": float(a["noul"]), "false": 1 - float(a["noul"])}
+            else: probs[qid] = {str(k): float(v) for k, v in a["probabilities"].items()}
+        return {"probabilities": probs, "latency_ms": latency, "input_tokens": (body.get("usage") or {}).get("input_tokens")}
+
+
+PRICE_PER_MILLION = 0.042   # Jev list price per million input tokens, for the budget cap
+
+
+class RotationAveraged:
+    """Test-time order averaging (PLAN.md round 4, item 4.4): score the record under the first `rotations` cyclic
+    rotations of every Choice question's options (rotation r of a K-option question is r mod K; Noul and Score keep their
+    order, which is part of their meaning) and average the logits per option key. The geometric mean of softmax
+    probabilities is the softmax of the mean logits, so the returned probabilities and logits stay consistent; predictors
+    that return no logits are averaged in log-probability space. Costs `rotations` forward passes per record."""
+
+    def __init__(self, predictor, rotations):
+        if rotations < 2:
+            raise ValueError("rotation averaging needs at least 2 rotations")
+        self.predictor, self.rotations = predictor, rotations
+        self.temperature = getattr(predictor, "temperature", None)
+        self.concurrency = getattr(predictor, "concurrency", 1)
+
+    @staticmethod
+    def rotated(record, r):
+        def rotate(q):
+            if q["type"] != "choice": return q
+            keys = list(q["criteria"]); k = r % len(keys)
+            return {**q, "criteria": {key: q["criteria"][key] for key in keys[k:] + keys[:k]}}
+        return {**record, "questions": {qid: rotate(q) for qid, q in record["questions"].items()}}
+
+    def __call__(self, record):
+        widest = max((len(q["criteria"]) for q in record["questions"].values() if q["type"] == "choice"), default=1)
+        preds = [self.predictor(self.rotated(record, r)) for r in range(min(self.rotations, widest))]
+        field = "logits" if all("logits" in p for p in preds) else "probabilities"
+        out = {"probabilities": {}, "latency_ms": sum(p["latency_ms"] for p in preds), "rotations": len(preds)}
+        if field == "logits":
+            out["logits"] = {}; out["inference_temperature"] = preds[0].get("inference_temperature", 1.0)
+        for qid in record["questions"]:
+            keys = list(preds[0]["probabilities"][qid])
+            if field == "logits":
+                z = {k: sum(p["logits"][qid][k] for p in preds) / len(preds) for k in keys}
+            else:
+                z = {k: sum(math.log(max(p["probabilities"][qid][k], 1e-12)) for p in preds) / len(preds) for k in keys}
+            top = max(z.values()); e = {k: math.exp(v - top) for k, v in z.items()}; s = sum(e.values())
+            out["probabilities"][qid] = {k: v / s for k, v in e.items()}
+            if field == "logits": out["logits"][qid] = z
+        return out
+
+
+class JevRefused(ContextOverflow):
+    """Jev declined a request because of the input itself (REFUSAL_STATUSES, or a hosted-side error on an oversize request;
+    see JevPredictor). With count_refusals, kev.benchmark counts such a record as rejected (rejected.json) instead of stopping."""
+
+
+# Jev's context in tokens (documented ~32k). Past it the AI Gateway answers with HTTP 400 *or* a 503 GatewayInternalServerError
+# (evals/longdoc-v1: 212 x 400 and 28 x 503 on the 240 requests of ~57k-61k tokens; every request of <= ~31k tokens answered),
+# so a hosted-side error on a request estimated past OVERSIZE x JEV_CONTEXT_TOKENS is the size, not an outage. The estimate is
+# characters / 4 of the request JSON; the 1.5 margin keeps a ~30k-token legal text (~144k characters) out of it.
+JEV_CONTEXT_TOKENS = 32768
+OVERSIZE = 1.5
+
+
+def oversize(request):
+    """Whether a Jev request is well past Jev's context (estimated tokens = characters / 4)."""
+    return len(json.dumps(request, ensure_ascii=False)) / 4 > OVERSIZE * JEV_CONTEXT_TOKENS
+
+
+class JevPredictor:
+    """Jev through the AI SDK worker (playground/scripts/jev-evaluate.mjs); every call is counted against a token budget.
+    count_refusals: a request answered with a REFUSAL_STATUSES error, or with a hosted-side error (5xx, no status) while
+    oversize(), raises JevRefused at once (counted in accounting()["refusals"], not fatal, not retried); any other client error
+    (401, 403, 429, ...) still stops the read. attempts: tries per request on hosted-side failures of requests that are not
+    oversize, backing off 1, 2, 4, ... seconds up to 30 between them."""
+    def __init__(self, key, budget=0.1, max_calls=700, count_refusals=False, attempts=4):
+        self.budget, self.max_calls, self.count_refusals, self.attempts = budget, max_calls, count_refusals, attempts
+        self.calls, self.input_tokens, self.output_tokens, self.retries = 0, 0, 0, 0
+        self.refusals = {}   # HTTP status -> count
+        self.started_at = datetime.now(timezone.utc).isoformat()
+        worker = Path(__file__).resolve().parents[1] / "playground/scripts/jev-evaluate.mjs"
+        self.process = subprocess.Popen(["node", str(worker)], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                                        stderr=subprocess.DEVNULL, text=True, bufsize=1,
+                                        env={**os.environ, "AI_GATEWAY_API_KEY": key})
+
+    def close(self):
+        self.process.stdin.close()
+        try:
+            self.process.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            self.process.terminate()
+            self.process.wait(timeout=5)
+
+    def __call__(self, record):
+        if self.calls >= self.max_calls or (self.input_tokens + 65536) * PRICE_PER_MILLION / 1e6 > self.budget:
+            raise RuntimeError("Jev evaluation reached the request/token cost cap")
+        request = api_request(record)
+        for attempt in range(self.attempts):
+            self.process.stdin.write(json.dumps(request) + "\n")
+            self.process.stdin.flush()
+            line = self.process.stdout.readline()
+            if not line:
+                raise RuntimeError("Jev SDK worker exited without a response")
+            result = json.loads(line)
+            self.calls += 1
+            if "error" not in result:
+                break
+            status = result["error"]["status"]
+            # bounded retry for hosted-side failures only; client errors (4xx) are real and must surface
+            hosted = status is None or status >= 500
+            if self.count_refusals and (status in REFUSAL_STATUSES or hosted and oversize(request)):
+                kind = str(status) if status in REFUSAL_STATUSES else f"{status} (oversize)"
+                self.refusals[kind] = self.refusals.get(kind, 0) + 1
+                raise JevRefused(f"Jev refused the request: {result['error']['name']} (HTTP {kind})")
+            if attempt == self.attempts - 1 or (status is not None and status < 500):
+                raise RuntimeError(f"Jev request failed: {result['error']['name']} (HTTP {status})")
+            self.retries += 1
+            time.sleep(min(2 ** attempt, 30))
+        usage = result["usage"]
+        if usage.get("inputTokens") is None:
+            raise RuntimeError("Jev returned no input token usage; cannot account for cost")
+        self.input_tokens += usage["inputTokens"]
+        self.output_tokens += usage.get("outputTokens") or 0
+        probabilities = {}
+        for qid, q in record["questions"].items():
+            answer = result["answers"][qid]
+            if q["type"] == "noul":
+                probabilities[qid] = {"false": 1 - answer["probability"], "true": answer["probability"]}
+            else:
+                probabilities[qid] = answer["probabilities"]
+        return {**result, "probabilities": probabilities}
+
+    def accounting(self):
+        return {"model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result",
+                "started_at": self.started_at, "calls": self.calls, "input_tokens": self.input_tokens,
+                "output_tokens": self.output_tokens, "retries_after_5xx": self.retries, "listed_input_usd_per_million": PRICE_PER_MILLION,
+                **({"refusals": dict(sorted(self.refusals.items()))} if self.count_refusals else {}),
+                "estimated_usd": self.input_tokens * PRICE_PER_MILLION / 1e6,
+                "budget_usd": self.budget, "sdk": "ai@7.0.105", "zero_data_retention_requested": True}

runtime/kev/publish.py

@@ -0,0 +1,106 @@
+"""Publish a trained run to the Hugging Face Hub.
+
+Repo naming: jaredpalmer/kev-<base size>, e.g. kev-0.5b (Qwen2.5-0.5B), kev-0.6b (Qwen3-0.6B), kev-4b, kev-8b.
+The exact base checkpoint is recorded in the model card's `base_model` field and in head.pt.
+
+    uv run python -m kev.publish --run runs/kev --repo jaredpalmer/kev-0.5b
+    uv run python -m kev.publish --run runs/kev2 --repo jaredpalmer/kev-0.5b --message "v0.2: none-of-the-above fix"
+
+Uploads: the adapter (or, for a full-weight run, config.json and every model*.safetensors shard with its index), head.pt,
+tokenizer files, eval.json, training log (if found), and the model card (--card) as README.md with the repo id and run
+name filled in. Requires `hf auth login`. With --private the repo is created private if missing and an existing repo
+that is not private is refused (kev.mirror.ensure_private), so a candidate never lands in a public repo. Full-weight
+shards are linked into the staging directory, not copied (modal_app.py::release_publish uploads a 27B from a CPU container).
+
+--replace makes the target revision hold exactly this upload: every file the upload does not carry (other than
+.gitattributes) is deleted in the same commit. A full-weight upload into a repo that still holds an adapter (or an adapter
+upload into one that holds backbone shards) is refused without it, because the loader rule (kev.checkpoint) would pick
+the stale layout.
+"""
+import argparse, os, re, shutil, tempfile
+from pathlib import Path
+from huggingface_hub import HfApi
+from .checkpoint import Checkpoint
+from .mirror import ensure_private
+from .suite import read_json, write_json
+
+FILES = ["head.pt", "tokenizer.json", "tokenizer_config.json", "chat_template.jinja",
+         "vocab.json", "merges.txt", "added_tokens.json", "special_tokens_map.json", "eval.json"]
+WEIGHTS = {False: ["adapter_config.json", "adapter_model.safetensors"], True: ["config.json", "model.safetensors.index.json"]}   # by Checkpoint.full
+
+
+def stale_layout(api, repo, revision, full):
+    """Files of the other checkpoint layout on the target revision (adapter files for a full-weight upload, backbone shards
+    for an adapter upload), sorted; [] for a missing branch or an empty repo."""
+    from huggingface_hub.utils import RevisionNotFoundError
+    try: files = api.list_repo_files(repo, repo_type="model", revision=revision)
+    except RevisionNotFoundError: return []
+    other = (lambda f: f in WEIGHTS[False]) if full else (lambda f: re.fullmatch(r"model.*\.safetensors", f) or f == "model.safetensors.index.json")
+    return sorted(f for f in files if other(f))
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--run", required=True)
+    ap.add_argument("--repo", required=True, help="e.g. jaredpalmer/kev-0.5b")
+    ap.add_argument("--message", default=None)
+    ap.add_argument("--card", required=True, help="model card markdown, e.g. docs/model-cards/kev-4b.md")
+    ap.add_argument("--private", action="store_true")
+    ap.add_argument("--tag", help="create this Hub tag on the uploaded commit (versioned release, e.g. v0.2)")
+    ap.add_argument("--revision", help="upload to this branch instead of main (created if missing); for candidates that must not replace the released weights")
+    ap.add_argument("--replace", action="store_true", help="delete every file of the target revision this upload does not carry, in the same commit (tag the previous version first)")
+    a = ap.parse_args()
+
+    checkpoint, run_name = Checkpoint(a.run), os.path.basename(a.run.rstrip("/"))
+    base, full = checkpoint.meta.base, checkpoint.full
+    api = HfApi()
+    if a.private: ensure_private(api, a.repo)   # creates a missing repo private; refuses one that exists public
+    else: api.create_repo(a.repo, repo_type="model", exist_ok=True, private=False)
+    stale = stale_layout(api, a.repo, a.revision, full)
+    if stale and not a.replace:
+        raise SystemExit(f"{a.repo}{'@' + a.revision if a.revision else ''} holds {stale}, the other checkpoint layout; the loader would pick it. "
+                         "Tag the previous version, then publish with --replace to delete it in the same commit")
+
+    with tempfile.TemporaryDirectory() as tmp:
+        shards = {p.name for p in checkpoint.shards()} if full else set()
+        for f in FILES + WEIGHTS[full] + sorted(shards):
+            src = f"{a.run}/{f}"
+            if not os.path.exists(src): print(f"skip {f} (not found)")
+            elif f in shards: os.symlink(os.path.abspath(src), f"{tmp}/{f}")   # read-only and ~5 GB each: linked, not copied (upload_folder follows links)
+            else: shutil.copy(src, tmp)
+        if not full:   # runs before task_type was set saved null; the Hub warns about it and PEFT treats both the same for a bare backbone
+            cfg_path = f"{tmp}/adapter_config.json"; cfg = read_json(cfg_path)
+            if not cfg.get("task_type"): cfg["task_type"] = "FEATURE_EXTRACTION"; write_json(cfg_path, cfg)
+        if os.path.exists(f"runs/logs/train_{run_name}.log"):   # standalone runs keep their log there (see .gitignore)
+            shutil.copy(f"runs/logs/train_{run_name}.log", f"{tmp}/train.log")
+        # research trials: runs/<study>/<trial>/checkpoint -> ship the trial's result, provenance and training log too
+        trial = os.path.dirname(a.run.rstrip("/")) if run_name == "checkpoint" else None
+        if trial:
+            run_name = os.path.relpath(trial, "runs")
+            for f in ("result.json", "provenance.json", "train.log", "training_config.json", "training_metrics.json", "interpolation.json"):
+                for src in (f"{trial}/{f}", f"{a.run}/{f}"):
+                    if os.path.exists(src): shutil.copy(src, f"{tmp}/{f}"); break
+
+        # the card's prose names the Hub repo and trial itself; only the frontmatter is filled from the checkpoint
+        card = re.sub(r"^base_model: .*$", f"base_model: {base}", Path(a.card).read_text(encoding="utf-8"), flags=re.M)
+        if "base_model_relation:" not in card: card = card.replace(f"base_model: {base}", f"base_model: {base}\nbase_model_relation: {'finetune' if full else 'adapter'}")
+        Path(tmp, "README.md").write_text(card, encoding="utf-8")
+
+        ev = read_json(f"{tmp}/eval.json") if os.path.exists(f"{tmp}/eval.json") else {}
+        acc = ev.get("accuracy_calibration", {}).get("ALL", {})
+        if os.path.exists(f"{tmp}/result.json"):
+            clean = read_json(f"{tmp}/result.json").get("clean", {})   # research trials; other result files (kev-finetune runs) just skip the figures
+            acc = {"acc": clean.get("acc", float("nan")), "ece": clean.get("ece", float("nan"))}
+        msg = a.message or f"Upload {run_name} (base {base}; acc {acc.get('acc', float('nan')):.3f}, ECE {acc.get('ece', float('nan')):.3f})"
+        if a.revision: api.create_branch(a.repo, branch=a.revision, repo_type="model", exist_ok=True)
+        # delete_patterns="*": remote files not in this upload are deleted in the same commit (huggingface_hub keeps .gitattributes and every file being uploaded)
+        info = api.upload_folder(folder_path=tmp, repo_id=a.repo, repo_type="model", commit_message=msg, revision=a.revision,
+                                 delete_patterns="*" if a.replace else None)
+        print(info)
+        if a.tag:
+            api.create_tag(a.repo, tag=a.tag, repo_type="model", tag_message=msg, exist_ok=False, revision=a.revision)
+            print(f"tagged {a.repo}@{a.tag}")
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/rounds.py

@@ -0,0 +1,1364 @@
+"""Registered research rounds as data: one spec per round, one engine for every stage.
+
+A round (PLAN.md, rounds 5-20) always has the same shape. Arms are config-only trials (or checkpoints made from them,
+round 20's interpolations) compared with a parent checkpoint;
+each arm and its parent are read on a list of suites; a rule compares every arm with its parent on paired,
+record-clustered bootstraps (primaries with a lower bound, guards with thresholds, pooled panels); the passing arm with
+the best score is the size's candidate; confirmation stages (test partitions, the locked read) are read once. The spec
+(`experiments/rounds/r<N>.json`, schema below) is committed before any training or read, like the PLAN registration it
+encodes.
+
+    uv run python -m kev.rounds validate      experiments/rounds/r18.json   # well-formed, every parent read exists, plans validate
+    uv run python -m kev.rounds launch        experiments/rounds/r18.json   # modal_app.py::study for each study (after `modal deploy`)
+    uv run python -m kev.rounds watch         experiments/rounds/r18.json   # pull + read each finished trial once, then the read-out
+    uv run python -m kev.rounds launch-reads  experiments/rounds/r18.json [--arms a,b] [--stage tests --arm a]
+    uv run python -m kev.rounds readout       experiments/rounds/r18.json   # -> runs/r18-readout/round18.json + a table
+    uv run python -m kev.rounds confirm       experiments/rounds/r18.json --stage tests [--arm a]   # -> runs/r18-verdict/<size>-<stage>.json
+
+Every comparison is served-vs-served: each side's rows are served at the temperature fitted on its own trial's
+development rows (`kev.metrics.served`), or, for the arms of a round that registers a `temperature` pool, on that pool;
+unknowable records are scored only by `unknowable_report`, and every delta is `kev.metrics.paired_bootstrap` (2,000
+resamples, seed 0, micro), the registered read since round 5.
+
+Spec (paths are relative to the repo root; templates take {round}, {arm}, {size}, {tag}):
+    round, registered                 the round number and where its registration lives
+    archive                           a recorded round: the git tag holding the plans, runs and data it names (validate lists
+                                      what this checkout lacks instead of failing; launch refuses a record)
+    gpu, app                          Modal GPU and KEV_APP_NAME for launches (optional)
+    studies   {name: {plan, suite, transfer, gpu, timeout, budget}}   one modal_app.py::study call each (budget >= its admission bound)
+    reads     {tag: {suite, flags?} | {entrypoint: "locked_test", decision}}  what each read tag scores (entrypoint: ENTRYPOINTS)
+    read_timeout {size: seconds}      overrides modal_app.READ_TIMEOUTS for one size (a 27B's fp32 reads)
+    locked_args  {size: [args]}       extra modal_app.py::locked_test switches for one size (a 27B's GPU memory)
+    parents   {name: {trial, checkpoint?, reads: {tag: dir}}}   checkpoint (Hub id[@rev]) only when /<trial>/checkpoint is not on the volume
+    arms      {name: {trial?, checkpoint?, parent, reads?, select?, transfer_read?, trained_on?}}   name = "<size>-<label>"; reads
+                                      default to arm_reads; select false = reported, never the candidate (attribution arms); an
+                                      arm without a trial (an interpolated checkpoint) names its /runs/... checkpoint, needs the
+                                      round's `temperature` pool (it has no development rows) and, if the rule reads
+                                      "transfer", a transfer_read; trained_on [suite dirs] names what such a checkpoint was
+                                      trained on (default: covered by the round's trial arms, e.g. the trials it interpolates)
+    arm_reads template of an arm's read directory (default "runs/r{round}-{arm}-{tag}")
+    transfer_read  a read tag (round level, or per arm, which wins; null = the trial's own) whose rows are an arm's "transfer"
+                                      rows instead of its trial's in-trial transfer read (a checkpoint without a trial has none)
+    temperature {reads: [tags], sources?: {tag: [source]}, exclude_reads?: [tags]}   every ARM (parents keep their trial's
+                                      development rows) is served at the temperature fitted, as everywhere (kev.metrics.served),
+                                      on the knowable rows of `reads` pooled (a read listed in `sources` gives only those
+                                      sources' rows), minus every record whose id is in the `exclude_reads` rows; the arm's own
+                                      rule-stage reads are used at every stage; no pooled tag may sit in a panel that a
+                                      temperature-dependent criterion reads (see pool_problems), and no pooled read may share
+                                      data with any arm's training data (pool_training_problems: round 19's failure mode)
+    drop_ids  record ids dropped on both sides of every comparison (devtools-v1's duplicated ids)
+    rule      {panels, unknowable?, criteria, rank}
+    confirm   {stage: {candidate_reads, parent_reads?, panels, criteria}}
+A panel is {reads: [tags], metrics: [bootstrapped], report: [value only], source?: filter, versus?: {name: dir}, by_length?,
+exclude_sources?, exclude_tasks?, exclude_file?, optional?}; the tag "transfer" is the trial's own in-trial transfer read (or
+the arm's transfer_read). source keeps one source (or a list of them); exclude_sources / exclude_tasks drop rows by source /
+task, and exclude_file ({path, sha256}: a JSON object of `ids` (question or record ids), `sources` and `tasks` lists, or a
+bare list of any of them) drops what it lists, checked against its registered hash, so a private list (round 24's
+tasksource-heldout families) enters a public spec only as a path and a sha256 (panel_filter; the same rows leave both
+sides and any `versus` reference; the panel records how many questions they removed as `excluded`). optional: true marks
+a report-only panel no criterion or rank may read; its absent reads leave the comparison complete (`missing_optional`).
+The `temperature` pool may carry ci: {level, samples, seed}, which adds each arm's bootstrap interval of its pooled
+temperature to the read-out (`temperature_ci`, report only). by_length (true, or {edges?:
+[tokens], tokenizer?: [name, revision]}) adds acc / ece / brier / confident_error_rate per state-token bucket
+(kev.metrics.calibration_by_length: "<metric>_<bucket>" entries, e.g. ece_16k_plus, value only) with state tokens counted
+from the reads' suite records. A criterion is {left, op, right, plus?} where left is a path
+("<panel>.<metric>.<candidate|parent|delta|lower|upper>", "<panel>.<metric>_<bucket>.<candidate|parent>",
+"unknowable.<candidate|parent>") or a list [a, b] meaning a - b, and right is a number or a path (plus is added to it).
+rank is a list of {by: path | [paths summed], order}.
+
+Every arm's read-out records where its temperature came from (`temperature_source`: the pool with its reads and question
+count, or its trial's development rows with their suite) and its parent's (`parent_temperature_source`: its trial's
+development rows, plus `shipped`, the head.pt temperature when it is here), and the printed table warns when an arm's rows
+are a training corpus's (in distribution: round 19's failure mode). From round 21 (POOL_REQUIRED_FROM) a rule or
+confirmation with a temperature-dependent criterion must register a `temperature` pool: validate and launch refuse it
+otherwise; earlier rounds get a warning, so their recorded specs keep validating.
+
+Removed suites (kev.suite.REMOVED_SUITES: evals/external/scienthoon-v1, removed 2026-09-27, last read by round 22;
+evals/external/wanli-v2 and typesafe-v1, removed 2026-09-30, last read by round 26; wanli-v1, last read by round 5). A round
+up to the suite's `last_round` that reads one validates with the read listed under `archived`; its read-out and
+confirmations are computed from the committed rows as before (they need rows, not the suite). A later round that names
+one is refused with the reason, and no read of one is launched again (read_commands skips it; kev.suite.load_split refuses it).
+"""
+import argparse
+import functools
+import operator
+import os
+import socket
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import NamedTuple
+
+from kev.metrics import (LENGTH_EDGES, LENGTH_METRICS, calibration_by_length, length_buckets, metrics, paired_bootstrap, raw_row, recorded,
+                         scored_rows, served, served_at, tempered_row, unknowable_report)
+from kev.suite import ADMISSION_TOKENIZER, digest, file_lock, load_split, read_json, read_manifest, removed_suite, suite_key, write_json
+
+ROOT = Path(__file__).resolve().parents[1]
+SAMPLES = 2000   # the registered resample count since round 5
+OPS = {">": operator.gt, ">=": operator.ge, "<": operator.lt, "<=": operator.le}
+FIELDS = ("candidate", "parent", "delta", "lower", "upper")
+TRANSFER = "transfer"   # the in-trial transfer read (runs/<study>/<trial>/transfer)
+STAGGER = 60            # seconds between `modal run` launches: Modal refuses more than ~3 app creations a minute
+ENTRYPOINTS = ("benchmarks", "locked_test")   # the modal_app.py entrypoints a read can go through
+IN_FLIGHT = 7200        # seconds an arm's launched reads are presumed running (the longest suite timeout in modal_app.READ_TIMEOUTS)
+
+
+# --- rows ------------------------------------------------------------------------------------------------------------
+
+def paired(candidate, reference, metric):
+    """The registered paired read: record-clustered bootstrap of candidate - reference, micro, 2,000 resamples, seed 0."""
+    x = paired_bootstrap(candidate, reference, samples=SAMPLES, seed=0, metric=metric, aggregation="micro")
+    return {"delta": x[f"micro_{metric}_delta"], "ci95": x["ci95"]}
+
+
+def temperature(trial, root=ROOT):
+    """The temperature fitted on a trial's own development rows (decision-v7's, or its suite's; how every checkpoint here is
+    served unless the round registers a `temperature` pool for its arms)."""
+    return served(read_json(Path(root) / trial / "development/rows.json"), [])[0]
+
+
+def served_clean(rows, t):
+    """Every clean row served at t, unknowable records included (what unknowable_report scores; served_at drops them)."""
+    return [tempered_row(raw_row(recorded(r)), t) for r in rows if r["variant"] == "clean"]
+
+
+class Pool(NamedTuple):
+    """Where a pooled temperature is fitted (spec `temperature`, resolved to one arm's read directories)."""
+    reads: list     # directories whose rows are pooled
+    sources: dict   # {directory: [source, ...]}: the only sources pooled from that read (absent = every source)
+    exclude: list   # directories whose record ids are removed from the pool
+    ci: dict = None # {level, samples, seed}: also bootstrap the fitted temperature's interval (report only; round 24)
+
+
+def select_rows(reads, exclude=()):
+    """(pooled rows, rows before the exclusion): every rows.json of reads, as (path, sources or None), concatenated with
+    each read limited to its sources, minus every record whose id appears in an exclude rows.json. The one row selection
+    of a pooled temperature, here and in scripts/calibrate_checkpoint.py (which writes it into a checkpoint)."""
+    excluded = {r["id"] for path in exclude for r in read_json(path)}
+    rows = [r for path, sources in reads for r in read_json(path) if sources is None or r["source"] in sources]
+    return [r for r in rows if r["id"] not in excluded], rows
+
+
+def pooled_temperature(pool, root=ROOT):
+    """(temperature, fit report): the temperature() objective (kev.metrics.served: knowable clean rows, TEMPERATURE_FIT)
+    on select_rows of the pool."""
+    kept, rows = select_rows([(Path(root) / d / "rows.json", pool.sources.get(d)) for d in pool.reads], [Path(root) / d / "rows.json" for d in pool.exclude])
+    return served(kept, [])[0], {"reads": pool.reads, **({"sources": pool.sources} if pool.sources else {}), "exclude_reads": pool.exclude,
+                                 "questions": len(scored_rows(kept)), "excluded_questions": len(scored_rows(rows)) - len(scored_rows(kept))}
+
+
+def pooled_temperature_ci(pool, root=ROOT):
+    """The pooled temperature's bootstrap interval (report only; the audit of 2026-09-27: the served T of a 648-question pool
+    moves by ~±0.15, enough to move an ECE criterion by more than its margin): the same fit (TEMPERATURE_FIT's grid and
+    objective, knowable clean rows of select_rows) on `samples` resamples of the pool's (source, group) clusters within each
+    source (kev.metrics.cluster_resamples, seed), percentile interval at `level`. Each question's NLL on the grid is computed
+    once; a resample's fit is the grid point with the least mean NLL over its questions."""
+    import numpy as np
+    from kev.metrics import TEMPERATURE_FIT, cluster_resamples, nll_at_temperature
+    kept, _ = select_rows([(Path(root) / d / "rows.json", pool.sources.get(d)) for d in pool.reads], [Path(root) / d / "rows.json" for d in pool.exclude])
+    raw = served_at(kept, 1.0)
+    grid = np.exp(np.linspace(np.log(0.25), np.log(4), TEMPERATURE_FIT["points"]))
+    nll = np.asarray([[nll_at_temperature(r, float(t)) for t in grid] for r in raw])
+    level, samples, seed = pool.ci.get("level", 0.9), pool.ci.get("samples", SAMPLES), pool.ci.get("seed", 0)
+    fits = np.asarray([grid[int(np.argmin(nll[idx].mean(axis=0)))] for idx in cluster_resamples(raw, samples, seed)])
+    lower, upper = np.quantile(fits, [(1 - level) / 2, 1 - (1 - level) / 2])
+    return {"level": level, "samples": samples, "seed": seed, "lower": float(lower), "upper": float(upper), "questions": len(raw),
+            "resampled": "(source, group) clusters within each source (kev.metrics.cluster_resamples)"}
+
+
+class Side:
+    """One checkpoint of a comparison: its trial, where each read tag's rows live and its served temperature (fitted on
+    the trial's development rows, or on `pool` when the round registers one for its arms)."""
+
+    def __init__(self, trial, dirs, root=ROOT, drop=(), pool=None, checkpoint=None, suite=None, shipped=None):
+        self.trial, self.dirs, self.root, self.drop, self.pool, self.checkpoint = trial, dirs, Path(root), set(drop), pool, checkpoint
+        self.suite = suite       # the suite the trial trained and was scored on (its development rows'), when known
+        self.shipped = shipped   # a parent's: () -> the temperature its checkpoint ships (head.pt), or None when not here
+        self._t, self._served, self.fit = None, {}, None
+
+    def rows_path(self, tag):
+        return self.root / self.dirs[tag] / "rows.json"
+
+    def has(self, tag):
+        return tag in self.dirs and self.rows_path(tag).exists()
+
+    def absent_fit(self):
+        """The rows the temperature needs that are not here (pool directories, or the trial's development rows)."""
+        needed = [*self.pool.reads, *self.pool.exclude] if self.pool else [f"{self.trial}/development"]
+        return [d for d in needed if not (self.root / d / "rows.json").exists()]
+
+    @property
+    def t(self):
+        if self._t is None:
+            if self.pool: self._t, self.fit = pooled_temperature(self.pool, self.root)
+            else: self._t = temperature(self.trial, self.root)
+        return self._t
+
+    def temperature_source(self):
+        """Where the served temperature was fitted (read-out provenance): the pool (its reads and knowable question count),
+        or the trial's development rows, their suite and whether that suite is a training corpus (then the temperature is
+        in distribution: round 19's failure mode)."""
+        if self.pool: return {"kind": "pool", "reads": self.pool.reads, "questions": self.fit["questions"] if self.fit else None}
+        m = suite_manifest(self.suite) if self.suite else None
+        return {"kind": "trial development rows", "rows": f"{self.trial}/development", "suite": self.suite,
+                "training_corpus": bool(trainable_sources(m)) if m is not None else None, **({"shipped": self.shipped()} if self.shipped else {})}
+
+    def served(self, tag):
+        if tag not in self._served: self._served[tag] = served_at(read_json(self.rows_path(tag)), self.t)
+        return self._served[tag]
+
+    def panel(self, spec):
+        """A panel's served, knowable rows, reads concatenated in the order listed, minus drop_ids and whatever the
+        panel's filters leave out (panel_filter: `source`, `exclude_sources`, `exclude_tasks`, `exclude_file`)."""
+        keep = panel_filter(spec, self.root)
+        return [r for tag in spec["reads"] for r in self.served(tag) if r["id"] not in self.drop and keep(r)]
+
+    def unknowable_share(self, tag):
+        return unknowable_report(served_clean(read_json(self.rows_path(tag)), self.t))["share_at_0_9"] if self.has(tag) else None
+
+
+# --- panel filters ---------------------------------------------------------------------------------------------------
+
+EXCLUDE_KEYS = ("exclude_sources", "exclude_tasks", "exclude_file")   # round 24: the audit's per-suite fixes, applied to both sides
+EXCLUDE_FILE_KEYS = ("ids", "sources", "tasks")                       # what an exclude file may list (other keys, e.g. a salt, are ignored)
+
+
+def exclude_file_path(entry, root=ROOT):
+    return Path(root) / entry["path"]
+
+
+@functools.cache
+def _exclusion(path, sha256):
+    if digest(path) != sha256:
+        raise ValueError(f"exclude_file {path} does not match the sha256 the spec registers ({sha256[:12]}...): a changed exclusion list is a new spec")
+    data = read_json(path)
+    if isinstance(data, list): return {k: frozenset(data) for k in EXCLUDE_FILE_KEYS}   # a bare list: ids, sources or tasks alike
+    return {k: frozenset(data.get(k, ())) for k in EXCLUDE_FILE_KEYS}
+
+
+def exclusion(entry, root=ROOT):
+    """{ids, sources, tasks} of a panel's exclude_file ({path, sha256}: a JSON object with any of those lists, or a bare list
+    matched against all three), checked against its registered sha256. The file may be private (never in git: e.g. round 24's
+    tasksource-heldout families, restored with scripts/private_rows.py); the spec carries only its path and hash."""
+    return _exclusion(exclude_file_path(entry, root), entry["sha256"])
+
+
+def panel_filter(panel, root=ROOT):
+    """row -> kept? for a panel: `source` (a name, or a list of names) keeps only those sources; `exclude_sources` and
+    `exclude_tasks` drop rows by source and task; `exclude_file` drops rows whose id, record (group), source or task it lists.
+    The same filter serves both sides of a comparison (and a `versus` reference), so paired deltas stay paired."""
+    source = panel.get("source")
+    only = None if source is None else {source} if isinstance(source, str) else set(source)
+    sources, tasks = set(panel.get("exclude_sources", ())), set(panel.get("exclude_tasks", ()))
+    listed = exclusion(panel["exclude_file"], root) if panel.get("exclude_file") else None
+    def keep(r):
+        if only is not None and r["source"] not in only: return False
+        if r["source"] in sources or r.get("task") in tasks: return False
+        return not (listed and (r["id"] in listed["ids"] or r.get("group") in listed["ids"] or r["source"] in listed["sources"] or r.get("task") in listed["tasks"]))
+    return keep
+
+
+def filter_problems(panel, spec, root=ROOT, rows=True):
+    """What is wrong with a panel's filters: malformed values, an exclude_file without {path, sha256}, and (rows: the local
+    data a read-out needs) an exclude_file here that does not match its hash. `exclude_sources` names are checked
+    against the manifests of the panel's reads (a typo would silently keep a source the rule means to drop)."""
+    problems, source = [], panel.get("source")   # an absent exclude_file is not a problem here: validate lists it with the absent rows
+    if source is not None and not (isinstance(source, str) or (isinstance(source, list) and source and all(isinstance(s, str) for s in source))):
+        problems.append("source is a source name or a non-empty list of names")
+    for key in ("exclude_sources", "exclude_tasks"):
+        if key in panel and not (isinstance(panel[key], list) and panel[key] and all(isinstance(s, str) for s in panel[key])):
+            problems.append(f"{key} is a non-empty list of names")
+    if "optional" in panel and not isinstance(panel["optional"], bool): problems.append("optional is true or false")
+    entry = panel.get("exclude_file")
+    if entry is not None:
+        if not (isinstance(entry, dict) and isinstance(entry.get("path"), str) and isinstance(entry.get("sha256"), str) and len(entry["sha256"]) == 64):
+            problems.append("exclude_file is {path, sha256}")
+        elif rows and exclude_file_path(entry, root).exists() and digest(exclude_file_path(entry, root)) != entry["sha256"]:
+            problems.append(f"exclude_file {entry['path']} does not match its registered sha256")
+    if isinstance(panel.get("exclude_sources"), list):
+        suites = [spec["reads"][t].get("suite") for t in panel["reads"] if t in spec["reads"]]
+        if TRANSFER in panel["reads"]:
+            tag = spec.get("transfer_read")
+            suites.append(spec["reads"].get(tag, {}).get("suite") if tag else None)
+        manifests = [suite_manifest(suite_dir(s)) if s and suite_dir(s) else None for s in suites]
+        if manifests and all(m is not None for m in manifests):
+            listed = set().union(*(listed_sources(m) for m in manifests))
+            if unknown := sorted(set(panel["exclude_sources"]) - listed): problems.append(f"exclude_sources name {unknown}, which no read of the panel lists")
+    return problems
+
+
+def gating_panels(rule):
+    """Panels a criterion or the rank reads: an `optional` panel (report only) must not be one of them."""
+    paths = [p for c in rule["criteria"].values() for p in _paths(c)]
+    paths += [p for key in rule.get("rank", []) for p in (key["by"] if isinstance(key["by"], list) else [key["by"]])]
+    return {p.split(".")[0] for p in paths}
+
+
+# --- spec ------------------------------------------------------------------------------------------------------------
+
+def load(path):
+    spec = read_json(path)
+    spec.setdefault("arm_reads", "runs/r{round}-{arm}-{tag}")
+    return spec
+
+
+def size_of(arm):
+    return arm.split("-")[0]
+
+
+def locations(where, spec, **values):
+    """{tag: dir} from a template string (one for every tag of the spec) or an explicit {tag: template} mapping."""
+    tags = [*spec.get("reads", {}), TRANSFER]
+    if isinstance(where, str):
+        return {tag: where.format(round=spec["round"], tag=tag, **values) for tag in tags}
+    return {tag: d.format(round=spec["round"], tag=tag, **values) for tag, d in where.items()}
+
+
+def transfer_read(spec, arm):
+    """The read tag whose rows are the arm's "transfer" rows, or None for its trial's in-trial transfer read."""
+    a = spec["arms"][arm]
+    return a["transfer_read"] if "transfer_read" in a else spec.get("transfer_read")
+
+
+def arm_side(spec, arm, root=ROOT, stage=None):
+    """The arm's side. Its "transfer" rows and its temperature pool come from its rule-stage reads at every stage (the
+    development reads it was selected on); a stage's candidate_reads locate the stage's own panels."""
+    a = spec["arms"][arm]
+    development = locations(a.get("reads", spec["arm_reads"]), spec, arm=arm, size=size_of(arm))
+    where = spec["confirm"][stage]["candidate_reads"] if stage else a.get("reads", spec["arm_reads"])
+    tag = transfer_read(spec, arm)
+    transfer = development.get(tag) if tag else f"{a['trial']}/transfer" if a.get("trial") else None
+    dirs = {**locations(where, spec, arm=arm, size=size_of(arm)), **({TRANSFER: transfer} if transfer else {})}
+    pool, registered = None, spec.get("temperature")
+    if registered:
+        fit = {**development, **({TRANSFER: transfer} if transfer else {})}
+        pool = Pool([fit[r] for r in registered["reads"]], {fit[r]: s for r, s in registered.get("sources", {}).items()}, [fit[r] for r in registered.get("exclude_reads", [])], registered.get("ci"))
+    suite = trial_suite(spec, a["trial"], root) if a.get("trial") and not pool else None
+    return Side(a.get("trial"), dirs, root, spec.get("drop_ids", ()), pool, a.get("checkpoint"), suite)
+
+
+def parent_side(spec, arm, root=ROOT, stage=None):
+    p = spec["parents"][spec["arms"][arm]["parent"]]
+    where = (spec["confirm"][stage].get("parent_reads") if stage else None) or p["reads"]
+    dirs = {**locations(where, spec, arm=arm, size=size_of(arm)), TRANSFER: f"{p['trial']}/transfer"}
+    return Side(p["trial"], dirs, root, spec.get("drop_ids", ()), suite=trial_suite(spec, p["trial"], root), shipped=lambda: shipped_temperature(p, root))
+
+
+def rule_tags(rule):
+    return {tag for panel in rule["panels"].values() for tag in panel["reads"]} | ({rule["unknowable"]} if rule.get("unknowable") else set())
+
+
+def _paths(criterion):
+    left = criterion["left"] if isinstance(criterion["left"], list) else [criterion["left"]]
+    return [*left, *([criterion["right"]] if isinstance(criterion["right"], str) else [])]
+
+
+class Validation(NamedTuple):
+    problems: list   # the spec is malformed, or something a launch or a read-out needs is missing
+    archived: list   # files a recorded round names that this checkout does not carry (they live on spec["archive"])
+
+
+def validate(spec, root=ROOT, rows=True, plans=True, partitions=False):
+    """Validation(problems, archived). rows: every parent's development rows and every read the rule needs from it exist
+    under root (what a read-out, and so a launch, needs); plans: every read suite exists, every plan trial passes
+    kev.experiment.validated_trial against its suite manifest and fits its budget; partitions: plans go through load_plan
+    instead (verifies the partitions, may fetch them from the Hub). A recorded round (`archive`: the git tag holding its
+    evidence) lists absent files under `archived` instead of failing; a round without it must have them all. A read of a
+    removed suite (kev.suite.REMOVED_SUITES) is archived (with plans) for a round up to the suite's last_round, whether or
+    not the round carries `archive`, and a problem for any later round."""
+    problems, archived, root = [], [], Path(root)
+    absent = (archived if spec.get("archive") else problems).append
+    for key in ("round", "registered", "parents", "arms", "reads", "rule"):
+        if key not in spec: problems.append(f"missing key {key!r}")
+    if problems: return Validation(problems, archived)
+    tags = {*spec["reads"], TRANSFER}
+    for name, r in spec["reads"].items():
+        if r.get("entrypoint", "benchmarks") not in ENTRYPOINTS: problems.append(f"read {name}: entrypoint {r['entrypoint']!r} is not one of {ENTRYPOINTS}"); continue
+        if r.get("entrypoint", "benchmarks") == "benchmarks" and not r.get("suite"): problems.append(f"read {name}: no suite")
+        if r.get("entrypoint") == "locked_test" and not r.get("decision"): problems.append(f"read {name}: locked_test needs a decision suite")
+        if r.get("suite") and (gone := removed_suite(r["suite"])):   # kev.suite.REMOVED_SUITES: archived up to its last round, refused after
+            if isinstance(spec["round"], int) and spec["round"] <= gone["last_round"]:
+                if plans: archived.append(f"read {name}: {r['suite']} not in this checkout: removed on {gone['removed']} (kev.suite.REMOVED_SUITES); "
+                                          f"round {spec['round']} read it before the removal, and the rows it made are the record")
+            else:
+                problems.append(f"read {name}: {r['suite']} was removed on {gone['removed']} and may not be read after round {gone['last_round']}: {gone['reason']}")
+            continue
+        if plans and r.get("suite") and not (root / r["suite"]).exists(): absent(f"read {name}: {r['suite']} not in this checkout")
+    stages = {None: spec["rule"], **spec.get("confirm", {})}
+    reads_transfer = any(TRANSFER in panel["reads"] for rule in stages.values() for panel in rule["panels"].values())
+    for name, a in spec["arms"].items():
+        if a.get("parent") not in spec["parents"]: problems.append(f"arm {name}: unknown parent {a.get('parent')!r}")
+        if "-" not in name: problems.append(f"arm {name}: name must be <size>-<label>")
+        tag = transfer_read(spec, name)
+        if tag is not None and (tag not in spec["reads"] or spec["reads"][tag].get("entrypoint", "benchmarks") != "benchmarks"):
+            problems.append(f"arm {name}: transfer_read {tag!r} is not a benchmarks read of this spec")
+        if tag is not None and isinstance(a.get("reads"), dict) and tag not in a["reads"]: problems.append(f"arm {name}: its reads do not locate its transfer_read {tag!r}")
+        if not a.get("trial"):
+            if not str(a.get("checkpoint", "")).startswith("/runs/"): problems.append(f"arm {name}: an arm without a trial needs a checkpoint on the runs volume (/runs/...)")
+            if not spec.get("temperature"): problems.append(f"arm {name}: no trial, so no development rows to fit its temperature on; register a `temperature` pool")
+            if reads_transfer and tag is None: problems.append(f"arm {name}: no trial, so no in-trial transfer read; give it a transfer_read")
+    problems += pool_problems(spec, stages)
+    conflicts, unknown_training = pool_training_problems(spec, root)
+    problems += conflicts + pool_required_problems(spec)
+    for line in unknown_training: absent(line)
+    for stage, rule in stages.items():
+        where = f"confirm.{stage}" if stage else "rule"
+        for pname, panel in rule["panels"].items():
+            unknown = set(panel["reads"]) - tags
+            if unknown: problems.append(f"{where} panel {pname}: unknown read tags {sorted(unknown)}")
+            if not panel.get("metrics") and not panel.get("report"): problems.append(f"{where} panel {pname}: nothing to compute")
+            problems += [f"{where} panel {pname}: {p}" for p in by_length_problems(panel, spec)]
+            problems += [f"{where} panel {pname}: {p}" for p in filter_problems(panel, spec, root, rows)]
+            if panel.get("optional") and pname in gating_panels(rule): problems.append(f"{where} panel {pname}: optional (report only), but a criterion or the rank reads it")
+        for cname, c in rule["criteria"].items():
+            if c.get("op") not in OPS: problems.append(f"{where} criterion {cname}: op {c.get('op')!r}")
+            for path in _paths(c):
+                if not _known_path(path, rule): problems.append(f"{where} criterion {cname}: unknown path {path!r}")
+        for key in rule.get("rank", []):
+            for path in key["by"] if isinstance(key["by"], list) else [key["by"]]:
+                if not _known_path(path, rule): problems.append(f"{where} rank: unknown path {path!r}")
+        if stage and "candidate_reads" not in rule: problems.append(f"{where}: no candidate_reads")
+    if rule_tags(spec["rule"]) - tags: problems.append(f"rule.unknowable: unknown read tag {spec['rule']['unknowable']!r}")
+    for study, s in spec.get("studies", {}).items() if plans else ():
+        if not (root / s["plan"]).exists(): absent(f"study {study}: plan {s['plan']} not in this checkout")
+        elif (root / s["suite"] / "manifest.json").exists(): problems += _validate_plan(study, s, root, partitions)
+        else: absent(f"study {study}: suite {s['suite']} not in this checkout")
+    if rows:
+        for where, pname, entry in [(s or "rule", n, p["exclude_file"]) for s, rule in stages.items() for n, p in rule["panels"].items() if isinstance(p.get("exclude_file"), dict)]:
+            if isinstance(entry.get("path"), str) and not exclude_file_path(entry, root).exists():
+                absent(f"{where} panel {pname}: exclude_file {entry['path']} not in this checkout (a private list: scripts/private_rows.py restore)")
+        for pname, p in spec["parents"].items():
+            if not (root / p["trial"] / "development/rows.json").exists(): absent(f"parent {pname}: {p['trial']}/development/rows.json not in this checkout")
+        for arm in spec["arms"]:
+            side = parent_side(spec, arm, root)
+            required = {t for panel in spec["rule"]["panels"].values() if not panel.get("optional") for t in panel["reads"]}   # an optional panel's reads may be absent
+            for tag in sorted(required - {spec["rule"].get("unknowable")}):   # the parent's unknowable read is reported, not required
+                if not side.has(tag): absent(f"arm {arm}: parent read {tag} not in this checkout ({side.dirs.get(tag, 'no location')})")
+    return Validation(problems, archived)
+
+
+def _known_path(path, rule):
+    parts = path.split(".")
+    if parts[0] == "unknowable": return len(parts) == 2 and parts[1] in ("candidate", "parent") and bool(rule.get("unknowable"))
+    panel = rule["panels"].get(parts[0])
+    if panel is None or len(parts) != 3 or parts[2] not in FIELDS: return False
+    value_only = parts[1] in panel.get("report", ()) or parts[1] in by_length_names(panel)   # report and by-length metrics have no interval
+    return parts[1] in panel.get("metrics", ()) or (value_only and parts[2] in ("candidate", "parent"))
+
+
+# --- calibration by state length -------------------------------------------------------------------------------------
+
+def by_length_options(panel):
+    """(edges, tokenizer) of a panel's by_length option: true = kev.metrics.LENGTH_EDGES and kev.suite.ADMISSION_TOKENIZER."""
+    option = panel.get("by_length")
+    option = {} if option is True else option
+    return tuple(option.get("edges", LENGTH_EDGES)), tuple(option.get("tokenizer", ADMISSION_TOKENIZER))
+
+
+def by_length_names(panel):
+    """{"<metric>_<bucket>": metric} for the entries a by_length panel reports (empty for any other panel, or invalid edges)."""
+    if not panel.get("by_length"): return {}
+    try:
+        return {f"{m}_{name}": m for m in LENGTH_METRICS for name, _, _ in length_buckets(by_length_options(panel)[0])}
+    except (ValueError, AttributeError, TypeError):
+        return {}
+
+
+def by_length_problems(panel, spec):
+    option = panel.get("by_length")
+    if not option: return []
+    if option is not True and (not isinstance(option, dict) or set(option) - {"edges", "tokenizer"}):
+        return ["by_length is true or {edges?, tokenizer?}"]
+    problems = []
+    try:
+        edges, tokenizer = by_length_options(panel)   # tuple() of a non-iterable raises: a malformed option is a problem, not a crash
+        length_buckets(edges)
+        if len(tokenizer) != 2 or not all(isinstance(x, str) for x in tokenizer): problems.append("by_length: tokenizer is [name, revision]")
+    except (ValueError, TypeError) as error:
+        problems.append(f"by_length: {error}" if isinstance(error, ValueError) else f"by_length: edges are a list of token counts and tokenizer is [name, revision] ({error})")
+    if TRANSFER in panel["reads"]: problems.append("by_length counts state tokens from a read's suite; 'transfer' names no single suite")
+    problems += [f"by_length: read {tag!r} has no suite" for tag in panel["reads"] if tag in spec["reads"] and not spec["reads"][tag].get("suite")]
+    return problems
+
+
+@functools.cache
+def state_lengths(suite, split, tokenizer=ADMISSION_TOKENIZER):
+    """{record id: state tokens} of a suite partition with the tokenizer (name, revision): the counts a by_length panel
+    buckets both sides by. A state's token count is its encoded state segment, as kev.model.encode builds it: the <state>
+    token plus user_tokens of the materialised state (len + 1, what encode checks against max_state and kev.serve reports
+    as `state_tokens`, seg.count(0)); the one definition behind a row's `state_tokens` (kev.metrics.calibration_by_length)."""
+    from kev.data import materialize
+    from kev.model import load_tokenizer, user_tokens
+    tok = load_tokenizer(*tokenizer)
+    return {r["_meta"]["id"]: 1 + len(user_tokens(tok, materialize(r)["state"])) for r in load_split(ROOT / suite, split, allow_test=split == "test")}
+
+
+def panel_lengths(spec, panel):
+    """{record id: state tokens} over a by_length panel's reads (each read's suite and partition)."""
+    tokenizer, out = by_length_options(panel)[1], {}
+    for tag in panel["reads"]:
+        read = spec["reads"][tag]
+        for rid, n in state_lengths(read["suite"], read_split(read), tokenizer).items():
+            if out.setdefault(rid, n) != n: raise ValueError(f"record {rid!r} has different states in the reads of one by_length panel")
+    return out
+
+
+# --- temperature pools against training data (round 19's failure mode) ------------------------------------------------
+
+ROUND_19 = ("round 19's failure mode: a temperature fitted on held-out items of a checkpoint's own training corpus is in distribution "
+            "(round 19 served its SFT arms at T 0.955 fitted on sft-v1 development rows: breadth-v1 ECE 0.059); fit on a pool of "
+            "held-out datasets instead (round 20: breadth-v1 ECE 0.0085)")
+FIT_SPLITS = ("calibration", "development")   # a training corpus's held-out partitions: never a temperature pool
+
+
+def read_split(read):
+    """The partition a read scores: kev.benchmark's development, `--split X`, or the test under --allow-test / locked_test."""
+    flags = read.get("flags", "").split()
+    if read.get("entrypoint") == "locked_test" or "--allow-test" in flags: return "test"
+    for i, flag in enumerate(flags):
+        if flag.startswith("--split="): return flag.split("=", 1)[1]
+        if flag == "--split" and i + 1 < len(flags): return flags[i + 1]
+    return "development"
+
+
+def suite_dir(path):
+    """'evals/...' for a suite path given relative, absolute or as a container saw it (/root/kev/evals/sft-v1); None if none."""
+    return suite_key(path)
+
+
+@functools.cache
+def _suites():
+    """({manifest sha256: suite dir}, {directory name: [suite dirs]}) of every suite in this checkout."""
+    by_digest, by_name = {}, {}
+    for m in sorted((ROOT / "evals").rglob("manifest.json")):
+        d = str(m.parent.relative_to(ROOT))
+        by_digest[digest(m)] = d
+        by_name.setdefault(m.parent.name, []).append(d)
+    return by_digest, by_name
+
+
+def suite_by_digest(sha256):
+    """The suite dir whose manifest.json hashes to sha256 (what provenance, head.pt and report.json record), or None."""
+    return _suites()[0].get(sha256)
+
+
+def suite_manifest(suite):
+    return read_manifest(ROOT / suite) if (ROOT / suite / "manifest.json").exists() else None
+
+
+def listed_sources(manifest):
+    """Every source a manifest names: trainable, held-out, eval-only and its `sources` table (a dict or a list)."""
+    return {*manifest.get("trainable_sources", []), *manifest.get("holdout_sources", []), *manifest.get("eval_only_sources", []), *manifest.get("sources", [])}
+
+
+def trainable_sources(manifest):
+    """The sources a suite offers for training (`trainable_sources`, or `sources` entries marked trainable): a suite with
+    any is a training corpus, whose calibration and development partitions are held-out items of training sources."""
+    table = manifest.get("sources", {})
+    return {*manifest.get("trainable_sources", []), *(k for k, v in table.items() if isinstance(v, dict) and v.get("trainable"))} if isinstance(table, dict) else set(manifest.get("trainable_sources", []))
+
+
+class Training(NamedTuple):
+    """What a checkpoint was trained on, as far as this checkout can tell."""
+    suites: frozenset     # suite dirs: its training suite, the components that suite names, a plan's `data` directory
+    sources: frozenset    # the sources those suites train
+    unlisted: tuple       # training suites whose sources this checkout cannot list
+
+
+def training_data(suites, data=None):
+    """Training over suite dirs plus a `data` file: each suite's trainable sources (every listed source for a data-only
+    manifest without `trainable_sources`, e.g. round6/b1v2) plus the suites its manifest names as components (sft-v1's
+    `inputs.components`: documents-v1-train -> evals/documents-v1, hard-v1-extra-train -> evals/hard-v1). A `data` file
+    counts through its directory's suite; one outside evals/ (a `kev.train --data` run) cannot be checked, so it is
+    `unlisted` like any training this checkout cannot place, never silently dropped."""
+    data_dir = suite_dir(Path(data).parent) if data else None
+    seen, sources, unlisted, todo = set(), set(), [] if data_dir or not data else [f"data {data} (outside evals/)"], [d for d in (*suites, data_dir) if d]
+    while todo:
+        d = todo.pop(0)
+        if d in seen: continue
+        seen.add(d)
+        m = suite_manifest(d)
+        own = (trainable_sources(m) if "trainable_sources" in m else listed_sources(m)) if m is not None else set()
+        if not own: unlisted.append(d)
+        sources |= own
+        inputs = (m or {}).get("inputs")
+        for component in inputs.get("components", {}) if isinstance(inputs, dict) else ():
+            found = _suites()[1].get(component.removesuffix("-train").removesuffix("-extra"))
+            if found: todo += found
+            else: unlisted.append(f"{d} component {component}")
+    return Training(frozenset(seen), frozenset(sources), tuple(unlisted))
+
+
+def recorded_training(suite_sha256=None, suite=None, data=None):
+    """Training of a checkpoint from what its trainer recorded (provenance.json, head.pt): the suite whose manifest hashes to
+    suite_sha256 (else the suite path it was given) and its `data` file's directory; None when no suite of this checkout matches."""
+    trained = suite_by_digest(suite_sha256)
+    if trained is None and suite and suite_dir(suite) and suite_manifest(suite_dir(suite)) is not None: trained = suite_dir(suite)
+    return training_data([trained], data) if trained else None
+
+
+def _study(spec, trial):
+    return next((s for name, s in spec.get("studies", {}).items() if trial.startswith(f"runs/{name}/")), None)
+
+
+def trial_suite(spec, trial, root=ROOT):
+    """The suite a trial trained on and was scored on (its development rows'): its study's in this spec, else the manifest
+    its provenance.json hashes; None when neither is here."""
+    if study := _study(spec, trial): return suite_dir(study["suite"])
+    provenance = Path(root) / trial / "provenance.json"
+    return suite_by_digest(read_json(provenance).get("suite_sha256")) if provenance.exists() else None
+
+
+def trial_training(spec, trial, root=ROOT):
+    """Training of a trial: its study's suite plus its plan entry's `data` (trial <NN>-<label> is entry NN), else what its
+    provenance.json records; None when neither is here."""
+    study, provenance = _study(spec, trial), Path(root) / trial / "provenance.json"
+    if study:
+        plan = next((p / study["plan"] for p in (Path(root), ROOT) if (p / study["plan"]).exists()), None)
+        entries, index = (read_json(plan) if plan else []), Path(trial).name.split("-")[0]
+        data = entries[int(index)].get("data") if isinstance(entries, list) and index.isdigit() and int(index) < len(entries) else None
+        if plan or not provenance.exists(): return training_data([suite_dir(study["suite"])], data)
+    if not provenance.exists(): return None
+    p = read_json(provenance)
+    return recorded_training(p.get("suite_sha256"), data=p.get("config", {}).get("data"))
+
+
+def pool_conflicts(pooled, training):
+    """[(label, why)] for every way a pooled read shares data with a checkpoint's Training. pooled = [(label, suite, split,
+    sources or None)]: (a) the read's suite is training data of the checkpoint (its training suite, or a component or `data`
+    suite of it); (b) the sources it pools (its allowlist, else every source its manifest lists) are sources the checkpoint
+    trained on; (c) it scores the calibration or development partition of a training corpus (any suite with trainable
+    sources). A read whose suite or sources cannot be established is refused too. The one check, for kev.rounds pools and
+    scripts/calibrate_checkpoint.py alike."""
+    out = []
+    for label, suite, split, sources in pooled:
+        d = suite_dir(suite) if suite else None
+        m = suite_manifest(d) if d else None
+        if m is None:
+            out.append((label, f"its suite ({suite!r}) has no manifest in this checkout, so what it pools cannot be checked")); continue
+        if d in training.suites: out.append((label, f"{d} is training data of the checkpoint"))
+        pooled_sources = set(sources) if sources else listed_sources(m)
+        if not pooled_sources: out.append((label, f"{d} lists no sources; give the read a `sources` allowlist"))
+        if shared := sorted(pooled_sources & training.sources):
+            out.append((label, f"it pools {len(shared)} training source(s) {shared[:8]}" + (" ..." if len(shared) > 8 else "")))
+        if split in FIT_SPLITS and trainable_sources(m): out.append((label, f"it reads the {split} partition of {d}, a training corpus"))
+    return out
+
+
+def pool_training_problems(spec, root=ROOT):
+    """(problems, unknown) of the round's `temperature` pool against every arm's training data (pool_conflicts; message
+    names round 19's failure mode). An arm's training comes from `trained_on`, else from its trial (trial_training); an arm
+    without either (a checkpoint made from the round's trials) is covered by the trial arms. unknown: arms whose training
+    this checkout cannot establish or list (a trial without provenance, a manifest without sources, a `data` file outside
+    evals/): a problem for a new round, reported under archived for a recorded one."""
+    pool = spec.get("temperature")
+    if not pool or not pool.get("reads"): return [], []
+    pooled = [(tag, spec["reads"][tag].get("suite"), read_split(spec["reads"][tag]), pool.get("sources", {}).get(tag)) for tag in pool["reads"] if tag in spec["reads"]]
+    found, problems, unknown = {}, [], []
+    for arm, a in spec["arms"].items():
+        if "trained_on" in a:
+            if not isinstance(a["trained_on"], list) or not a["trained_on"] or not all(isinstance(s, str) for s in a["trained_on"]):
+                problems.append(f"arm {arm}: trained_on is a non-empty list of suite directories"); continue
+            training = training_data([suite_dir(s) or s for s in a["trained_on"]])
+        elif a.get("trial"):
+            training = trial_training(spec, a["trial"], root)
+            if training is None:
+                unknown.append(f"arm {arm}: what {a['trial']} trained on is unknown (no study of this spec, no provenance.json), so the temperature pool cannot be checked against it"); continue
+        else:
+            continue
+        unknown += [f"arm {arm}: cannot list the sources of {d}, part of its training, so the temperature pool cannot be checked against it" for d in training.unlisted]
+        by_tag = {}
+        for tag, why in pool_conflicts(pooled, training): by_tag.setdefault(tag, []).append(why)
+        for tag, whys in by_tag.items(): found.setdefault((tag, "; ".join(whys)), []).append(arm)
+    for (tag, why), arms in found.items():
+        problems.append(f"temperature: pooled read {tag!r} shares data with the training of arm(s) {', '.join(arms)}: {why}. This is {ROUND_19}")
+    if not any("trained_on" in a or a.get("trial") for a in spec["arms"].values()):
+        problems.append("temperature: no arm names its training (a trial or `trained_on`), so the pool cannot be checked against it")
+    return problems, unknown
+
+
+POOL_REQUIRED_FROM = 21   # from this round on, a temperature-dependent criterion without a `temperature` pool is refused
+
+
+def temperature_criteria(spec):
+    """["rule <name>" | "confirm.<stage> <name>"] of every criterion that reads a number the temperature moves."""
+    stages = {None: spec["rule"], **spec.get("confirm", {})}
+    return sorted({f"{'confirm.' + s if s else 'rule'} {name}" for s, rule in stages.items() for name, c in rule["criteria"].items()
+                   if any(_moves_with_temperature(path, rule) for path in _paths(c))})
+
+
+def _listed(names):
+    return f"{len(names)} ({', '.join(names[:3])}{', ...' if len(names) > 3 else ''})"
+
+
+def pool_required_problems(spec):
+    """Round >= POOL_REQUIRED_FROM with a temperature-dependent criterion and no `temperature` pool: refused (its arms would
+    be served at their trials' development rows, round 19's failure mode). Earlier rounds only warn (calibration_warnings),
+    so their recorded specs keep validating."""
+    number, moved = spec.get("round"), temperature_criteria(spec)
+    if spec.get("temperature") or not moved or not isinstance(number, int) or number < POOL_REQUIRED_FROM: return []
+    return [f"temperature: round {number} registers no `temperature` pool, but {_listed(moved)} criteria depend on the temperature, "
+            f"so its arms would be served at their trials' development rows. This is {ROUND_19}; register a pool of held-out datasets (copy r20)"]
+
+
+def shipped_temperature(entry, root=ROOT):
+    """The temperature a parent's checkpoint ships (head.pt), read locally only: <trial>/checkpoint/head.pt, else its Hub
+    checkpoint's head.pt in the local Hugging Face cache (no network); None when neither is here."""
+    path = Path(root) / entry["trial"] / "checkpoint" / "head.pt"
+    if not path.exists() and entry.get("checkpoint") and not str(entry["checkpoint"]).startswith("/"):
+        from huggingface_hub import try_to_load_from_cache
+        repo, _, revision = entry["checkpoint"].partition("@")
+        cached = try_to_load_from_cache(repo, "head.pt", revision=revision or None)
+        path = Path(cached) if isinstance(cached, str) else path
+    if not path.exists(): return None
+    from kev.checkpoint import read_meta
+    return read_meta(path.parent).temperature
+
+
+SHIPPED_TOLERANCE = 0.05   # a parent served this far from its shipped temperature, fitted in distribution, is warned about
+
+
+def continuation_warnings(spec, root=ROOT):
+    """Report only: a full-weight study runs longer than one attempt only while something continues it. Its trials spawn
+    with Modal's retries off (kev.budget.FULL_FT_RETRIES says why), so a timed-out trial gets its next attempt from
+    `kev.rounds watch` (modal_app.continue_full_trial), or by hand with modal_app.py::resume --trial; an attached launch
+    keeps no attempt ledger and is never continued."""
+    out = []
+    for name, study in spec.get("studies", {}).items():
+        plan = Path(root) / study.get("plan", "")
+        trials = read_json(plan) if plan.is_file() else []
+        if any(isinstance(t, dict) and t.get("full_ft") for t in (trials if isinstance(trials, list) else [])):
+            out.append(f"study {name} is full-weight: a trial that times out is continued only while `kev.rounds watch` runs (or by hand: "
+                       f"modal_app.py::resume --study {name} --trial <label>); an attached launch (study --detached False) keeps no ledger and is never continued")
+    return out
+
+
+def calibration_warnings(spec, root=ROOT):
+    """Report-only warnings that `validate` and `launch` print. (1) For a round before POOL_REQUIRED_FROM without a
+    `temperature` pool whose criteria depend on the temperature (from then on it is a problem: pool_required_problems): each
+    trial arm whose development rows are a training corpus's is served in distribution, round 19's failure mode. (2) Every
+    parent served at a temperature fitted on a training corpus's development rows that differs from its shipped head.pt
+    temperature by more than SHIPPED_TOLERANCE (when both are here)."""
+    out, moved = [], temperature_criteria(spec)
+    early = isinstance(spec.get("round"), int) and spec["round"] < POOL_REQUIRED_FROM
+    for arm, a in spec["arms"].items() if moved and early and not spec.get("temperature") else ():
+        suite = trial_suite(spec, a["trial"], root) if a.get("trial") else None
+        if suite and trainable_sources(suite_manifest(suite) or {}):
+            out.append(f"arm {arm} will be served at a temperature fitted on its trial's {suite} development rows, a training corpus, and "
+                       f"{_listed(moved)} criteria depend on it: {ROUND_19}; register a `temperature` pool")
+    for name, p in spec["parents"].items():
+        suite = trial_suite(spec, p["trial"], root)
+        if not (suite and trainable_sources(suite_manifest(suite) or {}) and (Path(root) / p["trial"] / "development/rows.json").exists()): continue
+        shipped = shipped_temperature(p, root)
+        served_t = temperature(p["trial"], root) if shipped is not None else None
+        if shipped is not None and abs(served_t - shipped) > SHIPPED_TOLERANCE:
+            out.append(f"parent {name} is served at T {served_t:.3f} fitted on {p['trial']}/development ({suite}, a training corpus), "
+                       f"{abs(served_t - shipped):.3f} from the T {shipped:.3f} its checkpoint ships: its reads are not what it serves (report only)")
+    return out
+
+
+TEMPERATURE_FREE = ("acc",)   # the one bootstrapped metric a temperature cannot move (the argmax is invariant)
+
+
+def _moves_with_temperature(path, rule):
+    """Whether a criterion path reads a number the temperature moves: any panel metric but accuracy (a by-length bucket's
+    included: acc_16k_plus is accuracy); the unknowable share is scored on records a pool never fits on."""
+    panel, metric = (path.split(".") + [""])[:2]
+    return panel != "unknowable" and by_length_names(rule["panels"].get(panel, {})).get(metric, metric) not in TEMPERATURE_FREE
+
+
+def allowlist_problems(tag, suite, sources):
+    """A pool read's `sources` allowlist against the sources its suite's manifest lists: a name the suite does not contain
+    (a typo) would silently shrink the pool, so it is a problem, as is a suite whose sources cannot be listed."""
+    m = suite_manifest(suite_dir(suite)) if suite and suite_dir(suite) else None
+    listed = listed_sources(m) if m is not None else set()
+    if not listed: return [f"temperature: cannot check the sources allowlist of {tag!r}: {suite!r} lists no sources in this checkout"]
+    return [f"temperature: sources for {tag!r} name {unknown} which {suite} does not contain" for unknown in [sorted(set(sources) - listed)] if unknown]
+
+
+def pool_problems(spec, stages):
+    """What is wrong with the round's `temperature` pool: tags that are not reads of the spec, tags an arm cannot locate,
+    and pooled tags inside a panel that a temperature-dependent criterion reads (any metric but accuracy, at any stage):
+    the temperature would be fitted on the rows it is judged on. The unknowable share is scored on unknowable records,
+    which a pool never fits on (knowable rows only), so the unknowable read may be pooled."""
+    pool = spec.get("temperature")
+    if not pool: return []
+    reads, exclude = pool.get("reads") or [], pool.get("exclude_reads", [])
+    problems = ["temperature: no reads to pool"] if not reads else []
+    ci = pool.get("ci")
+    if ci is not None and not (isinstance(ci, dict) and not set(ci) - {"level", "samples", "seed"} and 0 < ci.get("level", 0.9) < 1
+                               and isinstance(ci.get("samples", SAMPLES), int) and ci.get("samples", SAMPLES) > 0 and isinstance(ci.get("seed", 0), int)):
+        problems.append("temperature: ci is {level (0-1), samples (> 0), seed}")
+    problems += [f"temperature: {tag!r} is not a read tag of this spec" for tag in reads if tag not in spec["reads"]]
+    problems += [f"temperature: exclude_reads {tag!r} is not a read tag of this spec" for tag in exclude if tag not in spec["reads"] and tag != TRANSFER]
+    for tag, sources in pool.get("sources", {}).items():
+        if tag not in reads: problems.append(f"temperature: sources for {tag!r}, which is not pooled")
+        if not sources or not isinstance(sources, list) or not all(isinstance(s, str) for s in sources):
+            problems.append(f"temperature: sources for {tag!r} must be a non-empty list of source names"); continue
+        problems += allowlist_problems(tag, spec["reads"].get(tag, {}).get("suite"), sources)
+    for stage, rule in stages.items():
+        for cname, c in rule["criteria"].items():
+            for path in _paths(c):
+                shared = sorted(set(reads) & set(rule["panels"].get(path.split(".")[0], {}).get("reads", [])))
+                if shared and _moves_with_temperature(path, rule):
+                    problems.append(f"temperature: {shared} pooled, but {'confirm.' + stage if stage else 'rule'} criterion {cname} reads {path}, which the temperature moves")
+    for arm, a in spec["arms"].items():
+        where = a.get("reads", spec["arm_reads"])
+        unplaced = [t for t in [*reads, *exclude] if (t == TRANSFER and not a.get("trial") and transfer_read(spec, arm) is None)
+                    or (t != TRANSFER and not isinstance(where, str) and t not in where)]
+        if unplaced: problems.append(f"arm {arm}: its reads do not locate the temperature pool's {unplaced}")
+    return problems
+
+
+def _validate_plan(study, s, root, partitions):
+    from kev.experiment import load_plan, validated_trial
+    from kev.suite import read_manifest
+    from kev.budget import compute_bound   # the admission bound modal_app.admit_study refuses a study over
+    problems = []
+    try:
+        trials = load_plan(root / s["suite"], root / s["plan"]) if partitions else [validated_trial(t, read_manifest(root / s["suite"])) for t in read_json(root / s["plan"])]
+    except (ValueError, KeyError) as error:
+        return [f"study {study}: plan {s['plan']} does not validate ({type(error).__name__}: {error})"]
+    bound = compute_bound(s["gpu"], s["timeout"], len(trials), any(t.get("full_ft") for t in trials))
+    if bound > s["budget"]: problems.append(f"study {study}: admission bound ${bound:.2f} exceeds budget ${s['budget']:.2f} (modal_app.admit_study would refuse it)")
+    return problems
+
+
+# --- rule ------------------------------------------------------------------------------------------------------------
+
+def compare(candidate, parent, rule, lengths=None):
+    """Every panel, the unknowable share and every criterion of one candidate against its parent. Panels whose reads are
+    missing on either side are listed under "missing" and their criteria are None; `passed` needs every criterion. A
+    candidate served at a pooled temperature records the fit (`temperature_fit`); every candidate records where its
+    temperature came from (`temperature_source`); one whose pool rows are missing is reported incomplete with nothing
+    computed. lengths(panel) -> {record id: state tokens} serves by_length panels whose rows record no state_tokens."""
+    out = {"trial": candidate.trial, "parent": parent.trial, **({"checkpoint": candidate.checkpoint} if candidate.checkpoint else {})}
+    if absent := [f"candidate:{d}" for d in candidate.absent_fit()]:
+        return {**out, "temperature": None, "missing": absent, "complete": False, "passed": None}
+    out.update(temperature=candidate.t, parent_temperature=parent.t, panels={}, missing=[])
+    if candidate.fit: out["temperature_fit"] = candidate.fit
+    if candidate.pool and candidate.pool.ci: out["temperature_ci"] = pooled_temperature_ci(candidate.pool, candidate.root)
+    out["temperature_source"] = candidate.temperature_source()
+    out["parent_temperature_source"] = parent.temperature_source()   # parents are served at their trial's development rows; `shipped` is head.pt's T
+    for name, spec in rule["panels"].items():
+        absent = [f"{who}:{side.dirs[t] if t in side.dirs else t}" for who, side in (("candidate", candidate), ("parent", parent)) for t in spec["reads"] if not side.has(t)]
+        if isinstance(spec.get("exclude_file"), dict) and not exclude_file_path(spec["exclude_file"], candidate.root).exists():
+            absent.append(f"exclude_file:{spec['exclude_file']['path']}")
+        if absent:   # an optional (report-only) panel's absent reads leave the comparison complete
+            if spec.get("optional"): out.setdefault("missing_optional", []).extend(absent)
+            else: out["missing"] += absent
+            continue
+        c, p = candidate.panel(spec), parent.panel(spec)
+        mc, mp = metrics(c), metrics(p)
+        panel = {"n": len(c)}
+        if any(k in spec for k in EXCLUDE_KEYS):   # questions the panel's exclusions removed (candidate side; the parent's are the same records)
+            panel["excluded"] = len(candidate.panel({k: v for k, v in spec.items() if k not in EXCLUDE_KEYS})) - len(c)
+        for m in spec.get("metrics", ()):
+            panel[m] = {"candidate": mc[m], "parent": mp[m], **paired(c, p, m)}
+        for m in spec.get("report", ()):
+            panel.setdefault(m, {"candidate": mc[m], "parent": mp[m]})
+        if spec.get("by_length"):
+            panel.update(_by_length(spec, c, p, lengths))
+        for ref, d in spec.get("versus", {}).items():
+            keep = panel_filter(spec, candidate.root)
+            rows = [r for r in read_json(candidate.root / d / "rows.json") if r["variant"] == "clean" and keep(r)]   # a reference as it served itself
+            panel.setdefault("versus", {})[ref] = {m: {"reference": metrics(rows)[m], **paired(c, rows, m)} for m in spec.get("metrics", ())}
+        out["panels"][name] = panel
+    if rule.get("unknowable"):
+        tag = rule["unknowable"]
+        if not candidate.has(tag): out["missing"].append(f"candidate:{candidate.dirs[tag]}")
+        out["unknowable"] = {"candidate": candidate.unknowable_share(tag), "parent": parent.unknowable_share(tag)}
+    out["criteria"] = {name: _criterion(out, c) for name, c in rule["criteria"].items()}
+    out["complete"] = not out["missing"]
+    out["passed"] = (out["complete"] and all(out["criteria"].values())) if out["criteria"] else None
+    return out
+
+
+def _by_length(spec, candidate, parent, lengths):
+    """A by_length panel's entries: "<metric>_<bucket>" -> {candidate, parent, n} (kev.metrics.calibration_by_length of
+    each side's served rows, bucketed by the same counts) and "by_length" -> how the tokens were counted."""
+    edges, tokenizer = by_length_options(spec)
+    counted = all(r.get("state_tokens") is not None for r in (*candidate, *parent))
+    if not counted and lengths is None: raise ValueError("a by_length panel needs state-token counts: rows with state_tokens, or lengths")
+    counts = None if counted else lengths(spec)
+    c, p = calibration_by_length(candidate, counts, edges), calibration_by_length(parent, counts, edges)
+    out = {"by_length": {"edges": list(edges), "tokens": "rows" if counted else {"records": "suite", "tokenizer": list(tokenizer)}}}
+    for bucket in c:
+        for m in LENGTH_METRICS: out[f"{m}_{bucket}"] = {"candidate": c[bucket][m], "parent": p[bucket][m], "n": c[bucket]["n"]}
+    return out
+
+
+def value(report, path):
+    """A number from a comparison by path; None if its panel was not read."""
+    parts = path.split(".")
+    if parts[0] == "unknowable": return (report.get("unknowable") or {}).get(parts[1])
+    entry = report["panels"].get(parts[0], {}).get(parts[1])
+    if entry is None: return None
+    if parts[2] in ("lower", "upper"): return entry["ci95"][0 if parts[2] == "lower" else 1]
+    return entry[parts[2]]
+
+
+def _criterion(report, c):
+    left = [value(report, p) for p in (c["left"] if isinstance(c["left"], list) else [c["left"]])]
+    right = value(report, c["right"]) if isinstance(c["right"], str) else c["right"]
+    if right is None or any(v is None for v in left): return None
+    lhs = left[0] - left[1] if len(left) == 2 else left[0]
+    return OPS[c["op"]](lhs, right + c["plus"] if "plus" in c else right)
+
+
+def rank(arms, keys, selectable):
+    """Passing selectable arms per size, best first by the rank keys (a key summing several paths adds them left to right)."""
+    def score(report):
+        out = []
+        for key in keys:
+            paths = key["by"] if isinstance(key["by"], list) else [key["by"]]
+            total = value(report, paths[0])
+            for p in paths[1:]: total = total + value(report, p)
+            out.append(-total if key.get("order", "desc") == "desc" else total)
+        return tuple(out)
+    by_size = {}
+    for arm, report in arms.items():
+        by_size.setdefault(size_of(arm), [])
+        if report.get("passed") and arm in selectable: by_size[size_of(arm)].append(arm)
+    return {size: sorted(names, key=lambda a: score(arms[a])) for size, names in by_size.items()}
+
+
+def readout(spec, root=ROOT):
+    """The registered rule applied to every arm that has finished training (development rows present); an arm without a
+    trial (a checkpoint made from others) is compared once its pool and reads are there (compare lists what is missing)."""
+    arms = {}
+    for arm, a in spec["arms"].items():
+        if a.get("trial") and not (Path(root) / a["trial"] / "development/rows.json").exists():
+            arms[arm] = {"trial": a["trial"], "parent": spec["parents"][a["parent"]]["trial"], "missing": [f"candidate:{a['trial']}/development/rows.json"], "complete": False, "passed": None}
+            continue
+        arms[arm] = compare(arm_side(spec, arm, root), parent_side(spec, arm, root), spec["rule"], lambda panel: panel_lengths(spec, panel))
+    ranking = rank(arms, spec["rule"].get("rank", []), {a for a, x in spec["arms"].items() if x.get("select", True)})
+    return {"round": spec["round"], "registered": spec["registered"], "drop_ids": sorted(spec.get("drop_ids", [])), "arms": arms,
+            "ranking": ranking, "candidates": {size: names[0] if names else None for size, names in ranking.items()}}
+
+
+def confirm(spec, stage, arm, root=ROOT):
+    """One confirmation stage for the chosen arm against its parent, on the stage's reads."""
+    out = compare(arm_side(spec, arm, root, stage), parent_side(spec, arm, root, stage), spec["confirm"][stage], lambda panel: panel_lengths(spec, panel))
+    return {"round": spec["round"], "stage": stage, "arm": arm, **out}
+
+
+# --- printing --------------------------------------------------------------------------------------------------------
+
+RATES = ("acc", "confident_error_rate", "coverage_at_5pct_error", "coverage_at_1pct_error")   # printed in percentage points
+
+
+def _interval(metric, entry):
+    scale, fmt = (100, "+.1f") if metric in RATES else (1, "+.3f")
+    return f"{scale * entry['delta']:{fmt}} [{scale * entry['ci95'][0]:{fmt}}, {scale * entry['ci95'][1]:{fmt}}]"
+
+
+def table(report):
+    """One line per comparison: every bootstrapped panel metric (rates in pp), the unknowable share, the verdict, what failed;
+    under it, each by_length panel's buckets, and a warning when the arm is served at a temperature fitted on its trial's
+    development rows of a training corpus (in distribution: round 19's failure mode)."""
+    lines = []
+    for arm, r in report.get("arms", {report.get("arm"): report}).items():
+        cells = [f"{p}.{m} {_interval(m, e)}" for p, panel in r.get("panels", {}).items() for m, e in panel.items() if isinstance(e, dict) and "ci95" in e]
+        if r.get("unknowable"): cells.append(f"unk {r['unknowable']['candidate']}")
+        failed = [k for k, v in r.get("criteria", {}).items() if v is False]
+        verdict = "incomplete" if not r.get("complete") else {True: "PASS", None: "reported", False: "fail: " + ", ".join(failed)}[r["passed"]]
+        lines.append(f"{arm:16} {' | '.join(cells)} -> {verdict}" + (f" (missing {len(r['missing'])})" if r.get("missing") else "")
+                     + (f" (report-only reads missing {len(r['missing_optional'])})" if r.get("missing_optional") else ""))
+        if r.get("temperature_ci"):
+            ci = r["temperature_ci"]
+            lines.append(f"{'':16} T {r['temperature']:.3f}, {100 * ci['level']:.0f} % bootstrap interval [{ci['lower']:.3f}, {ci['upper']:.3f}] ({ci['questions']} pool questions)")
+        for p, panel in r.get("panels", {}).items():
+            if "by_length" not in panel: continue
+            buckets = [name for name, _, _ in length_buckets(tuple(panel["by_length"]["edges"])) if panel[f"ece_{name}"]["n"]]
+            lines.append(f"{'':16} {p} by state length (candidate/parent): " + " | ".join(
+                f"{b} n={panel[f'ece_{b}']['n']} acc {panel[f'acc_{b}']['candidate']:.3f}/{panel[f'acc_{b}']['parent']:.3f} ece {panel[f'ece_{b}']['candidate']:.4f}/{panel[f'ece_{b}']['parent']:.4f}" for b in buckets))
+        source = r.get("temperature_source") or {}
+        if source.get("kind") == "trial development rows" and source.get("training_corpus"):
+            lines.append(f"{'':16} !!! {arm} is served at T {r['temperature']:.3f} fitted on {source['rows']}, held-out items of its training corpus {source['suite']}: "
+                         f"in distribution, {ROUND_19.split(': ', 1)[0]}; not a temperature to ship (register a held-out-datasets `temperature` pool)")
+    if "candidates" in report: lines.append(f"candidates {report['candidates']}")
+    return "\n".join(lines)
+
+
+# --- Modal orchestration ---------------------------------------------------------------------------------------------
+
+def bench_job(run, suite, name, flags=""):
+    """One modal_app.py::benchmarks entry. modal_app.parse_jobs reads the run as everything before the suite, so a pinned
+    Hub revision (repo@sha) is safe; the suite and the name must not contain '@' or ',' and flags must start with '--'."""
+    if any(c in s for s in (suite, name) for c in "@,") or "," in run or (flags and not flags.startswith("--")):
+        raise ValueError(f"cannot encode benchmark job {(run, suite, name, flags)}")
+    return "@".join([run, suite, name, *([flags] if flags else [])])
+
+
+def checkpoint_of(entry):
+    """Where a read runs from: the trial's checkpoint on the runs volume (runs/X -> /runs/X/checkpoint) unless declared."""
+    return entry["checkpoint"] if "checkpoint" in entry else f"/{entry['trial']}/checkpoint"
+
+
+def side_reads(spec, arm, rule, who, side, stage=None):
+    """[(read tag, directory)] a side needs for a stage, by tag: the rule's reads; "transfer" as the arm's transfer_read (a
+    trial's in-trial transfer read is never launched); and, for an arm at the rule stage, its temperature pool."""
+    tags = rule_tags(rule)
+    if who == "candidate" and stage is None and spec.get("temperature"):
+        tags |= {*spec["temperature"]["reads"], *spec["temperature"].get("exclude_reads", [])}
+    reads = {}
+    for tag in tags:
+        if tag != TRANSFER: reads.setdefault(side.dirs[tag], tag)
+        elif who == "candidate" and transfer_read(spec, arm): reads.setdefault(side.dirs[tag], transfer_read(spec, arm))
+    return sorted((read, d) for d, read in reads.items())
+
+
+def read_commands(spec, arm, stage=None, root=ROOT, sides=("candidate",)):
+    """The `modal run` commands for the missing reads of one arm (and, with sides, its parent): one benchmarks call per side
+    with every suite batched, plus one locked_test call per locked read. Reads that exist locally are skipped, and so are
+    reads of a removed suite (kev.suite.REMOVED_SUITES; the panels they feed stay missing)."""
+    rule = spec["confirm"][stage] if stage else spec["rule"]
+    size, commands = size_of(arm), []
+    for who in sides:
+        side = (arm_side if who == "candidate" else parent_side)(spec, arm, root, stage)
+        run = checkpoint_of(spec["arms"][arm] if who == "candidate" else spec["parents"][spec["arms"][arm]["parent"]])
+        jobs = []
+        for tag, d in side_reads(spec, arm, rule, who, side, stage):
+            if (root / d / "rows.json").exists(): continue
+            r = spec["reads"][tag]
+            if r.get("entrypoint") == "locked_test":
+                if not run.startswith("/runs/"): raise ValueError(f"locked_test reads a trial on the runs volume, not {run}")
+                commands.append(["modal", "run", "modal_app.py::locked_test", "--trial", run.removeprefix("/runs/").removesuffix("/checkpoint"),
+                                 "--name", Path(d).relative_to("runs/locked").parts[0], "--decision", r["decision"], "--gpu", spec.get("gpu", "H100"),
+                                 *spec.get("locked_args", {}).get(size, [])])
+                continue
+            if not d.startswith("runs/") or "/" in d.removeprefix("runs/"): raise ValueError(f"{tag}: benchmarks writes runs/<name>, not {d}")
+            if gone := removed_suite(r["suite"]):   # never made again; skipped, not raised, so a watcher does not retry it forever
+                print(f"!!! {arm}: not launching read {tag} ({d}): {r['suite']} was removed on {gone['removed']} (kev.suite.REMOVED_SUITES)", flush=True)
+                continue
+            jobs.append(bench_job(run, r["suite"], d.removeprefix("runs/"), r.get("flags", "")))
+        if jobs:
+            timeout = spec.get("read_timeout", {}).get(size)
+            commands.append(["modal", "run", "--detach", "modal_app.py::benchmarks", "--jobs", ",".join(jobs), "--gpu", spec.get("gpu", "H100"),
+                             *(["--timeout", str(timeout)] if timeout else [])])
+    return commands
+
+
+def modal_env(spec):
+    return {**os.environ, **({"KEV_APP_NAME": spec["app"]} if spec.get("app") else {}), **({"KEV_GPU": spec["gpu"]} if spec.get("gpu") else {})}
+
+
+def launch_commands(commands, spec, log_stem, stagger=STAGGER, run=subprocess.Popen, sleep=time.sleep):
+    """Start each command in the background (log under runs/), `stagger` seconds apart; returns the processes."""
+    procs = []
+    for i, cmd in enumerate(commands):
+        if i: sleep(stagger)
+        log = ROOT / "runs" / f"{log_stem}-{i}.log"
+        print("launch:", " ".join(cmd[:4]), "->", log.relative_to(ROOT), flush=True)
+        with log.open("w", encoding="utf-8") as f:
+            procs.append(run([sys.executable, "-m", *cmd], stdout=f, stderr=subprocess.STDOUT, cwd=ROOT, env=modal_env(spec)))
+    return procs
+
+
+class InFlight(Exception):
+    """An arm's reads were launched recently and may still be running; they are not launched again yet."""
+
+
+def launch_arm_reads(spec, arm, stage=None, sides=("candidate",), stagger=STAGGER, launch=None, now=time.time, log=print):
+    """Launch one arm's missing reads once. Under a per-arm lock (runs/.reads-r<N>-<arm>.lock: a watcher and a
+    `launch-reads` cannot both launch the arm), the intent is written first (runs/r<N>-reads-<arm>[-<stage>].json: when and
+    what), then the commands start. Detached benchmarks cannot be seen from here, so while an earlier intent is younger
+    than the reads' timeout the arm counts as in flight: InFlight is raised (the watcher retries on a later pass) instead of
+    a second launch whose job would find /runs/bench/<name> taken. Reads whose rows landed are never relaunched."""
+    tag = f"r{spec['round']}-reads-{arm}" + (f"-{stage}" if stage else "")
+    grace = max(IN_FLIGHT, spec.get("read_timeout", {}).get(size_of(arm), 0))
+    with file_lock(ROOT / "runs" / f".reads-r{spec['round']}-{arm}.lock"):
+        commands = read_commands(spec, arm, stage, ROOT, sides)
+        if not commands: log(f"{arm}: every read has landed"); return []
+        intent = ROOT / "runs" / f"{tag}.json"
+        if intent.exists() and now() - read_json(intent)["launched_at"] < grace:
+            since = read_json(intent)["launched_at"]
+            raise InFlight(f"{arm}: reads launched {int(now() - since)} s ago may still be running; not relaunching for another "
+                           f"{int(since + grace - now())} s (if they finished on Modal but were never pulled: modal volume get kev-runs /bench/<name> runs/)")
+        write_json(intent, {"launched_at": now(), "commands": commands}, atomic=True)
+        return (launch or launch_commands)(commands, spec, tag, stagger)
+
+
+def pull(study, spec):
+    """modal_app.py::pull for one study (modal_app.pull_study holds a per-study lock, so concurrent pulls wait)."""
+    subprocess.run([sys.executable, "-m", "modal", "run", "modal_app.py::pull", "--name", study], check=True, cwd=ROOT, env=modal_env(spec))
+
+
+def continue_command(spec, study, label):
+    s = spec["studies"][study]
+    return ["modal", "run", "modal_app.py::resume", "--study", study, "--suite", s["suite"], "--transfer", s["transfer"], "--trial", label]
+
+
+def continue_trial(spec, study, label, run=subprocess.run):
+    """The next attempt of a timed-out trial: modal_app.py::resume --trial (modal_app.continue_full_trial) spawns
+    run_full_trial again, which continues from the trial's last resume point, with the GPU and timeout the study was
+    admitted for, and records the call in runs/<study>.spawn.json; for a pending ledger entry it first adopts the call
+    from the trial's lease (or waits). Returns once the record names a new current call (spawned or adopted). Raises
+    NoContinuation when the study has no attempt ledger, when the ledger has no attempt left (kev.budget.trial_attempts,
+    the count compute_bound admitted; a pending last entry is resolved by the command first) or when the command
+    succeeded without recording a call, and RuntimeError when it refused (a live lease, a young pending entry, the
+    network: the watcher asks again on its next pass). Output in runs/<study>.continue-<label>-<attempt>.log."""
+    from kev.budget import trial_attempts
+    path = ROOT / "runs" / f"{study}.spawn.json"
+    record = read_json(path)
+    if record.get("modal_retries") != 0: raise NoContinuation(f"{path.name} has no attempt ledger (spawned with Modal's retries on, before the ledger)")
+    used, allowed = trial_attempts(record, label)
+    last = record["attempts"][label][-1]
+    if used >= allowed and (last["call"] is not None or last.get("abandoned")):
+        raise NoContinuation(f"{used} of {allowed} attempts used; the admission bound (${record.get('bound_usd')}) counts no more")
+    log = ROOT / "runs" / f"{study}.continue-{label}-{used + 1}.log"
+    print("continue:", " ".join(continue_command(spec, study, label)), "->", log.relative_to(ROOT), flush=True)
+    with log.open("a", encoding="utf-8") as f:
+        done = run([sys.executable, "-m", *continue_command(spec, study, label)], stdout=f, stderr=subprocess.STDOUT, cwd=ROOT, env=modal_env(spec))
+    if read_json(path)["calls"][label] != record["calls"][label]: return
+    if getattr(done, "returncode", 0): raise RuntimeError(f"modal_app.py::resume --trial exited {done.returncode} without a new call (see {log.relative_to(ROOT)})")
+    raise NoContinuation(f"modal_app.py::resume recorded no new call (see {log.relative_to(ROOT)})")
+
+
+# --- watch -----------------------------------------------------------------------------------------------------------
+
+NETWORK_MARKERS = ("nodename nor servname", "name or service not known", "temporary failure in name resolution", "unavailable", "connection reset",
+                   "connection refused", "network is unreachable", "deadline exceeded")
+
+
+def transient(error):
+    """Whether polling a call failed because of this machine's network (DNS drop, reset connection, gRPC UNAVAILABLE)
+    rather than because the trial failed. A trial's own exception is re-raised by FunctionCall.get with its original
+    type, so only network types and gRPC transport messages count; anything else is the trial's failure."""
+    try:
+        import modal.exception as me
+        modal_types = (me.ConnectionError, me.ClientClosed)
+    except ImportError:
+        modal_types = ()
+    if isinstance(error, (socket.gaierror, ConnectionError, *modal_types)): return True
+    return type(error).__module__.startswith(("grpclib", "modal")) and any(m in str(error).lower() for m in NETWORK_MARKERS)
+
+
+class TrialFailed(Exception):
+    """A full-weight trial that failed with an error returns {"failed": ...} (modal_app.failed_trial) instead of raising;
+    poll_modal raises this for it, which watch_studies marks failed like any trial error (never continued)."""
+
+
+class NoContinuation(Exception):
+    """A timed-out trial that is not continued (no attempt left under its study's admission bound, no attempt ledger, or
+    the continuation spawned no call): watch_studies marks it failed."""
+
+
+def poll_modal(call_id):
+    """'running' | 'done' | 'timeout' (the call ran out of time: its container was cancelled at the timeout) | 'refused'
+    (a full-weight attempt that found another attempt's fresh lease and did not start: modal_app.TrialLease) for a spawned
+    trial; the trial's exception (or a network error) propagates, and a returned failure is raised as TrialFailed."""
+    import modal
+    try:
+        result = modal.FunctionCall.from_id(call_id).get(timeout=0.5)
+    except TimeoutError:            # builtin: no output yet (modal.exception.FunctionTimeoutError is not a builtin TimeoutError)
+        return "running"
+    except modal.exception.FunctionTimeoutError:
+        return "timeout"
+    except modal.exception.OutputExpiredError:   # finished long ago; the result is on the volume
+        return "done"
+    if isinstance(result, dict) and "failed" in result: raise TrialFailed(result["failed"])
+    if isinstance(result, dict) and "refused" in result: return "refused"
+    return "done"
+
+
+class Unmapped(Exception):
+    """A finished call trained no arm of the spec: its reads cannot be launched."""
+
+
+def watch_studies(studies, on_done, poll=poll_modal, interval=120, max_transient=60, sleep=time.sleep, log=print, root=ROOT, now=time.time, on_timeout=None):
+    """Poll every spawned trial of the studies until each is settled, calling on_done(study, label) for a finished trial
+    until it succeeds; returns the finished calls that map to no arm. State lives in runs/<study>.watch.json, replaced
+    atomically after every change, so a restarted watcher resumes: finished trials are not polled again and launched reads
+    are not launched again. `launching_at` is written before on_done runs; a restart that finds it logs the interrupted
+    launch and calls on_done again, which must check what already happened (kev.rounds.launch_arm_reads does). Network
+    errors while polling are retried (max_transient in a row marks the call failed); an on_done that raises (a pull that
+    lost the network, reads still in flight) is retried on the next pass; one that raises Unmapped leaves the call
+    unlaunched, logged, and settled so the watch can end.
+    A call that ends by a timeout goes to on_timeout(study, label) (watch's: the trial's next attempt, which records its call
+    as the label's current call in runs/<study>.spawn.json); the spawn record is read on every pass, and a label whose
+    call changed (a continuation, from here or by hand) is polled again from its new call. Without on_timeout, or when it
+    raises NoContinuation, the timeout is the trial's failure; any other error from it is retried on the next pass
+    (max_transient in a row marks the trial failed). `continuing_at` is written before on_timeout runs; a restart that
+    finds it and a changed call adopts the call, one that finds the call unchanged asks again (modal_app.continue_full_trial
+    refuses a second attempt while one runs, waits for the old container's lease and counts every attempt in the ledger).
+    A call refused by the trial's lease, and a current attempt with no call recorded (a pending ledger entry), go to
+    on_timeout the same way."""
+    runs = Path(root) / "runs"
+    while True:
+        settled = True
+        calls = {study: read_json(runs / f"{study}.spawn.json")["calls"] for study in studies}   # a continuation replaces a label's call
+        for study, study_calls in calls.items():
+            path = runs / f"{study}.watch.json"
+            state = read_json(path) if path.exists() else {"calls": {}}
+            for label, call_id in study_calls.items():
+                s = state["calls"].setdefault(label, {"status": "running", "launched": False, "transient": 0})
+                if s.setdefault("call", call_id) != call_id and s["status"] != "done":   # the trial's next attempt
+                    log(f"{study}/{label}: polling its continuation {call_id} (was {s['call']})")
+                    s.update(call=call_id, status="running", transient=0); s.pop("error", None); s.pop("continuing_at", None)
+                if s["status"] == "running" and call_id is None:   # a pending ledger entry: spawned, or about to be, with no call recorded
+                    s["status"] = "timeout"; log(f"{study}/{label}: no call recorded for its current attempt (pending); a continuation adopts it from its lease or waits")
+                elif s["status"] == "running":
+                    try:
+                        s["status"], s["transient"] = poll(call_id), 0
+                    except Exception as error:   # noqa: BLE001 - a trial's own failure or this machine's network, told apart here
+                        if transient(error) and s["transient"] + 1 < max_transient:
+                            s["transient"] += 1; log(f"{study}/{label}: network error, retrying ({type(error).__name__}: {str(error)[:120]})")
+                        else:
+                            s["status"], s["error"] = "failed", f"{type(error).__name__}: {str(error)[:300]}"; log(f"{study}/{label}: FAILED {s['error']}")
+                if s["status"] == "refused":   # the attempt found another attempt's fresh lease and never started: it needs a next attempt too
+                    s["status"] = "timeout"; log(f"{study}/{label}: its call {call_id} was refused by the trial's lease (another attempt's container was alive)")
+                if s["status"] == "timeout":
+                    if on_timeout is None:
+                        s["status"], s["error"] = "failed", "FunctionTimeoutError (not continued: no on_timeout)"; log(f"{study}/{label}: FAILED {s['error']}")
+                    else:
+                        if "continuing_at" in s: log(f"{study}/{label}: a continuation asked for at {s['continuing_at']:.0f} spawned no call it recorded; asking again")
+                        s["continuing_at"] = now(); write_json(path, state, atomic=True)
+                        try:
+                            on_timeout(study, label); log(f"{study}/{label}: timed out; its next attempt was spawned")
+                        except NoContinuation as error:
+                            s["status"], s["error"] = "failed", f"FunctionTimeoutError, not continued: {error}"; s.pop("continuing_at"); log(f"!!! {study}/{label}: FAILED {s['error']}")
+                        except Exception as error:   # noqa: BLE001 - the network, or a refusal that may clear; retried on the next pass
+                            s["transient"] += 1; s.pop("continuing_at")
+                            if s["transient"] >= max_transient:
+                                s["status"], s["error"] = "failed", f"FunctionTimeoutError, continuation failed {s['transient']} times: {type(error).__name__}: {str(error)[:300]}"; log(f"!!! {study}/{label}: FAILED {s['error']}")
+                            else:
+                                log(f"{study}/{label}: continuing failed, retrying next pass ({type(error).__name__}: {str(error)[:300]})")
+                if s["status"] == "done" and not s["launched"] and not s.get("unmapped"):
+                    if "launching_at" in s: log(f"{study}/{label}: a launch started at {s['launching_at']:.0f} was interrupted; checking the arm's reads before any relaunch")
+                    s["launching_at"] = now(); write_json(path, state, atomic=True)
+                    try:
+                        on_done(study, label); s["launched"] = True; s.pop("launching_at"); log(f"{study}/{label}: done, reads launched")
+                    except Unmapped as error:
+                        s["unmapped"] = True; s.pop("launching_at"); log(f"!!! {study}/{label}: finished but {error}; its reads were NOT launched")
+                    except Exception as error:   # noqa: BLE001 - retried on the next pass; only a dead process leaves launching_at behind
+                        s.pop("launching_at"); log(f"{study}/{label}: launching reads failed, retrying next pass ({type(error).__name__}: {str(error)[:300]})")
+                write_json(path, state, atomic=True)
+                settled = settled and (s["status"] == "failed" or s["launched"] or bool(s.get("unmapped")))
+        if settled:
+            return [f"{study}/{label}" for study in calls for label, s in read_json(runs / f"{study}.watch.json")["calls"].items() if s.get("unmapped")]
+        sleep(interval)
+
+
+def arm_of(spec, study, label):
+    """The arm a spawned call trained (trial directories are <NN>-<label>)."""
+    return next((a for a, x in spec["arms"].items() if x.get("trial", "").startswith(f"runs/{study}/") and Path(x["trial"]).name.split("-", 1)[1] == label), None)
+
+
+def watch(spec, interval=120, stagger=STAGGER, reads_timeout=6 * 3600):
+    """The round from spawned trials to read-out: when a trial finishes, pull its study and launch that arm's reads
+    (`stagger` apart); once every trial is settled, wait for the reads to land and write the read-out. The wait ends when
+    every read is there, when every benchmarks process this watcher started has exited (a failed read never lands), or
+    after reads_timeout (a restarted watcher has no processes to follow)."""
+    last, procs = [0.0], []
+    def on_done(study, label):
+        arm = arm_of(spec, study, label)
+        if arm is None: raise Unmapped(f"no arm of round {spec['round']} has a trial runs/{study}/<NN>-{label}")
+        pull(study, spec)
+        time.sleep(max(0.0, last[0] + stagger - time.time()))
+        procs.extend(launch_arm_reads(spec, arm, stagger=stagger))
+        last[0] = time.time()
+    unmapped = watch_studies(list(spec.get("studies", {})), on_done, interval=interval, on_timeout=lambda study, label: continue_trial(spec, study, label))
+    deadline = time.time() + reads_timeout
+    finished = [a for a, x in spec["arms"].items() if x.get("trial") and (ROOT / x["trial"] / "result.json").exists()]
+    landed = lambda a: all((ROOT / d / "rows.json").exists() for tag, d in side_reads(spec, a, spec["rule"], "candidate", arm_side(spec, a))
+                           if not removed_suite(spec["reads"][tag].get("suite") or ""))   # a removed suite's read is never launched
+    while (waiting := [a for a in finished if not landed(a)]) and time.time() < deadline:
+        if procs and all(p.poll() is not None for p in procs): break
+        print(f"waiting for the reads of {waiting}", flush=True); time.sleep(interval)
+    report = write_readout(spec)
+    if unmapped: raise SystemExit(f"finished calls that map to no arm, never read: {unmapped} (fix the spec's arms, then launch-reads)")
+    return report
+
+
+def launchable(spec, rows=True):
+    """Problems that stop a launch: a recorded round is never relaunched (register a new one), and a new round needs its
+    plans, suites and (for trials) its parents' reads, or its read-out could not be computed."""
+    if spec.get("archive"): return [f"round {spec['round']} is a record (evidence on {spec['archive']}); register a new round to train or read again"]
+    return validate(spec, rows=rows).problems
+
+
+def study_commands(spec):
+    return {name: ["modal", "run", "modal_app.py::study", "--suite", s["suite"], "--plan", s["plan"], "--name", name, "--transfer", s["transfer"],
+                   "--budget", str(s["budget"]), "--timeout", str(s["timeout"]), "--gpu", s["gpu"]] for name, s in spec["studies"].items()}
+
+
+def launch_studies(spec, stagger=STAGGER, run=subprocess.run, sleep=time.sleep):
+    """modal_app.py::study for every study of the round (each spawns its trials on the deployed app and returns), `stagger`
+    apart, output in runs/<study>.log (a filter on this output can hide the SystemExit that explains a refusal)."""
+    for i, (name, cmd) in enumerate(study_commands(spec).items()):
+        if i: sleep(stagger)
+        print("launch:", " ".join(cmd), flush=True)
+        with (ROOT / "runs" / f"{name}.log").open("w", encoding="utf-8") as f:
+            run([sys.executable, "-m", *cmd], stdout=f, stderr=subprocess.STDOUT, cwd=ROOT, env=modal_env(spec), check=True)
+
+
+def write_readout(spec, root=ROOT, out=None):
+    report = readout(spec, root)
+    out = Path(out or Path(root) / f"runs/r{spec['round']}-readout"); out.mkdir(parents=True, exist_ok=True)
+    write_json(out / f"round{spec['round']}.json", report); print(table(report))
+    return report
+
+
+# --- CLI -------------------------------------------------------------------------------------------------------------
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="python -m kev.rounds", description=__doc__.split("\n\n")[0])
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    for name in ("validate", "launch", "launch-reads", "watch", "readout", "confirm"):
+        p = sub.add_parser(name); p.add_argument("spec")
+        if name in ("validate", "readout", "confirm"): p.add_argument("--root", default=str(ROOT), help="checkout whose runs/ holds the rows (default: this one)")
+        if name == "validate": p.add_argument("--partitions", action="store_true", help="verify the suites' partitions through load_plan (may fetch from the Hub)")
+        if name in ("launch", "launch-reads", "watch"): p.add_argument("--stagger", type=int, default=STAGGER)
+        if name in ("launch", "launch-reads"): p.add_argument("--dry-run", action="store_true", help="print the modal commands only")
+        if name == "launch-reads": p.add_argument("--arms", help="comma-separated arms (default: every arm with a pulled result, and every arm that is a checkpoint without a trial)"); p.add_argument("--parents", action="store_true", help="also the parents' missing reads")
+        if name in ("launch-reads", "confirm"): p.add_argument("--stage", required=name == "confirm"); p.add_argument("--arm", help="the candidate (default: the one the readout names)")
+        if name == "watch": p.add_argument("--interval", type=int, default=120)
+        if name in ("readout", "confirm"): p.add_argument("--out")
+    a = ap.parse_args(argv)
+    spec, root = load(a.spec), Path(getattr(a, "root", ROOT))
+    for line in [*calibration_warnings(spec, root), *continuation_warnings(spec, root)] if a.cmd in ("validate", "launch") else ():
+        print(f"!!! warning: {line}")
+    if a.cmd == "validate":
+        problems, archived = validate(spec, root, partitions=a.partitions)
+        where = f"on {spec['archive']}, not in this checkout" if spec.get("archive") else "not in this checkout"
+        if archived: print(f"archived ({where}):\n  " + "\n  ".join(archived))
+        print("\n".join(problems) or f"round {spec['round']}: ok ({len(spec['arms'])} arms, {len(spec.get('studies', {}))} studies)")
+        raise SystemExit(1 if problems else 0)
+    if a.cmd == "readout":
+        write_readout(spec, root, a.out); return
+    if a.cmd == "confirm":
+        arm = a.arm or _candidate(spec, root)
+        report = confirm(spec, a.stage, arm, root)
+        out = Path(a.out or root / f"runs/r{spec['round']}-verdict"); out.mkdir(parents=True, exist_ok=True)
+        write_json(out / f"{size_of(arm)}-{a.stage}.json", report); print(table(report)); return
+    problems = launchable(spec, rows=a.cmd != "launch-reads")   # launch-reads is how a missing parent read gets made
+    if problems: raise SystemExit("spec does not validate for a launch:\n" + "\n".join(problems))
+    if a.cmd == "launch":
+        if a.dry_run: print("\n".join(" ".join(c) for c in study_commands(spec).values()))
+        else: launch_studies(spec, a.stagger)
+        return
+    if a.cmd == "watch":
+        watch(spec, a.interval, a.stagger); return
+    arms = [a.arm or _candidate(spec, root)] if a.stage else (a.arms.split(",") if a.arms else [x for x, v in spec["arms"].items() if not v.get("trial") or (root / v["trial"] / "result.json").exists()])
+    sides = ("candidate", "parent") if a.parents or a.stage else ("candidate",)
+    if a.dry_run:
+        for arm in arms: print("\n".join(" ".join(c) for c in read_commands(spec, arm, a.stage, root, sides)))
+        return
+    for i, arm in enumerate(arms):
+        if i: time.sleep(a.stagger)
+        try: launch_arm_reads(spec, arm, a.stage, sides, a.stagger)
+        except InFlight as error: print(f"!!! {error}")
+
+
+def _candidate(spec, root):
+    report = read_json(Path(root) / f"runs/r{spec['round']}-readout/round{spec['round']}.json")
+    chosen = [arm for arm in report["candidates"].values() if arm]
+    if len(chosen) != 1: raise SystemExit(f"the readout names {len(chosen)} candidates ({report['candidates']}); pass --arm")
+    return chosen[0]
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/serve.py

@@ -0,0 +1,343 @@
+"""FastAPI sidecar for the playground: loads one checkpoint, exposes prefill-only decisions.
+
+Run: uv run --extra serve python -m kev.serve --run runs/kev --port 8008
+
+TypeSafe-compatible: POST /v1/systemone, GET /v1/models, the `x-typesafe-request-id` response header, and bearer auth
+when KEV_API_KEY is set (unset = open server, the local default). Demo extras: POST /v1/systemone/permute (one Choice
+under several option orders) and POST /v1/systemone/separate (each question in its own pass, for the packed-vs-separate
+comparison). KEV_PREFIX_CACHE / KEV_PREFIX_MIN_TOKENS / KEV_PREFIX_MAX_TOKENS size the state-prefix cache; KEV_DATE_FACTS=1 opts into the
+date preprocessing (api.with_date_facts). A state over kev.model.SERVE_MAX_STATE tokens is refused with a 422 (kev.model.admit);
+KEV_TRUNCATE_STATES=1 reads its first SERVE_MAX_STATE tokens instead, and then every response says whether it did. Backend
+and precision follow LoadOptions (KEV_BACKEND, KEV_DTYPE, ...): on Apple Silicon the hybrid Qwen3.5 checkpoints run on MLX
+by default, elsewhere on torch in bf16.
+"""
+import argparse, asyncio, atexit, hmac, os, queue, random, sys, threading, time, traceback, uuid
+from concurrent.futures import Future
+import torch
+from dataclasses import dataclass, field, replace
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+from .api import SystemOneRequest, to_record, to_answers, output_tokens, with_date_facts
+from .checkpoint import Checkpoint, LoadOptions, fused_available, is_hub_id
+from .device import default_device, empty_cache, out_of_memory, sync
+from .model import SERVE_MAX_STATE, ContextOverflow, admit
+
+PREFIX_CACHE_SIZE = int(os.environ.get("KEV_PREFIX_CACHE", "4"))          # states kept (KV + DeltaNet states; attention-only backbones also the state's hidden states); 0 disables
+PREFIX_MIN_TOKENS = os.environ.get("KEV_PREFIX_MIN_TOKENS")               # states shorter than this are not cached; default = the model's prefix_min_tokens (0 for hybrid backbones and MLX, 384 for attention-only torch models)
+PREFIX_MAX_TOKENS = int(os.environ.get("KEV_PREFIX_MAX_TOKENS", "65536"))  # state tokens the cache holds in all (least recently used evicted first); a longer state is not cached.
+                                                                         # One 64k state (Kev-27B: ~1.3 GB of keys, values and DeltaNet states), or four 16k ones, not four 64k ones
+DATE_FACTS = os.environ.get("KEV_DATE_FACTS", "0") == "1"
+TRUNCATE_STATES = os.environ.get("KEV_TRUNCATE_STATES", "0") == "1"    # unset = a state over SERVE_MAX_STATE tokens gets a 422; 1 = read its first SERVE_MAX_STATE tokens, and every response says whether it did (truncated, usage.state_tokens / state_tokens_used)
+API_KEY = os.environ.get("KEV_API_KEY")                                  # unset = open server; set = require Authorization: Bearer <key>, as the TypeSafe clients always send
+MAX_BATCH = 64                                                           # requests the model thread takes at once (kev.cuda_graphs splits them to fit its buffers)
+MODEL_NAMES = ("kev-latest", "jev-latest")                               # both names serve this checkpoint; jev-latest is the TypeSafe SDK default model, so an unconfigured client works
+
+
+@dataclass
+class PrefixCache:
+    """State prefixes kept across requests, least recently used first: (state token ids, option_isolation) -> prefix.
+    At most `size` states and `max_tokens` state tokens in all; states shorter than min_tokens or longer than max_tokens
+    are not cached. A batch keeps (copies) only the new states that will still be here after it, its last distinct ones
+    within both bounds: the rest would be evicted by the batch itself."""
+    size: int
+    min_tokens: int
+    max_tokens: int = PREFIX_MAX_TOKENS
+    entries: dict = field(default_factory=dict)
+    hits: int = 0
+    misses: int = 0
+    oom_retries: int = 0   # batches that ran out of device memory with states cached, dropped them and ran again (Server._run)
+
+    def plan(self, encs):
+        """-> (key per request, None when its state is not cached; its cached prefix or None; whether to keep a new one)."""
+        lengths = [enc["seg"].count(0) for enc in encs]
+        keys = [(tuple(enc["ids"][:n]), bool(enc.get("option_isolation"))) if self.size and self.min_tokens <= n <= self.max_tokens else None
+                for enc, n in zip(encs, lengths)]
+        survivors, tokens = set(), 0
+        for key in dict.fromkeys(k for k in reversed(keys) if k is not None):   # most recent first, as store() keeps them
+            if len(survivors) == self.size or tokens + len(key[0]) > self.max_tokens: break
+            survivors.add(key); tokens += len(key[0])
+        return keys, [self.entries.get(k) if k is not None else None for k in keys], [k in survivors for k in keys]
+
+    def over(self, keys):
+        """Whether these keys (states) exceed either bound: more than `size` states or `max_tokens` state tokens."""
+        return len(keys) > self.size or sum(len(k[0]) for k in keys) > self.max_tokens
+
+    def make_room(self, keys, cached, keep):
+        """Drop now, before the batch runs, the entries this batch's store() will evict anyway, so an old state does not
+        stay resident through the passes of the new one that replaces it. store() (re)inserts every hit and every kept new
+        state as most recent and evicts the least recently used while over a bound; evicting the same keys first leaves
+        the same cache (tests/test_unit.py). A hit dropped here is still held by `cached` for this batch. Kev-4B on MLX:
+        a 64k state's cache is 2.2 GB, so a new 64k state after another one peaked with both (runs/mlx-long-states)."""
+        order = dict.fromkeys(self.entries)                     # least recently used first, as store() evicts
+        for key, old, k in zip(keys, cached, keep):
+            if key is not None and (old is not None or k):
+                order.pop(key, None); order[key] = None
+        while self.over(order):
+            key = next(iter(order)); del order[key]; self.entries.pop(key, None)
+
+    def store(self, keys, cached, prefixes):
+        """Record hits and misses, and (re)insert the batch's prefixes in order: most recently used last."""
+        for key, old, new in zip(keys, cached, prefixes):
+            if key is None: continue
+            self.hits += old is not None; self.misses += old is None
+            if new is None: continue
+            self.entries.pop(key, None); self.entries[key] = new
+            while self.over(self.entries): self.entries.pop(next(iter(self.entries)))
+
+    def clear(self):
+        self.entries.clear()
+
+
+@dataclass
+class Server:
+    """The loaded checkpoint, the state-prefix cache (PrefixCache), and the one model thread that runs every forward pass.
+
+    Request threads encode their record and queue it; the model thread takes everything queued when it becomes free and
+    runs it as one batch (model.probs_batch: with CUDA graphs, shared state and row passes; otherwise one request at a
+    time), then answers each request. It also captures pending CUDA graphs when the graphs say so (capture_due). `lock` is
+    held around each batch and capture: hold it to use the model directly."""
+    checkpoint: Checkpoint
+    tok: object
+    model: object
+    device: str
+    lock: threading.Lock = field(default_factory=threading.Lock)
+    batches: int = 0
+    batched_requests: int = 0
+    release_date: str = field(default="")   # for the TypeSafe model card; resolved once (may ask the Hub)
+    truncate_states: bool = field(default_factory=lambda: TRUNCATE_STATES)   # KEV_TRUNCATE_STATES; off: an over-length state is refused (kev.model.admit)
+
+    def __post_init__(self):
+        self.release_date = self.release_date or self.checkpoint.release_date()
+        self.prefix_cache = PrefixCache(PREFIX_CACHE_SIZE, int(PREFIX_MIN_TOKENS) if PREFIX_MIN_TOKENS else self.model.prefix_min_tokens)
+        self.queue, self.stopping = queue.Queue(), threading.Event()
+        # the model thread gives up the GIL at every CUDA sync and waits to get it back while the event loop parses and
+        # answers requests; at Python's default 5 ms switch interval those waits stretched a batch's model time ~2x.
+        # Process-wide, so close() puts it back.
+        self.switch_interval = sys.getswitchinterval()
+        sys.setswitchinterval(0.0005)
+        self.thread = threading.Thread(target=self._work, name="kev-model", daemon=True)
+        self.thread.start()
+        atexit.register(self.close)   # a daemon thread killed inside a CUDA call at interpreter exit aborts the process
+
+    def close(self):
+        """Stop the model thread after its current batch; requests still queued fail, and so do later ones (submit)."""
+        if self.stopping.is_set(): return
+        self.stopping.set(); self.thread.join()
+        while not self.queue.empty():
+            self.queue.get_nowait()[1].set_exception(RuntimeError("the server stopped")); self.queue.task_done()
+        sys.setswitchinterval(self.switch_interval)
+
+    def submit(self, rec):
+        """Queue one record for the model thread. -> a Future of (probabilities, stats). The state prefix (tokens up to the
+        first question) is cached across requests, so a repeated state only pays for its question rows. latency_ms is the
+        model time of the batch the request ran in (not its wait in the queue). A record kev.model.admit refuses (a state
+        over SERVE_MAX_STATE tokens unless truncate_states, a question row over SERVE_MAX_BRANCH) is a 422 here, before
+        it reaches the model."""
+        if self.stopping.is_set(): raise HTTPException(503, "the server is stopping")
+        try: enc = admit(self.model, self.tok, rec, truncate=self.truncate_states)
+        except ValueError as e:   # ContextOverflow, or a record the encoder cannot take
+            state = isinstance(e, ContextOverflow) and e.max_state is not None
+            raise HTTPException(422, str(e) + (f"; or start the server with KEV_TRUNCATE_STATES=1 to read only its first {e.max_state:,} tokens (responses then say truncated: true)" if state else ""))
+        done = Future()
+        self.queue.put((enc, done))
+        return done
+
+    def probs(self, rec):
+        return self.submit(rec).result()
+
+    def _work(self):
+        graphs = getattr(self.model, "graphs", None)
+        while not self.stopping.is_set():
+            try: batch = [self.queue.get(timeout=0.05)]
+            except queue.Empty:
+                if graphs is not None and graphs.capture_due(idle=True):
+                    with self.lock: graphs.capture_pending(limit=1)
+                continue
+            while len(batch) < MAX_BATCH:
+                try: batch.append(self.queue.get_nowait())
+                except queue.Empty: break
+            try:
+                with self.lock: results = self._run([enc for enc, _ in batch])
+            except Exception as e:   # every request of the batch gets the error; the thread lives on
+                traceback.clear_frames(e.__traceback__)   # the batch's futures keep the exception, and its frames held the failed pass's tensors (142 MiB, a 3,578-token state on Kev-0.8B); the traceback keeps its lines
+                results = [e] * len(batch)
+            for (_, done), result in zip(batch, results):
+                (done.set_exception if isinstance(result, Exception) else done.set_result)(result)
+                self.queue.task_done()
+            if graphs is not None and graphs.capture_due(idle=False):
+                with self.lock: graphs.capture_pending(limit=1)
+
+    def _run(self, encs):
+        """One batch through model.probs_batch, with the prefix cache. -> per request (probs, stats). A pass out of device
+        memory while states are cached drops the cache and runs once more: the cache only saves time, and kept resident
+        it failed every later batch of that size (#75, @vtxyer)."""
+        sync(self.device); t = time.time()
+        for retry in (False, True):
+            keys, cached, keep = self.prefix_cache.plan(encs)
+            self.prefix_cache.make_room(keys, cached, keep)
+            try: ps, prefixes = self.model.probs_batch(encs, cached, keep); break
+            except Exception as e:   # states held: the cache's, and this batch's hits make_room already dropped from it
+                held = self.prefix_cache.entries or any(c is not None for c in cached)
+                if retry or not held or not out_of_memory(e): raise
+            cached = None; self.prefix_cache.clear(); self.prefix_cache.oom_retries += 1   # after the except: its traceback holds the failed pass's tensors
+            empty_cache(self.device)
+        sync(self.device); dt = round((time.time() - t) * 1000, 1)
+        self.prefix_cache.store(keys, cached, prefixes)
+        self.batches += 1; self.batched_requests += len(encs)
+        return [([q.tolist() for q in p], {"tokens": len(enc["ids"]), "state_tokens": enc["state_tokens"], "state_tokens_used": enc["seg"].count(0),
+                                           "latency_ms": dt, "prefix_cache_hit": c is not None})   # state_tokens: the request's state; _used: what the model read (less only when truncated)
+                for enc, p, c in zip(encs, ps, cached)]
+
+    def wait_idle(self):
+        """Block until every submitted request is answered and no CUDA graph waits to be captured (benchmarks, warm-up)."""
+        self.queue.join()
+        graphs = getattr(self.model, "graphs", None)
+        while graphs is not None and graphs.capture_due(idle=True): time.sleep(0.01)
+        with self.lock: pass                                   # a capture in progress finishes
+
+    def answer(self, req):
+        """The /v1/systemone response body for one request."""
+        rec, meta = to_record(prepare(req))
+        return self._body(req, meta, *self.probs(rec))
+
+    async def answer_async(self, req):
+        """answer() for the event loop: a request waiting on the model thread holds no worker thread, so a container takes
+        as many concurrent requests as its batches can absorb (FastAPI runs sync endpoints on a 40-thread pool)."""
+        rec, meta = to_record(prepare(req))
+        return self._body(req, meta, *await asyncio.wrap_future(self.submit(rec)))
+
+    def _body(self, req, meta, ps, m):
+        """The TypeSafe body. A server that may truncate (truncate_states) also says, on every response, whether it did:
+        `truncated`, and usage.state_tokens (the request's state) / state_tokens_used (what the model read), both counting
+        the <state> token. The TypeSafe SDKs ignore fields they do not model, so clients keep parsing."""
+        answers = to_answers(ps, meta)
+        body = {"model": req.model, "answers": answers, "usage": {"input_tokens": m["tokens"], "output_tokens": output_tokens(self.tok, answers)}, "latency_ms": m["latency_ms"]}
+        if self.truncate_states:
+            body["usage"].update(state_tokens=m["state_tokens"], state_tokens_used=m["state_tokens_used"])
+            body["truncated"] = m["state_tokens"] > m["state_tokens_used"]
+        return body
+
+
+def prepare(req):
+    """Opt-in preprocessing applied to every request before the model sees it."""
+    return req.model_copy(update={"state": with_date_facts(req.state)}) if DATE_FACTS else req
+
+
+app = FastAPI(title="kev")
+app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], expose_headers=["x-typesafe-request-id", "server-timing"])
+
+
+@app.middleware("http")
+async def typesafe(request, call_next):
+    """Bearer auth (when API_KEY is set) and the request id every TypeSafe client reads off the response."""
+    started = time.perf_counter()
+    if API_KEY and request.url.path.startswith("/v1") and not hmac.compare_digest(request.headers.get("authorization", ""), f"Bearer {API_KEY}"):
+        resp = JSONResponse({"detail": "missing or invalid API key; send Authorization: Bearer <KEV_API_KEY>"}, 401, {"www-authenticate": "Bearer"})
+    else:
+        resp = await call_next(request)
+    resp.headers["x-typesafe-request-id"] = request.headers.get("x-typesafe-request-id") or uuid.uuid4().hex
+    resp.headers["server-timing"] = f"app;dur={(time.perf_counter() - started) * 1000:.1f}"   # time inside this process, for telling it from the network
+    return resp
+
+
+def server() -> Server:
+    return app.state.server
+
+
+@app.post("/v1/systemone")
+async def systemone(req: SystemOneRequest):
+    """TypeSafe-compatible endpoint: typed questions in, typed answers out, one prefill pass."""
+    return await server().answer_async(req)
+
+
+class PermuteSystemOne(BaseModel):
+    request: SystemOneRequest
+    question: str
+    n_perm: int = Field(default=6, ge=1, le=64)   # each order is a forward pass; 0 divided by nothing, unbounded counts ran forever (#30)
+    seed: int = 0
+
+
+@app.post("/v1/systemone/permute")
+def systemone_permute(r: PermuteSystemOne):
+    """Re-run one Choice question under n_perm option orders. Returns per-order probabilities keyed by option name."""
+    q = r.request.questions.get(r.question)
+    if q is None or q.type != "choice": raise HTTPException(422, "question must be an existing choice question")
+    rng = random.Random(r.seed); keys = list(q.criteria); runs = []
+    for i in range(r.n_perm):
+        order = list(keys)
+        if i > 0: rng.shuffle(order)
+        one = r.request.model_copy(update={"questions": {r.question: q.model_copy(update={"criteria": {k: q.criteria[k] for k in order}})}})
+        resp = server().answer(one); a = resp["answers"][r.question]
+        runs.append({"order": order, "probabilities": a["probabilities"], "choice": a["choice"], "latency_ms": resp["latency_ms"]})
+    spread = {k: max(x["probabilities"][k] for x in runs) - min(x["probabilities"][k] for x in runs) for k in keys}
+    return truncation_marks({"runs": runs, "argmax_stable": len({x["choice"] for x in runs}) == 1, "spread": spread}, resp)
+
+
+@app.post("/v1/systemone/separate")
+def systemone_separate(req: SystemOneRequest):
+    """Answer each question in its own request against the same state (N passes). For packed-vs-separate comparison."""
+    parts = [server().answer(req.model_copy(update={"questions": {qid: q}})) for qid, q in req.questions.items()]
+    answers = {qid: a for p in parts for qid, a in p["answers"].items()}
+    body = {"model": req.model, "answers": answers,
+            "usage": {"input_tokens": sum(p["usage"]["input_tokens"] for p in parts), "output_tokens": output_tokens(server().tok, answers)},
+            "latency_ms": round(sum(p["latency_ms"] for p in parts), 1)}
+    return truncation_marks(body, parts[0])
+
+
+def truncation_marks(body, part):
+    """A demo endpoint built from several /v1/systemone answers on one state carries a truncating server's marks
+    (Server._body: `truncated`, usage.state_tokens / state_tokens_used) from any one of them: each read the same tokens."""
+    if "truncated" in part:
+        body["truncated"] = part["truncated"]
+        body.setdefault("usage", {}).update({k: part["usage"][k] for k in ("state_tokens", "state_tokens_used")})
+    return body
+
+
+@app.get("/v1/models")
+def models():
+    """One TypeSafe model card (name, description, release_date) per accepted model name, plus the Kev serving details
+    a client may ignore: the run, the base, the device, the backend and precision, the temperature, prefix-cache stats."""
+    s = server()
+    ck, meta = s.checkpoint, s.checkpoint.meta
+    card = {"description": f"Kev pointer head on {meta.base}, serving {ck.requested} at temperature {s.model.head.temperature:.2f}",
+            "release_date": s.release_date,
+            "run": ck.requested, "base": meta.base, "lora": meta.lora, "device": s.device, "backend": s.model.backend, "dtype": s.model.dtype,
+            "temperature": s.model.head.temperature, "max_state_tokens": SERVE_MAX_STATE, "truncate_states": s.truncate_states,
+            "cuda_graphs": graphs.stats() if (graphs := getattr(s.model, "graphs", None)) else None,
+            "prefix_cache": {"size": s.prefix_cache.size, "min_state_tokens": s.prefix_cache.min_tokens, "max_tokens": s.prefix_cache.max_tokens, "hits": s.prefix_cache.hits,
+                             "misses": s.prefix_cache.misses, "cached_states": len(s.prefix_cache.entries), "oom_retries": s.prefix_cache.oom_retries},
+            "batches": {"count": s.batches, "requests": s.batched_requests, "queued": s.queue.qsize()}}
+    return {"models": [{"name": name, **card} for name in MODEL_NAMES]}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--run", default="runs/kev")
+    ap.add_argument("--fallback", default="runs/smoke")
+    ap.add_argument("--host", default="127.0.0.1", help="interface to bind; 0.0.0.0 to serve beyond this machine (a container, a VM behind a proxy)")
+    ap.add_argument("--port", type=int, default=8008)
+    a = ap.parse_args()
+    run = a.run if is_hub_id(a.run) or os.path.exists(f"{a.run}/head.pt") else a.fallback
+    if run != a.run: print(f"{a.run} not found, falling back to {run}")
+    dev = default_device()
+    opts = LoadOptions.from_env()
+    if dev == "mps" and opts.attn is None: opts = replace(opts, attn="sdpa")   # serving default on Apple GPUs (parity measured)
+    if dev != "cpu" and opts.dtype is None: opts = replace(opts, dtype=torch.bfloat16)   # serving default: 2-4.5x faster than fp32 on an L4, same answers (LoadOptions.dtype); KEV_DTYPE=fp32 for the exact path
+    if dev == "cuda" and opts.cuda_graphs is None: opts = replace(opts, cuda_graphs=True)   # serving default: a pass is ~2,000 kernel launches, so replaying graphs cuts warm latency several-fold (kev.cuda_graphs); KEV_CUDA_GRAPHS=0 to decline
+    fused_default = dev == "cuda" and opts.fused is None
+    if fused_default: opts = replace(opts, fused=fused_available())   # serving default: fused Qwen3.5 kernels, ~1/3 less GPU time per batch (kev.fused_qwen35), when fla is installed; KEV_FUSED=0 to decline, KEV_FUSED=1 to insist
+    if opts.backend is None: opts = replace(opts, backend="auto")   # serving default: MLX for the hybrid Qwen3.5 checkpoints on Apple Silicon (LoadOptions.backend); KEV_BACKEND=torch to decline
+    ck = Checkpoint(run)
+    tok, model = ck.load(dev, opts)
+    if fused_default and not opts.fused and model.hybrid: print("fused Qwen3.5 kernels off: install the flash-linear-attention version kev/fused_qwen35.py pins (FLA_VERSION) to turn them on")
+    app.state.server = Server(ck, tok, model, dev)
+    print(f"serving {ck.requested} ({ck.path}) on {dev} via {model.backend} ({model.dtype}) {a.host}:{a.port}; states over {SERVE_MAX_STATE:,} tokens "
+          f"{'truncated (KEV_TRUNCATE_STATES=1)' if app.state.server.truncate_states else 'refused (422)'}")   # /v1/models reports the run as given, not the resolved cache path
+    import uvicorn
+    uvicorn.run(app, host=a.host, port=a.port)
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/shared_prefix.py

@@ -0,0 +1,141 @@
+"""Training forward through a shared state prefix, for the hybrid Qwen3.5 backbones (kev.train --shared_prefix).
+
+The row form (kev.model.forward_rows_batch) trains a record with q questions as q causal rows, state + one branch each,
+so the state runs q times. Here each record's state runs once and its branches continue from it, which costs
+state + the branches instead of q x (state + branch). The two are the same computation: a branch token attends to the
+same tokens at the same positions, and it reaches the state through what each layer leaves behind at the end of the
+state, as the layer's own cache path reads it for inference (kev.model._branch_rows_from_prefix): the attention layers'
+keys and values, and the Gated DeltaNet layers' short-convolution window (the conv's left context) and recurrent state
+(the chunked delta rule's `initial_state`, which fla differentiates). Two things differ from serving:
+- `Prefix` holds that per-layer state functionally (transformers' cache layers copy into static buffers in place, which
+  autograd cannot see through), so every branch's gradient flows back into the state and accumulates there;
+- each decoder layer runs its state pass and its branch pass inside one step, and gradient checkpointing (when the
+  backbone has it on) recomputes the step as a whole, so no cache outlives its layer or is written twice on replay.
+States of a batch are left-padded (the padded positions are zeroed and stay zero: a zero input leaves the DeltaNet state,
+the conv window and the residual stream unchanged), branches right-padded.
+Exactness: fp32 with transformers' reference kernels matches the row form to ~1e-5 (logits and every gradient; tiny
+random Qwen3.5 in tests/test_unit.py, Qwen3.5-0.8B-Base in tests/test_model.py). On CUDA fla's Triton kernels round their
+fp32 dots like TF32 and the shared prefix left-pads states, so it rounds differently from the row form: measured against the
+exact kernels, it is as far from the exact answer as the row form's own batchings are, with no systematic shift.
+"""
+import types
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.distributed.fsdp import FSDPModule, register_fsdp_forward_method
+from torch.utils.checkpoint import checkpoint
+
+
+class Prefix:
+    """One decoder layer's view of the shared state, duck-typing the cache calls a Qwen3.5 layer makes. The state pass
+    stores what the layer leaves (keys and values, or the conv window and the recurrent state); `branch` gathers it once
+    per branch row; the branch pass reads it. Nothing is written in place."""
+    record_past = True   # keeps a one-token pass off the in-place decode kernel
+
+    def __init__(self):
+        self.keys = self.values = self.conv = None
+        self.recurrent_states = {}
+
+    @property
+    def layers(self):   # layer code reads `cache.layers[layer_idx].recurrent_states[0]`: every index is this prefix
+        return self
+
+    def __getitem__(self, layer_idx):
+        return self
+
+    def has_previous_state(self, layer_idx=None, state_idx=0):
+        return self.conv is not None
+
+    def update_conv_state(self, conv_input, layer_idx, conv_kernel_size, **kwargs):
+        if self.conv is not None: return torch.cat([self.conv, conv_input], -1)   # the branch sees the state's last inputs
+        self.conv = F.pad(conv_input, (max(0, conv_kernel_size - conv_input.shape[-1]), 0))[..., -conv_kernel_size:]
+        return conv_input
+
+    def update_recurrent_state(self, recurrent_state, layer_idx, **kwargs):
+        self.recurrent_states.setdefault(0, recurrent_state)   # the state's final state; a branch's own is discarded
+        return recurrent_state
+
+    def update(self, keys, values, layer_idx, *args, **kwargs):
+        if self.keys is None:
+            self.keys, self.values = keys, values
+            return keys, values
+        return torch.cat([self.keys, keys], -2), torch.cat([self.values, values], -2)
+
+    def branch(self, owner):
+        """Gather the stored state per branch row (owner[i] = the record of branch i)."""
+        pick = lambda t: None if t is None else t.index_select(0, owner)
+        self.keys, self.values, self.conv = pick(self.keys), pick(self.values), pick(self.conv)
+        self.recurrent_states = {k: pick(v) for k, v in self.recurrent_states.items()}
+
+
+def _masks(allow, dtype, attn):
+    """Additive float mask for eager attention, boolean for SDPA (which also takes a float mask, but only in the query's
+    dtype, and autocast changes that)."""
+    allow = allow[:, None]
+    return allow if attn == "sdpa" else torch.zeros(allow.shape, dtype=dtype, device=allow.device).masked_fill(~allow, torch.finfo(dtype).min)
+
+
+def branch_hidden(lm, splits, pad_id, device):
+    """splits[b] = kev.model.rows_of(record b's encoding). -> per record, per question, the branch's final hidden states
+    [branch length, d] in fp32 (what the pointer head reads; the state's own hidden states are never needed)."""
+    base = lm.get_base_model() if hasattr(lm, "get_base_model") else lm
+    owner = [b for b, (_, _, rows) in enumerate(splits) for _ in rows]
+    branches = [r for _, _, rows in splits for r in rows]
+    Ls, Lb = max(len(s) for s, _, _ in splits), max(len(r["ids"]) for r in branches)
+    s_ids = torch.full((len(splits), Ls), pad_id, device=device); s_pos = torch.zeros_like(s_ids); s_real = torch.zeros_like(s_ids, dtype=torch.bool)
+    for b, (ids, pos, _) in enumerate(splits):   # left-padded
+        s_ids[b, Ls - len(ids):] = torch.tensor(ids, device=device); s_pos[b, Ls - len(ids):] = torch.tensor(pos, device=device); s_real[b, Ls - len(ids):] = True
+    b_ids = torch.full((len(branches), Lb), pad_id, device=device); b_pos = torch.zeros_like(b_ids); b_real = torch.zeros_like(b_ids, dtype=torch.bool)
+    for i, r in enumerate(branches):   # right-padded; their positions already continue the state
+        b_ids[i, :len(r["ids"])] = torch.tensor(r["ids"], device=device); b_pos[i, :len(r["pos"])] = torch.tensor(r["pos"], device=device); b_real[i, :len(r["ids"])] = True
+    run = _method(base, "kev_shared_prefix", _forward)   # on an FSDP2 root: embeddings and final norm gathered as for base(...)
+    h_b = run(s_ids, s_pos, s_real, b_ids, b_pos, b_real, torch.tensor(owner, device=device))
+    out = [[] for _ in splits]
+    for i, b in enumerate(owner): out[b].append(h_b[i, :len(branches[i]["ids"])])
+    return out
+
+
+def _forward(base, s_ids, s_pos, s_real, b_ids, b_pos, b_real, owner):
+    """The decoder stack over left-padded states and right-padded branches (owner[i] = the state of branch i)."""
+    device, (Ls, Lb) = s_ids.device, (s_ids.shape[1], b_ids.shape[1])
+    h_s = base.embed_tokens(s_ids) * s_real[..., None]; h_b = base.embed_tokens(b_ids) * b_real[..., None]
+    dtype, attn = h_s.dtype, base.config._attn_implementation
+    # attention: a state token sees the real state tokens up to itself; a branch token its record's real state and its
+    # own branch so far; a padded query keeps its diagonal so no softmax row is empty (its output is zeroed or unused)
+    causal = lambda n: torch.ones(n, n, dtype=torch.bool, device=device).tril()
+    eye = lambda n: torch.eye(n, dtype=torch.bool, device=device)
+    # States without padding (one per micro-batch, or all of one length) need no state mask under SDPA: None makes it
+    # causal (is_causal), the flash kernel's case. A 64k-token state's explicit mask would be 4 GB, kept for the whole
+    # backward, and would rule the flash kernel out.
+    plain = attn == "sdpa" and bool(s_real.all())
+    allow_b = torch.cat([s_real[owner][:, None, :].expand(-1, Lb, -1), (causal(Lb)[None] & b_real[:, None, :]) | eye(Lb)[None]], -1)
+    masks_s = {"full_attention": None if plain else _masks((causal(Ls)[None] & s_real[:, None, :]) | eye(Ls)[None], dtype, attn), "linear_attention": s_real.to(dtype)}
+    masks_b = {"full_attention": _masks(allow_b, dtype, attn), "linear_attention": b_real.to(dtype)}
+    rope_s, rope_b = base.rotary_emb(h_s, s_pos[None].expand(3, -1, -1)), base.rotary_emb(h_b, b_pos[None].expand(3, -1, -1))
+
+    recompute = base.gradient_checkpointing and base.training and torch.is_grad_enabled()
+    for layer in base.layers:
+        step = _method(layer, "kev_shared_prefix_step", _step)
+        args = (h_s, h_b, (rope_s, rope_b), (masks_s[layer.block_type], masks_b[layer.block_type]), (s_pos, b_pos), s_real, owner)
+        h_s, h_b = checkpoint(step, *args, use_reentrant=False) if recompute else step(*args)
+    return base.norm(h_b).float()
+
+
+def _step(layer, h_s, h_b, ropes, masks, positions, s_real, owner):
+    """One decoder layer: the state pass, then the branch pass reading what it left (`Prefix`). A single call on the layer
+    (both passes call its plain `forward`), so an FSDP2 unit is gathered once for both and checkpointing replays both."""
+    prefix = Prefix()
+    h_s = layer.forward(h_s, position_embeddings=ropes[0], attention_mask=masks[0], position_ids=positions[0], past_key_values=prefix)
+    prefix.branch(owner)
+    h_b = layer.forward(h_b, position_embeddings=ropes[1], attention_mask=masks[1], position_ids=positions[1], past_key_values=prefix)
+    return h_s * s_real[..., None], h_b
+
+
+def _method(module, name, fn):
+    """`fn` bound to `module` as `name`; on an FSDP2 unit also registered as a forward method, so the unit's parameters
+    are gathered around the call and resharded after it, and backward re-gathers them from its outputs' hooks."""
+    if not hasattr(module, name):
+        setattr(module, name, types.MethodType(fn, module))
+        if isinstance(module, FSDPModule): register_fsdp_forward_method(module, name)
+    return getattr(module, name)

runtime/kev/study_v3.py

@@ -0,0 +1,232 @@
+import argparse
+import copy
+import json
+import random
+from collections import Counter, defaultdict
+from pathlib import Path
+
+from huggingface_hub import HfApi
+
+from kev.composition import DEV_SHAPES, HELD_OUT_KEYS, TEST_SHAPES, TRAIN_SHAPES, canonical, check_group, generate as compose, sample_trees
+from kev.contrastive import ORDINAL_FAMILIES, generate
+from kev.data import materialize
+from kev.model import fits, load_tokenizer
+from kev.suite import SPLITS, digest, load_split, read_manifest, semantic_hash, validate_training, write_json, write_jsonl
+
+BASES = ("Qwen/Qwen3-0.6B-Base", "Qwen/Qwen3-4B-Base")
+FAMILIES = ("return_window", "spend_threshold", "age_eligibility", "quantity_limit")
+
+
+def grouped_split(records, calibration_groups):
+    groups = defaultdict(dict)
+    for r in records:
+        m = r["_meta"]
+        groups[m["family"]].setdefault(m["group_id"], []).append(r)
+    train, calibration = [], []
+    for family in sorted(groups):
+        units = list(groups[family].values())
+        if len(units) <= calibration_groups:
+            raise ValueError("not enough groups per family")
+        calibration.extend(r for group in units[:calibration_groups] for r in group)
+        train.extend(r for group in units[calibration_groups:] for r in group)
+    return train, calibration
+
+
+def legacy(pairs, seed, families=FAMILIES, source="legacy_policy", excluded=()):
+    candidates, _ = generate(3 * pairs, seed, families)
+    records, seen, counts = [], set(excluded), Counter()
+    for a, b in zip(candidates[::2], candidates[1::2]):
+        family = a["_meta"]["family"]
+        hashes = {semantic_hash(a), semantic_hash(b)}
+        if counts[family] >= pairs or hashes & seen:
+            continue
+        for r in (a, b):
+            m = r["_meta"]
+            m.update(source=source, group_id=m["pair_id"], variant="clean")
+            m["text_sha256"] = semantic_hash(r)
+        records.extend((a, b)); seen.update(hashes); counts[family] += 1
+    if any(counts[f] != pairs for f in families):
+        raise ValueError("insufficient unique legacy pairs")
+    return records
+
+
+def freeze(out, source="evals/decision-v2", transfer="evals/transfer-v2", public_train=None, synthetic_scale=1, inherit_eval=None,
+           random_structures=0, groups_per_structure=8, train_styles=(0, 1), matched_arms=True, legacy_families=FAMILIES, structures_seed=None):
+    """random_structures > 0: the compositional arm is generated from that many random rule trees (negation anywhere,
+    held-out and locked structures excluded by canonical key) instead of the eight fixed TRAIN_SHAPES."""
+    """inherit_eval: a frozen decision suite whose development/test bytes are reused verbatim (only train/calibration are
+    regenerated), so results stay comparable across suite versions that differ in training data only."""
+    """public_train: optional larger public training pool (a frozen suite dir). Its train+calibration partitions replace
+    the source suite's public train/calibration; development/test still come from `source` so results stay comparable."""
+    out, source, transfer = Path(out), Path(source), Path(transfer)
+    if out.exists():
+        raise FileExistsError("v3 destination already exists; choose a new version")
+    original = read_manifest(source)
+    old_transfer = read_manifest(transfer)
+    revisions = {base: HfApi().model_info(base).sha for base in BASES}
+    tokenizers = [load_tokenizer(base, revision=sha) for base, sha in revisions.items()]
+    parts = {s: [] for s in SPLITS}
+    for split in ("train", "calibration", "development"):
+        parts[split] = [r for r in load_split(source, split) if r["_meta"]["source"] != "contrastive"]
+    if public_train:
+        pool = Path(public_train)
+        dev_test_states = {r["_meta"]["text_sha256"] for split in ("development", "test") for r in load_split(source, split, allow_test=True)}
+        for split in ("train", "calibration"):
+            fresh = [r for r in load_split(pool, split) if r["_meta"]["source"] != "contrastive"]
+            leaked = [r for r in fresh if r["_meta"]["text_sha256"] in dev_test_states]
+            if leaked: raise ValueError(f"{len(leaked)} public {split} records collide with development/test states")
+            parts[split] = fresh
+        source_dirs = [pool]
+    else:
+        source_dirs = []
+    reserved = set()
+    for split in ("train", "calibration", "development"):
+        reserved.update(semantic_hash(r) for r in parts[split])
+    sources = [p for p in source.glob("*.json*")] + [p for p in transfer.glob("*.json*")] + [p for d in source_dirs for p in d.glob("*.json*")]
+    parent_hashes = {str(p): digest(p) for p in sources}
+
+    def admit_groups(records):
+        groups = defaultdict(list)
+        for r in records:
+            groups[r["_meta"]["group_id"]].append(r)
+        for group in groups.values():
+            if group[0]["_meta"]["source"] in ("compositional", "composition_holdout"):
+                check_group(group)
+            hashes = {semantic_hash(r) for r in group}
+            if hashes & reserved:
+                raise ValueError("semantic state collision across groups; choose a new generation seed")
+            for r in group:
+                if not fits(materialize(r), *tokenizers):
+                    raise ValueError("synthetic record exceeds the training context")
+            reserved.update(hashes)
+        return records
+
+    # development synthetic records are generated first, against public states only, so they do not depend on the size
+    # of the synthetic training arms (dev/test bytes stay identical across suite versions); training groups that would
+    # collide with a development state are rejected by admit_groups
+    if inherit_eval:
+        inherited_dev = load_split(inherit_eval, "development")
+        parts["development"] = inherited_dev
+        reserved.update(semantic_hash(r) for r in inherited_dev)
+        reserved.update(semantic_hash(r) for r in load_split(inherit_eval, "test", allow_test=True))
+        dev_synthetic = []
+    else:
+        dev_synthetic = admit_groups(legacy(12, "v3-control-dev", excluded=reserved)) + admit_groups(compose(4, "v3-composition-dev"))
+    old_train, old_cal = grouped_split(admit_groups(legacy(64 * synthetic_scale, "v3-control", families=legacy_families, excluded=reserved)), 8 * synthetic_scale)
+    if random_structures:
+        trees = {f"rand:{canonical(t)}": t for t in sample_trees(random_structures, f"{structures_seed or out.name}-structures")}
+        new_train, new_cal = grouped_split(admit_groups(compose(groups_per_structure, "v7-composition", styles=train_styles, trees=trees)), 1)
+    else:
+        new_train, new_cal = grouped_split(admit_groups(compose(16 * synthetic_scale, "v3-composition")), 2 * synthetic_scale)
+    if matched_arms and (len(old_train) != len(new_train) or len(old_cal) != len(new_cal)):
+        raise ValueError("synthetic arms have unequal record budgets")
+    parts["train"] += old_train + new_train
+    parts["calibration"] += old_cal + new_cal
+    parts["development"] += dev_synthetic
+    transfer_dev = [r for r in load_split(transfer, "development") if r["_meta"]["source"] != "contrastive"]
+    transfer_dev += admit_groups(legacy(20, "v3-legacy-transfer", ("authorization", "deadline"), source="legacy_holdout"))
+    transfer_dev += admit_groups(compose(8, "v3-composition-transfer", DEV_SHAPES, styles=(1,), source="composition_holdout"))
+    final_extra = admit_groups(compose(8, "v3-locked", TEST_SHAPES, styles=(2,), source="composition_holdout"))
+    for split in ("train", "calibration"):
+        random.Random(f"v3-{split}").shuffle(parts[split])
+    manifest = {"version": 3, "base_revisions": revisions, "dataset_revisions": original["dataset_revisions"],
+        "parent_files": parent_hashes, "holdout_sources": [],
+        "trainable_sources": sorted({s for s in original["trainable_sources"] if s != "contrastive"}
+                                    | ({s for s in read_manifest(public_train)["trainable_sources"]} if public_train else set())) + ["legacy_policy", "compositional"],
+        "eval_only_sources": old_transfer["eval_only_sources"] + ["legacy_holdout", "composition_holdout"],
+        "context": original["context"],
+        "protocol": {"train_shapes": TRAIN_SHAPES, "transfer_shapes": DEV_SHAPES, "locked_shapes": TEST_SHAPES,
+                     "train_render_styles": list(train_styles), "locked_render_styles": [2],
+                     "random_structures": random_structures, "groups_per_structure": groups_per_structure if random_structures else None,
+                     "structures_seed": f"{structures_seed or out.name}-structures" if random_structures else None,
+                     "excluded_structure_keys": sorted(HELD_OUT_KEYS), "legacy_families_trainable": list(legacy_families),
+                     "public_train_records": len(parts["train"]) - len(old_train) - len(new_train),
+                     "public_train_pool": str(public_train) if public_train else str(source), "synthetic_scale": synthetic_scale,
+                     "inherited_eval": str(inherit_eval) if inherit_eval else None,
+                     "synthetic_records_per_arm": len(old_train) if matched_arms else {"legacy_policy": len(old_train), "compositional": len(new_train)},
+                     "calibration": "shared; stratified by family and group",
+                     "arm_selection": "train_sources selects public sources plus exactly one synthetic arm",
+                     "primary": "macro development NLL; transfer and confident-error checks required; no automatic release",
+                     "legacy_test": "Inherited v2 locked test bytes retained without inspecting examples; only new structures added to transfer test",
+                     "contamination": "Exact semantic checks among new groups, not fuzzy or pretraining decontamination. Inherited legacy test overlap with new synthetic controls is not certified."},
+        "files": {}, "code_hashes": {p.name: digest(p) for p in Path(__file__).parent.glob("*.py")}}
+    validate_training(parts["train"], manifest)
+    for name, partitions, inherited in (("decision-v3", parts, source),
+            ("transfer-v3", {"train": [], "calibration": [], "development": transfer_dev, "test": final_extra}, transfer)):
+        folder = out / name
+        folder.mkdir(parents=True, exist_ok=False)
+        m = copy.deepcopy(manifest)
+        if name.startswith("transfer"):
+            m.update(trainable_sources=[], holdout_sources=manifest["eval_only_sources"], eval_only=True)
+        for split, records in partitions.items():
+            path = folder / f"{split}.jsonl"
+            payload = b""
+            inherited_records = inherited_questions = 0
+            if split == "test":
+                test_src = Path(inherit_eval) if (inherit_eval and name.startswith("decision")) else inherited
+                old = read_manifest(test_src)["files"]["test.jsonl"]
+                if digest(test_src / "test.jsonl") != old["sha256"]:
+                    raise ValueError("inherited test checksum mismatch")
+                payload = (test_src / "test.jsonl").read_bytes()
+                inherited_records, inherited_questions = old["records"], old["questions"]
+                if inherit_eval and name.startswith("decision"): records = []   # the inherited test already contains the locked composition groups
+            payload += "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records).encode()
+            path.write_bytes(payload)
+            m["files"][path.name] = {"sha256": digest(path), "records": inherited_records + len(records),
+                                     "questions": inherited_questions + sum(len(r["questions"]) for r in records)}
+        write_json(folder / "manifest.json", m)
+        print(name, {s: v["records"] for s, v in m["files"].items()}, flush=True)
+    if any(digest(p) != h for p, h in parent_hashes.items()):
+        raise ValueError("parent artifacts changed")
+    return manifest
+
+
+def smoke_subset(source, out):
+    source, out = Path(source), Path(out)
+    manifest = read_manifest(source)
+    out.mkdir(parents=True, exist_ok=False)
+    manifest["files"] = {}
+    for split in SPLITS:
+        records = []
+        if split != "test":
+            groups = defaultdict(list)
+            for r in load_split(source, split):
+                groups[(r["_meta"]["source"], r["_meta"]["group_id"])].append(r)
+            taken = Counter()
+            for (name, _), group in groups.items():
+                if taken[name] < 2:
+                    records.extend(group); taken[name] += 1
+        path = out / f"{split}.jsonl"
+        write_jsonl(path, records)
+        manifest["files"][path.name] = {"sha256": digest(path), "records": len(records),
+                                        "questions": sum(len(r["questions"]) for r in records)}
+    manifest["smoke_only"] = True
+    write_json(out / "manifest.json", manifest)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--smoke-from")
+    ap.add_argument("--public-train", help="frozen suite whose train/calibration public partitions replace the source's (larger pool)")
+    ap.add_argument("--synthetic-scale", type=int, default=1, help="multiply synthetic training pairs/groups per arm (both arms stay equal)")
+    ap.add_argument("--inherit-eval", help="decision suite whose development/test bytes are reused verbatim")
+    ap.add_argument("--random-structures", type=int, default=0, help="compositional arm from N random rule trees (held-out structures excluded)")
+    ap.add_argument("--groups-per-structure", type=int, default=8)
+    ap.add_argument("--train-styles", default="0,1", help="rendering styles for training compositional records (2 is locked-test only)")
+    ap.add_argument("--unmatched-arms", action="store_true", help="allow the two synthetic arms to differ in size")
+    ap.add_argument("--legacy-families", default="v3", choices=["v3", "all"], help="'all' adds the ordinal Score threshold families (deadline stays held out)")
+    ap.add_argument("--structures-seed", help="reuse another version's random structures (e.g. v7) so only the legacy arm differs")
+    a = ap.parse_args()
+    styles = tuple(int(x) for x in a.train_styles.split(","))
+    if 2 in styles: ap.error("rendering style 2 is reserved for the locked test")
+    if a.smoke_from:
+        smoke_subset(a.smoke_from, a.out)
+    else:
+        freeze(a.out, public_train=a.public_train, synthetic_scale=a.synthetic_scale, inherit_eval=a.inherit_eval,
+               random_structures=a.random_structures, groups_per_structure=a.groups_per_structure, train_styles=styles, matched_arms=not a.unmatched_arms,
+               legacy_families=FAMILIES + ORDINAL_FAMILIES if a.legacy_families == "all" else FAMILIES, structures_seed=a.structures_seed)
+
+
+if __name__ == "__main__":
+    main()

runtime/kev/suite.py

@@ -0,0 +1,450 @@
+import argparse
+import copy
+import fcntl
+import hashlib
+import json
+import os
+import random
+import shutil
+from collections import Counter
+from contextlib import contextmanager
+from pathlib import Path
+
+from kev.composition import DEV_SHAPES, HELD_OUT_KEYS, TEST_SHAPES, TRAIN_SHAPES
+from kev.contrastive import FAMILIES, generate
+from kev.data import ALL_REPOS, ALL_SOURCES, EVAL_ONLY, REPOS, SOURCES, TRAINABLE, TRANSFER_REPOS, TRANSFER_SOURCES, build, dataset_ref, materialize, source_seed
+from kev.model import (MAX_BRANCH, SERVE_MAX_BRANCH, SERVE_MAX_BRANCH_8K, SERVE_MAX_PACKED, SERVE_MAX_STATE, SERVE_MAX_STATE_8K, fits, load_tokenizer,
+                       training_context)
+
+SPLITS = ("train", "calibration", "development", "test")
+BASES = ("Qwen/Qwen2.5-0.5B", "Qwen/Qwen3-0.6B-Base")
+# the encoder limits every frozen record satisfies, as written into manifests ("context")
+CONTEXT = {**training_context(), "truncate": False}
+# what a manifest records for an eval-only suite frozen as published rather than admitted to the training context
+SERVING_CONTEXT = {"max_state": SERVE_MAX_STATE, "max_branch": SERVE_MAX_BRANCH, "max_packed": SERVE_MAX_PACKED, "truncate": False}
+# the serving context before 64k states: what the suites frozen until then record and were admitted under; their builders
+# (hard-v1, devtools-v1, breadth-v1, long states, semif/typesafe, documents-v1) keep it and kev.model.MAX_TRAIN_STATE_8K,
+# so they still rebuild byte for byte
+SERVING_CONTEXT_8K = {"max_state": SERVE_MAX_STATE_8K, "max_branch": SERVE_MAX_BRANCH_8K, "max_packed": SERVE_MAX_STATE_8K + SERVE_MAX_BRANCH_8K, "truncate": False}
+# Clean records are admitted with this many branch tokens to spare, so the variants that add an option (contrast_cases'
+# none-of-these, training-time none/distractor augmentation) still encode under MAX_BRANCH.
+ADMISSION_BRANCH_HEADROOM = 64
+# Frozen suites are mirrored on the Hub. Manifests (with the sha256 of every partition) and the development/test
+# partitions live in git; large training partitions are fetched from this dataset on first use and verified against
+# the manifest, so the suite hash and every provenance record stay unchanged. A suite whose partitions must never be
+# public (a held-out test set) names its own mirror in the manifest, {"mirror": {"dataset": ..., "revision": ...}},
+# usually the private PRIVATE_DATASET; only its manifest is in git, which publishes the hashes but not the text.
+SUITES_DATASET = "jaredpalmer/kev-suites"
+PRIVATE_DATASET = "jaredpalmer/kev-private-evals"
+SUITES_REVISION = "cc4bac803e73112689ec327ffa481c519cbc7a05"
+# partitions larger than this stay out of git (gitignored; the manifest's sha256 still pins them)
+GIT_LIMIT = 10 * 1024 * 1024
+# the pinned tokenizer suites built for the Qwen3.5 family are admitted and length-counted under (hard-v1, devtools-v1, long states)
+ADMISSION_TOKENIZER = ("Qwen/Qwen3.5-4B-Base", "1001bb4d826a52d1f399e183466143f4da7b741b")
+# programmatic policy sources (kev.study_v3 / kev.contrastive); the trainer's mix ablations treat them as one group
+SYNTHETIC_SOURCES = ("legacy_policy", "compositional", "contrastive")
+# Suites deleted from the repo because they are unsound as a gate: {suite dir: why, when, the last round that read it}.
+# load_split / read_manifest refuse them with the reason; kev.rounds.validate lists a read of one as archived for rounds up
+# to `last_round` (their committed rows under runs/ are the record and still reproduce) and refuses it after that.
+REMOVED_SUITES = {
+    "evals/external/scienthoon-v1": {
+        "removed": "2026-09-27",
+        "last_round": 22,
+        "reason": ("unsound as a gate: 291 templated synthetic support tickets x 3 questions; `queue` is saturated (0.948-0.952 "
+                   "for every 27B), `priority` is unlearnable by construction (its own manifest: the label follows an org rule "
+                   "absent from the text), and `angry` has 15 of 291 gold labels that contradict the text and turns on ~12 stock "
+                   "closing phrases with disputed conventions"),
+        "record": "PLAN.md (Standing rules; 2026-09-27 note); committed rows under runs/ (e.g. runs/r20-scienthoon)",
+    },
+    "evals/external/wanli-v2": {
+        "removed": "2026-09-30",
+        "last_round": 26,
+        "reason": ("unsound as a gate: 271 of its 1,002 WANLI test pairs (27 %) are ones WANLI's two crowd annotators labelled "
+                   "differently, and the published gold is one of the two labels; every Kev scores 49-62 % on those against "
+                   "64-81 % on the 731 the annotators agreed on, and the 2026-09-27 audit measured split-half r 0.04 across 23 "
+                   "checkpoints (all within 0.735-0.763), a half-width as wide as its 2 pp bar and ~11 % invalid labels"),
+        "record": "PLAN.md (Standing rules; 2026-09-30 note); committed rows under runs/ (e.g. runs/r23-27b-k-w85-wanli2)",
+    },
+    "evals/external/wanli-v1": {
+        "removed": "2026-09-30",
+        "last_round": 5,
+        "reason": ("the same WANLI test pairs as wanli-v2 (SemIf's 256): 63 of 256 (25 %) are ones WANLI's two crowd annotators "
+                   "labelled differently, and the published gold is one of the two labels"),
+        "record": "PLAN.md (2026-09-30 note); README; committed rows under runs/ (e.g. runs/kev-9b-wanli-v1, runs/jev-wanli-v1)",
+    },
+    "evals/external/typesafe-v1": {
+        "removed": "2026-09-30",
+        "last_round": 26,
+        "reason": ("unsound as a gate: its gold is not a ground truth but the argmax of the average of two closed frontier models' "
+                   "answers (evals.typesafe.ai: GPT-6 Astra and Claude Fable 5.1), 13 of its 102 reference distributions put "
+                   "the answer below 0.75 (the two references split), and on 89 answered questions from 20 cases the split-half "
+                   "correlation across 20 Kev-27B checkpoints is -0.01 (2026-09-27 audit: -0.27): its differences between "
+                   "checkpoints are noise"),
+        "record": "PLAN.md (Standing rules; 2026-09-30 note); README; committed rows under runs/ (e.g. runs/kev-9b-typesafe-v1)",
+    },
+}
+
+
+class RemovedSuite(ValueError):
+    """A read of a suite in REMOVED_SUITES."""
+
+
+def suite_key(path):
+    """'evals/...' for a suite path given relative, absolute or as a container saw it (/root/kev/evals/x); None if none."""
+    parts = Path(str(path)).parts
+    return str(Path(*parts[parts.index("evals"):])) if "evals" in parts else None
+
+
+def removed_suite(path):
+    """The REMOVED_SUITES entry of a suite path (any form suite_key accepts), or None."""
+    key = suite_key(path)
+    return REMOVED_SUITES.get(key) if key else None
+
+
+def refuse_removed(path):
+    if entry := removed_suite(path):
+        raise RemovedSuite(f"{suite_key(path)} was removed on {entry['removed']}: {entry['reason']}. Rounds up to "
+                           f"{entry['last_round']} read it; their committed rows are the record ({entry['record']}). Do not read it again.")
+
+
+def digest(path):
+    h = hashlib.sha256()
+    with Path(path).open("rb") as stream:
+        for block in iter(lambda: stream.read(1024 * 1024), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def record_digest(record):
+    return hashlib.sha256(json.dumps(record, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest()
+
+
+def normalise_text(text):
+    """Casefolded, whitespace runs collapsed to one space: the text two states are compared on for exact deduplication."""
+    return " ".join(text.casefold().split())
+
+
+def text_digest(text):
+    """sha256 of normalise_text(text): the `text_sha256` of suite builders (kev.data computes the same inline; it cannot
+    import this module). scripts/screen_overlap.py tokenises differently on purpose (words only, for n-gram overlap)."""
+    return hashlib.sha256(normalise_text(text).encode()).hexdigest()
+
+
+# Every JSON/JSONL file this repo writes is UTF-8 with LF line endings, whatever the platform's locale says (issue #12:
+# frozen partitions are sha256-checked byte for byte, and they contain non-ASCII text). Read them the same way.
+ENCODING = "utf-8"
+
+
+def read_json(path):
+    return json.loads(Path(path).read_text(encoding=ENCODING))
+
+
+def write_json(path, value, atomic=False):
+    """atomic: write a sibling temp file and os.replace it over `path`, so a reader (or a restart after a crash mid-write)
+    sees the old file or the new one, never a torn one. For state files rewritten in place (kev.rounds' watcher)."""
+    text = json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
+    if not atomic:
+        Path(path).write_text(text, encoding=ENCODING); return
+    tmp = Path(path).with_name(f".{Path(path).name}.tmp")
+    tmp.write_text(text, encoding=ENCODING)
+    os.replace(tmp, path)
+
+
+@contextmanager
+def file_lock(path):
+    """Hold an exclusive advisory lock on `path` (created if absent) for the block; a second holder waits. Local
+    orchestration only: one pull of a study (modal_app.pull_lock), one launch of an arm's reads (kev.rounds)."""
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with Path(path).open("a", encoding=ENCODING) as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        yield
+
+
+def read_jsonl(path):
+    # split on "\n" only: str.splitlines() also breaks on U+2028, U+2029 and U+0085, which write_jsonl leaves unescaped
+    return [json.loads(line) for line in Path(path).read_text(encoding=ENCODING).split("\n") if line.strip()]
+
+
+def write_jsonl(path, records):
+    Path(path).write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records), encoding=ENCODING)
+
+
+def semantic_hash(r):
+    """State hash that ignores sentence order for policy/case records (contrastive, compositional), so the same case
+    told in a different order counts as the same state; record_digest of the state otherwise."""
+    state = r["state"]
+    if isinstance(state, dict) and "policy" in state and "case" in state:
+        state = {"policy": state["policy"], "sentences": sorted(s.rstrip(".") for s in state["case"].split(". "))}
+    return record_digest(state)
+
+
+def validate_training(records, manifest):
+    """Every training record must come from a source the manifest declares trainable, never from an eval-only or held-out
+    one, and compositional records must not use a held-out rule structure."""
+    allowed = set(manifest.get("trainable_sources", []))
+    forbidden = set(EVAL_ONLY) | set(manifest.get("eval_only_sources", [])) | set(manifest.get("holdout_sources", []))
+    for r in records:
+        m = r["_meta"]
+        if m["source"] in forbidden or (allowed and m["source"] not in allowed):
+            raise ValueError(f"eval-only or undeclared training source: {m['source']}")
+        if m["source"] == "compositional":
+            held_shape = m["family"] in DEV_SHAPES + TEST_SHAPES
+            held_structure = m.get("structure") in HELD_OUT_KEYS
+            if held_shape or held_structure or (m["family"] not in TRAIN_SHAPES and not m["family"].startswith("rand:")):
+                raise ValueError("held-out compositional structure in training")
+    if not records:
+        raise ValueError("empty training partition")
+
+
+def read_manifest(directory):
+    refuse_removed(directory)
+    return read_json(Path(directory) / "manifest.json")
+
+
+def load_split(directory, split, allow_test=False):
+    refuse_removed(directory)   # a removed suite is refused with its reason, not a missing-file error
+    if split not in SPLITS:
+        raise ValueError(f"unknown split: {split}")
+    if split == "test" and not allow_test:
+        raise ValueError("locked test requires explicit --allow-test; never use it for search")
+    directory = Path(directory)
+    manifest = read_manifest(directory)
+    path = directory / f"{split}.jsonl"
+    if not path.exists():
+        fetch_partition(directory, path.name)
+    if digest(path) != manifest["files"][path.name]["sha256"]:
+        raise ValueError(f"suite checksum mismatch: {path}")
+    records = read_jsonl(path)
+    if len(records) != manifest["files"][path.name]["records"]:
+        raise ValueError("suite record count mismatch")
+    return records
+
+
+def fetch_partition(directory, filename):
+    """Download one partition of a frozen suite from its Hub mirror into place: the manifest's own "mirror" if it names
+    one, else SUITES_DATASET@SUITES_REVISION. The caller verifies the sha256."""
+    from huggingface_hub import hf_hub_download
+    from huggingface_hub.errors import GatedRepoError, RepositoryNotFoundError
+    directory = Path(directory).resolve()
+    evals_root = next((p for p in directory.parents if p.name == "evals"), None)
+    if evals_root is None:
+        raise FileNotFoundError(f"{directory / filename} is missing and is not under an evals/ tree")
+    relative = directory.relative_to(evals_root) / filename
+    mirror = read_manifest(directory).get("mirror")
+    repo, revision = (mirror["dataset"], mirror["revision"]) if mirror else (SUITES_DATASET, SUITES_REVISION)   # a named mirror pins its own revision
+    try:
+        cached = hf_hub_download(repo, str(relative), repo_type="dataset", revision=revision)
+    except (RepositoryNotFoundError, GatedRepoError) as e:   # a private mirror answers "not found" to anyone without access
+        raise PermissionError(f"{relative} is only in {repo}, which is missing or private to this account; `hf auth login` "
+                              "(or HF_TOKEN) with access to it, or ask for it") from e
+    shutil.copyfile(cached, directory / filename)
+    print(f"fetched {relative} from {repo}@{revision[:10]}", flush=True)
+
+
+def case_copy(record, variant):
+    result = copy.deepcopy(record)
+    result["_meta"]["group_id"] = record["_meta"].get("group_id", record["_meta"]["id"])   # bootstrap unit (pair for contrastive)
+    result["_meta"]["parent_id"] = record["_meta"]["id"]                                   # the clean record this variant perturbs
+    result["_meta"]["id"] += "/" + variant
+    result["_meta"]["variant"] = variant
+    return result
+
+
+def contrast_cases(record, seed=0):
+    candidates = [(qid, q) for qid, q in record["questions"].items() if q["type"] == "choice" and len(q["criteria"]) >= 3]
+    if not candidates:
+        return []
+    qid, q = candidates[0]
+    nk = "none_of_these"
+    if nk in q["criteria"]:
+        raise ValueError("reserved contrast option collision")
+    out = []
+    for variant in ("none_present", "none_absent"):
+        r = case_copy(record, variant)
+        r["questions"] = {qid: copy.deepcopy(q)}
+        rq = r["questions"][qid]
+        rq["criteria"][nk] = "None of these options describes the answer"
+        if variant == "none_absent":
+            rq["criteria"].pop(rq["label"])
+            rq["label"] = nk
+        keys = list(rq["criteria"])
+        random.Random(source_seed(seed, record["_meta"]["id"])).shuffle(keys)
+        rq["criteria"] = {k: rq["criteria"][k] for k in keys}
+        r["_meta"]["none_key"] = nk
+        out.append(r)
+    r = case_copy(record, "permuted")
+    rng = random.Random(source_seed(seed, record["_meta"]["id"]))
+    for q in r["questions"].values():
+        if q["type"] == "choice":
+            keys = list(q["criteria"])
+            rng.shuffle(keys)
+            q["criteria"] = {k: q["criteria"][k] for k in keys}
+    out.append(r)
+    return out
+
+
+def select_unique(records, count, seen, tokenizers, report):
+    selected = []
+    for record in sorted(records, key=lambda r: r["_meta"]["row_sha256"]):
+        report["considered"] += 1
+        key = record["_meta"]["text_sha256"]
+        if key in seen:
+            report["duplicate_state"] += 1
+            continue
+        rec = materialize(record)   # a record that fails request validation is a converter bug: let it abort the freeze
+        if not fits(rec, *tokenizers, max_branch=MAX_BRANCH - ADMISSION_BRANCH_HEADROOM):
+            report["context_rejected"] += 1
+            continue
+        record["_meta"].update(group_id=record["_meta"]["id"], variant="clean")
+        seen.add(key)
+        selected.append(record)
+        report["accepted"] += 1
+        if len(selected) == count:
+            return selected
+    raise ValueError(f"only {len(selected)}/{count} records fit the common context policy")
+
+
+def training_state_hashes(suite_dir):
+    """Normalized state hashes of a frozen suite's training + calibration partitions, for exact-match contamination checks."""
+    hashes = set()
+    for split in ("train", "calibration"):
+        for record in load_split(suite_dir, split):
+            hashes.add(record["_meta"]["text_sha256"])
+    return hashes
+
+
+def freeze(directory, train=300, calibration=40, development=80, test=80, seed=20260918, holdout=("mnli", "sst5"),
+           sources=None, repos=None, exclude_states_from=None, contrastive_pairs=0, contrastive_holdout_families=(), contrastive_eval_pairs=40):
+    from huggingface_hub import HfApi
+
+    sources = SOURCES if sources is None else sources
+    repos = REPOS if repos is None else repos
+    eval_only = set(holdout) >= set(sources)
+    directory = Path(directory)
+    if directory.exists():
+        raise FileExistsError(f"refusing to overwrite frozen suite {directory}")
+    hub = HfApi()
+    revisions = {repo: hub.dataset_info(*dataset_ref(repo)[:1], revision=dataset_ref(repo)[1]).sha for repo in set(repos.values())}
+    base_revisions = {base: hub.model_info(base).sha for base in BASES}
+    tokenizers = [load_tokenizer(base, revision=revision) for base, revision in base_revisions.items()]
+    trainable_here = [x for x in sources if x not in holdout]
+    if set(trainable_here) & set(EVAL_ONLY):
+        raise ValueError(f"eval-only sources cannot be trainable: {sorted(set(trainable_here) & set(EVAL_ONLY))}")
+    manifest = {
+        "version": 2, "seed": seed, "holdout_sources": list(holdout), "eval_only": eval_only,
+        "trainable_sources": trainable_here, "eval_only_sources": [x for x in sources if x in holdout],
+        "policy": {"trainable": list(TRAINABLE), "eval_only": list(EVAL_ONLY)},
+        "dataset_revisions": revisions, "base_revisions": base_revisions,
+        "context": {**CONTEXT, "admission_branch_headroom": ADMISSION_BRANCH_HEADROOM},
+        "selection": "Normalized exact-state deduplication across partitions; common tokenizer context admission; no fuzzy decontamination or pretraining-contamination claim.",
+        "legacy_checkpoints": "Training/calibration overlap for pre-manifest checkpoints is unknown; exploratory only.",
+        "objective": "Negative macro-average clean development NLL, equal weight per task; raw probabilities.",
+        "excluded_training_states_from": str(exclude_states_from) if exclude_states_from else None,
+        "files": {}, "admission": {},
+    }
+    partitions = {split: [] for split in SPLITS}
+    seen = set(training_state_hashes(exclude_states_from)) if exclude_states_from else set()
+    manifest["excluded_training_state_hashes"] = len(seen)
+    for source in sources:
+        report = Counter()
+        test_pool = build(max(3 * (development + test), 600), "test", seed, only=[source], revisions=revisions, sources=sources, repos=repos)
+        chosen = select_unique(test_pool, development + test, seen, tokenizers, report)
+        partitions["development"].extend(chosen[:development])
+        partitions["test"].extend(chosen[development:])
+        if source not in holdout:
+            train_pool = build(max(3 * (train + calibration), 800), "train", seed, only=[source], revisions=revisions, sources=sources, repos=repos)
+            chosen = select_unique(train_pool, train + calibration, seen, tokenizers, report)
+            partitions["calibration"].extend(chosen[:calibration])
+            partitions["train"].extend(chosen[calibration:])
+        manifest["admission"][source] = dict(report)
+        print(f"froze {source}: {dict(report)}", flush=True)
+    if contrastive_pairs:
+        families = list(FAMILIES)
+        held = list(contrastive_holdout_families)
+        unknown = set(held) - set(families)
+        if unknown: raise ValueError(f"unknown contrastive families: {sorted(unknown)}")
+        trainable_fams = [f for f in families if f not in held]
+        # training/calibration pairs and development/test pairs come from disjoint seeds within trainable families;
+        # held-out families appear only in development/test (never trained anywhere)
+        train_recs, rep_train = generate(contrastive_pairs, seed=f"{seed}-train", families=trainable_fams) if trainable_fams and not eval_only else ([], {})
+        # eval-only suites carry the held-out families only; trainable suites carry their trainable families only
+        eval_families = (held or families) if eval_only else trainable_fams
+        eval_recs, rep_eval = generate(contrastive_eval_pairs, seed=f"{seed}-eval", families=eval_families)
+        for r in train_recs + eval_recs:
+            r["_meta"].update(group_id=r["_meta"]["family_id"], variant="clean")   # siblings are one bootstrap unit
+            if r["_meta"]["text_sha256"] in seen: raise ValueError("contrastive state collides with an existing state")
+        seen.update(r["_meta"]["text_sha256"] for r in train_recs + eval_recs)
+        n_cal = 2 * max(len(train_recs) // 20, 1) if train_recs else 0
+        partitions["calibration"].extend(train_recs[:n_cal]); partitions["train"].extend(train_recs[n_cal:])
+        # stratified by family: alternate pairs (siblings adjacent) between development and test
+        for i in range(0, len(eval_recs), 2):
+            partitions["development" if (i // 2) % 2 == 0 else "test"].extend(eval_recs[i : i + 2])
+        manifest["contrastive"] = {"pairs_per_family_train": contrastive_pairs, "pairs_per_family_eval": contrastive_eval_pairs,
+                                   "eval_families": eval_families, "trainable_families": trainable_fams, "eval_only_families": held,
+                                   "train_report": rep_train, "eval_report": rep_eval,
+                                   "note": "labels from rule evaluators; ablation + invariance checks passed for every kept pair; no LLM"}
+        if trainable_fams and not eval_only: manifest["trainable_sources"].append("contrastive")
+        else: manifest["eval_only_sources"].append("contrastive")
+        print(f"contrastive: {len(train_recs)} train/cal records, {len(eval_recs)} dev/test records; held-out families {held}", flush=True)
+    for split in ("development", "test"):
+        extras = []
+        per_source = Counter()
+        for record in partitions[split]:
+            source = record["_meta"]["source"]
+            if per_source[source] < 12:
+                variants = contrast_cases(record, seed)
+                for variant in variants:
+                    if not fits(materialize(variant), *tokenizers):
+                        raise ValueError(f"variant of {record['_meta']['id']} exceeds the training context")
+                extras.extend(variants)
+                per_source[source] += bool(variants)
+        partitions[split].extend(extras)
+    directory.mkdir(parents=True)
+    for split, records in partitions.items():
+        path = directory / f"{split}.jsonl"
+        write_jsonl(path, records)
+        manifest["files"][path.name] = {"sha256": digest(path), "records": len(records),
+                                        "questions": sum(len(r["questions"]) for r in records)}
+    manifest["code_hashes"] = {name: digest(Path(__file__).parent / name) for name in ("data.py", "api.py", "model.py", "suite.py")}
+    write_json(directory / "manifest.json", manifest)
+    print(json.dumps(manifest["files"], indent=2), flush=True)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--train", type=int, default=300)
+    ap.add_argument("--calibration", type=int, default=40)
+    ap.add_argument("--development", type=int, default=80)
+    ap.add_argument("--test", type=int, default=80)
+    ap.add_argument("--seed", type=int, default=20260918)
+    ap.add_argument("--transfer", action="store_true", help="eval-only suite from the eight transfer-v1 sources (no train/calibration partitions)")
+    ap.add_argument("--sources", help="comma-separated source names (any of ALL_SOURCES); trainable ones get train/calibration partitions")
+    ap.add_argument("--holdout", help="comma-separated sources kept eval-only within --sources (eval-only policy sources are always held out)")
+    ap.add_argument("--exclude-states-from", help="frozen suite whose train+calibration states must not appear here")
+    ap.add_argument("--contrastive-pairs", type=int, default=0, help="programmatic contrastive pairs per trainable family for training (0 = none)")
+    ap.add_argument("--contrastive-holdout", default="", help="comma-separated contrastive families kept eval-only")
+    a = ap.parse_args()
+    cfam = tuple(x for x in a.contrastive_holdout.split(",") if x)
+    if min(a.development, a.test) < 1:
+        ap.error("split sizes must be positive")
+    if a.transfer:
+        freeze(a.out, 0, 0, a.development, a.test, a.seed, holdout=tuple(TRANSFER_SOURCES), sources=TRANSFER_SOURCES,
+               repos=TRANSFER_REPOS, exclude_states_from=a.exclude_states_from, contrastive_pairs=a.contrastive_pairs, contrastive_holdout_families=cfam)
+    elif a.sources:
+        names = [x for x in a.sources.split(",") if x]
+        unknown = set(names) - set(ALL_SOURCES)
+        if unknown: ap.error(f"unknown sources: {sorted(unknown)}")
+        holdout = tuple(x for x in names if x in EVAL_ONLY or x in (a.holdout or "").split(","))
+        if any(x not in holdout for x in names) and min(a.train, a.calibration) < 1:
+            ap.error("train and calibration sizes must be positive when a trainable source is included")
+        freeze(a.out, a.train, a.calibration, a.development, a.test, a.seed, holdout=holdout,
+               sources={k: ALL_SOURCES[k] for k in names}, repos={k: ALL_REPOS[k] for k in names}, exclude_states_from=a.exclude_states_from,
+               contrastive_pairs=a.contrastive_pairs, contrastive_holdout_families=cfam)
+    else:
+        if min(a.train, a.calibration) < 1: ap.error("split sizes must be positive")
+        freeze(a.out, a.train, a.calibration, a.development, a.test, a.seed, exclude_states_from=a.exclude_states_from)
+
+
+if __name__ == "__main__":
+    main()