span-02: single-source release folder (engine final path, soft blend)

README.md

@@ -159,7 +159,7 @@ The capacity errors carry the Decision Index capacity markers, so the DI client
 ## Serving settings (Decision Index submission)
 ```
 --max-len 16384 --budget-ms 0 --tau 0.8 --cap 1024 --think-ms 750 --think-cost think_cost.json
---image-soft-tokens 560 --serial --no-prefix-cache --spec-max-branches 16
+--image-soft-tokens 560 --serial --no-prefix-cache --spec-max-branches 16 --pages 12000 --kv-gb 24
 ```
 * `--budget-ms 0`: there is no wall-clock limit; the clock is never read for thinking decisions.
 * `--think-ms 750 --think-cost think_cost.json`: deterministic thought budget. Each request's per-question thought
@@ -176,6 +176,10 @@ The capacity errors carry the Decision Index capacity markers, so the DI client
 * `--serial --no-prefix-cache --spec-max-branches 16`: each answer is a deterministic function of its request.
   It does not depend on load or on earlier requests (for up to 16 questions of one request thinking at once).
   `--serial` runs one request at a time; concurrent clients queue.
+* `--pages 12000 --kv-gb 24`: KV cache pools (full-attention layers 12000 x 16 = 192K tokens, 3.9 GB;
+  sliding-window layers get the rest of `--kv-gb 24`, ~20 GB), sized so that every Decision Index request fits on its own under `--serial` (the
+  largest needs about 3.7K full pages: many questions with long tails). The pool size does not change any answer.
+  The server uses about 41 GB of GPU memory.
 
 ## Latency
 Measured with the submission settings on one RTX PRO 6000 Blackwell, one request at a time, as client wall time
@@ -204,6 +208,8 @@ Requests over these limits are rejected with the DI capacity markers. Nothing is
 * Context: the prompt tokens + 562 per image + the reserved thought tokens + 3 must fit in 16384 tokens. The
   reservation is the deterministic cap of a lone thinking question of that size (about 330 tokens for a short
   prompt, about 145 at 16k prompt tokens). That allows up to about 16240 prompt tokens without images.
+* KV cache: a request whose prompt pages (shared prefix + every question's own tail) exceed the whole KV pool is
+  rejected with 422 "maximum context length"; none of the DI 0.3 public requests does with the pools above.
 * Request body: at most 64 MB.
 
 ## License

engine/crates/runtime/src/engine.rs

@@ -883,6 +883,11 @@ impl Sched {
         let start_pages: Vec<usize> = (0..n).map(|b| sp_pages.max(matched[b].len())).collect();
         let need: usize = need_shared + (0..n).map(|b| total_pages[b] - start_pages[b]).sum::<usize>();
         let _ = r;
+        // a request that needs more pages than the WHOLE pool can never be admitted (not a transient shortage): fail it as a capacity limit
+        if need > self.pool.rc.len() {
+            rollback(self, &matched, &nodes);
+            bail!("KV pool exhausted: KV cache capacity exceeded ({} pages needed, {} pages total)", need, self.pool.rc.len());
+        }
         // 3. allocate
         let alloc = match self.alloc(need) {
             Ok(a) => a,
@@ -899,6 +904,14 @@ impl Sched {
             let sw = self.swa.as_ref().unwrap();
             let bound = sw.wpages() + 2 * SWA_SLACK_PAGES + self.model.max_rows() / PAGE + 4;
             let p_req = need_shared.min(bound) + (0..n).map(|b| (total_pages[b] - start_pages[b]).min(bound)).sum::<usize>();
+            if p_req > sw.n {
+                for &p in &alloc {
+                    self.pool.decref(p);
+                }
+                rollback(self, &matched, &nodes);
+                let total = self.swa.as_ref().unwrap().n;
+                bail!("sliding-window KV pool exhausted: KV cache capacity exceeded ({} pages needed, {} pages total)", p_req, total);
+            }
             if !self.sl_make_free(p_req) {
                 for &p in &alloc {
                     self.pool.decref(p);

engine/crates/runtime/src/tests/engine.rs

@@ -95,6 +95,8 @@ fn kv_exhaustion_is_an_error_not_a_panic_and_recovers() {
     let big = text_item("big", &toks(1, 400), &toks(2, 10), 2);
     let err = e.run(&[big], &[], &think_pol()).unwrap_err();
     assert!(format!("{err:#}").contains("KV pool exhausted"), "{err:#}");
+    // larger than the whole pool: reported as a capacity limit (server: 422 "maximum context length"), not a transient shortage
+    assert!(format!("{err:#}").contains("KV cache capacity exceeded"), "{err:#}");
     let small = text_item("s", &toks(1, 20), &toks(2, 5), 2);
     assert!(e.run(&[small], &[], &Policy { tau: 0.0, budget_ms: 0, ..Policy::default() }).is_ok());
     let st = e.stats();
@@ -113,6 +115,8 @@ fn decode_exhaustion_midway_fails_only_that_request() {
         match e.run(&[it], &[], &pol) {
             Err(err) => {
                 assert!(format!("{err:#}").contains("KV pool exhausted"), "{err:#}");
+                // the prompt fits the pool; running out while thinking is not the admission capacity error
+                assert!(!format!("{err:#}").contains("KV cache capacity exceeded"), "{err:#}");
                 found = true;
                 break;
             }

engine/crates/server/src/lib.rs

@@ -239,14 +239,7 @@ async fn systemone(State(st): State<Arc<AppState>>, headers: HeaderMap, body: ax
         Ok(Ok(_)) => return err(ApiError::new(500, "inference failed: result count mismatch")),
         Ok(Err(e)) => {
             let m = format!("{e:#}");
-            return err(if m.contains("cannot decode image") {
-                ApiError::new(422, "cannot decode image as a base64 data URI")
-            } else if m.contains("KV pool exhausted") {
-                // transient overload (concurrent requests hold the KV cache): TypeSafe's 529, clients retry after a short delay
-                ApiError::new(529, format!("overloaded, retry after a short delay ({m})"))
-            } else {
-                ApiError::new(500, format!("inference failed: {m}"))
-            });
+            return err(backend_error(&m));
         }
         Err(e) => return err(ApiError::new(500, format!("inference failed: {e}"))),
     };
@@ -388,6 +381,9 @@ pub struct Args {
     /// hard row limit of one forward (workspace, ~0.45 MB per row); no-think prefill packs fill it
     #[arg(long, default_value_t = 8192)]
     pub max_rows: usize,
+    /// KV pages (16 tokens) of the full-attention layers; default max(2500, (max_len + cap) / 16 + 1024)
+    #[arg(long)]
+    pub pages: Option<usize>,
     /// KV memory budget (GB): the full-attention pool holds one max-length request plus 16K tokens, the sliding-window pool gets the rest
     #[arg(long, default_value_t = 9.0)]
     pub kv_gb: f64,
@@ -446,7 +442,7 @@ pub fn start_engine(a: &Args) -> anyhow::Result<EngineBackend> {
     // KV pools: bytes per 16-token page of the full-attention / sliding-window layers
     let pb = |k: LayerKind| -> usize { cfg.text.layers.iter().filter(|s| s.kind == k).map(|s| 2 * s.n_kv_heads * 16 * s.head_dim * 2).sum() };
     let (full_pb, sl_pb) = (pb(LayerKind::Full), pb(LayerKind::Sliding));
-    let num_pages = 2500usize.max((a.max_len + a.cap + 15) / 16 + 1024);
+    let num_pages = a.pages.unwrap_or_else(|| 2500usize.max((a.max_len + a.cap + 15) / 16 + 1024));
     let sliding_pages = ((a.kv_gb * 1e9) as usize).saturating_sub(num_pages * full_pb) / sl_pb.max(1);
     anyhow::ensure!(sliding_pages >= 256, "sliding-window pool too small ({sliding_pages} pages): raise --kv-gb");
     eprintln!("KV pools: {} full-attention pages ({:.2} GB, {} tokens), {} sliding-window pages ({:.2} GB)", num_pages, (num_pages * full_pb) as f64 / 1e9, num_pages * 16, sliding_pages, ((sliding_pages + 1) * sl_pb) as f64 / 1e9);
@@ -526,6 +522,19 @@ mod tests {
     use axum::body::Body;
     use tower::ServiceExt;
 
+    #[test]
+    fn kv_capacity_is_422_with_di_marker_and_transient_shortage_is_529() {
+        let cap = backend_error("KV pool exhausted: KV cache capacity exceeded (3652 pages needed, 2500 pages total)");
+        assert_eq!(cap.code, 422);
+        assert!(cap.msg.contains("maximum context length"), "{}", cap.msg);
+        let sw = backend_error("sliding-window KV pool exhausted: KV cache capacity exceeded (3000 pages needed, 2496 pages total)");
+        assert_eq!(sw.code, 422);
+        let tr = backend_error("KV pool exhausted (2500 pages total, 300 needed, 100 free)");
+        assert_eq!(tr.code, 529);
+        assert!(!tr.msg.contains("maximum context length"));
+        assert_eq!(backend_error("boom").code, 500);
+    }
+
     /// uniform distributions, never thinks
     struct UniformBackend;
     impl Backend for UniformBackend {
@@ -806,3 +815,18 @@ mod tests {
         assert_eq!(post_json(app, b, None).await.1["model"], "span-02");
     }
 }
+
+/// HTTP error for a failed backend run. A request larger than the whole KV pool can never fit: 422 with the DI capacity marker
+/// ("maximum context length"), so clients do not retry it. A transient shortage (concurrent requests hold the KV cache) stays 529.
+pub fn backend_error(m: &str) -> ApiError {
+    if m.contains("cannot decode image") {
+        ApiError::new(422, "cannot decode image as a base64 data URI")
+    } else if m.contains("KV cache capacity exceeded") {
+        ApiError::new(422, format!("This model's maximum context length is exceeded: the request does not fit in the server's KV cache ({m})"))
+    } else if m.contains("KV pool exhausted") {
+        // transient overload (concurrent requests hold the KV cache): TypeSafe's 529, clients retry after a short delay
+        ApiError::new(529, format!("overloaded, retry after a short delay ({m})"))
+    } else {
+        ApiError::new(500, format!("inference failed: {m}"))
+    }
+}

launch.sh

@@ -44,6 +44,9 @@ dl "$DRAFT" "$D/draft"
 #  --spec-draft           MTP speculative decoding of the thought (lossless: token-identical to greedy)
 #  --image-soft-tokens 560  Gemma-4 image budget (max_soft_tokens 560, about 1.3 MP)
 #  --serial --no-prefix-cache --spec-max-branches 16   determinism: answers do not depend on load or earlier requests
+#  --pages 12000 --kv-gb 24   KV pools (full attention 192K tokens, 3.9 GB; sliding window ~20 GB from --kv-gb 24) sized so every DI request
+#                         fits on its own under --serial (largest DI 0.3 request: ~3.7K full pages); the pool size does not change answers
 exec "$BIN" --bind "${HOST:-0.0.0.0}:${PORT:-8000}" --model "$M" --spec-draft "$D/draft" \
   --max-len 16384 --budget-ms 0 --tau 0.8 --cap 1024 --think-ms 750 --think-cost "$M/think_cost.json" \
-  --image-soft-tokens 560 --serial --no-prefix-cache --spec-max-branches 16 "$@"
+  --image-soft-tokens 560 --serial --no-prefix-cache --spec-max-branches 16 \
+  --pages 12000 --kv-gb 24 "$@"