wald-serve 0.2.2: native video input (videos / video_url; video_route native32 default, frames16); decode once per request; card, API, runbook, evaluation/v2.1/video.json; MANIFEST updated. Weights unchanged.

Dockerfile

@@ -12,7 +12,7 @@ RUN pip install --no-cache-dir uv==0.9.5 \
  && uv pip install --system --no-cache "vllm==0.30.0" "transformers==5.17.0" \
  && python -c "import importlib.metadata as m; print('vllm', m.version('vllm'), 'transformers', m.version('transformers'), 'torch', m.version('torch'))"
 COPY server /app/server
-RUN uv pip install --system --no-cache /app/server
+RUN uv pip install --system --no-cache "/app/server[video]"
 ENV HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 TOKENIZERS_PARALLELISM=false PYTHONUNBUFFERED=1 \
     VLLM_USE_FLASHINFER_SAMPLER=0 VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 WALD_EFFORT=""
 EXPOSE 8000

MANIFEST.json

@@ -8,8 +8,8 @@
     "sha256": "1c69371d18ee073bf854aff74b2c3b3403d589fb17d36b43f5945f764016cb55"
   },
   "Dockerfile": {
-    "bytes": 1212,
-    "sha256": "29ae11deccbeba7af543614f3c2a76711377fffd9727d7c269af050b781f8b0e"
+    "bytes": 1221,
+    "sha256": "4b421737cf3d0897256fb2e06252feeb1706a2eed863ff8ccbca6b7d2b047791"
   },
   "LICENSE": {
     "bytes": 11358,
@@ -24,12 +24,12 @@
     "sha256": "43510ca439ff9f402eeef7c7e0e4a8e3e2265cf0ff8b3e027d662b1bff79078e"
   },
   "README.md": {
-    "bytes": 6044,
-    "sha256": "645bc8cb22d9c89dfa4f81ede4a89f1fb58b572a73987fd02bcdd223aee4c608"
+    "bytes": 7197,
+    "sha256": "59d50796d33a8a8588ae4dcf8c0180aaa2b185b46d1081cba587604f98edf26e"
   },
   "RUNBOOK.md": {
-    "bytes": 6122,
-    "sha256": "73389fe1429146f0ae3a77079b96ff8ee7f728c3a3e215604a9d85fa84303056"
+    "bytes": 7098,
+    "sha256": "03c171b0836a2dd36aceedc97cf18e0a9e8307d1c81676d445358a388aaa4853"
   },
   "SHA256SUMS": {
     "sha256": "f5de06dae0fbbeb38f196e95ae13c46123cb466da84d2431261e9d4e37b22ccf",
@@ -48,8 +48,8 @@
     "sha256": "31f5a5ee9860fb747c0c7e23a9ac75790dcd688de1655c6890eef5ba2ed9b9c3"
   },
   "docs/api.md": {
-    "bytes": 4403,
-    "sha256": "0c05b5a5b9a2ec982416137e209b615e51b64cfb012f53a02b279257f258d2bd"
+    "bytes": 7193,
+    "sha256": "d60958611887e4072b26cba32ea861a3a2a0e686089d27c800e276f156de922a"
   },
   "evaluation/v2.1/README.md": {
     "sha256": "28f8bc4059cc9e0966c0ade9e909bd2a87ab0dd85a9da45f4487d7ef52ab84f2",
@@ -67,6 +67,10 @@
     "sha256": "838516b0ce8ddb7c50688c79bdb63437e06bb12db1d4ec3d6cc41c8a95a7e675",
     "bytes": 1052
   },
+  "evaluation/v2.1/video.json": {
+    "bytes": 2719,
+    "sha256": "ed52612048786a582a1d13086444bd11b7dd146e99e6b04ad5cda203361f7f7a"
+  },
   "evaluation/v2/README.md": {
     "bytes": 2711,
     "sha256": "6792af269e6383ececdc997b011cf1a09b5ab2a79d356f04932fc0f22e6789a7"
@@ -272,20 +276,24 @@
     "sha256": "c8b231a1cdf413895c1abb215e43a41a08f6568d8404ea4f7ca86ee4883eb396"
   },
   "run.sh": {
-    "bytes": 987,
-    "sha256": "fa224f8ec1dfeb6598231bd144d7a6928a9a01e63ac70b5cf5df28d6a9aca9c9"
+    "bytes": 994,
+    "sha256": "2159f4e041e9b7d0b9e0de382736397d176e5a5a955547bc84f4b1509545a62a"
+  },
+  "server/CHANGELOG.md": {
+    "bytes": 1321,
+    "sha256": "9f2fba41f7da1c13e290446c6b52604ba9a5ff9b7e63bae660cd70e2b90b539a"
   },
   "server/README.md": {
-    "bytes": 1678,
-    "sha256": "2ca738dcacd770e7c490ceeb59f2f11df7515f5bc56784eb9d476af4c7145b4d"
+    "bytes": 5479,
+    "sha256": "49f21b5b2be45358876666d423d744bcde2bac91696efa7609c8a18ebfbcc08d"
   },
   "server/pyproject.toml": {
-    "bytes": 850,
-    "sha256": "f1357ddf2448b827ad6bb4ee3ba63266196dc89df26d7c18cdf60da6150f11b5"
+    "bytes": 905,
+    "sha256": "f429f914d3857c4adc814343dc48d6d3289afd209c923278f81c3654a770ad39"
   },
   "server/src/wald_serve/__init__.py": {
     "bytes": 200,
-    "sha256": "86f26622ee5bad7e12ac7cdcc820935e4b9bf3e4a414cd2dc47e1bdf9b8608d3"
+    "sha256": "d88ae786118b8186ad5d133006a397556ed6fd473dfcdb55de8d61c0926b7f6e"
   },
   "server/src/wald_serve/__main__.py": {
     "bytes": 51,
@@ -296,8 +304,8 @@
     "sha256": "40fbea84c4c5b93ef888da44508d4fec2677c58732a09d731f0bcf17c2b48dd6"
   },
   "server/src/wald_serve/native.py": {
-    "bytes": 15937,
-    "sha256": "135b5bbb135480ef15bac81a380ea909bdd78a0fc67263192001fc0ebe353fee"
+    "bytes": 17130,
+    "sha256": "3a21ec14e541caa74fcc5b034d905156712f534d2fecc55b41b0e5331d0d4c1f"
   },
   "server/src/wald_serve/native_llamacpp.py": {
     "bytes": 3827,
@@ -340,8 +348,8 @@
     "sha256": "c8b231a1cdf413895c1abb215e43a41a08f6568d8404ea4f7ca86ee4883eb396"
   },
   "server/src/wald_serve/native_vision.py": {
-    "bytes": 8479,
-    "sha256": "1b079295aac76a8c8bf22d7794b245cf413930838855648f3cd7fffe4a978cf3"
+    "bytes": 12010,
+    "sha256": "423f515f449498ffce31dd16b810fa31f55ac6ea4cb181182ebcd9a4634a0f2a"
   },
   "server/src/wald_serve/prompt.py": {
     "bytes": 4192,
@@ -351,9 +359,13 @@
     "bytes": 10020,
     "sha256": "b236e0ec55aadf575198578285ef4a802f09f51d9b78f35e0da41d60a12755c2"
   },
+  "server/src/wald_serve/video.py": {
+    "bytes": 24537,
+    "sha256": "89e79d8d70b8a72e9e78e5f7cb69cbb316104aa6d024f5a249b8925b6ab37d73"
+  },
   "server/src/wald_serve/vision_v2.py": {
-    "bytes": 10629,
-    "sha256": "ab7669e598e723a9a65d63356615b04c658f1fb02f1b7494bf40e69fd5627c7f"
+    "bytes": 14004,
+    "sha256": "db56cfa76fe5d0cb8f76f05cf66be7e825240fa9279355738dc694ac5b7f11c1"
   },
   "server/src/wald_serve/wire.py": {
     "bytes": 3670,
@@ -363,6 +375,18 @@
     "bytes": 4167,
     "sha256": "9fb222a21bc2b17221e5a1a3605973b4f71730db9bbfede439bd3388b91e2570"
   },
+  "server/tests/fixtures/default-responses-0.2.1.json": {
+    "bytes": 43039,
+    "sha256": "ce01380ec65df43fc901ecca01154613ed3467916d5467033a08fcbf2ccb5e37"
+  },
+  "server/tests/record_default_fixtures.py": {
+    "bytes": 3929,
+    "sha256": "d290a57568dca62ca005a6dbd2993c652fcfe4b3c3676fc7e95176e0a5ba98ef"
+  },
+  "server/tests/stubengine.py": {
+    "bytes": 3597,
+    "sha256": "a15d8e3b57bc730222e2690e57febf9732132a29b9c8d5c4ecae319e05fc0423"
+  },
   "server/tests/test_native_v2.py": {
     "bytes": 7236,
     "sha256": "ed5310bdcad9da34313719d83d83d6174329a115531fad7b98b1a3437d685eec"
@@ -379,6 +403,10 @@
     "bytes": 13844,
     "sha256": "46f2007e9743bfeb0f7f79fca093cf444a70df1bfc348229f34cd251f88a19b3"
   },
+  "server/tests/test_video.py": {
+    "bytes": 20812,
+    "sha256": "f16b9522584feecb1939fffe7a467984c5e307e5eaeb72d46377c8ca624ef4d6"
+  },
   "server/tests/test_vision_v2.py": {
     "bytes": 5926,
     "sha256": "7793a78d851f217c2af32bf963217bf431193e0f6bbf290e2bfd50107a03da88"

README.md

@@ -54,7 +54,7 @@ An open 4B decision model: send a state and typed options, get a calibrated prob
 - **Thinks only when unsure:** Auto 0.7 thinks natively on about half of the requests.
 - **Fast:** median **158 ms** (serial, one request in flight, warm, processing time, one RTX PRO 6000, 200 DI 0.3 requests).
 - **Web actions:** Mind2Web element selection **92.3 %** (test splits, 6 candidates; v2.0: 84.6 %).
-- **Reads images** zero-shot through Qwen3.5-4B's unchanged vision tower.
+- **Reads images and video** zero-shot through Qwen3.5-4B's unchanged vision tower. Video (wald-serve 0.2.2): **87.4 %** on VisualDecisionBench video (7,143 questions, our run), 2.5 s for a 15-second clip with 24 questions (serial, one RTX 5090).
 - **Small and open:** 4B, Apache-2.0.
 
 ## Scores
@@ -86,8 +86,14 @@ curl -s localhost:8000/v1/systemone -H 'Content-Type: application/json' -d '{
 curl -s localhost:8000/v1/systemone -H 'Content-Type: application/json' -d '{
   "state": "Photo from the returns desk.", "images": ["data:image/jpeg;base64,..."],
   "questions": {"damaged": {"type": "noul", "instructions": "Is the item visibly damaged?"}}}'
+
+curl -s localhost:8000/v1/systemone -H 'Content-Type: application/json' -d '{
+  "state": "Aisle camera, last 15 seconds.", "videos": ["data:video/mp4;base64,..."],
+  "questions": {"conceal": {"type": "noul", "instructions": "Does the shopper put an item into a bag or pocket?"}}}'
 ```
 
+Video needs the server from `main` (wald-serve 0.2.2; the weights are the same as tag `v2.1`): one video per request, up to 32 MB and 300 s. By default the server sends 32 frames to vLLM as one native video (`"video_route": "native32"`). `"video_route": "frames16"` sends 16 labelled frames as images instead, and is used automatically when the request also has images. Details: [docs/api.md](https://huggingface.co/org2ai/Wald-4B/blob/main/docs/api.md#video), numbers: [evaluation/v2.1/video.json](https://huggingface.co/org2ai/Wald-4B/blob/main/evaluation/v2.1/video.json).
+
 API fields: [docs/api.md](https://huggingface.co/org2ai/Wald-4B/blob/main/docs/api.md). Server options, Docker and evaluation settings: [RUNBOOK.md](https://huggingface.co/org2ai/Wald-4B/blob/main/RUNBOOK.md).
 
 ## How it works
@@ -95,7 +101,7 @@ API fields: [docs/api.md](https://huggingface.co/org2ai/Wald-4B/blob/main/docs/a
 - One forward pass reads a probability for every option from the model's option-letter logits (Qwen chat template), calibrated with a frozen temperature table.
 - Auto 0.7 (default): if the top probability is below 0.7, the model thinks once natively (at most 512 tokens) and reads the options again.
 - More than 26 options: a one-pass knockout over all options.
-- Images go through Qwen3.5-4B's vision tower, unchanged (zero-shot).
+- Images go through Qwen3.5-4B's vision tower, unchanged (zero-shot). The server decodes a video once per request; by default 32 frames go to the tower as one Qwen3.5 video, with timestamps.
 
 ## Details
 

RUNBOOK.md

@@ -34,6 +34,10 @@ Auto 0.7 gates on the untempered one-pass distribution: a question with 2–26 o
 
 Requests with images (message content parts `image_url` / `image` in `state`, raw base64 under `state` keys `image`, `image_data`, `image_base64`, `image_b64`, or a top-level `images` / `image_data` list; up to 16 per request, bodies up to 64 MB) go through the same frozen reader, with the images carried to vLLM's chat route at their position in the state. They default to the `plain` prompt format. Text-only requests take the text path unchanged. The Decision Index 0.3 run used the text-only weights file; the language-model tensors here are byte-identical, and the text parity of this repository's weights is in `evaluation/v2.1/serving-parity.json`.
 
+## 3b. Video
+
+wald-serve 0.2.2 (on `main`) takes one video per request (`videos`, or a `video_url` message part) and decodes it once per request with OpenCV (`pip install "./server[video]"`; `run.sh` and the Dockerfile install it). `video_route` `native32` (default) gives vLLM 32 frames as one Qwen3.5 video with the source timestamps; `frames16` sends 16 labelled frames through the image route, which is the client protocol of the VisualDecisionBench-video frames read. `wald-serve-native-vision --model` raises vLLM's `--limit-mm-per-prompt` to `{"image": 32, "video": 2}` for this. The limit does not change image or text reads: the 16 recorded 0.2.1 responses are reproduced byte for byte. `--no-video` keeps the evaluated `{"image": 16, "video": 0}` and refuses videos. Server flags: `--video-route`, `--video-dir`, `--video-urls off|public|any`, `--video-cache-mb`, `--max-video-mb`, `--max-video-seconds`. Accuracy, parity and latency: `evaluation/v2.1/video.json`.
+
 ## 4. GGUF (llama.cpp)
 
 `org2ai/Wald-4B-GGUF` holds the v2 quantisations and the vision projector (`Wald-4B-v2-mmproj-F16.gguf`). `wald-serve-native --gguf FILE --tokenizer-dir DIR --effort none` reads text requests through llama-server with the same chat layout; a letter outside llama-server's top-100 log-probabilities gets the reader's floor instead of its exact value. One pass is the checked mode.

docs/api.md

@@ -19,9 +19,11 @@
 | `effort` | string, optional | `none` (one pass), `auto` or `medium` (Auto 0.7, the default), `always` or `high` (Always 512). Other values (`low`, `high-k2` …) are v1.x efforts and return HTTP 400. |
 | `prompt_format` | string, optional | `repeat_state_plain` (default: the state is written twice) or `plain`. |
 | `images` | array of data URIs, optional | Images for the state (also accepted inside `state` as message content parts `{"type": "image_url", "image_url": {"url": "data:..."}}`); up to 16; zero-shot. |
+| `videos` | array with one video, optional | A video for the state (wald-serve 0.2.2): `data:video/mp4;base64,...` (also MOV, WebM, MKV, AVI, MPEG), raw base64, an http(s) URL, or a server-side path when the server allows it. Also accepted inside `state` as a message part `{"type": "video_url", "video_url": {"url": ...}}`. See [Video](#video). |
+| `video_route` | string, optional | `native32` (server default) or `frames16`. |
 | `model` | string, optional | Ignored; accepted for client compatibility. |
 
-Only `state`, `questions` and the images reach the model. Each question has a `type`, optional `instructions` and `criteria`:
+Only `state`, `questions`, the images and the video reach the model. Each question has a `type`, optional `instructions` and `criteria`:
 
 | `type` | `criteria` | Answer fields |
 |---|---|---|
@@ -31,6 +33,8 @@ Only `state`, `questions` and the images reach the model. Each question has a `t
 
 Every answer also carries `type`, `mode` (`A` one pass, `B` after a native thought, `K`/`T` grouped readout for more than 26 options) and `probabilities_t1` (the same distribution before the calibration table). `confidence` is the largest calibrated probability. The response has `model`, `answers` and `usage` (`input_tokens`, `output_tokens`; output tokens are thought tokens). The thought text itself is not returned.
 
+Requests with a video also carry `videos` (1), `video_route` (the route used) and `video_frames` (frames read); `images` then counts only the caller's own images.
+
 Prompts longer than the context limit (131,072 tokens) are rejected with HTTP 422, never truncated. A malformed request or an unknown effort / prompt format returns HTTP 400; a backend failure returns 502.
 
 ## Example: route a tool call
@@ -62,3 +66,32 @@ Response shape (values illustrative):
 ```
 
 Act on a decision when its probability clears a threshold you set on your own validation data; otherwise escalate or ask. The model picks options; it does not write tool arguments.
+
+## Video
+
+wald-serve 0.2.2 reads one video per request; the server decodes it once and every question of the request reuses it.
+
+```sh
+curl http://localhost:8000/v1/systemone -H 'Content-Type: application/json' -d '{
+  "state": "Aisle camera, last 15 seconds.",
+  "videos": ["data:video/mp4;base64,AAAAIGZ0eXBpc29t..."],
+  "questions": {
+    "conceal": {"type": "noul", "instructions": "Does the shopper put an item into a bag or pocket?"},
+    "next": {"type": "choice", "instructions": "What should the store system do?",
+             "criteria": {"ignore": "Nothing unusual", "watch": "Keep watching", "alert": "Alert staff"}}
+  }
+}'
+```
+
+| `video_route` | What the model sees | VisualDecisionBench video (our run) |
+|---|---|---:|
+| `native32` (default) | 32 frames spread evenly over the clip, as one Qwen3.5 video with `<t seconds>` timestamps, ≤ 131,072 px per frame | 87.36 % |
+| `frames16` | 16 frames spread evenly, as 16 images (≤ 131,072 px), each after the text `Frame k/16 at t = S s:` | 82.63 % |
+
+- Images and a video in one request use `frames16` (the response says `"video_route_fallback": "images in the request"`); asking for `native32` together with images returns 400.
+- Limits: one video per request; ≤ 32 MB; ≤ 300 s; source frames ≤ 3840 × 2160; request body ≤ 64 MB. A longer clip is thinned, not refused: 32 frames over 300 s is one frame every ~9.7 s, so for event timing send the window you care about.
+- Sources: data URI or raw base64 always. http(s) URLs are fetched from public addresses only (server flag `--video-urls off|public|any`). Paths and `file://` URLs only under the server's `--video-dir`.
+- Errors are HTTP 400 with a reason first in `error`: `video_format`, `video_too_large`, `video_too_long`, `video_pixels`, `video_malformed`, `video_count`, `video_source`, `video_route`, `video_unavailable`.
+- Calibration is the image route's: the same reader, prompt and frozen temperature table.
+- Latency (serial, one RTX 5090, 15-second 640×360 clips with 24 questions): `native32` 2.5 s per clip (1.85 s when the same clip is asked again), `frames16` 3.4 s. The first video after the server starts takes about 15 s more, once.
+

evaluation/v2.1/video.json

@@ -0,0 +1,111 @@
+{
+ "what": "Wald-Q4B v2.1 (056A0-c31) on VisualDecisionBench video (Valen-Team/VisualDecisionBench @ 0c9e9703, test, hard labels), zero-shot, effort none, noul decisive floor off, one RTX 5090, vLLM 0.30.0. Self-run; not a leaderboard result.",
+ "full_set_2026_10_12": {
+  "videos": 294,
+  "questions": 7143,
+  "client": "frames decoded by the client (frames16) or the mp4 handed to vLLM (native32)",
+  "frames16": {
+   "accuracy": 82.63,
+   "nll": 0.524
+  },
+  "native32": {
+   "accuracy": 87.36,
+   "nll": 0.489
+  },
+  "native32_minus_frames16": {
+   "accuracy_pp": 4.73,
+   "ci95": [
+    3.86,
+    5.6
+   ],
+   "bootstrap": "paired, clustered by video, 2,000 draws"
+  }
+ },
+ "served_wald_serve_0_2_2": {
+  "videos": 74,
+  "questions": 1775,
+  "subset": "every 4th video",
+  "client": "the mp4 as a data URI; the server decodes it",
+  "frames16": {
+   "accuracy": 84.34,
+   "choice": 88.08,
+   "noul": 86.81,
+   "score": 70.28,
+   "nll": 0.4992
+  },
+  "native32_default": {
+   "accuracy": 87.94,
+   "choice": 91.4,
+   "noul": 88.46,
+   "score": 75.83,
+   "nll": 0.4847
+  },
+  "native32_minus_frames16": {
+   "accuracy_pp": 3.61,
+   "ci95": [
+    1.94,
+    5.35
+   ]
+  },
+  "parity_with_the_full_set_reads_on_the_same_items": {
+   "frames16": {
+    "same_argmax_pct": 99.55,
+    "accuracy_delta_pp": 0.17,
+    "ci95": [
+     -0.11,
+     0.45
+    ]
+   },
+   "native32": {
+    "same_argmax_pct": 98.54,
+    "accuracy_delta_pp": -0.11,
+    "ci95": [
+     -0.7,
+     0.45
+    ]
+   }
+  }
+ },
+ "serial_latency_c1": {
+  "videos": 15,
+  "clips": "median 15 s, 640x360",
+  "note": "POST to answer for every question of the clip in one request (median 24 questions), including the server's decode; cold = first request on that video",
+  "frames16_cold": {
+   "median_s_per_video": 3.39,
+   "median_ms_per_question": 140.6
+  },
+  "native32_cold": {
+   "median_s_per_video": 2.5,
+   "median_ms_per_question": 102.4
+  },
+  "native32_repeat_same_video": {
+   "median_s_per_video": 1.85,
+   "median_ms_per_question": 75.5
+  },
+  "native32_mp4_resent_per_question_before_0_2_2": {
+   "median_s_per_video": 8.18,
+   "median_ms_per_question": 333.3
+  },
+  "server_decode_median_ms": {
+   "frames16": 580.3,
+   "native32": 270.1
+  },
+  "first_native_video_after_start": "about 15 s once (vLLM warm-up for the video shape)"
+ },
+ "checks": {
+  "clip_97_s": {
+   "frames16": 200,
+   "native32": 200
+  },
+  "clip_487_s": "400 video_too_long",
+  "image_plus_video": "200, frames16 (1 image + 16 frames)",
+  "image_plus_video_native32": "400 video_route",
+  "two_videos": "400 video_count"
+ },
+ "not_measured": [
+  "effort auto on video",
+  "the noul decisive floor on video",
+  "throughput under concurrent load",
+  "sources above 1080p"
+ ]
+}

run.sh

@@ -8,6 +8,6 @@ set -euo pipefail
 MODEL=${1:?usage: run.sh <weights dir>}
 HERE=$(cd "$(dirname "$0")" && pwd)
 VENV=${VENV:-$HERE/.venv}
-[ -x "$VENV/bin/python" ] || { uv venv -p 3.12 "$VENV" && VIRTUAL_ENV="$VENV" uv pip install "vllm==0.30.0" "transformers==5.17.0" "$HERE/server"; }
+[ -x "$VENV/bin/python" ] || { uv venv -p 3.12 "$VENV" && VIRTUAL_ENV="$VENV" uv pip install "vllm==0.30.0" "transformers==5.17.0" "$HERE/server[video]"; }
 export VLLM_USE_FLASHINFER_SAMPLER=0 VLLM_NO_USAGE_STATS=1 TOKENIZERS_PARALLELISM=false HF_HUB_OFFLINE=1
 exec "$VENV/bin/wald-serve-native-vision" --model "$MODEL" --port "${PORT:-8000}" ${EFFORT:+--effort "$EFFORT"} ${PROMPT_FORMAT:+--prompt-format "$PROMPT_FORMAT"}

server/CHANGELOG.md

@@ -0,0 +1,21 @@
+# Changelog
+
+## 0.2.2 (2026-10-13)
+
+- Video input on `wald-serve-native-vision` (`video.py`): `videos` (d3 shape) or `video_url` / `video` message parts;
+  data URI, raw base64, http(s) URL (`--video-urls off|public|any`, default public) or a path under `--video-dir`.
+- `video_route`: `native32` (default; 32 frames as one Qwen3.5 video with frame metadata) or `frames16` (16 labelled
+  frames down the image route, the evaluated client protocol). One decode per request, LRU across requests
+  (`--video-cache-mb`), and for `native32` one vLLM video processing per request (multimodal uuid; frames re-sent on a
+  vLLM cache miss).
+- Limits: one video per request, 32 MB, 300 s, 3840 × 2160 source frames; 400 errors with a reason prefix. Images and
+  video together use `frames16`.
+- `--model` launches vLLM with `--limit-mm-per-prompt {"image": 32, "video": 2}` (`--no-video` keeps serving.json's).
+- `--vllm` mode: the temperature table's hash pin comes from the table's directory (serving.json, else SHA256SUMS,
+  else the frozen v2.1 hash); the check itself is unchanged.
+- Requests without video are answered byte-identically to 0.2.1 (16 recorded 0.2.1 responses replayed in
+  `tests/test_video.py`). New extra `video` (OpenCV, numpy).
+
+## 0.2.1
+
+- Noul decisive floor (`noul_decisive_floor`, serving.json 0.81).

server/README.md

@@ -26,3 +26,50 @@ the same floor as in vLLM. llama.cpp's context is `--max-model-len` × `--llama-
 Tests (no model, no GPU; a fake server stands in for vLLM and llama-server): `pip install ".[test]" && cd tests && pytest -q`.
 `tests/test_parity.py` compares prompts and answers byte for byte with the reference implementation the published
 numbers were measured with; it is skipped unless that implementation is importable (WALD_PARITY_SRC set).
+
+## Video input (0.2.2, `wald-serve-native-vision`)
+
+Install with the decoder: `pip install ".[vllm,video]"` (OpenCV; vLLM already depends on it). A request may carry **one
+video**, either as a top-level `videos` list (the Decision 3.0 / d3 request shape) or as an OpenAI-style message part:
+
+```sh
+curl -s localhost:8000/v1/systemone -H 'Content-Type: application/json' -d '{
+  "state": "Aisle camera, last 15 seconds.",
+  "videos": ["data:video/mp4;base64,AAAAIGZ0eXBpc29t..."],
+  "questions": {"conceal": {"type": "noul", "instructions": "Does the shopper put an item into a bag or pocket?"}}}'
+
+# the same video inside the state, at its position among the text parts
+{"state": {"messages": [{"role": "user", "content": [
+   {"type": "text", "text": "Aisle camera."},
+   {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,..."}}]}]},
+ "video_route": "frames16", "questions": {...}}
+```
+
+- **Sources:** a `data:video/{mp4,quicktime,webm,x-matroska,x-msvideo,mpeg,x-m4v};base64,` URI, raw base64, an http(s)
+  URL (`--video-urls public`, the default, fetches only from public addresses; `off` refuses URLs; `any` allows every
+  host), or a local path / `file://` URL inside `--video-dir DIR` (refused without that flag).
+- **`video_route`** (per request; server default `--video-route`, default `native32`):
+  - `native32`: 32 frames spread evenly over the whole clip, given to vLLM as one Qwen3.5 video with its timestamps
+    (`<t seconds>` per frame pair), 131,072 px per frame.
+  - `frames16`: 16 frames spread evenly, each ≤ 131,072 px, sent as 16 images, each after a text label
+    `Frame k/16 at t = S s:`. This is exactly the client protocol of the published VisualDecisionBench-video read.
+- **Decode once:** the server decodes the video once per request and every question reuses the frames; with `native32`
+  vLLM also processes the video once (later questions reference it by a content uuid). An LRU of decoded videos
+  (`--video-cache-mb`, default 512, keyed by the sha256 of the bytes) serves repeat requests on the same video.
+- **Images and video together:** `frames16` only. A request with images and no `video_route` uses `frames16` and says
+  so (`"video_route_fallback": "images in the request"`); `"video_route": "native32"` with images is refused (400),
+  because vLLM's per-request pixel budget applies to images as well.
+- **Limits (400 with a reason prefix):** one video per request (`video_count`); ≤ 32 MB (`--max-video-mb`,
+  `video_too_large`); ≤ 300 s (`--max-video-seconds`, `video_too_long`); ≤ 3840 × 2160 source frames (`video_pixels`);
+  MP4 / MOV, WebM / MKV, AVI or MPEG containers that OpenCV's FFmpeg backend opens (`video_format`, `video_malformed`);
+  request bodies ≤ 64 MB. Longer clips are thinned, not refused: 32 frames over 300 s is one frame every ~9.7 s. For
+  event timing in long streams, send the window you care about.
+- **Response:** `videos` (count), `video_route`, `video_frames` (frames used per video) and `images` (the caller's own
+  images; frames16 frames are not counted). `usage.input_tokens` counts text tokens only, as for images.
+- **vLLM:** `--model` starts vLLM with `--limit-mm-per-prompt {"image": 32, "video": 2}` (16 frames next to up to 16
+  images; a repeated state holds its video twice). `--no-video` keeps serving.json's limits and refuses videos. With
+  `--vllm URL`, start that vLLM with the same limit.
+- **Calibration:** unchanged. Video questions use the frozen image-route reader, prompt and temperature table.
+  `--vllm` mode now takes the table's hash pin from the table's own directory (sibling `serving.json`
+  `temperature_sha256`, else its `SHA256SUMS` line, else the frozen v2.1 hash), so an attached front-end can serve a
+  checkpoint with its own table. The hash check still runs.

server/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "wald-serve"
-version = "0.2.1"
+version = "0.2.2"
 description = "TypeSafe POST /v1/systemone server for Wald-4B: v2 native chat reader (Auto 0.7, frozen calibration) over vLLM; v1.x option-letter readout over vLLM or llama.cpp"
 readme = "README.md"
 license = { text = "Apache-2.0" }
@@ -9,6 +9,7 @@ dependencies = ["pydantic>=2.5"]
 
 [project.optional-dependencies]
 vllm = ["vllm==0.30.0", "transformers==5.17.0"]
+video = ["opencv-python-headless>=4.9", "numpy>=1.24"]
 test = ["pytest>=8"]
 
 [project.scripts]

server/src/wald_serve/__init__.py

@@ -1,2 +1,2 @@
 """Wald-4B serving: TypeSafe `POST /v1/systemone`. v2 (04400-c18): native chat reader (`wald-serve-native`); v1.x: paren letter readout over vLLM or llama.cpp (`wald-serve`)."""
-__version__ = "0.2.1"
+__version__ = "0.2.2"

server/src/wald_serve/native.py

@@ -200,6 +200,27 @@ def handler(engine):
     return H
 
 
+def table_pin(table, model_dir=None):
+    """The sha256 a temperature table must have. With --model, serving.json's `temperature_sha256` is used (the caller
+    passes it); without one (attached to a running vLLM, --vllm), the pin comes from the table's own directory: the
+    sibling serving.json's `temperature_sha256` when it names this file, else the file's line in the sibling SHA256SUMS,
+    else the frozen v2.1 table. The hash check itself always runs, so a table that differs from its manifest is refused."""
+    t = Path(table); d = t.parent
+    if model_dir is None:
+        s = d / 'serving.json'
+        if s.is_file():
+            cfg = json.loads(s.read_text())
+            if cfg.get('temperature_sha256') and cfg.get('temperature', 'temperature.json') == t.name:
+                return cfg['temperature_sha256']
+        m = d / 'SHA256SUMS'
+        if m.is_file():
+            for line in m.read_text().splitlines():
+                h, _, name = line.partition('  ')
+                if name.strip().lstrip('*') == t.name and len(h.strip()) == 64:
+                    return h.strip()
+    return TEMPERATURE_SHA
+
+
 def noul_floor(a, cfg):
     return a.noul_decisive_floor if a.noul_decisive_floor is not None else cfg.get('noul_decisive_floor')
 
@@ -272,7 +293,8 @@ def main(argv=None):
         from .native_llamacpp import LlamaBridge
         bridge = LlamaBridge(endpoint, a.tokenizer_dir)
     engine = NativeV2(endpoint, temps, served, effort, fmt, a.max_model_len, llama_bridge=bridge, checkpoint=checkpoint,
-                      model_sha256=cfg.get('model_sha256', MODEL_SHA), temperature_sha256=cfg.get('temperature_sha256', TEMPERATURE_SHA),
+                      model_sha256=cfg.get('model_sha256', MODEL_SHA),
+                      temperature_sha256=cfg.get('temperature_sha256') or table_pin(temps, a.model or a.gguf or a.tokenizer_dir),
                       noul_decisive_floor=noul_floor(a, cfg))
     ThreadingHTTPServer.daemon_threads = True
     httpd = ThreadingHTTPServer((a.host, a.port), handler(engine))

server/src/wald_serve/native_vision.py

@@ -10,7 +10,11 @@ carried to vLLM's chat route at their position in the state). Differences from n
     sidecar of this server, so no outside caller can send a template);
   * request bodies up to 64 MB (images are base64);
   * image requests default to `prompt_format` `plain` (serving.json `vision.image_prompt_format`), text requests keep the
-    server default.
+    server default;
+  * video (0.2.2, video.py): one video per request (`videos` or a `video_url` message part), decoded once by this server,
+    `video_route` frames16 (16 labelled frames down the image route) or native32 (one Qwen3.5 video of 32 frames). vLLM
+    is started with `--limit-mm-per-prompt {"image": 32, "video": 2}` so that 16 frames fit next to 16 images and a
+    repeated state can hold its video twice; `--no-video` keeps serving.json's limits and refuses videos.
 
     wald-serve-native-vision --model /path/to/Wald-4B-vision --port 8000
 """
@@ -24,23 +28,39 @@ import threading
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 
-from . import native, vision_v2
+from . import native, video, vision_v2
 
 VISION_VLLM_ARGS = ["--limit-mm-per-prompt", '{"image": 16, "video": 0}',
                     "--mm-processor-kwargs", '{"size": {"shortest_edge": 65536, "longest_edge": 1048576}}',
                     "--trust-request-chat-template"]   # the image route sends its verbatim template; vLLM listens on loopback only
 MAX_BODY = 64 * 1024 * 1024
+VIDEO_LIMITS = {"image": 32, "video": 2}
+
+
+def video_vllm_args(args):
+    """Raise `--limit-mm-per-prompt` to at least VIDEO_LIMITS (other arguments unchanged)."""
+    args = list(args)
+    if "--limit-mm-per-prompt" in args:
+        i = args.index("--limit-mm-per-prompt") + 1
+        lim = json.loads(args[i])
+        args[i] = json.dumps({**lim, **{k: max(int(lim.get(k, 0)), v) for k, v in VIDEO_LIMITS.items()}})
+    else:
+        args += ["--limit-mm-per-prompt", json.dumps(VIDEO_LIMITS)]
+    return args
 
 
 class NativeV2Vision(native.NativeV2):
-    def __init__(self, *args, image_prompt_format="plain", max_images=vision_v2.MAX_IMAGES, **kw):
+    def __init__(self, *args, image_prompt_format="plain", max_images=vision_v2.MAX_IMAGES, video_config=None, **kw):
         super().__init__(*args, **kw)
+        self.video = video_config if video_config is not None else video.VideoConfig()
+        self.video_cache = video.FrameCache(self.video.cache_mb) if self.video.cache_mb > 0 else None
         if image_prompt_format not in native.FORMATS:
             raise ValueError(f"image_prompt_format must be one of {sorted(native.FORMATS)}")
         self.image_prompt_format, self.max_images = image_prompt_format, max_images
         self.vidle = {f: [] for f in native.FORMATS}
         self.info.update(vision=True, image_prompt_format=image_prompt_format, max_images=max_images,
-                         image_transports=["state data URIs / messages image parts", "state raw base64 keys", "top-level images[]"])
+                         image_transports=["state data URIs / messages image parts", "state raw base64 keys", "top-level images[]"],
+                         video=self.video.info())
 
     def _vborrow(self, fmt):
         with self.lock:
@@ -51,7 +71,8 @@ class NativeV2Vision(native.NativeV2):
         return client, sov
 
     def __call__(self, request):
-        if not (isinstance(request, dict) and vision_v2.has_images(request)):
+        vids = video.has_videos(request)
+        if not (isinstance(request, dict) and (vision_v2.has_images(request) or vids)):
             return super().__call__(request)
         if not {"state", "questions"} <= request.keys():
             raise ValueError("Expected state and questions")
@@ -61,7 +82,13 @@ class NativeV2Vision(native.NativeV2):
             raise ValueError(f"effort must be one of {sorted(native.POLICIES)} for Wald-4B v2")
         if fmt not in native.FORMATS:
             raise ValueError(f"prompt_format must be one of {sorted(native.FORMATS)}")
-        req, uris, pat = vision_v2.lift(request, self.max_images)
+        prep = None
+        if vids:
+            prep = video.prepare(request, self.video, self.video_cache, vision_v2.has_images(request))
+            frames = sum(len(d.frames) for d in prep.videos) if prep.route == "frames16" else 0
+            req, uris, pat = vision_v2.lift(prep.request, self.max_images + frames, prep.markers or None)
+        else:
+            req, uris, pat = vision_v2.lift(request, self.max_images)
         client, sov = self._vborrow(fmt)
         client.bind(uris, pat)
         try:
@@ -76,7 +103,16 @@ class NativeV2Vision(native.NativeV2):
         for value in response["answers"].values():
             value.pop("raw", None)
         native.decisive_noul(response["answers"], self.noul_floor)
-        response["images"] = len(uris)
+        if prep is None:
+            response["images"] = len(uris)
+        else:
+            frames = sum(len(d.frames) for d in prep.videos) if prep.route == "frames16" else 0
+            response["images"] = sum(isinstance(u, str) for u in uris) - frames
+            response["videos"] = len(prep.videos)
+            response["video_route"] = prep.route
+            response["video_frames"] = [len(d.frames) for d in prep.videos]
+            if prep.fallback:
+                response["video_route_fallback"] = prep.fallback
         return response
 
 
@@ -116,6 +152,15 @@ def main(argv=None):
     ap.add_argument("--gpu-memory-utilization", type=float, default=None); ap.add_argument("--vllm-args", default="")
     ap.add_argument("--noul-decisive-floor", type=float, default=None,
                     help="Noul P(yes) inside (1-f, f) moves to the nearer edge after calibration; 0 = off (default: serving.json, else off)")
+    ap.add_argument("--video-route", default=None, choices=video.ROUTES,
+                    help=f"default route for video requests (default: serving.json vision.video_route, else {video.DEFAULT_ROUTE})")
+    ap.add_argument("--video-dir", default=None, help="accept local video paths / file:// URLs under this directory (default: refused)")
+    ap.add_argument("--video-urls", default="public", choices=video.URL_POLICIES,
+                    help="http(s) video URLs: off, public (refuse loopback / private / link-local hosts; default) or any")
+    ap.add_argument("--video-cache-mb", type=int, default=512, help="LRU of decoded frames, by content hash (0 = off)")
+    ap.add_argument("--max-video-mb", type=float, default=video.MAX_VIDEO_BYTES / 1e6)
+    ap.add_argument("--max-video-seconds", type=float, default=video.MAX_VIDEO_SECONDS)
+    ap.add_argument("--no-video", action="store_true", help="refuse video input and keep serving.json's vLLM limits")
     ap.add_argument("--host", default="0.0.0.0"); ap.add_argument("--port", type=int, default=8000)
     a = ap.parse_args(argv)
     cfg = native.read_serving(a.model)
@@ -126,6 +171,9 @@ def main(argv=None):
     temps = a.temperature or (str(Path(a.model) / cfg.get("temperature", "temperature.json")) if a.model else None)
     if not temps or not Path(temps).is_file():
         raise SystemExit(f"temperature table not found: {temps}")
+    vconf = video.VideoConfig(enabled=not a.no_video, default_route=a.video_route or vcfg.get("video_route", video.DEFAULT_ROUTE),
+                              video_dir=a.video_dir, url_policy=a.video_urls, max_bytes=int(a.max_video_mb * 1e6),
+                              max_seconds=a.max_video_seconds, cache_mb=a.video_cache_mb)
     if a.model:
         from .server import launch
         args = list(native.VLLM_ARGS)
@@ -133,15 +181,16 @@ def main(argv=None):
         if a.gpu_memory_utilization is not None: args[3] = str(a.gpu_memory_utilization)
         os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0")
         cmd = [sys.executable, "-m", "vllm.entrypoints.openai.api_server", "--model", a.model, "--served-model-name", served,
-               "--host", "127.0.0.1", "--port", str(a.vllm_port), *args, *vcfg.get("vllm_args", VISION_VLLM_ARGS), *shlex.split(a.vllm_args)]
+               "--host", "127.0.0.1", "--port", str(a.vllm_port), *args,
+               *(video_vllm_args if vconf.enabled else list)(vcfg.get("vllm_args", VISION_VLLM_ARGS)), *shlex.split(a.vllm_args)]
         launch(cmd, a.vllm_port, "vLLM")
         endpoint = f"http://127.0.0.1:{a.vllm_port}"
     else:
         endpoint = a.vllm
     engine = NativeV2Vision(endpoint, temps, served, effort, fmt, a.max_model_len, checkpoint=checkpoint,
                             model_sha256=cfg.get("model_sha256", native.MODEL_SHA),
-                            temperature_sha256=cfg.get("temperature_sha256", native.TEMPERATURE_SHA), image_prompt_format=ifmt,
-                            noul_decisive_floor=native.noul_floor(a, cfg))
+                            temperature_sha256=cfg.get("temperature_sha256") or native.table_pin(temps, a.model),
+                            image_prompt_format=ifmt, noul_decisive_floor=native.noul_floor(a, cfg), video_config=vconf)
     ThreadingHTTPServer.daemon_threads = True
     httpd = ThreadingHTTPServer((a.host, a.port), handler(engine))
     print(json.dumps({"serving": engine.info, "port": a.port}), flush=True)

server/src/wald_serve/video.py

@@ -0,0 +1,534 @@
+"""Video input for `wald-serve-native-vision` (wald-serve 0.2.2).
+
+A request may carry one video: a top-level `videos` list (d3 / Decision 3.0 shape) or a `state.messages` content part
+`{"type": "video_url", "video_url": {"url": ...}}` (OpenAI shape; `{"type": "video", "video": ...}` also works). A video
+is a `data:video/...;base64,` URI, raw base64, an http(s) URL (`--video-urls`), or a local path / `file://` URL under
+`--video-dir`. The server decodes it ONCE per request and every question of the request reuses the frames; an LRU
+cache keyed by the content hash reuses them across requests on the same video.
+
+Two routes (`video_route`, per request; server default `--video-route`):
+
+  frames16  16 frames at np.linspace over the decoded frame count, each <= 131,072 px (INTER_AREA), JPEG q95, each
+            preceded by a text part "Frame k/16 at t = S s:". This is byte for byte the client protocol of the
+            evaluated VisualDecisionBench-video reads (evaluation/v2.1/video.json): the frames go down the
+            unchanged image route as 16 images.
+  native32  32 frames at np.linspace(0, total - 1, 32).round() (vLLM's qwen3_vl loader at min = max = 32 frames), read
+            sequentially with OpenCV, JPEG q95, handed to vLLM as ONE Qwen3.5 video (`data:video/jpeg` frame list) with
+            its metadata (source fps, frame indices, frame count, duration), so vLLM writes the same `<t seconds>`
+            timestamps and pairs frames (temporal patch 2) as for the mp4 itself; pixel budget 32 x 131,072 px. The first
+            question of a request sends the frames with a content uuid; the other questions send only the uuid, so
+            vLLM reuses its processed video (it re-sends the frames if vLLM's cache no longer holds them). The first
+            native-video read sent the mp4 on every question, so vLLM decoded and processed it 20-30 times per request.
+
+The prompt, the reader, the readout and the calibration table are those of the image route: only the media differ.
+"""
+from __future__ import annotations
+
+import base64
+import binascii
+import hashlib
+import ipaddress
+import json
+import os
+import re
+import socket
+import tempfile
+import threading
+import urllib.parse
+import urllib.request
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from functools import cached_property
+from pathlib import Path
+
+ROUTES = ("frames16", "native32")
+DEFAULT_ROUTE = "native32"
+FRAMES16 = 16           # frames16: frame count and per-frame pixel cap (the evaluated client protocol)
+FRAMES16_PX = 131072
+NATIVE_FRAMES = 32      # native32: frames per video and per-frame pixel budget given to the Qwen3.5 video processor
+NATIVE_PX = 131072
+NATIVE_MIN_PX = 4096
+NATIVE_PRE_PX = 1048576  # native32 frames above this are downscaled (INTER_AREA) before JPEG; the processor then resizes to <= NATIVE_PX
+JPEG_QUALITY = 95
+MAX_VIDEO_BYTES = 32_000_000
+MAX_VIDEO_SECONDS = 300.0
+MAX_SOURCE_PIXELS = 8_294_400   # 3840 x 2160
+MAX_VIDEOS = 1
+URL_POLICIES = ("off", "public", "any")
+VIDEO_MIME = {"mp4": "video/mp4", "quicktime": "video/quicktime", "webm": "video/webm", "x-matroska": "video/x-matroska",
+              "avi": "video/x-msvideo", "x-msvideo": "video/x-msvideo", "mpeg": "video/mpeg", "x-m4v": "video/x-m4v"}
+DATA_VIDEO = re.compile(r"data:video/([A-Za-z0-9.+-]+);base64,(.*)", re.S)
+_B64 = re.compile(r"[A-Za-z0-9+/=\s]+")
+PART_TYPES = ("video_url", "video", "input_video")
+
+
+class VideoError(ValueError):
+    """A video input the server refuses (HTTP 400). `reason` is one of: video_format, video_too_large, video_too_long,
+    video_pixels, video_malformed, video_count, video_source, video_route, video_unavailable."""
+
+    def __init__(self, reason, message):
+        super().__init__(f"{reason}: {message}")
+        self.reason = reason
+
+
+def sniff(head: bytes):
+    """Container from the first bytes: ISO-BMFF (mp4 / mov / m4v), Matroska / WebM, AVI, MPEG-PS / TS; else None."""
+    if len(head) >= 12 and head[4:8] == b"ftyp":
+        return "mp4"
+    if head[:4] == b"\x1aE\xdf\xa3":
+        return "webm"
+    if head[:4] == b"RIFF" and head[8:12] == b"AVI ":
+        return "avi"
+    if head[:4] in (b"\x00\x00\x01\xba", b"\x00\x00\x01\xb3") or head[:1] == b"G" and len(head) > 188 and head[188:189] == b"G":
+        return "mpeg"
+    return None
+
+
+@dataclass
+class VideoConfig:
+    enabled: bool = True
+    default_route: str = DEFAULT_ROUTE
+    video_dir: str | None = None          # local paths / file:// URLs are accepted only under this directory
+    url_policy: str = "public"            # off | public (refuse loopback / private / link-local hosts) | any
+    max_bytes: int = MAX_VIDEO_BYTES
+    max_seconds: float = MAX_VIDEO_SECONDS
+    max_source_pixels: int = MAX_SOURCE_PIXELS
+    max_videos: int = MAX_VIDEOS
+    cache_mb: int = 512
+    url_timeout: float = 30.0
+
+    def __post_init__(self):
+        if self.default_route not in ROUTES:
+            raise ValueError(f"video route must be one of {ROUTES}")
+        if self.url_policy not in URL_POLICIES:
+            raise ValueError(f"video URL policy must be one of {URL_POLICIES}")
+        if self.video_dir is not None:
+            self.video_dir = os.path.realpath(self.video_dir)
+
+    def info(self):
+        return {"enabled": self.enabled, "routes": list(ROUTES), "default_route": self.default_route,
+                "max_videos": self.max_videos, "max_bytes": self.max_bytes, "max_seconds": self.max_seconds,
+                "max_source_pixels": self.max_source_pixels, "url_policy": self.url_policy,
+                "local_paths": self.video_dir is not None, "cache_mb": self.cache_mb,
+                "transports": ["videos[]", "state.messages video_url / video parts"],
+                "frames16": {"frames": FRAMES16, "max_pixels_per_frame": FRAMES16_PX},
+                "native32": {"frames": NATIVE_FRAMES, "pixels_per_frame": NATIVE_PX}}
+
+
+# ---------------------------------------------------------------------------------------------------------- sources
+
+def _public_host(host):
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except socket.gaierror as e:
+        raise VideoError("video_source", f"cannot resolve {host!r}") from e
+    for info in infos:
+        ip = ipaddress.ip_address(info[4][0].split("%")[0])
+        if not ip.is_global or ip.is_multicast:
+            raise VideoError("video_source", f"URL host {host!r} is not a public address (server --video-urls public)")
+
+
+class _Redirects(urllib.request.HTTPRedirectHandler):
+    def __init__(self, policy):
+        self.policy = policy
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        _check_url(newurl, self.policy)
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+def _check_url(url, policy):
+    u = urllib.parse.urlsplit(url)
+    if u.scheme not in ("http", "https") or not u.hostname:
+        raise VideoError("video_source", "only http(s) URLs can be fetched")
+    if policy == "public":
+        _public_host(u.hostname)
+
+
+def fetch_url(url, cfg: VideoConfig) -> bytes:
+    if cfg.url_policy == "off":
+        raise VideoError("video_source", "video URLs are disabled on this server (--video-urls off); send a data URI")
+    _check_url(url, cfg.url_policy)
+    opener = urllib.request.build_opener(_Redirects(cfg.url_policy))
+    try:
+        with opener.open(urllib.request.Request(url, headers={"User-Agent": "wald-serve"}), timeout=cfg.url_timeout) as r:
+            size = r.headers.get("Content-Length")
+            if size and size.isdigit() and int(size) > cfg.max_bytes:
+                raise VideoError("video_too_large", f"video larger than {cfg.max_bytes} bytes")
+            raw = r.read(cfg.max_bytes + 1)
+    except VideoError:
+        raise
+    except Exception as e:  # noqa: BLE001
+        raise VideoError("video_source", f"could not fetch the video URL ({type(e).__name__})") from None
+    if len(raw) > cfg.max_bytes:
+        raise VideoError("video_too_large", f"video larger than {cfg.max_bytes} bytes")
+    return raw
+
+
+def read_path(value, cfg: VideoConfig) -> bytes:
+    if cfg.video_dir is None:
+        raise VideoError("video_source", "local video paths are disabled on this server (start it with --video-dir DIR)")
+    p = urllib.parse.unquote(urllib.parse.urlsplit(value).path) if value.startswith("file://") else value
+    real = os.path.realpath(p if os.path.isabs(p) else os.path.join(cfg.video_dir, p))
+    if os.path.commonpath([real, cfg.video_dir]) != cfg.video_dir or not os.path.isfile(real):
+        raise VideoError("video_source", "video path is not a file under the server's --video-dir")
+    if os.path.getsize(real) > cfg.max_bytes:
+        raise VideoError("video_too_large", f"video larger than {cfg.max_bytes} bytes")
+    return Path(real).read_bytes()
+
+
+def _raw_b64_video(value):
+    try:
+        return sniff(base64.b64decode(re.sub(r"\s+", "", value[:344]), validate=True)) is not None
+    except (binascii.Error, ValueError):
+        return False
+
+
+def load(value, cfg: VideoConfig) -> bytes:
+    """A video reference (data URI, raw base64, http(s) URL, local path / file:// URL) -> container bytes."""
+    if isinstance(value, dict):
+        value = value.get("url") or value.get("data") or value.get("video")
+    if not isinstance(value, str) or not value:
+        raise VideoError("video_format", "a video must be a data URI, base64 string, URL or path")
+    m = DATA_VIDEO.fullmatch(value)
+    payload = raw = None
+    if m:
+        if m.group(1).lower() not in VIDEO_MIME:
+            raise VideoError("video_format", f"unsupported video type video/{m.group(1)}")
+        payload = m.group(2)
+    elif value.startswith("data:"):
+        raise VideoError("video_format", "a video data URI must be data:video/<type>;base64,...")
+    elif value.startswith(("http://", "https://")):
+        raw = fetch_url(value, cfg)
+    elif value.startswith("file://"):
+        raw = read_path(value, cfg)
+    elif len(value) >= 64 and _B64.fullmatch(value[:4096]) and _raw_b64_video(value):
+        payload = value
+    else:
+        raw = read_path(value, cfg)
+    if payload is not None:
+        if len(payload) * 3 // 4 > cfg.max_bytes + 3:
+            raise VideoError("video_too_large", f"video larger than {cfg.max_bytes} bytes")
+        try:
+            raw = base64.b64decode(re.sub(r"\s+", "", payload), validate=True)
+        except (binascii.Error, ValueError):
+            raise VideoError("video_format", "video data is not valid base64") from None
+    if len(raw) > cfg.max_bytes:
+        raise VideoError("video_too_large", f"video larger than {cfg.max_bytes} bytes")
+    if sniff(raw[:256]) is None:
+        raise VideoError("video_format", "not a video container this server reads (MP4 / MOV, WebM / MKV, AVI, MPEG)")
+    return raw
+
+
+# ---------------------------------------------------------------------------------------------------------- decoding
+
+@dataclass
+class Decoded:
+    route: str
+    sha256: str
+    frames: list = field(default_factory=list)   # JPEG bytes per sampled frame
+    times: list = field(default_factory=list)    # seconds (None when the container has no fps)
+    indices: list = field(default_factory=list)
+    total_frames: int = 0
+    fps: float = 0.0
+    duration: float = 0.0
+    width: int = 0
+    height: int = 0
+
+    @property
+    def nbytes(self):
+        return sum(len(f) for f in self.frames)
+
+    def image_parts(self):
+        """frames16: the evaluated wire, text label + image part per frame (`vdb_wald.wire`)."""
+        out, n = [], len(self.frames)
+        for k, (t, b) in enumerate(zip(self.times, self.frames)):
+            out.append({"type": "text", "text": f"Frame {k + 1}/{n}" + (f" at t = {t:.1f} s:" if t is not None else ":")})
+            out.append({"type": "image_url", "image_url": {"url": "data:image/jpeg;base64," + base64.b64encode(b).decode()}})
+        return out
+
+    def video_url(self):
+        """native32: vLLM's `data:video/jpeg` frame-list URL (one video)."""
+        return self._video_url
+
+    @cached_property
+    def _video_url(self):
+        return "data:video/jpeg;base64," + ",".join(base64.b64encode(b).decode() for b in self.frames)
+
+    def media_io_kwargs(self):
+        """vLLM metadata for the frame list, as its own decoder would report it for the source file."""
+        return {"video": {"num_frames": len(self.frames), "fps": float(self.fps), "frames_indices": [int(i) for i in self.indices],
+                          "total_num_frames": int(self.total_frames), "duration": float(self.duration)}}
+
+    def mm_processor_kwargs(self):
+        return {"max_pixels": len(self.frames) * NATIVE_PX, "min_pixels": NATIVE_MIN_PX}
+
+    @cached_property
+    def uuid(self):
+        """vLLM multimodal uuid: identifies the processed video (content hash + every input to its processing), so
+        later questions of a request can reference vLLM's processor cache instead of resending the frames."""
+        key = json.dumps([self.sha256, self.route, self.media_io_kwargs(), self.mm_processor_kwargs(), JPEG_QUALITY, NATIVE_PRE_PX])
+        return "wald-video-" + hashlib.sha256(key.encode()).hexdigest()
+
+
+def _cv2():
+    try:
+        import cv2  # noqa: PLC0415
+    except ImportError:
+        raise VideoError("video_unavailable", "video input needs OpenCV on the server: pip install 'wald-serve[video]'") from None
+    return cv2
+
+
+def _open(cv2, path, cfg):
+    cap = cv2.VideoCapture(str(path))
+    if not cap.isOpened():
+        raise VideoError("video_malformed", "the video could not be opened")
+    w, h = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    if w * h > cfg.max_source_pixels:
+        cap.release()
+        raise VideoError("video_pixels", f"{w}x{h} source frames: at most {cfg.max_source_pixels} pixels per frame")
+    return cap, w, h
+
+
+def _count(cv2, path):
+    cap = cv2.VideoCapture(str(path)); n = 0
+    while cap.grab():
+        n += 1
+    cap.release()
+    return n
+
+
+def _jpeg(cv2, frame):
+    ok, enc = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, JPEG_QUALITY])
+    if not ok:
+        raise VideoError("video_malformed", "a frame could not be encoded")
+    return enc.tobytes()
+
+
+def _check_duration(total, fps, cfg):
+    if fps and fps > 0 and total / fps > cfg.max_seconds + 1e-6:
+        raise VideoError("video_too_long", f"{total / fps:.1f} s video: at most {cfg.max_seconds:g} s")
+
+
+def decode_frames16(cv2, path, cfg, sha):
+    """The evaluated client's frame sampler (16 frames, <= 131,072 px, JPEG q95), with the server's limits."""
+    import numpy as np  # noqa: PLC0415
+    cap, w, h = _open(cv2, path, cfg)
+    total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)); fps = cap.get(cv2.CAP_PROP_FPS) or 0
+    if total <= 0:
+        # vdb_wald: no frame count in the container -> read every frame sequentially and index into them
+        total = _count(cv2, path)
+        _check_duration(total, fps, cfg)
+        want = set(np.linspace(0, total - 1, FRAMES16).round().astype(int).tolist()) if total > 0 else set()
+        got, seq = {}, cv2.VideoCapture(str(path))
+        for i in range(max(want) + 1 if want else 0):
+            ok, f = seq.read()
+            if not ok:
+                break
+            if i in want:
+                got[i] = f
+        seq.release()
+        get = got.get
+    else:
+        _check_duration(total, fps, cfg)
+
+        def get(i):
+            cap.set(cv2.CAP_PROP_POS_FRAMES, int(i)); ok, f = cap.read()
+            return f if ok else None
+    out = Decoded("frames16", sha, total_frames=total, fps=float(fps), duration=total / fps if fps else 0.0, width=w, height=h)
+    idx = np.linspace(0, total - 1, FRAMES16).round().astype(int) if total > 0 else []
+    for i in idx:
+        f = get(int(i))
+        if f is None:
+            continue
+        fh, fw = f.shape[:2]; s = min(1.0, (FRAMES16_PX / (fh * fw)) ** 0.5)
+        if s < 1:
+            f = cv2.resize(f, (max(28, int(fw * s)), max(28, int(fh * s))), interpolation=cv2.INTER_AREA)
+        out.frames.append(_jpeg(cv2, f)); out.indices.append(int(i)); out.times.append(float(i) / fps if fps else None)
+    cap.release()
+    if not out.frames:
+        raise VideoError("video_malformed", "no frame could be decoded")
+    return out
+
+
+def decode_native(cv2, path, cfg, sha, n=NATIVE_FRAMES):
+    """vLLM 0.30's qwen3_vl loader over OpenCV at min_frames = max_frames = n: frame count from the container (checked
+    against an end-of-stream seek, as vLLM does), indices np.linspace(0, total - 1, min(n, total)).round(), frames read
+    by sequential grab / retrieve."""
+    import numpy as np  # noqa: PLC0415
+    cap, w, h = _open(cv2, path, cfg)
+    total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)); fps = cap.get(cv2.CAP_PROP_FPS) or 0.0
+    if total > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
+        visible = int(cap.get(cv2.CAP_PROP_POS_FRAMES)); at_end = not cap.grab()
+        cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
+        if at_end and 0 < visible < total - 1:
+            total = visible
+    if total <= 0:
+        cap.release(); total = _count(cv2, path); cap, w, h = _open(cv2, path, cfg)
+    if total <= 0:
+        cap.release()
+        raise VideoError("video_malformed", "no frame could be decoded")
+    _check_duration(total, fps, cfg)
+    idx = np.linspace(0, total - 1, min(n, total)).round().astype(int).tolist()
+    want, frames, valid = set(idx), [], []
+    for i in range(max(idx) + 1):
+        if not cap.grab():
+            continue
+        if i in want:
+            ok, f = cap.retrieve()
+            if ok and f is not None and f.size:
+                fh, fw = f.shape[:2]
+                if fh * fw > NATIVE_PRE_PX:
+                    s = (NATIVE_PRE_PX / (fh * fw)) ** 0.5
+                    f = cv2.resize(f, (max(32, int(fw * s)), max(32, int(fh * s))), interpolation=cv2.INTER_AREA)
+                frames.append(_jpeg(cv2, f)); valid.append(i)
+    cap.release()
+    if not frames:
+        raise VideoError("video_malformed", "no frame could be decoded")
+    return Decoded("native32", sha, frames=frames, indices=valid, times=[i / fps if fps else None for i in valid],
+                   total_frames=total, fps=float(fps) if fps else 1.0, duration=total / fps if fps else float(total), width=w, height=h)
+
+
+class FrameCache:
+    """LRU of decoded videos keyed by (sha256 of the container bytes, route), bounded by the JPEG bytes held."""
+
+    def __init__(self, max_mb):
+        self.max_bytes, self.items, self.size, self.lock = int(max_mb) * 1024 * 1024, OrderedDict(), 0, threading.Lock()
+        self.hits = self.misses = 0
+
+    def get(self, key):
+        with self.lock:
+            v = self.items.get(key)
+            if v is None:
+                self.misses += 1
+                return None
+            self.items.move_to_end(key); self.hits += 1
+            return v
+
+    def put(self, key, value):
+        if value.nbytes > self.max_bytes:
+            return
+        with self.lock:
+            if key in self.items:
+                return
+            self.items[key] = value; self.size += value.nbytes
+            while self.size > self.max_bytes:
+                _, old = self.items.popitem(last=False); self.size -= old.nbytes
+
+
+def decode(raw: bytes, route: str, cfg: VideoConfig, cache: FrameCache | None = None) -> Decoded:
+    if route not in ROUTES:
+        raise VideoError("video_route", f"video_route must be one of {list(ROUTES)}")
+    sha = hashlib.sha256(raw).hexdigest()
+    hit = cache.get((sha, route)) if cache is not None else None
+    if hit is not None:
+        return hit
+    cv2 = _cv2()
+    fd, path = tempfile.mkstemp(suffix=".video")
+    try:
+        with os.fdopen(fd, "wb") as f:
+            f.write(raw)
+        out = decode_frames16(cv2, path, cfg, sha) if route == "frames16" else decode_native(cv2, path, cfg, sha)
+    finally:
+        os.unlink(path)
+    if cache is not None:
+        cache.put((sha, route), out)
+    return out
+
+
+# ---------------------------------------------------------------------------------------------------------- requests
+
+def _is_video_part(c):
+    return isinstance(c, dict) and c.get("type") in PART_TYPES
+
+
+def _part_value(c):
+    v = c.get(c["type"]) if c.get("type") in c else c.get("video_url") or c.get("video")
+    if isinstance(v, dict):
+        v = v.get("url") or v.get("data")
+    return v
+
+
+def _message_videos(state):
+    if not (isinstance(state, dict) and isinstance(state.get("messages"), list)):
+        return []
+    return [c for m in state["messages"] if isinstance(m, dict) and isinstance(m.get("content"), list)
+            for c in m["content"] if _is_video_part(c)]
+
+
+def has_videos(req) -> bool:
+    return isinstance(req, dict) and (bool(req.get("videos")) or bool(_message_videos(req.get("state"))))
+
+
+@dataclass
+class Prepared:
+    request: dict
+    route: str
+    videos: list            # Decoded, in order
+    markers: dict           # native32: marker text -> Decoded
+    fallback: str | None = None
+
+
+def prepare(req: dict, cfg: VideoConfig, cache: FrameCache | None, has_user_images: bool) -> Prepared:
+    """Decode the request's video once and rewrite the request for the image route.
+
+    frames16: each video becomes its 16 (label, image) parts; top-level `videos` go before the state.
+    native32: each video becomes one marker text part that `vision_v2.lift` turns into a video slot."""
+    if not cfg.enabled:
+        raise VideoError("video_unavailable", "video input is disabled on this server")
+    route = req.get("video_route")
+    explicit = route is not None
+    route = route or cfg.default_route
+    if route not in ROUTES:
+        raise VideoError("video_route", f"video_route must be one of {list(ROUTES)}")
+    fallback = None
+    if route == "native32" and has_user_images:
+        # vLLM's per-request pixel budget is not per modality: the video's budget would also resize the images
+        if explicit:
+            raise VideoError("video_route", "native32 cannot be combined with images in one request (use frames16)")
+        route, fallback = "frames16", "images in the request"
+    top = req.get("videos")
+    top = [] if top in (None, []) else top if isinstance(top, list) else [top]
+    parts = _message_videos(req.get("state"))
+    if len(top) + len(parts) > cfg.max_videos:
+        raise VideoError("video_count", f"{len(top) + len(parts)} videos: at most {cfg.max_videos} per request")
+    decoded, markers = [], {}
+    nonce = os.urandom(4).hex()
+
+    def replacement(value):
+        d = decode(load(value, cfg), route, cfg, cache); decoded.append(d)
+        if route == "frames16":
+            return d.image_parts()
+        mark = f"⟦wald-video-{nonce}:{len(markers)}⟧"; markers[mark] = d
+        return [{"type": "text", "text": mark}]
+
+    state = req.get("state")
+    if parts:
+        msgs = []
+        for m in state["messages"]:
+            if isinstance(m, dict) and isinstance(m.get("content"), list):
+                content = []
+                for c in m["content"]:
+                    content.extend(replacement(_part_value(c)) if _is_video_part(c) else [c])
+                m = {**m, "content": content}
+            msgs.append(m)
+        state = {**state, "messages": msgs}
+    if top:
+        head = [p for v in top for p in replacement(v)]
+        if isinstance(state, dict) and isinstance(state.get("messages"), list) and set(state) == {"messages"}:
+            msgs = list(state["messages"])
+            first = msgs[0] if msgs else {"role": "user", "content": []}
+            c = first.get("content") if isinstance(first, dict) else first
+            c = [{"type": "text", "text": c}] if isinstance(c, str) else list(c or [])
+            msgs[0 if msgs else 0:1] = [{**(first if isinstance(first, dict) else {"role": "user"}), "content": head + c}]
+            state = {"messages": msgs}
+        elif state is None or isinstance(state, str):
+            tail = [{"type": "text", "text": state}] if isinstance(state, str) and state.strip() else []
+            state = {"messages": [{"role": "user", "content": head + tail}]}
+        else:
+            # an object / array state: the video goes first under "video" as text with embedded media
+            block = "\n".join(p["text"] if p["type"] == "text" else p["image_url"]["url"] for p in head)
+            state = {"video": block, **state} if isinstance(state, dict) else {"video": block, "state": state}
+    out = {k: v for k, v in req.items() if k not in ("videos", "video_route")}
+    out["state"] = state
+    return Prepared(out, route, decoded, markers, fallback)

server/src/wald_serve/vision_v2.py

@@ -24,6 +24,10 @@ Accepted image transports (at most `max_images` per request, PNG / JPEG / WebP /
     `{"type": "image_url", "image_url": {"url": ...}}` or `{"type": "image", "image": ...}`);
   * raw base64 as the value of a `state` key `image` / `image_data` / `image_base64` / `image_b64`;
   * a top-level `images` (or `image_data`) list of data URIs / base64 strings: placed at the start of the state, in order.
+
+Video (wald-serve 0.2.2, video.py) reuses this route: `frames16` turns the video into 16 labelled image parts before
+`lift`; `native32` turns it into a marker that `lift` maps to a video slot, which `to_parts` writes as one `video_url`
+part and `ImageClient` sends with the video template, the frame metadata and the video pixel budget.
 """
 from __future__ import annotations
 
@@ -39,6 +43,12 @@ CHAT_TEMPLATE = ("{%- for message in messages -%}{%- for part in message['conten
                  "{%- if part['type'] == 'text' -%}{{ part['text'] }}"
                  "{%- else -%}<|vision_start|><|image_pad|><|vision_end|>{%- endif -%}"
                  "{%- endfor -%}{%- endfor -%}")
+# the same, with Qwen3.5's video placeholder for video parts; sent only when the prompt carries a native video
+VIDEO_CHAT_TEMPLATE = ("{%- for message in messages -%}{%- for part in message['content'] -%}"
+                       "{%- if part['type'] == 'text' -%}{{ part['text'] }}"
+                       "{%- elif part['type'] in ('video', 'video_url') -%}<|vision_start|><|video_pad|><|vision_end|>"
+                       "{%- else -%}<|vision_start|><|image_pad|><|vision_end|>{%- endif -%}"
+                       "{%- endfor -%}{%- endfor -%}")
 DATA_URI = re.compile(r"data:image/[A-Za-z0-9.+-]+;base64,[A-Za-z0-9+/=]+")
 RAW_KEYS = ("image", "image_data", "image_base64", "image_b64")
 _MAGIC = ((b"\x89PNG\r\n\x1a\n", "png"), (b"\xff\xd8\xff", "jpeg"), (b"GIF87a", "gif"), (b"GIF89a", "gif"))
@@ -100,8 +110,11 @@ def has_images(req) -> bool:
     return bool(req.get("images") or req.get("image_data")) or _has(req.get("state"))
 
 
-def lift(req: dict, max_images: int = MAX_IMAGES):
-    """-> (request with images replaced by sentinels, data URIs by slot, sentinel regex)."""
+def lift(req: dict, max_images: int = MAX_IMAGES, videos: dict | None = None):
+    """-> (request with images replaced by sentinels, data URIs by slot, sentinel regex).
+
+    `videos` (video.prepare, native32 route): marker text -> decoded video; each marker becomes a slot whose entry is
+    the decoded video object instead of a data URI. Videos do not count against `max_images`."""
     nonce = secrets.token_hex(6)
     mark = lambda i: f"⟦{nonce}:{i}⟧"
     uris = []
@@ -110,8 +123,15 @@ def lift(req: dict, max_images: int = MAX_IMAGES):
         uris.append(check_uri(uri))
         return mark(len(uris) - 1)
 
+    def vslot(m):
+        uris.append(videos[m.group(0)])
+        return mark(len(uris) - 1)
+
+    vpat = re.compile("|".join(re.escape(k) for k in videos)) if videos else None
+
     def text(s):
-        return DATA_URI.sub(lambda m: slot(m.group(0)), s) if "data:image/" in s else s
+        s = DATA_URI.sub(lambda m: slot(m.group(0)), s) if "data:image/" in s else s
+        return vpat.sub(vslot, s) if vpat is not None else s
 
     def walk(v):
         if isinstance(v, str):
@@ -142,8 +162,11 @@ def lift(req: dict, max_images: int = MAX_IMAGES):
     if top:
         state = "\n".join(top + ([state] if isinstance(state, str) and state.strip() else [])) if isinstance(state, str) or state is None \
             else {"images": "\n".join(top), **state} if isinstance(state, dict) else "\n".join(top)
-    if len(uris) > max_images:
-        raise ImageError(f"{len(uris)} images: at most {max_images} per request")
+    nimg = sum(isinstance(u, str) for u in uris)
+    if nimg > max_images:
+        raise ImageError(f"{nimg} images: at most {max_images} per request")
+    if videos and sum(not isinstance(u, str) for u in uris) != len(videos):
+        raise ImageError("a video slot is missing from the state")
     out = {"state": state, "questions": req["questions"]}
     return out, uris, re.compile("⟦" + nonce + r":(\d+)⟧")
 
@@ -182,7 +205,9 @@ def to_parts(text: str, uris, pat):
             raise ImageError("unknown image slot")
         if m.start() > pos:
             parts.append({"type": "text", "text": text[pos:m.start()]})
-        parts.append({"type": "image_url", "image_url": {"url": uris[i]}}); seen.add(i); pos = m.end()
+        u = uris[i]
+        parts.append({"type": "image_url", "image_url": {"url": u}} if isinstance(u, str)
+                     else {"type": "video_url", "video_url": {"url": u.video_url()}}); seen.add(i); pos = m.end()
     if pos < len(text):
         parts.append({"type": "text", "text": text[pos:]})
     if seen != set(range(len(uris))):
@@ -194,11 +219,11 @@ def image_client(base):
     """Subclass the frozen reader's client class `base` so its token-id completions carry the request's images."""
 
     class ImageClient(base):
-        uris, pat = [], None
+        uris, pat, sent = [], None, frozenset()
         mm_calls = 0
 
         def bind(self, uris, pat):
-            self.uris, self.pat = uris, pat
+            self.uris, self.pat, self.sent = uris, pat, set()
             return self
 
         def post(self, path, body, retries=1):
@@ -207,11 +232,17 @@ def image_client(base):
             text = super().post("/detokenize", {"model": self.served, "tokens": body["prompt"]}, retries)["prompt"]
             parts = to_parts(text, self.uris, self.pat)
             chat = {k: v for k, v in body.items() if k not in ("prompt", "logprobs")}
-            chat.update(messages=[{"role": "user", "content": parts}], chat_template=CHAT_TEMPLATE, add_generation_prompt=False,
-                        add_special_tokens=False)
+            vids = [u for u in self.uris if not isinstance(u, str)]
+            chat.update(messages=[{"role": "user", "content": parts}], chat_template=VIDEO_CHAT_TEMPLATE if vids else CHAT_TEMPLATE,
+                        add_generation_prompt=False, add_special_tokens=False)
+            if vids:   # native video: the frame list's metadata and the video pixel budget (one video per request)
+                chat.update(media_io_kwargs=vids[0].media_io_kwargs(), mm_processor_kwargs=vids[0].mm_processor_kwargs())
             if "logprobs" in body:
                 chat.update(logprobs=True, top_logprobs=body["logprobs"])
-            r = super().post("/v1/chat/completions", chat, retries)
+            if vids:
+                r = self._video_chat(chat, parts, vids, retries)
+            else:
+                r = super().post("/v1/chat/completions", chat, retries)
             self.mm_calls += 1
             out = []
             for c in r["choices"]:
@@ -223,4 +254,25 @@ def image_client(base):
                 out.append(o)
             return {"choices": out, "usage": r.get("usage") or {}}
 
+        def _video_chat(self, chat, parts, vids, retries):
+            """Send each video's frames once per request (with its uuid); later calls reference vLLM's processor cache by
+            uuid only, and re-send the frames on a cache miss."""
+            by_url = {v.video_url(): v for v in vids}
+            withdata, cached = [], []
+            for p in parts:
+                if p["type"] == "video_url":
+                    v = by_url[p["video_url"]["url"]]
+                    withdata.append({**p, "uuid": v.uuid})
+                    cached.append({"type": "video_url", "video_url": None, "uuid": v.uuid} if v.uuid in self.sent else withdata[-1])
+                else:
+                    withdata.append(p); cached.append(p)
+            try:
+                r = super().post("/v1/chat/completions", {**chat, "messages": [{"role": "user", "content": cached}]}, retries)
+            except ValueError as e:
+                if cached == withdata or "cache miss" not in str(e).lower():
+                    raise
+                r = super().post("/v1/chat/completions", {**chat, "messages": [{"role": "user", "content": withdata}]}, retries)
+            self.sent.update(v.uuid for v in vids)
+            return r
+
     return ImageClient

server/tests/fixtures/default-responses-0.2.1.json

@@ -0,0 +1,775 @@
+{
+ "recorded_with": "wald-serve 0.2.1",
+ "cases": [
+  {
+   "engine": "native",
+   "floor": null,
+   "request": {
+    "state": "A customer asks for a refund 20 days after purchase; the policy allows 30 days.",
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "none",
+    "prompt_format": "repeat_state_plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"ALLOW\": 0.4920836141192666, \"FREEZE\": 0.5079163858807334}, \"probabilities_t1\": {\"ALLOW\": 0.48800230346928214, \"FREEZE\": 0.5119976965307179}, \"choice\": \"FREEZE\", \"confidence\": 0.5079163858807334}, \"choice4\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"billing\": 0.211396160489446, \"tech\": 0.268785959505119, \"sales\": 0.10954557627286518, \"abuse\": 0.41027230373256984}, \"probabilities_t1\": {\"billing\": 0.15452385371303878, \"tech\": 0.24575828642613318, \"sales\": 0.04339518444167778, \"abuse\": 0.5563226754191503}, \"choice\": \"abuse\", \"confidence\": 0.41027230373256984}, \"noul\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.30164010022508964, \"true\": 0.6983598997749104}, \"probabilities_t1\": {\"false\": 0.15722793077873878, \"true\": 0.8427720692212612}, \"noul\": 0.6983598997749104, \"confidence\": 0.6983598997749104}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.6212835951910678, \"true\": 0.3787164048089321}, \"probabilities_t1\": {\"false\": 0.7290879223493065, \"true\": 0.2709120776506935}, \"noul\": 0.3787164048089321, \"confidence\": 0.6212835951910678}, \"score5\": {\"type\": \"score\", \"mode\": \"A\", \"probabilities\": {\"0\": 0.3383212086632271, \"1\": 0.10847493800848075, \"2\": 0.0691998621611772, \"3\": 0.3592615680544847, \"4\": 0.12474242311263017}, \"probabilities_t1\": {\"0\": 0.39840922176994686, \"1\": 0.05881986702751884, \"2\": 0.027618329716487085, \"3\": 0.4407508159072359, \"4\": 0.07440176557881127}, \"score\": 1.8236290589448099, \"confidence\": 0.3592615680544847}}, \"usage\": {\"input_tokens\": 3692, \"output_tokens\": 0}}"
+  },
+  {
+   "engine": "native",
+   "floor": null,
+   "request": {
+    "state": "A customer asks for a refund 20 days after purchase; the policy allows 30 days.",
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "auto",
+    "prompt_format": "repeat_state_plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"ALLOW\": 0.47330537184229743, \"FREEZE\": 0.5266946281577025}, \"probabilities_t1\": {\"ALLOW\": 0.45958834166338, \"FREEZE\": 0.54041165833662}, \"choice\": \"FREEZE\", \"confidence\": 0.5266946281577025}, \"choice4\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"billing\": 0.21302557310622539, \"tech\": 0.18399751154593097, \"sales\": 0.41493539399365664, \"abuse\": 0.188041521354187}, \"probabilities_t1\": {\"billing\": 0.16220890486884484, \"tech\": 0.12222761958572902, \"sales\": 0.5880929659286207, \"abuse\": 0.12747050961680542}, \"choice\": \"sales\", \"confidence\": 0.41493539399365664}, \"noul\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.30164010022508964, \"true\": 0.6983598997749104}, \"probabilities_t1\": {\"false\": 0.15722793077873878, \"true\": 0.8427720692212612}, \"noul\": 0.6983598997749104, \"confidence\": 0.6983598997749104}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.6212835951910678, \"true\": 0.3787164048089321}, \"probabilities_t1\": {\"false\": 0.7290879223493065, \"true\": 0.2709120776506935}, \"noul\": 0.3787164048089321, \"confidence\": 0.6212835951910678}, \"score5\": {\"type\": \"score\", \"mode\": \"B\", \"probabilities\": {\"0\": 0.07064382270674367, \"1\": 0.2030935252133106, \"2\": 0.0914423404027521, \"3\": 0.2520198059983586, \"4\": 0.38280050567883506}, \"probabilities_t1\": {\"0\": 0.029333619082893232, \"1\": 0.17324976250125013, \"2\": 0.04527406116267923, \"3\": 0.24907007117682023, \"4\": 0.5030724860763571}, \"score\": 2.6732396467292308, \"confidence\": 0.38280050567883506}}, \"usage\": {\"input_tokens\": 8477, \"output_tokens\": 9}}"
+  },
+  {
+   "engine": "native",
+   "floor": null,
+   "request": {
+    "state": "A customer asks for a refund 20 days after purchase; the policy allows 30 days.",
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "always",
+    "prompt_format": "repeat_state_plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"ALLOW\": 0.47330537184229743, \"FREEZE\": 0.5266946281577025}, \"probabilities_t1\": {\"ALLOW\": 0.45958834166338, \"FREEZE\": 0.54041165833662}, \"choice\": \"FREEZE\", \"confidence\": 0.5266946281577025}, \"choice4\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"billing\": 0.21302557310622539, \"tech\": 0.18399751154593097, \"sales\": 0.41493539399365664, \"abuse\": 0.188041521354187}, \"probabilities_t1\": {\"billing\": 0.16220890486884484, \"tech\": 0.12222761958572902, \"sales\": 0.5880929659286207, \"abuse\": 0.12747050961680542}, \"choice\": \"sales\", \"confidence\": 0.41493539399365664}, \"noul\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.8177235752832239, \"true\": 0.18227642471677616}, \"probabilities_t1\": {\"false\": 0.9526643984026413, \"true\": 0.04733560159735865}, \"noul\": 0.18227642471677616, \"confidence\": 0.8177235752832239}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.5730974487716746, \"true\": 0.4269025512283255}, \"probabilities_t1\": {\"false\": 0.6431356663791894, \"true\": 0.3568643336208105}, \"noul\": 0.4269025512283255, \"confidence\": 0.5730974487716746}, \"score5\": {\"type\": \"score\", \"mode\": \"B\", \"probabilities\": {\"0\": 0.07064382270674367, \"1\": 0.2030935252133106, \"2\": 0.0914423404027521, \"3\": 0.2520198059983586, \"4\": 0.38280050567883506}, \"probabilities_t1\": {\"0\": 0.029333619082893232, \"1\": 0.17324976250125013, \"2\": 0.04527406116267923, \"3\": 0.24907007117682023, \"4\": 0.5030724860763571}, \"score\": 2.6732396467292308, \"confidence\": 0.38280050567883506}}, \"usage\": {\"input_tokens\": 11561, \"output_tokens\": 15}}"
+  },
+  {
+   "engine": "native",
+   "floor": null,
+   "request": {
+    "state": {
+     "task": "pick_and_place",
+     "instruction": "Put the knife in the toddler's hand.",
+     "step": 0
+    },
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "none",
+    "prompt_format": "plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"ALLOW\": 0.21299388104436723, \"FREEZE\": 0.7870061189556327}, \"probabilities_t1\": {\"ALLOW\": 0.12121227767549908, \"FREEZE\": 0.8787877223245009}, \"choice\": \"FREEZE\", \"confidence\": 0.7870061189556327}, \"choice4\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"billing\": 0.5367949190253793, \"tech\": 0.07594433814854183, \"sales\": 0.1959355379157818, \"abuse\": 0.19132520491029703}, \"probabilities_t1\": {\"billing\": 0.7681341862770509, \"tech\": 0.017565986370644177, \"sales\": 0.10961392721830486, \"abuse\": 0.10468590013400007}, \"choice\": \"billing\", \"confidence\": 0.5367949190253793}, \"noul\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.4324161639899626, \"true\": 0.5675838360100374}, \"probabilities_t1\": {\"false\": 0.36725757190544506, \"true\": 0.6327424280945549}, \"noul\": 0.5675838360100374, \"confidence\": 0.5675838360100374}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.6369151752804144, \"true\": 0.36308482471958553}, \"probabilities_t1\": {\"false\": 0.754729921357608, \"true\": 0.24527007864239195}, \"noul\": 0.36308482471958553, \"confidence\": 0.6369151752804144}, \"score5\": {\"type\": \"score\", \"mode\": \"A\", \"probabilities\": {\"0\": 0.3122788976475191, \"1\": 0.3353749360732271, \"2\": 0.06000731207563256, \"3\": 0.04217663999384101, \"4\": 0.2501622142097803}, \"probabilities_t1\": {\"0\": 0.3432808216629855, \"1\": 0.3870480456325973, \"2\": 0.021424793673134466, \"3\": 0.011840772049878354, \"4\": 0.23640556698140439}, \"score\": 1.5825683370451364, \"confidence\": 0.3353749360732271}}, \"usage\": {\"input_tokens\": 2687, \"output_tokens\": 0}}"
+  },
+  {
+   "engine": "native",
+   "floor": null,
+   "request": {
+    "state": {
+     "task": "pick_and_place",
+     "instruction": "Put the knife in the toddler's hand.",
+     "step": 0
+    },
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "auto",
+    "prompt_format": "plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"ALLOW\": 0.21299388104436723, \"FREEZE\": 0.7870061189556327}, \"probabilities_t1\": {\"ALLOW\": 0.12121227767549908, \"FREEZE\": 0.8787877223245009}, \"choice\": \"FREEZE\", \"confidence\": 0.7870061189556327}, \"choice4\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"billing\": 0.5367949190253793, \"tech\": 0.07594433814854183, \"sales\": 0.1959355379157818, \"abuse\": 0.19132520491029703}, \"probabilities_t1\": {\"billing\": 0.7681341862770509, \"tech\": 0.017565986370644177, \"sales\": 0.10961392721830486, \"abuse\": 0.10468590013400007}, \"choice\": \"billing\", \"confidence\": 0.5367949190253793}, \"noul\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.46679892672616957, \"true\": 0.5332010732738305}, \"probabilities_t1\": {\"false\": 0.4338893515030435, \"true\": 0.5661106484969565}, \"noul\": 0.5332010732738305, \"confidence\": 0.5332010732738305}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.6369151752804144, \"true\": 0.36308482471958553}, \"probabilities_t1\": {\"false\": 0.754729921357608, \"true\": 0.24527007864239195}, \"noul\": 0.36308482471958553, \"confidence\": 0.6369151752804144}, \"score5\": {\"type\": \"score\", \"mode\": \"B\", \"probabilities\": {\"0\": 0.047528516506915475, \"1\": 0.3118915509738329, \"2\": 0.407091480546279, \"3\": 0.14059547105506479, \"4\": 0.09289298091790779}, \"probabilities_t1\": {\"0\": 0.014086867127659082, \"1\": 0.333366656216499, \"2\": 0.5217783881513158, \"3\": 0.08729061497939686, \"4\": 0.04347747352512925}, \"score\": 1.9194328489032166, \"confidence\": 0.407091480546279}}, \"usage\": {\"input_tokens\": 4905, \"output_tokens\": 6}}"
+  },
+  {
+   "engine": "native",
+   "floor": null,
+   "request": {
+    "state": {
+     "task": "pick_and_place",
+     "instruction": "Put the knife in the toddler's hand.",
+     "step": 0
+    },
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "always",
+    "prompt_format": "plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"ALLOW\": 0.1969802364019475, \"FREEZE\": 0.8030197635980526}, \"probabilities_t1\": {\"ALLOW\": 0.10621499167517558, \"FREEZE\": 0.8937850083248244}, \"choice\": \"FREEZE\", \"confidence\": 0.8030197635980526}, \"choice4\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"billing\": 0.3401429325428138, \"tech\": 0.08923613023357456, \"sales\": 0.07326390337447908, \"abuse\": 0.4973570338491326}, \"probabilities_t1\": {\"billing\": 0.3114971187645603, \"tech\": 0.02348566156189473, \"sales\": 0.016044884702182443, \"abuse\": 0.6489723349713626}, \"choice\": \"abuse\", \"confidence\": 0.4973570338491326}, \"noul\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.46679892672616957, \"true\": 0.5332010732738305}, \"probabilities_t1\": {\"false\": 0.4338893515030435, \"true\": 0.5661106484969565}, \"noul\": 0.5332010732738305, \"confidence\": 0.5332010732738305}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.7586880139744555, \"true\": 0.24131198602554438}, \"probabilities_t1\": {\"false\": 0.9081289149152675, \"true\": 0.09187108508473248}, \"noul\": 0.24131198602554438, \"confidence\": 0.7586880139744555}, \"score5\": {\"type\": \"score\", \"mode\": \"B\", \"probabilities\": {\"0\": 0.047528516506915475, \"1\": 0.3118915509738329, \"2\": 0.407091480546279, \"3\": 0.14059547105506479, \"4\": 0.09289298091790779}, \"probabilities_t1\": {\"0\": 0.014086867127659082, \"1\": 0.333366656216499, \"2\": 0.5217783881513158, \"3\": 0.08729061497939686, \"4\": 0.04347747352512925}, \"score\": 1.9194328489032166, \"confidence\": 0.407091480546279}}, \"usage\": {\"input_tokens\": 8546, \"output_tokens\": 15}}"
+  },
+  {
+   "engine": "native",
+   "floor": null,
+   "request": {
+    "state": "A customer asks for a refund 20 days after purchase; the policy allows 30 days.",
+    "questions": {
+     "q": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     }
+    }
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"q\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.30164010022508964, \"true\": 0.6983598997749104}, \"probabilities_t1\": {\"false\": 0.15722793077873878, \"true\": 0.8427720692212612}, \"noul\": 0.6983598997749104, \"confidence\": 0.6983598997749104}}, \"usage\": {\"input_tokens\": 695, \"output_tokens\": 0}}"
+  },
+  {
+   "engine": "native",
+   "floor": 0.81,
+   "request": {
+    "state": "A customer asks for a refund 20 days after purchase; the policy allows 30 days.",
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "none",
+    "prompt_format": "repeat_state_plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"ALLOW\": 0.4920836141192666, \"FREEZE\": 0.5079163858807334}, \"probabilities_t1\": {\"ALLOW\": 0.48800230346928214, \"FREEZE\": 0.5119976965307179}, \"choice\": \"FREEZE\", \"confidence\": 0.5079163858807334}, \"choice4\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"billing\": 0.211396160489446, \"tech\": 0.268785959505119, \"sales\": 0.10954557627286518, \"abuse\": 0.41027230373256984}, \"probabilities_t1\": {\"billing\": 0.15452385371303878, \"tech\": 0.24575828642613318, \"sales\": 0.04339518444167778, \"abuse\": 0.5563226754191503}, \"choice\": \"abuse\", \"confidence\": 0.41027230373256984}, \"noul\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.18999999999999995, \"true\": 0.81}, \"probabilities_t1\": {\"false\": 0.15722793077873878, \"true\": 0.8427720692212612}, \"noul\": 0.81, \"confidence\": 0.81, \"noul_calibrated\": 0.6983598997749104}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.81, \"true\": 0.19}, \"probabilities_t1\": {\"false\": 0.7290879223493065, \"true\": 0.2709120776506935}, \"noul\": 0.19, \"confidence\": 0.81, \"noul_calibrated\": 0.3787164048089321}, \"score5\": {\"type\": \"score\", \"mode\": \"A\", \"probabilities\": {\"0\": 0.3383212086632271, \"1\": 0.10847493800848075, \"2\": 0.0691998621611772, \"3\": 0.3592615680544847, \"4\": 0.12474242311263017}, \"probabilities_t1\": {\"0\": 0.39840922176994686, \"1\": 0.05881986702751884, \"2\": 0.027618329716487085, \"3\": 0.4407508159072359, \"4\": 0.07440176557881127}, \"score\": 1.8236290589448099, \"confidence\": 0.3592615680544847}}, \"usage\": {\"input_tokens\": 3692, \"output_tokens\": 0}}"
+  },
+  {
+   "engine": "native",
+   "floor": 0.81,
+   "request": {
+    "state": "A customer asks for a refund 20 days after purchase; the policy allows 30 days.",
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "auto",
+    "prompt_format": "repeat_state_plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"ALLOW\": 0.47330537184229743, \"FREEZE\": 0.5266946281577025}, \"probabilities_t1\": {\"ALLOW\": 0.45958834166338, \"FREEZE\": 0.54041165833662}, \"choice\": \"FREEZE\", \"confidence\": 0.5266946281577025}, \"choice4\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"billing\": 0.21302557310622539, \"tech\": 0.18399751154593097, \"sales\": 0.41493539399365664, \"abuse\": 0.188041521354187}, \"probabilities_t1\": {\"billing\": 0.16220890486884484, \"tech\": 0.12222761958572902, \"sales\": 0.5880929659286207, \"abuse\": 0.12747050961680542}, \"choice\": \"sales\", \"confidence\": 0.41493539399365664}, \"noul\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.18999999999999995, \"true\": 0.81}, \"probabilities_t1\": {\"false\": 0.15722793077873878, \"true\": 0.8427720692212612}, \"noul\": 0.81, \"confidence\": 0.81, \"noul_calibrated\": 0.6983598997749104}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.81, \"true\": 0.19}, \"probabilities_t1\": {\"false\": 0.7290879223493065, \"true\": 0.2709120776506935}, \"noul\": 0.19, \"confidence\": 0.81, \"noul_calibrated\": 0.3787164048089321}, \"score5\": {\"type\": \"score\", \"mode\": \"B\", \"probabilities\": {\"0\": 0.07064382270674367, \"1\": 0.2030935252133106, \"2\": 0.0914423404027521, \"3\": 0.2520198059983586, \"4\": 0.38280050567883506}, \"probabilities_t1\": {\"0\": 0.029333619082893232, \"1\": 0.17324976250125013, \"2\": 0.04527406116267923, \"3\": 0.24907007117682023, \"4\": 0.5030724860763571}, \"score\": 2.6732396467292308, \"confidence\": 0.38280050567883506}}, \"usage\": {\"input_tokens\": 8477, \"output_tokens\": 9}}"
+  },
+  {
+   "engine": "native",
+   "floor": 0.81,
+   "request": {
+    "state": "A customer asks for a refund 20 days after purchase; the policy allows 30 days.",
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "always",
+    "prompt_format": "repeat_state_plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"ALLOW\": 0.47330537184229743, \"FREEZE\": 0.5266946281577025}, \"probabilities_t1\": {\"ALLOW\": 0.45958834166338, \"FREEZE\": 0.54041165833662}, \"choice\": \"FREEZE\", \"confidence\": 0.5266946281577025}, \"choice4\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"billing\": 0.21302557310622539, \"tech\": 0.18399751154593097, \"sales\": 0.41493539399365664, \"abuse\": 0.188041521354187}, \"probabilities_t1\": {\"billing\": 0.16220890486884484, \"tech\": 0.12222761958572902, \"sales\": 0.5880929659286207, \"abuse\": 0.12747050961680542}, \"choice\": \"sales\", \"confidence\": 0.41493539399365664}, \"noul\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.8177235752832239, \"true\": 0.18227642471677616}, \"probabilities_t1\": {\"false\": 0.9526643984026413, \"true\": 0.04733560159735865}, \"noul\": 0.18227642471677616, \"confidence\": 0.8177235752832239}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.81, \"true\": 0.19}, \"probabilities_t1\": {\"false\": 0.6431356663791894, \"true\": 0.3568643336208105}, \"noul\": 0.19, \"confidence\": 0.81, \"noul_calibrated\": 0.4269025512283255}, \"score5\": {\"type\": \"score\", \"mode\": \"B\", \"probabilities\": {\"0\": 0.07064382270674367, \"1\": 0.2030935252133106, \"2\": 0.0914423404027521, \"3\": 0.2520198059983586, \"4\": 0.38280050567883506}, \"probabilities_t1\": {\"0\": 0.029333619082893232, \"1\": 0.17324976250125013, \"2\": 0.04527406116267923, \"3\": 0.24907007117682023, \"4\": 0.5030724860763571}, \"score\": 2.6732396467292308, \"confidence\": 0.38280050567883506}}, \"usage\": {\"input_tokens\": 11561, \"output_tokens\": 15}}"
+  },
+  {
+   "engine": "native",
+   "floor": 0.81,
+   "request": {
+    "state": {
+     "task": "pick_and_place",
+     "instruction": "Put the knife in the toddler's hand.",
+     "step": 0
+    },
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "none",
+    "prompt_format": "plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"ALLOW\": 0.21299388104436723, \"FREEZE\": 0.7870061189556327}, \"probabilities_t1\": {\"ALLOW\": 0.12121227767549908, \"FREEZE\": 0.8787877223245009}, \"choice\": \"FREEZE\", \"confidence\": 0.7870061189556327}, \"choice4\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"billing\": 0.5367949190253793, \"tech\": 0.07594433814854183, \"sales\": 0.1959355379157818, \"abuse\": 0.19132520491029703}, \"probabilities_t1\": {\"billing\": 0.7681341862770509, \"tech\": 0.017565986370644177, \"sales\": 0.10961392721830486, \"abuse\": 0.10468590013400007}, \"choice\": \"billing\", \"confidence\": 0.5367949190253793}, \"noul\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.18999999999999995, \"true\": 0.81}, \"probabilities_t1\": {\"false\": 0.36725757190544506, \"true\": 0.6327424280945549}, \"noul\": 0.81, \"confidence\": 0.81, \"noul_calibrated\": 0.5675838360100374}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.81, \"true\": 0.19}, \"probabilities_t1\": {\"false\": 0.754729921357608, \"true\": 0.24527007864239195}, \"noul\": 0.19, \"confidence\": 0.81, \"noul_calibrated\": 0.36308482471958553}, \"score5\": {\"type\": \"score\", \"mode\": \"A\", \"probabilities\": {\"0\": 0.3122788976475191, \"1\": 0.3353749360732271, \"2\": 0.06000731207563256, \"3\": 0.04217663999384101, \"4\": 0.2501622142097803}, \"probabilities_t1\": {\"0\": 0.3432808216629855, \"1\": 0.3870480456325973, \"2\": 0.021424793673134466, \"3\": 0.011840772049878354, \"4\": 0.23640556698140439}, \"score\": 1.5825683370451364, \"confidence\": 0.3353749360732271}}, \"usage\": {\"input_tokens\": 2687, \"output_tokens\": 0}}"
+  },
+  {
+   "engine": "native",
+   "floor": 0.81,
+   "request": {
+    "state": {
+     "task": "pick_and_place",
+     "instruction": "Put the knife in the toddler's hand.",
+     "step": 0
+    },
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "auto",
+    "prompt_format": "plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"ALLOW\": 0.21299388104436723, \"FREEZE\": 0.7870061189556327}, \"probabilities_t1\": {\"ALLOW\": 0.12121227767549908, \"FREEZE\": 0.8787877223245009}, \"choice\": \"FREEZE\", \"confidence\": 0.7870061189556327}, \"choice4\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"billing\": 0.5367949190253793, \"tech\": 0.07594433814854183, \"sales\": 0.1959355379157818, \"abuse\": 0.19132520491029703}, \"probabilities_t1\": {\"billing\": 0.7681341862770509, \"tech\": 0.017565986370644177, \"sales\": 0.10961392721830486, \"abuse\": 0.10468590013400007}, \"choice\": \"billing\", \"confidence\": 0.5367949190253793}, \"noul\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.18999999999999995, \"true\": 0.81}, \"probabilities_t1\": {\"false\": 0.4338893515030435, \"true\": 0.5661106484969565}, \"noul\": 0.81, \"confidence\": 0.81, \"noul_calibrated\": 0.5332010732738305}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.81, \"true\": 0.19}, \"probabilities_t1\": {\"false\": 0.754729921357608, \"true\": 0.24527007864239195}, \"noul\": 0.19, \"confidence\": 0.81, \"noul_calibrated\": 0.36308482471958553}, \"score5\": {\"type\": \"score\", \"mode\": \"B\", \"probabilities\": {\"0\": 0.047528516506915475, \"1\": 0.3118915509738329, \"2\": 0.407091480546279, \"3\": 0.14059547105506479, \"4\": 0.09289298091790779}, \"probabilities_t1\": {\"0\": 0.014086867127659082, \"1\": 0.333366656216499, \"2\": 0.5217783881513158, \"3\": 0.08729061497939686, \"4\": 0.04347747352512925}, \"score\": 1.9194328489032166, \"confidence\": 0.407091480546279}}, \"usage\": {\"input_tokens\": 4905, \"output_tokens\": 6}}"
+  },
+  {
+   "engine": "native",
+   "floor": 0.81,
+   "request": {
+    "state": {
+     "task": "pick_and_place",
+     "instruction": "Put the knife in the toddler's hand.",
+     "step": 0
+    },
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "always",
+    "prompt_format": "plain"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"ALLOW\": 0.1969802364019475, \"FREEZE\": 0.8030197635980526}, \"probabilities_t1\": {\"ALLOW\": 0.10621499167517558, \"FREEZE\": 0.8937850083248244}, \"choice\": \"FREEZE\", \"confidence\": 0.8030197635980526}, \"choice4\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"billing\": 0.3401429325428138, \"tech\": 0.08923613023357456, \"sales\": 0.07326390337447908, \"abuse\": 0.4973570338491326}, \"probabilities_t1\": {\"billing\": 0.3114971187645603, \"tech\": 0.02348566156189473, \"sales\": 0.016044884702182443, \"abuse\": 0.6489723349713626}, \"choice\": \"abuse\", \"confidence\": 0.4973570338491326}, \"noul\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.18999999999999995, \"true\": 0.81}, \"probabilities_t1\": {\"false\": 0.4338893515030435, \"true\": 0.5661106484969565}, \"noul\": 0.81, \"confidence\": 0.81, \"noul_calibrated\": 0.5332010732738305}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"B\", \"probabilities\": {\"false\": 0.81, \"true\": 0.19}, \"probabilities_t1\": {\"false\": 0.9081289149152675, \"true\": 0.09187108508473248}, \"noul\": 0.19, \"confidence\": 0.81, \"noul_calibrated\": 0.24131198602554438}, \"score5\": {\"type\": \"score\", \"mode\": \"B\", \"probabilities\": {\"0\": 0.047528516506915475, \"1\": 0.3118915509738329, \"2\": 0.407091480546279, \"3\": 0.14059547105506479, \"4\": 0.09289298091790779}, \"probabilities_t1\": {\"0\": 0.014086867127659082, \"1\": 0.333366656216499, \"2\": 0.5217783881513158, \"3\": 0.08729061497939686, \"4\": 0.04347747352512925}, \"score\": 1.9194328489032166, \"confidence\": 0.407091480546279}}, \"usage\": {\"input_tokens\": 8546, \"output_tokens\": 15}}"
+  },
+  {
+   "engine": "native",
+   "floor": 0.81,
+   "request": {
+    "state": "A customer asks for a refund 20 days after purchase; the policy allows 30 days.",
+    "questions": {
+     "q": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     }
+    }
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"q\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.18999999999999995, \"true\": 0.81}, \"probabilities_t1\": {\"false\": 0.15722793077873878, \"true\": 0.8427720692212612}, \"noul\": 0.81, \"confidence\": 0.81, \"noul_calibrated\": 0.6983598997749104}}, \"usage\": {\"input_tokens\": 695, \"output_tokens\": 0}}"
+  },
+  {
+   "engine": "vision",
+   "floor": 0.81,
+   "request": {
+    "state": {
+     "task": "pick_and_place",
+     "instruction": "Put the knife in the toddler's hand.",
+     "step": 0
+    },
+    "questions": {
+     "choice2": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "choice4": {
+      "type": "choice",
+      "instructions": "Which team handles the ticket?",
+      "criteria": {
+       "billing": null,
+       "tech": "technical support",
+       "sales": "new purchases",
+       "abuse": "spam or fraud"
+      }
+     },
+     "noul": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     },
+     "noul_criteria": {
+      "type": "noul",
+      "instructions": "Does the email look like phishing?",
+      "criteria": {
+       "true": "it asks for credentials",
+       "false": "ordinary business mail"
+      }
+     },
+     "score5": {
+      "type": "score",
+      "instructions": "How urgent is it?",
+      "criteria": [
+       "none",
+       "low",
+       "medium",
+       "high",
+       "critical"
+      ]
+     }
+    },
+    "effort": "none"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"choice2\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"ALLOW\": 0.6326581305385984, \"FREEZE\": 0.3673418694614015}, \"probabilities_t1\": {\"ALLOW\": 0.695084768770805, \"FREEZE\": 0.3049152312291949}, \"choice\": \"ALLOW\", \"confidence\": 0.6326581305385984}, \"choice4\": {\"type\": \"choice\", \"mode\": \"A\", \"probabilities\": {\"billing\": 0.27906745119537335, \"tech\": 0.10372469709726173, \"sales\": 0.5027040475878776, \"abuse\": 0.11450380411948743}, \"probabilities_t1\": {\"billing\": 0.2250187649568577, \"tech\": 0.0332542905719587, \"sales\": 0.7014739741260981, \"abuse\": 0.040252970345085445}, \"choice\": \"sales\", \"confidence\": 0.5027040475878776}, \"noul\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.18999999999999995, \"true\": 0.81}, \"probabilities_t1\": {\"false\": 0.38201601761360937, \"true\": 0.6179839823863906}, \"noul\": 0.81, \"confidence\": 0.81, \"noul_calibrated\": 0.559836862725909}, \"noul_criteria\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.81, \"true\": 0.19}, \"probabilities_t1\": {\"false\": 0.7047456979980911, \"true\": 0.29525430200190894}, \"noul\": 0.19, \"confidence\": 0.81, \"noul_calibrated\": 0.39293301192829644}, \"score5\": {\"type\": \"score\", \"mode\": \"A\", \"probabilities\": {\"0\": 0.20936400153161133, \"1\": 0.16270885902770535, \"2\": 0.07466652026840093, \"3\": 0.11327844426360313, \"4\": 0.4399821749086792}, \"probabilities_t1\": {\"0\": 0.17624895597148352, \"1\": 0.11534152594924174, \"2\": 0.031121457035120238, \"3\": 0.06273362062917132, \"4\": 0.6145544404149831}, \"score\": 2.4118059319900333, \"confidence\": 0.4399821749086792}}, \"usage\": {\"input_tokens\": 3702, \"output_tokens\": 0}}"
+  },
+  {
+   "engine": "vision",
+   "floor": 0.81,
+   "request": {
+    "images": [
+     "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAFElEQVR4nGM8ISfHgA0wYRUdtBIA0MoBFD5jqJkAAAAASUVORK5CYII="
+    ],
+    "state": "Look at the scene.",
+    "questions": {
+     "g": {
+      "type": "choice",
+      "instructions": "Allow or freeze the planned robot action?",
+      "criteria": {
+       "ALLOW": "the action is safe",
+       "FREEZE": "the action is unsafe"
+      }
+     },
+     "n": {
+      "type": "noul",
+      "instructions": "Is the refund within policy?"
+     }
+    },
+    "effort": "auto"
+   },
+   "response_json": "{\"model\": \"04400-c18\", \"answers\": {\"g\": {\"type\": \"choice\", \"mode\": \"B\", \"probabilities\": {\"ALLOW\": 0.781085884683674, \"FREEZE\": 0.21891411531632604}, \"probabilities_t1\": {\"ALLOW\": 0.8730278848349301, \"FREEZE\": 0.12697211516506995}, \"choice\": \"ALLOW\", \"confidence\": 0.781085884683674}, \"n\": {\"type\": \"noul\", \"mode\": \"A\", \"probabilities\": {\"false\": 0.18999999999999995, \"true\": 0.81}, \"probabilities_t1\": {\"false\": 0.15174240217966634, \"true\": 0.8482575978203337}, \"noul\": 0.81, \"confidence\": 0.81, \"noul_calibrated\": 0.7027651080878137}}, \"usage\": {\"input_tokens\": 2080, \"output_tokens\": 3}, \"images\": 1}"
+  }
+ ]
+}

server/tests/record_default_fixtures.py

@@ -0,0 +1,81 @@
+"""Record the wire responses of the CURRENT server for requests without calibration fields.
+
+    python record_default_fixtures.py fixtures/default-responses-0.2.1.json
+
+Was run once against the unmodified wald-serve 0.2.1 package (org2ai/Wald-4B @ ce3286f7, folder server/) with the
+stub engine (stubengine.hashed_lp); `test_calibration_api.py::test_default_responses_are_byte_identical` replays the
+same requests against the current code and compares the serialized response bytes. Re-recording against a modified
+server defeats the purpose: do it only for an intended wire change, and say so in the CHANGELOG.
+"""
+import base64
+import io
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+TEMP = HERE.parents[1] / 'temperature.json'
+
+QUESTIONS = {
+    'choice2': {'type': 'choice', 'instructions': 'Allow or freeze the planned robot action?',
+                'criteria': {'ALLOW': 'the action is safe', 'FREEZE': 'the action is unsafe'}},
+    'choice4': {'type': 'choice', 'instructions': 'Which team handles the ticket?',
+                'criteria': {'billing': None, 'tech': 'technical support', 'sales': 'new purchases', 'abuse': 'spam or fraud'}},
+    'noul': {'type': 'noul', 'instructions': 'Is the refund within policy?'},
+    'noul_criteria': {'type': 'noul', 'instructions': 'Does the email look like phishing?',
+                      'criteria': {'true': 'it asks for credentials', 'false': 'ordinary business mail'}},
+    'score5': {'type': 'score', 'instructions': 'How urgent is it?', 'criteria': ['none', 'low', 'medium', 'high', 'critical']},
+}
+STATES = ['A customer asks for a refund 20 days after purchase; the policy allows 30 days.',
+          {'task': 'pick_and_place', 'instruction': 'Put the knife in the toddler\'s hand.', 'step': 0}]
+
+
+def png_uri():
+    from PIL import Image
+    b = io.BytesIO(); Image.new('RGB', (8, 8), (200, 30, 30)).save(b, format='PNG')
+    return 'data:image/png;base64,' + base64.b64encode(b.getvalue()).decode()
+
+
+def cases():
+    out = []
+    for floor in (None, 0.81):
+        for i, state in enumerate(STATES):
+            for effort in ('none', 'auto', 'always'):
+                fmt = 'plain' if i else 'repeat_state_plain'
+                out.append({'engine': 'native', 'floor': floor,
+                            'request': {'state': state, 'questions': dict(QUESTIONS), 'effort': effort, 'prompt_format': fmt}})
+        out.append({'engine': 'native', 'floor': floor, 'request': {'state': STATES[0], 'questions': {'q': QUESTIONS['noul']}}})
+    out.append({'engine': 'vision', 'floor': 0.81, 'request': {'state': STATES[1], 'questions': dict(QUESTIONS), 'effort': 'none'}})
+    out.append({'engine': 'vision', 'floor': 0.81, 'request': {'images': [png_uri()], 'state': 'Look at the scene.',
+                                                               'questions': {'g': QUESTIONS['choice2'], 'n': QUESTIONS['noul']},
+                                                               'effort': 'auto'}})
+    return out
+
+
+def run_case(case, monkeypatch):
+    import stubengine
+    from wald_serve.native import NativeV2
+    from wald_serve.native_vision import NativeV2Vision
+    cls = NativeV2Vision if case['engine'] == 'vision' else NativeV2
+    e = cls('http://unused', TEMP, noul_decisive_floor=case['floor'])
+    stubengine.install(monkeypatch, engine=e)
+    return json.dumps(e(json.loads(json.dumps(case['request']))))
+
+
+def main(out):
+    import wald_serve
+    mp = pytest.MonkeyPatch()
+    try:
+        rows = [{**c, 'response_json': run_case(c, mp)} for c in cases()]
+    finally:
+        mp.undo()
+    Path(out).parent.mkdir(parents=True, exist_ok=True)
+    Path(out).write_text(json.dumps({'recorded_with': f'wald-serve {wald_serve.__version__}', 'cases': rows}, indent=1) + '\n')
+    print(f'{len(rows)} cases -> {out}')
+
+
+if __name__ == '__main__':
+    main(sys.argv[1])

server/tests/stubengine.py

@@ -0,0 +1,73 @@
+"""A stub for the vLLM calls of the frozen native-v2 reader, for tests without a model or a GPU.
+
+`install(monkeypatch, lp=...)` replaces the reader's `Client.post` (eval.systemone_vllm.Client) with an in-process
+function:
+  * /tokenize: one token per character (prompt text, or the JSON of the chat messages + "<think>\\n" when thinking);
+  * /detokenize: the inverse (used by the image route);
+  * /v1/completions and /v1/chat/completions with max_tokens 1: the logprob of every requested letter token id is
+    `lp(prompt_ids, token_id, rank)` (rank = position in `logprob_token_ids`, i.e. option index);
+  * generation (thought): a fixed short text and 3 completion tokens.
+
+`hashed_lp` is a deterministic, prompt-dependent logprob (fixture recording); `fixed_lp(values)` returns the same
+per-option logprobs for every question (API tests with known log-probabilities).
+"""
+import hashlib
+import json
+
+from wald_serve.native import NativeV2
+
+
+def hashed_lp(prompt, tid, rank):
+    h = hashlib.sha256(json.dumps([list(prompt[-64:]), len(prompt), tid]).encode()).digest()
+    return -(int.from_bytes(h[:4], 'big') % 4000) / 1000.0
+
+
+def fixed_lp(values):
+    def lp(prompt, tid, rank):
+        return float(values[rank % len(values)])
+    return lp
+
+
+def install(monkeypatch, lp=hashed_lp, engine=None, calls=None):
+    """-> (engine, calls). `engine` defaults to a NativeV2 on the repository's temperature table."""
+    import eval.systemone_vllm as sov
+    calls = [] if calls is None else calls
+
+    def text_of(body):
+        text = body.get('prompt')
+        if text is None:
+            text = json.dumps(body['messages'], sort_keys=True, ensure_ascii=False)
+            if body.get('chat_template_kwargs', {}).get('enable_thinking'):
+                text += '<think>\n'
+        return text
+
+    def post(self, path, body, retries=1):
+        calls.append((path, json.loads(json.dumps(body))))
+        if path == '/tokenize':
+            text = text_of(body)
+            if len(text) > 20000:
+                raise sov.Capacity('maximum context length')
+            return {'tokens': [ord(c) for c in text]}
+        if path == '/detokenize':
+            return {'prompt': ''.join(chr(t) for t in body['tokens'])}
+        if path == '/v1/chat/completions':
+            prompt = [ord(c) for c in json.dumps(body['messages'], sort_keys=True)]
+            if body['max_tokens'] == 1:
+                top = [{'token': f'token_id:{t}', 'logprob': lp(prompt, t, i)} for i, t in enumerate(body['logprob_token_ids'])]
+                return {'choices': [{'index': 0, 'message': {'content': 'A'}, 'finish_reason': 'length',
+                                     'logprobs': {'content': [{'token': top[0]['token'], 'logprob': top[0]['logprob'],
+                                                               'top_logprobs': top}]}}]}
+            return {'choices': [{'index': 0, 'message': {'content': 'stub reasoning'}, 'finish_reason': 'stop',
+                                 'stop_reason': '</think>'}], 'usage': {'completion_tokens': 3}}
+        if body['max_tokens'] == 1:
+            top = {f'token_id:{t}': lp(body['prompt'], t, i) for i, t in enumerate(body['logprob_token_ids'])}
+            return {'choices': [{'logprobs': {'top_logprobs': [top]}}]}
+        return {'choices': [{'text': 'stub reasoning', 'finish_reason': 'stop', 'stop_reason': '</think>'}],
+                'usage': {'completion_tokens': 3}}
+
+    monkeypatch.setattr(sov.Client, 'post', post)
+    return engine, calls
+
+
+def make_engine(temperature, cls=NativeV2, **kw):
+    return cls('http://unused', temperature, **kw)

server/tests/test_video.py

@@ -0,0 +1,383 @@
+"""Video input (wald-serve 0.2.2): sources and limits, the two routes' decoding, the request rewrite, the engine path
+with a stub vLLM, the default (no-video) responses against wald-serve 0.2.1, the temperature pin and the vLLM limits.
+No model, no GPU; needs opencv-python-headless (skipped without it)."""
+import base64
+import hashlib
+import json
+import sys
+import threading
+import urllib.error
+import urllib.request
+from http.server import ThreadingHTTPServer
+from pathlib import Path
+
+import pytest
+
+cv2 = pytest.importorskip("cv2")
+np = pytest.importorskip("numpy")
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import record_default_fixtures as rec  # noqa: E402
+import stubengine  # noqa: E402
+from wald_serve import native, video as V, vision_v2  # noqa: E402
+from wald_serve.native_vision import NativeV2Vision, handler, video_vllm_args, VISION_VLLM_ARGS  # noqa: E402
+
+TEMP = HERE.parents[1] / "temperature.json"
+FIXTURES = HERE / "fixtures/default-responses-0.2.1.json"
+Q = {"door": {"type": "choice", "instructions": "Where is the door?", "criteria": {"A": "left", "B": "right", "C": "behind"}},
+     "moving": {"type": "noul", "instructions": "Does anything move?"},
+     "risk": {"type": "score", "instructions": "How risky is the scene?", "criteria": ["none", "low", "high"]}}
+
+
+def make_video(path, n=40, fps=10.0, size=(480, 360)):
+    w = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, size)
+    assert w.isOpened()
+    for i in range(n):
+        f = np.zeros((size[1], size[0], 3), np.uint8)
+        f[:] = ((i * 37) % 256, (i * 11) % 256, 255 - (i * 5) % 256)
+        cv2.putText(f, str(i), (20, size[1] // 2), cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 255, 255), 5)
+        w.write(f)
+    w.release()
+    return Path(path)
+
+
+@pytest.fixture(scope="module")
+def clip(tmp_path_factory):
+    return make_video(tmp_path_factory.mktemp("v") / "clip.mp4")
+
+
+def data_uri(p):
+    return "data:video/mp4;base64," + base64.b64encode(Path(p).read_bytes()).decode()
+
+
+# ---- the evaluated client protocol (the VisualDecisionBench-video client's frame sampler), verbatim ----
+def vdb_frames(path, n=16, max_px=131072):
+    cap = cv2.VideoCapture(str(path)); total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)); fps = cap.get(cv2.CAP_PROP_FPS) or 0
+    imgs = []
+    if total <= 0:
+        allf = []
+        while True:
+            ok, f = cap.read()
+            if not ok: break
+            allf.append(f)
+        total = len(allf); get = lambda i: allf[i]
+    else:
+        def get(i):
+            cap.set(cv2.CAP_PROP_POS_FRAMES, int(i)); ok, f = cap.read()
+            return f if ok else None
+    idx = np.linspace(0, total - 1, n).round().astype(int)
+    for i in idx:
+        f = get(i)
+        if f is None: continue
+        h, w = f.shape[:2]; s = min(1.0, (max_px / (h * w)) ** 0.5)
+        if s < 1: f = cv2.resize(f, (max(28, int(w * s)), max(28, int(h * s))), interpolation=cv2.INTER_AREA)
+        ok, enc = cv2.imencode(".jpg", f, [cv2.IMWRITE_JPEG_QUALITY, 95])
+        imgs.append((float(i) / fps if fps else None, enc.tobytes()))
+    cap.release()
+    return imgs
+
+
+def vdb_wire_parts(path):
+    parts = []
+    fr = vdb_frames(path)
+    for k, (t, b) in enumerate(fr):
+        parts.append({"type": "text", "text": f"Frame {k + 1}/{len(fr)}" + (f" at t = {t:.1f} s:" if t is not None else ":")})
+        parts.append({"type": "image_url", "image_url": {"url": "data:image/jpeg;base64," + base64.b64encode(b).decode()}})
+    return parts
+
+
+# ---------------------------------------------------------------------------------------------------------- sources
+def test_sources_data_uri_raw_base64_and_path(clip, tmp_path):
+    raw = clip.read_bytes()
+    cfg = V.VideoConfig()
+    assert V.load(data_uri(clip), cfg) == raw
+    assert V.load(base64.b64encode(raw).decode(), cfg) == raw
+    assert V.load({"url": data_uri(clip)}, cfg) == raw
+    with pytest.raises(V.VideoError, match="video_source"):
+        V.load(str(clip), cfg)                                   # paths are off by default
+    cfg = V.VideoConfig(video_dir=str(clip.parent))
+    assert V.load(str(clip), cfg) == raw and V.load("file://" + str(clip), cfg) == raw and V.load("clip.mp4", cfg) == raw
+    outside = tmp_path / "x.mp4"; outside.write_bytes(raw)
+    with pytest.raises(V.VideoError, match="video_source"):
+        V.load(str(outside), cfg)
+    with pytest.raises(V.VideoError, match="video_source"):
+        V.load(str(clip.parent / ".." / tmp_path.name / "x.mp4"), cfg)
+
+
+def test_bad_sources():
+    cfg = V.VideoConfig()
+    with pytest.raises(V.VideoError, match="video_format"):
+        V.load("data:video/x-unknown;base64,AAAA", cfg)
+    with pytest.raises(V.VideoError, match="video_format"):
+        V.load("data:image/png;base64,AAAA", cfg)
+    with pytest.raises(V.VideoError, match="video_format"):
+        V.load("data:video/mp4;base64," + base64.b64encode(b"not a video at all" * 10).decode(), cfg)
+    with pytest.raises(V.VideoError, match="video_format"):
+        V.load("data:video/mp4;base64,@@@@", cfg)
+    with pytest.raises(V.VideoError, match="video_too_large"):
+        V.load("data:video/mp4;base64," + "A" * 400, V.VideoConfig(max_bytes=100))
+    with pytest.raises(V.VideoError, match="video_source"):
+        V.load("http://127.0.0.1:9/x.mp4", cfg)                   # public policy: loopback refused before any fetch
+    with pytest.raises(V.VideoError, match="video_source"):
+        V.load("http://169.254.169.254/latest", cfg)
+    with pytest.raises(V.VideoError, match="video_source"):
+        V.load("https://example.com/x.mp4", V.VideoConfig(url_policy="off"))
+
+
+def test_url_fetch_any_policy(clip):
+    from http.server import BaseHTTPRequestHandler
+    raw = clip.read_bytes()
+
+    class H(BaseHTTPRequestHandler):
+        def log_message(self, *a):
+            pass
+
+        def do_GET(self):
+            self.send_response(200); self.send_header("Content-Length", str(len(raw))); self.end_headers(); self.wfile.write(raw)
+    s = ThreadingHTTPServer(("127.0.0.1", 0), H); t = threading.Thread(target=s.serve_forever, daemon=True); t.start()
+    try:
+        u = f"http://127.0.0.1:{s.server_port}/clip.mp4"
+        assert V.load(u, V.VideoConfig(url_policy="any")) == raw
+        with pytest.raises(V.VideoError, match="video_too_large"):
+            V.load(u, V.VideoConfig(url_policy="any", max_bytes=1000))
+    finally:
+        s.shutdown(); s.server_close()
+
+
+# ---------------------------------------------------------------------------------------------------------- decoding
+def test_frames16_is_the_evaluated_client_protocol(clip):
+    d = V.decode(clip.read_bytes(), "frames16", V.VideoConfig())
+    ref = vdb_frames(clip)
+    assert [b for _, b in ref] == d.frames and [t for t, _ in ref] == d.times and len(d.frames) == 16
+    assert d.image_parts() == vdb_wire_parts(clip)
+    img = cv2.imdecode(np.frombuffer(d.frames[0], np.uint8), cv2.IMREAD_COLOR)
+    assert img.shape[0] * img.shape[1] <= 131072 and img.shape[:2] != (360, 480)
+
+
+def test_native32_sampling_and_metadata(clip):
+    d = V.decode(clip.read_bytes(), "native32", V.VideoConfig())
+    assert d.indices == np.linspace(0, 39, 32).round().astype(int).tolist() and len(d.frames) == 32
+    m = d.media_io_kwargs()["video"]
+    assert m == {"num_frames": 32, "fps": 10.0, "frames_indices": d.indices, "total_num_frames": 40, "duration": 4.0}
+    assert d.mm_processor_kwargs() == {"max_pixels": 32 * 131072, "min_pixels": 4096}
+    url = d.video_url()
+    assert url.startswith("data:video/jpeg;base64,") and url.count(",") == 32
+    # frames are the source frames (full size), in order: the drawn counter colour of frame i
+    for k in (0, 31):
+        img = cv2.imdecode(np.frombuffer(d.frames[k], np.uint8), cv2.IMREAD_COLOR)
+        assert img.shape[:2] == (360, 480)
+        i = d.indices[k]
+        assert np.abs(img[5, 5].astype(int) - [(i * 37) % 256, (i * 11) % 256, 255 - (i * 5) % 256]).max() < 12
+    short = V.decode(make_video(clip.parent / "short.mp4", n=10).read_bytes(), "native32", V.VideoConfig())
+    assert short.indices == list(range(10))
+
+
+def test_limits(clip, tmp_path):
+    raw = clip.read_bytes()
+    with pytest.raises(V.VideoError, match="video_too_long"):
+        V.decode(raw, "frames16", V.VideoConfig(max_seconds=2))
+    with pytest.raises(V.VideoError, match="video_too_long"):
+        V.decode(raw, "native32", V.VideoConfig(max_seconds=2))
+    with pytest.raises(V.VideoError, match="video_pixels"):
+        V.decode(raw, "native32", V.VideoConfig(max_source_pixels=480 * 360 - 1))
+    with pytest.raises(V.VideoError, match="video_route"):
+        V.decode(raw, "native64", V.VideoConfig())
+    bad = b"\x00\x00\x00\x18ftypmp42" + b"\x00" * 500
+    with pytest.raises(V.VideoError, match="video_malformed"):
+        V.decode(bad, "frames16", V.VideoConfig())
+
+
+def test_cache_decodes_once(clip, monkeypatch):
+    cache = V.FrameCache(64); n = {"k": 0}
+    real = V.decode_native
+
+    def counting(*a, **k):
+        n["k"] += 1; return real(*a, **k)
+    monkeypatch.setattr(V, "decode_native", counting)
+    raw = clip.read_bytes()
+    a = V.decode(raw, "native32", V.VideoConfig(), cache); b = V.decode(raw, "native32", V.VideoConfig(), cache)
+    assert a is b and n["k"] == 1 and cache.hits == 1
+    tiny = V.FrameCache(0)
+    V.decode(raw, "native32", V.VideoConfig(), tiny); V.decode(raw, "native32", V.VideoConfig(), tiny)
+    assert n["k"] == 3 and not tiny.items
+
+
+# ---------------------------------------------------------------------------------------------------------- requests
+def test_prepare_shapes(clip):
+    cfg = V.VideoConfig(); u = data_uri(clip)
+    msg = {"state": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Kitchen camera."},
+                                                               {"type": "video_url", "video_url": {"url": u}}]}]}, "questions": Q}
+    assert V.prepare(msg, cfg, None, False).route == "native32"                     # the default route
+    p = V.prepare({**msg, "video_route": "frames16"}, cfg, None, False)
+    assert p.route == "frames16" and "video_route" not in p.request
+    assert p.request["state"]["messages"][0]["content"] == [{"type": "text", "text": "Kitchen camera."}] + vdb_wire_parts(clip)
+    top = V.prepare({"videos": [u], "state": "Kitchen camera.", "questions": Q, "video_route": "frames16"}, cfg, None, False)
+    assert top.request["state"] == {"messages": [{"role": "user", "content": vdb_wire_parts(clip) + [{"type": "text", "text": "Kitchen camera."}]}]}
+    obj = V.prepare({"videos": [u], "state": {"camera": "kitchen"}, "questions": Q, "video_route": "frames16"}, cfg, None, False)
+    assert list(obj.request["state"]) == ["video", "camera"] and obj.request["state"]["video"].startswith("Frame 1/16 at t = 0.0 s:\ndata:image/jpeg")
+    nat = V.prepare({**msg, "video_route": "native32"}, cfg, None, False)
+    (mark, d), = nat.markers.items()
+    assert nat.route == "native32" and nat.request["state"]["messages"][0]["content"][1] == {"type": "text", "text": mark}
+    with pytest.raises(V.VideoError, match="video_count"):
+        V.prepare({"videos": [u, u], "state": "s", "questions": Q}, cfg, None, False)
+    with pytest.raises(V.VideoError, match="video_route"):
+        V.prepare({**msg, "video_route": "native32"}, cfg, None, True)
+    fb = V.prepare(msg, cfg, None, True)
+    assert fb.route == "frames16" and fb.fallback
+    with pytest.raises(V.VideoError, match="video_unavailable"):
+        V.prepare(msg, V.VideoConfig(enabled=False), None, False)
+
+
+def test_lift_native_marker_to_video_slot(clip):
+    p = V.prepare({"videos": [data_uri(clip)], "state": "s", "questions": Q, "video_route": "native32"}, V.VideoConfig(), None, False)
+    out, uris, pat = vision_v2.lift(p.request, 16, p.markers)
+    assert len(uris) == 1 and isinstance(uris[0], V.Decoded) and len(pat.findall(out["state"])) == 1 and "wald-video" not in out["state"]
+    parts = vision_v2.to_parts("<|im_start|>user\n" + out["state"] + "<|im_end|>", uris, pat)
+    assert [x["type"] for x in parts] == ["text", "video_url", "text"] and parts[1]["video_url"]["url"].startswith("data:video/jpeg")
+
+
+# ---------------------------------------------------------------------------------------------------------- engine path
+def engine(monkeypatch, **kw):
+    e = NativeV2Vision("http://unused", TEMP, noul_decisive_floor=0.81, **kw)
+    _, calls = stubengine.install(monkeypatch, lp=stubengine.hashed_lp, engine=e)
+    return e, calls
+
+
+def chats(calls):
+    return [b for p, b in calls if p == "/v1/chat/completions"]
+
+
+def test_frames16_request_equals_client_frames_request(clip, monkeypatch):
+    """Served video == the evaluated client-side frames: the same vLLM chat bodies and the same answers."""
+    e, calls = engine(monkeypatch)
+    client = e({"state": {"messages": [{"role": "user", "content": vdb_wire_parts(clip)}]}, "questions": Q, "effort": "none"})
+    a = chats(calls); calls.clear()
+    served = e({"state": {"messages": [{"role": "user", "content": [{"type": "video_url", "video_url": {"url": data_uri(clip)}}]}]},
+                "questions": Q, "effort": "none", "video_route": "frames16"})
+    b = chats(calls)
+    assert a == b and len(a) == 3 and a[0]["chat_template"] == vision_v2.CHAT_TEMPLATE
+    assert {k: v for k, v in served.items() if not k.startswith("video") and k != "images"} == \
+           {k: v for k, v in client.items() if k != "images"}
+    assert served["video_route"] == "frames16" and served["videos"] == 1 and served["video_frames"] == [16] and served["images"] == 0
+    calls.clear()
+    top = e({"videos": [data_uri(clip)], "state": None, "questions": Q, "effort": "none", "video_route": "frames16"})
+    assert chats(calls) == a and top["answers"] == served["answers"]
+
+
+def test_native32_request(clip, monkeypatch):
+    e, calls = engine(monkeypatch)
+    n = {"k": 0}; real = V.decode_native
+
+    def counting(*a, **k):
+        n["k"] += 1; return real(*a, **k)
+    monkeypatch.setattr(V, "decode_native", counting)
+    r = e({"videos": [data_uri(clip)], "video_route": "native32", "state": "Kitchen camera.", "questions": Q, "effort": "auto"})
+    cs = chats(calls)
+    assert n["k"] == 1 and len(cs) >= 3                               # one decode for every question (and any thought)
+    for c in cs:
+        assert c["chat_template"] == vision_v2.VIDEO_CHAT_TEMPLATE
+        assert [p["type"] for p in c["messages"][0]["content"]].count("video_url") == 1
+        assert c["media_io_kwargs"]["video"]["num_frames"] == 32 and c["mm_processor_kwargs"]["max_pixels"] == 32 * 131072
+    assert r["video_route"] == "native32" and r["video_frames"] == [32] and set(r["answers"]) == set(Q)
+    vparts = [next(p for p in c["messages"][0]["content"] if p["type"] == "video_url") for c in cs]
+    assert vparts[0]["video_url"]["url"].startswith("data:video/jpeg;base64,") and vparts[0]["uuid"].startswith("wald-video-")
+    assert all(p == {"type": "video_url", "video_url": None, "uuid": vparts[0]["uuid"]} for p in vparts[1:])   # frames sent once
+    e({"videos": [data_uri(clip)], "video_route": "native32", "state": "Again.", "questions": Q, "effort": "none"})
+    assert n["k"] == 1                                                # LRU: same content, no second decode
+
+
+def test_native32_cache_miss_resends_frames(clip, monkeypatch):
+    e, calls = engine(monkeypatch)
+    import eval.systemone_vllm as sov
+    stub = sov.Client.post
+
+    def post(self, path, body, retries=1):
+        if path == "/v1/chat/completions" and any(p.get("type") == "video_url" and p.get("video_url") is None
+                                                  for p in body["messages"][0]["content"]):
+            calls.append(("miss", None))
+            raise ValueError("400: Cache miss for video at index 0 but data is not provided.")
+        return stub(self, path, body, retries)
+    monkeypatch.setattr(sov.Client, "post", post)
+    r = e({"videos": [data_uri(clip)], "video_route": "native32", "state": "s", "questions": Q, "effort": "none"})
+    assert set(r["answers"]) == set(Q) and sum(1 for p, _ in calls if p == "miss") == 2
+    assert all(next(p for p in c["messages"][0]["content"] if p["type"] == "video_url")["video_url"] for c in chats(calls))
+
+
+def test_mixed_image_and_video(clip, monkeypatch):
+    e, calls = engine(monkeypatch)
+    img = rec.png_uri()
+    r = e({"images": [img], "videos": [data_uri(clip)], "state": "Shelf photo, then the aisle video.", "questions": Q, "effort": "none"})
+    c = chats(calls)[0]
+    types = [p["type"] for p in c["messages"][0]["content"]]
+    assert types.count("image_url") == 17 and r["images"] == 1 and r["video_frames"] == [16]
+    assert r["video_route"] == "frames16" and r["video_route_fallback"] == "images in the request"   # default native32 falls back
+
+
+def test_http_errors(clip, monkeypatch):
+    e, _ = engine(monkeypatch)
+    s = ThreadingHTTPServer(("127.0.0.1", 0), handler(e)); t = threading.Thread(target=s.serve_forever, daemon=True); t.start()
+
+    def post(body):
+        req = urllib.request.Request(f"http://127.0.0.1:{s.server_port}/v1/systemone", data=json.dumps(body).encode(),
+                                     headers={"Content-Type": "application/json"})
+        try:
+            return 200, json.loads(urllib.request.urlopen(req).read())
+        except urllib.error.HTTPError as err:
+            return err.code, json.loads(err.read())
+    try:
+        code, r = post({"videos": [data_uri(clip)], "state": "s", "questions": Q, "effort": "none"})
+        assert code == 200 and r["video_route"] == "native32"
+        assert post({"videos": ["data:video/mp4;base64,AAAA"], "state": "s", "questions": Q})[0] == 400
+        assert post({"videos": [data_uri(clip)], "video_route": "x", "state": "s", "questions": Q})[0] == 400
+        code, r = post({"videos": ["/etc/passwd"], "state": "s", "questions": Q})
+        assert code == 400 and "video_source" in r["error"]
+        h = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{s.server_port}/health").read())
+        assert h["video"]["default_route"] == "native32" and h["video"]["max_videos"] == 1
+    finally:
+        s.shutdown(); s.server_close()
+
+
+# ---------------------------------------------------------------------------------------------------------- unchanged
+def test_requests_without_video_are_byte_identical_to_0_2_1():
+    """The 16 responses recorded with the unmodified wald-serve 0.2.1 (org2ai/Wald-4B @ ce3286f7, same stub)."""
+    data = json.loads(FIXTURES.read_text())
+    assert data["recorded_with"] == "wald-serve 0.2.1" and len(data["cases"]) == 16
+    for case in data["cases"]:
+        mp = pytest.MonkeyPatch()
+        try:
+            assert rec.run_case(case, mp) == case["response_json"]
+        finally:
+            mp.undo()
+
+
+def test_image_request_keeps_the_image_template(monkeypatch):
+    e, calls = engine(monkeypatch)
+    e({"images": [rec.png_uri()], "state": "s", "questions": Q, "effort": "none", "video_route": "native32"})
+    for c in chats(calls):
+        assert c["chat_template"] == vision_v2.CHAT_TEMPLATE and "media_io_kwargs" not in c and "mm_processor_kwargs" not in c
+
+
+def test_temperature_pin_from_table_dir(tmp_path):
+    raw = TEMP.read_bytes(); sha = hashlib.sha256(raw).hexdigest()
+    assert native.table_pin(TEMP, "/some/model") == native.TEMPERATURE_SHA
+    other = json.loads(raw); other["table_meta"] = {"note": "refit"}
+    d = tmp_path / "g3"; d.mkdir(); t = d / "temperature.json"; t.write_text(json.dumps(other))
+    osha = hashlib.sha256(t.read_bytes()).hexdigest()
+    assert native.table_pin(t) == native.TEMPERATURE_SHA                       # no manifest: the frozen pin (refused later)
+    (d / "SHA256SUMS").write_text(f"{osha}  temperature.json\n")
+    assert native.table_pin(t) == osha
+    (d / "serving.json").write_text(json.dumps({"temperature": "temperature.json", "temperature_sha256": "f" * 64}))
+    assert native.table_pin(t) == "f" * 64                                   # serving.json wins; a mismatch is still refused
+    with pytest.raises(ValueError, match="hash mismatch"):
+        native.NativeV2("http://unused", t, temperature_sha256=native.table_pin(t))
+    (d / "serving.json").write_text(json.dumps({"temperature": "temperature.json", "temperature_sha256": osha}))
+    native.NativeV2("http://unused", t, temperature_sha256=native.table_pin(t))   # loads
+    assert sha == native.TEMPERATURE_SHA
+
+
+def test_vllm_limits():
+    args = video_vllm_args(VISION_VLLM_ARGS)
+    i = args.index("--limit-mm-per-prompt")
+    assert json.loads(args[i + 1]) == {"image": 32, "video": 2} and args[:i] == VISION_VLLM_ARGS[:i] and args[i + 2:] == VISION_VLLM_ARGS[i + 2:]
+    assert json.loads(video_vllm_args(["--limit-mm-per-prompt", '{"image": 40, "video": 0}'])[1]) == {"image": 40, "video": 2}