"""Local playground HTTP API for the imajev decision model — see docs/playground-spec.md ("HTTP API"). Run it directly: .venv/bin/python scripts/playground/server.py --backend auto The model loads once at startup and stays resident; a lock keeps requests strictly sequential because there is only one GPU. """ from __future__ import annotations import argparse import base64 import binascii import json import logging import mimetypes import re import sys import threading from datetime import datetime, timezone from pathlib import Path from time import perf_counter ROOT = Path(__file__).resolve().parents[2] for extra in (ROOT / "src", ROOT / "scripts", Path(__file__).resolve().parent): if str(extra) not in sys.path: sys.path.insert(0, str(extra)) from fastapi import FastAPI, HTTPException # noqa: E402 from fastapi.exceptions import RequestValidationError # noqa: E402 from fastapi.responses import FileResponse, HTMLResponse, JSONResponse # noqa: E402 from fastapi.staticfiles import StaticFiles # noqa: E402 from pydantic import ValidationError # noqa: E402 from starlette.concurrency import run_in_threadpool # noqa: E402 from starlette.requests import Request as HttpRequest # noqa: E402 from vision_decision.images import MAX_BYTES, load_image_bytes # noqa: E402 from vision_decision.jev_api import MAX_OPTIONS, to_request_with_plan, to_response # noqa: E402 from vision_decision.thinking import ThinkingPolicy # noqa: E402 from vision_decision.scoring import (check_prompt_layout, check_readout_codes, combine_rotations, compile_question, # noqa: E402 cyclic_offsets, result_from_logits, rotate) try: # package import (PYTHONPATH=scripts) or plain script run from .examples import load_examples except ImportError: # pragma: no cover - exercised only when run as a script from examples import load_examples # Public model names. The base checkpoint behind the adapter is an implementation detail and is # deliberately not part of the API or the UI; `/v1/models` still reports the adapter directory. MODEL_NAME = "imajev-v1.1" BASE_MODEL_NAME = "imajev-v1.1-base" MLX_ADAPTER = ROOT / "reports/decision-v1/runs/h100x4-full/best-mlx" TORCH_ADAPTER = ROOT / "reports/decision-v1/runs/h100x4-full/best" BUNDLE = ROOT / "artifacts/model.json" STATIC = Path(__file__).resolve().parent / "static" DATA_URL = re.compile(r"^data:(?P[\w.+-]+/[\w.+-]+)?(?P;base64)?,(?P.*)$", re.DOTALL) log = logging.getLogger("playground") class PlaygroundError(Exception): """A request the server refuses; `status` is the HTTP code and `detail` the human message.""" def __init__(self, status, error, detail): super().__init__(detail) self.status, self.error, self.detail = status, error, detail def _image_status(message): """20 MiB / 20 Mpixel limits are 413; everything else about an image is a 422.""" return 413 if "exceeds 20 MiB" in message or "20 million decoded pixels" in message else 422 # --------------------------------------------------------------------------------------- backends class MLXBackend: """MLX path: one image prefill shared by every question of the request.""" name = "mlx" def __init__(self, bundle=BUNDLE, adapter=None, rotations=1, max_input_tokens=4096, readout_codes=None, prompt_layout=None): from vision_decision.backend import MLXDirect self.engine = MLXDirect(str(bundle), adapter=None if adapter is None else str(adapter), max_input_tokens=max_input_tokens, readout_codes=readout_codes, prompt_layout=prompt_layout) self.readout_codes, self.prompt_layout = self.engine.codes, self.engine.prompt_layout self.max_options = self.engine.max_options _warn_layout(self.engine.trained_prompt_layout, self.prompt_layout) self.adapter = None if adapter is None else str(adapter) self.model = MODEL_NAME if adapter else BASE_MODEL_NAME self.load_seconds = self.engine.load_seconds self.rotations = rotations def score(self, images, request): # One image is passed on its own, a pair as a list; [] is a text-only request. target = images[0] if len(images) == 1 else list(images) results, run = self.engine.score_request(target, request.fields, request.state, self.rotations) passes = [row for detail in run.get("questions", []) for row in detail["rotations"]] return results, { "prefill_ms": round(run.get("prefill_seconds", 0.0) * 1000, 1), "questions_ms": round(sum(row["forward_seconds"] for row in passes) * 1000, 1), "input_tokens": max((row["input_tokens"] for row in passes), default=0), } class TorchBackend: """PyTorch fallback: one full forward per question, never a batch (left padding NaNs on MPS).""" name = "torch" supports_thinking = True def __init__(self, bundle=BUNDLE, adapter=None, device=None, rotations=1, max_input_tokens=4096, readout_codes=None, prompt_layout=None, fast=False, graph_lengths=None, merge_lora=False, float32=False): import torch from torch_decision import GRAPH_LENGTHS, TorchDecision self.torch = torch self.rotations = max(1, int(rotations)) self.fast = bool(fast) if device is None: # CUDA on a pod or Space, Metal on a Mac, CPU otherwise device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" self.bundle = json.loads(Path(bundle).read_text()) if not Path(self.bundle["path"]).is_dir(): raise ValueError("Local model snapshot is missing; run scripts/download_model.py") start = perf_counter() dtype = torch.bfloat16 if device == "cuda" and not float32 else torch.float32 # --merge-lora loads in float32 so the merge is formed at full precision, then rounds once to the serving dtype. # In bf16 that rounding still moves near-tie decisions (and calibration), so it is opt-in; --fast alone keeps # the LoRA unmerged and computes exactly the benchmarked function. merge = bool(merge_lora) and adapter is not None # The float32 merge runs on the CPU (no float32 copy on the GPU); the rounded weights then move to the device. self.engine = TorchDecision(self.bundle["path"], "cpu" if merge else device, dtype=torch.float32 if merge else dtype, max_length=int(max_input_tokens)) if adapter is not None: if merge: self.engine.merge_adapter(adapter, dtype) self.engine.model.to(device); self.engine.device = device else: from peft import PeftModel self.engine.model = PeftModel.from_pretrained(self.engine.model, str(adapter)).eval() self.engine.enable_readout(adapter, trainable=False, codes=readout_codes) self.graph_lengths = [] if self.fast and device == "cuda": lengths = [n for n in (graph_lengths or GRAPH_LENGTHS) if n <= int(max_input_tokens)] try: self.graph_lengths = self.engine.capture_graphs(lengths).lengths except Exception: # the eager path computes the same function log.exception("CUDA graph capture failed; serving the eager path") self.engine.graphs = None if self.engine.readout is None and readout_codes is not None: # base model / legacy adapter: LM-head rows self.engine.codes = check_readout_codes(readout_codes) trained = self.engine.prompt_layout # the adapter's recorded layout (standard when absent) self.prompt_layout = trained if prompt_layout is None else check_prompt_layout(prompt_layout) _warn_layout(trained, self.prompt_layout) self.readout_codes, self.max_options = self.engine.codes, self.engine.max_options self.adapter = None if adapter is None else str(adapter) self.model = MODEL_NAME if adapter else BASE_MODEL_NAME self.load_seconds = perf_counter() - start def score(self, images, request, thinking=None): """One forward per question and per presentation order; with rotations > 1 the per-candidate log-probabilities are averaged over cyclic option orders exactly as the MLX path does (combine_rotations), which drops the engine's vocabulary tie break; with rotations == 1 the single pass keeps it. thinking (a ThinkingPolicy, None = off): a question whose single-pass answer the policy calls unsure gets a greedy thought of at most max_tokens, and its decision is read again right after the thought (docs/think-if-unsure-plan.md).""" if thinking is not None and thinking.active and self.rotations != 1: raise PlaygroundError(422, "invalid_request", "thinking needs a server running --rotations 1") results, seconds, tokens, thoughts = [], 0.0, 0, [] for field in request.fields: header, choices, texts = compile_question(field, request.state, getattr(self, "prompt_layout", "standard")) labels = self.engine.labels(len(choices), len(images)) start = perf_counter() passes = [] for offset in cyclic_offsets(len(choices), self.rotations): prompt = header + "\n".join(f"{label}: {text}" for label, text in zip(labels, rotate(texts, offset))) if getattr(self, "fast", False): with self.torch.inference_mode(): _, inputs, token_ids = self.engine.prepare_fast(images, prompt, labels) logits = [float(x) for x in self.engine.candidate_logits_fast(inputs, token_ids).cpu().tolist()] else: with self.torch.no_grad(): _, inputs, token_ids = self.engine.prepare(images, prompt, labels) logits = [float(x) for x in self.engine.candidate_logits(inputs, token_ids).cpu().tolist()] tokens = max(tokens, int(inputs["input_ids"].shape[-1])) passes.append((offset, logits, token_ids)) seconds += perf_counter() - start if len(passes) == 1: result = result_from_logits(choices, passes[0][1], token_ids=passes[0][2]) else: result = combine_rotations(choices, [(offset, logits) for offset, logits, _ in passes]) if thinking is not None and thinking.should_think(result): start = perf_counter() thought_result, note = self._think(images, prompt, choices, labels, thinking) result = thought_result or result note["think_ms"] = round((perf_counter() - start) * 1000, 1) thoughts.append({"question": field.id, **note}) results.append(result) # No shared prefill on this path: the whole cost is reported per question. usage = {"prefill_ms": 0.0, "questions_ms": round(seconds * 1000, 1), "input_tokens": tokens, "rotations": self.rotations} if thinking is not None and thinking.active: usage["thinking"] = {"mode": thinking.mode, "max_tokens": thinking.max_tokens, "thought": thoughts} return results, usage def _think(self, images, prompt, choices, labels, thinking): """-> (the Result read after a greedy thought, a usage note). `prompt` is the rotations == 1 single-pass prompt.""" engine = getattr(self, "thoughts", None) # a VLLMThoughts client (--think-engine vllm), else transformers generate try: tokens, closed = engine.think(images, prompt, thinking.max_tokens)[:2] if engine else self.engine.generate_thought(images, prompt, thinking.max_tokens) except Exception as exc: # no thought (e.g. a prompt longer than the thought engine's context): keep the single pass log.warning("thought failed, keeping the single-pass answer: %s", exc) return None, {"thought_tokens": 0, "closed": False, "error": f"{type(exc).__name__}: {str(exc)[:200]}"} inputs, token_ids = self.engine.inputs_after_thought(images, prompt, labels, tokens, closed) with self.torch.inference_mode(): scored = self.engine.candidate_logits_fast(inputs, token_ids) if getattr(self, "fast", False) else self.engine.candidate_logits(inputs, token_ids) logits = [float(x) for x in scored.cpu().tolist()] note = {"thought_tokens": len(tokens), "closed": closed} if thinking.return_thought: note["text"] = self.engine.processor.tokenizer.decode(tokens) return result_from_logits(choices, logits, token_ids=token_ids), note def _warn_layout(trained, served): if trained != served: log.warning("serving prompt layout %r but the adapter was trained with %r", served, trained) def build_backend(kind, adapter=None, no_adapter=False, bundle=BUNDLE, rotations=1, max_input_tokens=4096, readout_codes=None, prompt_layout=None, fast=False, merge_lora=False, float32=False): """`auto` prefers MLX with the converted adapter and falls back to torch + the PEFT adapter. readout_codes None / prompt_layout None follow the adapter (its readout rows; its decision_readout.json layout).""" if kind == "auto": kind = "mlx" if MLX_ADAPTER.is_dir() else "torch" default = MLX_ADAPTER if kind == "mlx" else TORCH_ADAPTER if no_adapter: chosen = None elif adapter is not None: chosen = Path(adapter) else: chosen = default if default.is_dir() else None if chosen is None: log.warning("adapter %s is missing; serving the base model", default) if chosen is not None and not Path(chosen).is_dir(): raise ValueError(f"Adapter directory {chosen} does not exist") if kind == "mlx": return MLXBackend(bundle, chosen, rotations=rotations, max_input_tokens=max_input_tokens, readout_codes=readout_codes, prompt_layout=prompt_layout) if kind == "torch": return TorchBackend(bundle, chosen, rotations=rotations, max_input_tokens=max_input_tokens, readout_codes=readout_codes, prompt_layout=prompt_layout, fast=fast, merge_lora=merge_lora, float32=float32) raise ValueError(f"Unknown backend {kind!r}") # ----------------------------------------------------------------------------------- request body def decode_data_url(value, position): if not isinstance(value, str) or not value: raise PlaygroundError(422, "bad_image", f"images[{position}] must be a data URL string") match = DATA_URL.match(value.strip()) if match is None: raise PlaygroundError(422, "bad_image", f"images[{position}] is not a data: URL") if not match.group("b64"): raise PlaygroundError(422, "bad_image", f"images[{position}] must be base64-encoded (data:...;base64,...)") try: return base64.b64decode(match.group("payload"), validate=True) except (binascii.Error, ValueError): raise PlaygroundError(422, "bad_image", f"images[{position}] is not valid base64") IMAGE_DATA_URL = re.compile(r"data:image/[\w.+-]+;base64,[A-Za-z0-9+/=]+") def extract_state_images(payload): """Image JevBench-style requests carry the photo as a data:image URI inside the state instead of the `images` field. Pull every such URI out of the state (a string, a dict value, a list item or a messages[].content), in reading order, replace it with "[image N]" and return the data URLs. The state keeps its structure; nothing else is touched.""" found = [] def take(value): if not isinstance(value, str) or "data:image/" not in value: return value def repl(match): found.append(match.group(0)); return f"[image {len(found)}]" return IMAGE_DATA_URL.sub(repl, value) def walk(node): if isinstance(node, str): return take(node) if isinstance(node, list): return [walk(x) for x in node] if isinstance(node, dict): return {k: walk(v) for k, v in node.items()} return node if "state" in payload and payload["state"] is not None: payload["state"] = walk(payload["state"]) return found async def read_payload(http_request): """-> (payload dict without images, [image bytes]) for multipart or JSON encodings.""" content_type = (http_request.headers.get("content-type") or "").split(";")[0].strip().lower() if content_type == "multipart/form-data": try: form = await http_request.form() except Exception as exc: # malformed multipart body raise PlaygroundError(422, "bad_request", f"Malformed multipart body: {exc}") raw = form.get("request") if raw is None: raise PlaygroundError(422, "bad_request", "Missing the 'request' form field") if not isinstance(raw, str): raw = (await raw.read()).decode("utf-8", "replace") try: payload = json.loads(raw) except json.JSONDecodeError as exc: raise PlaygroundError(422, "bad_json", f"The 'request' field is not valid JSON: {exc}") blobs = [] for key in ("image", "images", "image[]"): for upload in form.getlist(key): if isinstance(upload, str): raise PlaygroundError(422, "bad_image", f"Form field {key!r} must be an uploaded file") blobs.append(await upload.read()) if isinstance(payload, dict): blobs += [decode_data_url(value, i) for i, value in enumerate(extract_state_images(payload))] elif content_type == "application/json": body = await http_request.body() try: payload = json.loads(body) except json.JSONDecodeError as exc: raise PlaygroundError(422, "bad_json", f"Request body is not valid JSON: {exc}") if not isinstance(payload, dict): raise PlaygroundError(422, "bad_request", "Request body must be a JSON object") images = payload.get("images", []) if not isinstance(images, list): raise PlaygroundError(422, "bad_image", "'images' must be a list of data URLs") images = list(images) + extract_state_images(payload) # data:image URIs embedded in the state count as images too blobs = [decode_data_url(value, i) for i, value in enumerate(images)] else: raise PlaygroundError(422, "bad_request", "Use multipart/form-data (field 'request' + files 'image') or application/json") if not isinstance(payload, dict): raise PlaygroundError(422, "bad_request", "The request must be a JSON object") payload = {k: v for k, v in payload.items() if k not in ("images", "model")} return payload, blobs def decode_images(blobs): """Zero images is a text-only request; one or two go to the image model. Three is a 422.""" if len(blobs) > 2: raise PlaygroundError(422, "bad_image", f"Provide at most two images (first = reference, second = target); got {len(blobs)}") loaded = [] for position, data in enumerate(blobs): if len(data) > MAX_BYTES: raise PlaygroundError(413, "image_too_large", f"Image {position} exceeds 20 MiB") try: loaded.append(load_image_bytes(data)) except ValueError as exc: raise PlaygroundError(_image_status(str(exc)), "bad_image", f"Image {position}: {exc}") except OSError as exc: raise PlaygroundError(422, "bad_image", f"Image {position} could not be decoded: {exc}") return loaded def compile_payload(payload, max_options=MAX_OPTIONS): """-> (internal Request, multi plan). max_options is the loaded readout's limit (254, or 255 with 256 codes).""" try: return to_request_with_plan(payload, request_id="playground", max_options=max_options) except ValidationError as exc: raise PlaygroundError(422, "invalid_request", "; ".join( f"{'.'.join(str(p) for p in e['loc'])}: {e['msg']}" for e in exc.errors()) or str(exc)) except (ValueError, TypeError) as exc: raise PlaygroundError(422, "invalid_request", str(exc)) # ------------------------------------------------------------------------------------------- app def create_app(backend, examples=None, static=STATIC, calibration=None, thinking=None): """`backend` needs .name, .model, .adapter, .load_seconds and .score(images, request); a backend with .supports_thinking also takes .score(images, request, thinking=ThinkingPolicy). `thinking` = the server's default policy (off unless the server was started with --thinking); a request's "thinking" object overrides it.""" thinking = thinking or ThinkingPolicy() app = FastAPI(title="imajev playground", docs_url="/docs", redoc_url=None) app.state.backend = backend app.state.examples = load_examples() if examples is None else examples app.state.lock = threading.Lock() @app.exception_handler(PlaygroundError) async def playground_error(request, exc): return JSONResponse({"error": exc.error, "detail": exc.detail}, status_code=exc.status) @app.exception_handler(RequestValidationError) async def validation_error(request, exc): return JSONResponse({"error": "invalid_request", "detail": str(exc)}, status_code=422) @app.exception_handler(HTTPException) async def http_error(request, exc): return JSONResponse({"error": "http_error", "detail": exc.detail}, status_code=exc.status_code) @app.get("/v1/models") def models(): body = {"model": backend.model, "adapter": backend.adapter, "backend": backend.name, "loaded": True, "load_seconds": round(backend.load_seconds, 3)} for extra in ("readout_codes", "prompt_layout", "max_options"): if hasattr(backend, extra): body[extra] = getattr(backend, extra) return body @app.get("/examples") def examples_index(): return app.state.examples @app.get("/examples/{example}/image/{index}") def example_image(example: int, index: int): items = app.state.examples if not 0 <= example < len(items) or not 0 <= index < len(items[example]["images"]): raise PlaygroundError(404, "not_found", "No such example image") path = ROOT / items[example]["images"][index]["path"] if not path.is_file(): raise PlaygroundError(404, "not_found", "Example image is missing from this checkout") media = mimetypes.guess_type(path.name)[0] or "application/octet-stream" return FileResponse(path, media_type=media, headers={"Cache-Control": "no-cache"}) @app.post("/v1/systemone") async def systemone(http_request: HttpRequest): started = perf_counter() payload, blobs = await read_payload(http_request) loaded = decode_images(blobs) request, plan = compile_payload(payload, getattr(backend, "max_options", MAX_OPTIONS)) images = [image for image, _ in loaded] # [] is a text-only request; the model handles it try: policy = thinking.with_request(payload.get("thinking")) except (TypeError, ValueError) as exc: raise PlaygroundError(422, "invalid_request", str(exc)) if policy.active and not getattr(backend, "supports_thinking", False): raise PlaygroundError(422, "invalid_request", f"thinking is not available on the {backend.name} backend") def run(): with app.state.lock: # one GPU: strictly sequential return backend.score(images, request, thinking=policy) if policy.active else backend.score(images, request) try: results, usage = await run_in_threadpool(run) except PlaygroundError: raise except Exception as exc: log.exception("scoring failed") raise PlaygroundError(500, type(exc).__name__, str(exc)) if calibration is not None: photo_only = bool(images) and not request.state # images with no record: schema-1.2 files carry their own temperature results = [calibration.calibrate_result(result, field.type, len(result.scores) - 1, image=bool(images), photo_only=photo_only) for field, result in zip(request.fields, results)] body = to_response(request, results, model=backend.model, plan=plan) total_ms = round((perf_counter() - started) * 1000, 1) body["usage"] = {**usage, "total_ms": total_ms, "images": [{k: meta[k] for k in ("sha256", "width", "height")} for _, meta in loaded]} abstained = sum(1 for answer in body["answers"].values() if answer["abstained"]) log.info("%s images=%d questions=%d total_ms=%.1f abstained=%d", datetime.now(timezone.utc).isoformat(timespec="milliseconds"), len(images), len(request.fields), total_ms, abstained) return body static = Path(static) if static.is_dir(): app.mount("/", StaticFiles(directory=str(static), html=True), name="static") else: @app.get("/", response_class=HTMLResponse) def placeholder(): return ("imajev playground" "

Playground UI not built yet — the API is at POST /v1/systemone.

") return app def main(argv=None): parser = argparse.ArgumentParser(prog="playground-server", description=__doc__) parser.add_argument("--backend", choices=("auto", "mlx", "torch"), default="auto") parser.add_argument("--adapter", help="override the adapter directory") parser.add_argument("--no-adapter", action="store_true", help="serve the base model") parser.add_argument("--model-bundle", default=str(BUNDLE)) parser.add_argument("--rotations", type=int, default=1, help="candidate orders averaged per question (MLX and torch)") parser.add_argument("--calibration", help="held-out temperature calibration artifact") parser.add_argument("--max-input-tokens", type=int, default=4096, help="refuse requests longer than this many processed tokens (training used <= 4096)") parser.add_argument("--model-name", help="public model name reported by the API and the UI (e.g. imajev-2b)") parser.add_argument("--readout-codes", type=int, choices=(255, 256), help="decision readout size; default = the adapter's. 256 extends a 255-code adapter by one " "LM-head-initialised code so choice questions may have 255 options") parser.add_argument("--prompt-layout", choices=("auto", "standard", "compact"), default="auto", help="prompt layout; auto = the layout recorded in the adapter (standard when absent)") parser.add_argument("--fast", action="store_true", help="torch only: tokenize once and, on CUDA, replay recorded graphs of the language model " "(captured at load, about a minute); same decisions as without it") parser.add_argument("--merge-lora", action="store_true", help="torch only: fold the LoRA into the weights at load (a little faster; bf16 rounding can move " "near-tie decisions)") parser.add_argument("--thinking", choices=("off", "auto", "always"), default="off", help="default thinking mode (torch only); a request's \"thinking\" object overrides it. auto thinks only " "when the single-pass answer is unsure (docs/think-if-unsure-plan.md)") parser.add_argument("--think-max-tokens", type=int, default=256, help="default thought budget in tokens") parser.add_argument("--think-threshold", type=float, default=0.7, help="auto: think when the top option's share of the non-unknown mass is below this") parser.add_argument("--think-unknown-threshold", type=float, default=None, help="auto: also think when the unknown mass is above this (default: not used)") parser.add_argument("--think-engine", choices=("hf", "vllm"), default="hf", help="who writes thoughts: transformers generate in this process (hf), or a vLLM OpenAI server running " "the same merged weights (vllm; see scripts/export_merged.py)") parser.add_argument("--think-url", default="http://127.0.0.1:8000", help="--think-engine vllm: the vLLM server") parser.add_argument("--think-model", default="imajev-4b", help="--think-engine vllm: its served model name") parser.add_argument("--float32", action="store_true", help="torch only: serve in float32 on CUDA too (a slow reference for checking bf16 serving paths)") parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8765) args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) backend = build_backend(args.backend, args.adapter, args.no_adapter, Path(args.model_bundle), args.rotations, args.max_input_tokens, readout_codes=args.readout_codes, prompt_layout=None if args.prompt_layout == "auto" else args.prompt_layout, fast=args.fast, merge_lora=args.merge_lora, float32=args.float32) if args.model_name: backend.model = args.model_name log.info("backend=%s model=%s adapter=%s load_seconds=%.1f readout_codes=%s prompt_layout=%s", backend.name, backend.model, backend.adapter, backend.load_seconds, backend.readout_codes, backend.prompt_layout) calibration = None if args.calibration: from vision_decision.calibration import TemperatureCalibrator calibration = TemperatureCalibrator.load(args.calibration) import uvicorn thinking = ThinkingPolicy(mode=args.thinking, max_tokens=args.think_max_tokens, threshold=args.think_threshold, unknown_threshold=args.think_unknown_threshold) if thinking.active and not getattr(backend, "supports_thinking", False): parser.error(f"--thinking needs the torch backend (this server runs {backend.name})") if args.think_engine == "vllm": if not getattr(backend, "supports_thinking", False): parser.error("--think-engine vllm needs the torch backend") from thought_engine import VLLMThoughts backend.thoughts = VLLMThoughts(args.think_url, args.think_model, backend.engine.processor.tokenizer) uvicorn.run(create_app(backend, calibration=calibration, thinking=thinking), host=args.host, port=args.port, log_level="warning") return 0 if __name__ == "__main__": raise SystemExit(main())