Download upstream/scripts/playground/server.py from loveaa/OneAlign: direct link, hf CLI and curl.
- Browser
- Download file 31.7 kB
-
https://huggingface.co/spaces/loveaa/OneAlign/resolve/main/upstream/scripts/playground/server.py
- Command line
-
hf download hf://spaces/loveaa/OneAlign/upstream/scripts/playground/server.py
-
curl -L -o server.py https://huggingface.co/spaces/loveaa/OneAlign/resolve/main/upstream/scripts/playground/server.py
31.7 kB
| """Local playground HTTP API for the imajev decision model — see docs/playground-spec.md ("HTTP API"). | |
| Run it directly: | |
| .venv/bin/python scripts/playground/server.py --backend auto | |
| The model loads once at startup and stays resident; a lock keeps requests strictly sequential | |
| because there is only one GPU. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import base64 | |
| import binascii | |
| import json | |
| import logging | |
| import mimetypes | |
| import re | |
| import sys | |
| import threading | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from time import perf_counter | |
| ROOT = Path(__file__).resolve().parents[2] | |
| for extra in (ROOT / "src", ROOT / "scripts", Path(__file__).resolve().parent): | |
| if str(extra) not in sys.path: | |
| sys.path.insert(0, str(extra)) | |
| from fastapi import FastAPI, HTTPException # noqa: E402 | |
| from fastapi.exceptions import RequestValidationError # noqa: E402 | |
| from fastapi.responses import FileResponse, HTMLResponse, JSONResponse # noqa: E402 | |
| from fastapi.staticfiles import StaticFiles # noqa: E402 | |
| from pydantic import ValidationError # noqa: E402 | |
| from starlette.concurrency import run_in_threadpool # noqa: E402 | |
| from starlette.requests import Request as HttpRequest # noqa: E402 | |
| from vision_decision.images import MAX_BYTES, load_image_bytes # noqa: E402 | |
| from vision_decision.jev_api import MAX_OPTIONS, to_request_with_plan, to_response # noqa: E402 | |
| from vision_decision.thinking import ThinkingPolicy # noqa: E402 | |
| from vision_decision.scoring import (check_prompt_layout, check_readout_codes, combine_rotations, compile_question, # noqa: E402 | |
| cyclic_offsets, result_from_logits, rotate) | |
| try: # package import (PYTHONPATH=scripts) or plain script run | |
| from .examples import load_examples | |
| except ImportError: # pragma: no cover - exercised only when run as a script | |
| from examples import load_examples | |
| # Public model names. The base checkpoint behind the adapter is an implementation detail and is | |
| # deliberately not part of the API or the UI; `/v1/models` still reports the adapter directory. | |
| MODEL_NAME = "imajev-v1.1" | |
| BASE_MODEL_NAME = "imajev-v1.1-base" | |
| MLX_ADAPTER = ROOT / "reports/decision-v1/runs/h100x4-full/best-mlx" | |
| TORCH_ADAPTER = ROOT / "reports/decision-v1/runs/h100x4-full/best" | |
| BUNDLE = ROOT / "artifacts/model.json" | |
| STATIC = Path(__file__).resolve().parent / "static" | |
| DATA_URL = re.compile(r"^data:(?P<mime>[\w.+-]+/[\w.+-]+)?(?P<b64>;base64)?,(?P<payload>.*)$", re.DOTALL) | |
| log = logging.getLogger("playground") | |
| class PlaygroundError(Exception): | |
| """A request the server refuses; `status` is the HTTP code and `detail` the human message.""" | |
| def __init__(self, status, error, detail): | |
| super().__init__(detail) | |
| self.status, self.error, self.detail = status, error, detail | |
| def _image_status(message): | |
| """20 MiB / 20 Mpixel limits are 413; everything else about an image is a 422.""" | |
| return 413 if "exceeds 20 MiB" in message or "20 million decoded pixels" in message else 422 | |
| # --------------------------------------------------------------------------------------- backends | |
| class MLXBackend: | |
| """MLX path: one image prefill shared by every question of the request.""" | |
| name = "mlx" | |
| def __init__(self, bundle=BUNDLE, adapter=None, rotations=1, max_input_tokens=4096, readout_codes=None, prompt_layout=None): | |
| from vision_decision.backend import MLXDirect | |
| self.engine = MLXDirect(str(bundle), adapter=None if adapter is None else str(adapter), | |
| max_input_tokens=max_input_tokens, readout_codes=readout_codes, prompt_layout=prompt_layout) | |
| self.readout_codes, self.prompt_layout = self.engine.codes, self.engine.prompt_layout | |
| self.max_options = self.engine.max_options | |
| _warn_layout(self.engine.trained_prompt_layout, self.prompt_layout) | |
| self.adapter = None if adapter is None else str(adapter) | |
| self.model = MODEL_NAME if adapter else BASE_MODEL_NAME | |
| self.load_seconds = self.engine.load_seconds | |
| self.rotations = rotations | |
| def score(self, images, request): | |
| # One image is passed on its own, a pair as a list; [] is a text-only request. | |
| target = images[0] if len(images) == 1 else list(images) | |
| results, run = self.engine.score_request(target, request.fields, request.state, self.rotations) | |
| passes = [row for detail in run.get("questions", []) for row in detail["rotations"]] | |
| return results, { | |
| "prefill_ms": round(run.get("prefill_seconds", 0.0) * 1000, 1), | |
| "questions_ms": round(sum(row["forward_seconds"] for row in passes) * 1000, 1), | |
| "input_tokens": max((row["input_tokens"] for row in passes), default=0), | |
| } | |
| class TorchBackend: | |
| """PyTorch fallback: one full forward per question, never a batch (left padding NaNs on MPS).""" | |
| name = "torch" | |
| supports_thinking = True | |
| def __init__(self, bundle=BUNDLE, adapter=None, device=None, rotations=1, max_input_tokens=4096, readout_codes=None, | |
| prompt_layout=None, fast=False, graph_lengths=None, merge_lora=False, float32=False): | |
| import torch | |
| from torch_decision import GRAPH_LENGTHS, TorchDecision | |
| self.torch = torch | |
| self.rotations = max(1, int(rotations)) | |
| self.fast = bool(fast) | |
| if device is None: # CUDA on a pod or Space, Metal on a Mac, CPU otherwise | |
| device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" | |
| self.bundle = json.loads(Path(bundle).read_text()) | |
| if not Path(self.bundle["path"]).is_dir(): | |
| raise ValueError("Local model snapshot is missing; run scripts/download_model.py") | |
| start = perf_counter() | |
| dtype = torch.bfloat16 if device == "cuda" and not float32 else torch.float32 | |
| # --merge-lora loads in float32 so the merge is formed at full precision, then rounds once to the serving dtype. | |
| # In bf16 that rounding still moves near-tie decisions (and calibration), so it is opt-in; --fast alone keeps | |
| # the LoRA unmerged and computes exactly the benchmarked function. | |
| merge = bool(merge_lora) and adapter is not None | |
| # The float32 merge runs on the CPU (no float32 copy on the GPU); the rounded weights then move to the device. | |
| self.engine = TorchDecision(self.bundle["path"], "cpu" if merge else device, dtype=torch.float32 if merge else dtype, | |
| max_length=int(max_input_tokens)) | |
| if adapter is not None: | |
| if merge: | |
| self.engine.merge_adapter(adapter, dtype) | |
| self.engine.model.to(device); self.engine.device = device | |
| else: | |
| from peft import PeftModel | |
| self.engine.model = PeftModel.from_pretrained(self.engine.model, str(adapter)).eval() | |
| self.engine.enable_readout(adapter, trainable=False, codes=readout_codes) | |
| self.graph_lengths = [] | |
| if self.fast and device == "cuda": | |
| lengths = [n for n in (graph_lengths or GRAPH_LENGTHS) if n <= int(max_input_tokens)] | |
| try: | |
| self.graph_lengths = self.engine.capture_graphs(lengths).lengths | |
| except Exception: # the eager path computes the same function | |
| log.exception("CUDA graph capture failed; serving the eager path") | |
| self.engine.graphs = None | |
| if self.engine.readout is None and readout_codes is not None: # base model / legacy adapter: LM-head rows | |
| self.engine.codes = check_readout_codes(readout_codes) | |
| trained = self.engine.prompt_layout # the adapter's recorded layout (standard when absent) | |
| self.prompt_layout = trained if prompt_layout is None else check_prompt_layout(prompt_layout) | |
| _warn_layout(trained, self.prompt_layout) | |
| self.readout_codes, self.max_options = self.engine.codes, self.engine.max_options | |
| self.adapter = None if adapter is None else str(adapter) | |
| self.model = MODEL_NAME if adapter else BASE_MODEL_NAME | |
| self.load_seconds = perf_counter() - start | |
| def score(self, images, request, thinking=None): | |
| """One forward per question and per presentation order; with rotations > 1 the per-candidate log-probabilities | |
| are averaged over cyclic option orders exactly as the MLX path does (combine_rotations), which drops the engine's | |
| vocabulary tie break; with rotations == 1 the single pass keeps it. | |
| thinking (a ThinkingPolicy, None = off): a question whose single-pass answer the policy calls unsure gets a greedy | |
| thought of at most max_tokens, and its decision is read again right after the thought (docs/think-if-unsure-plan.md).""" | |
| if thinking is not None and thinking.active and self.rotations != 1: | |
| raise PlaygroundError(422, "invalid_request", "thinking needs a server running --rotations 1") | |
| results, seconds, tokens, thoughts = [], 0.0, 0, [] | |
| for field in request.fields: | |
| header, choices, texts = compile_question(field, request.state, getattr(self, "prompt_layout", "standard")) | |
| labels = self.engine.labels(len(choices), len(images)) | |
| start = perf_counter() | |
| passes = [] | |
| for offset in cyclic_offsets(len(choices), self.rotations): | |
| prompt = header + "\n".join(f"{label}: {text}" for label, text in zip(labels, rotate(texts, offset))) | |
| if getattr(self, "fast", False): | |
| with self.torch.inference_mode(): | |
| _, inputs, token_ids = self.engine.prepare_fast(images, prompt, labels) | |
| logits = [float(x) for x in self.engine.candidate_logits_fast(inputs, token_ids).cpu().tolist()] | |
| else: | |
| with self.torch.no_grad(): | |
| _, inputs, token_ids = self.engine.prepare(images, prompt, labels) | |
| logits = [float(x) for x in self.engine.candidate_logits(inputs, token_ids).cpu().tolist()] | |
| tokens = max(tokens, int(inputs["input_ids"].shape[-1])) | |
| passes.append((offset, logits, token_ids)) | |
| seconds += perf_counter() - start | |
| if len(passes) == 1: | |
| result = result_from_logits(choices, passes[0][1], token_ids=passes[0][2]) | |
| else: | |
| result = combine_rotations(choices, [(offset, logits) for offset, logits, _ in passes]) | |
| if thinking is not None and thinking.should_think(result): | |
| start = perf_counter() | |
| thought_result, note = self._think(images, prompt, choices, labels, thinking) | |
| result = thought_result or result | |
| note["think_ms"] = round((perf_counter() - start) * 1000, 1) | |
| thoughts.append({"question": field.id, **note}) | |
| results.append(result) | |
| # No shared prefill on this path: the whole cost is reported per question. | |
| usage = {"prefill_ms": 0.0, "questions_ms": round(seconds * 1000, 1), "input_tokens": tokens, "rotations": self.rotations} | |
| if thinking is not None and thinking.active: | |
| usage["thinking"] = {"mode": thinking.mode, "max_tokens": thinking.max_tokens, "thought": thoughts} | |
| return results, usage | |
| def _think(self, images, prompt, choices, labels, thinking): | |
| """-> (the Result read after a greedy thought, a usage note). `prompt` is the rotations == 1 single-pass prompt.""" | |
| engine = getattr(self, "thoughts", None) # a VLLMThoughts client (--think-engine vllm), else transformers generate | |
| try: | |
| tokens, closed = engine.think(images, prompt, thinking.max_tokens)[:2] if engine else self.engine.generate_thought(images, prompt, thinking.max_tokens) | |
| except Exception as exc: # no thought (e.g. a prompt longer than the thought engine's context): keep the single pass | |
| log.warning("thought failed, keeping the single-pass answer: %s", exc) | |
| return None, {"thought_tokens": 0, "closed": False, "error": f"{type(exc).__name__}: {str(exc)[:200]}"} | |
| inputs, token_ids = self.engine.inputs_after_thought(images, prompt, labels, tokens, closed) | |
| with self.torch.inference_mode(): | |
| scored = self.engine.candidate_logits_fast(inputs, token_ids) if getattr(self, "fast", False) else self.engine.candidate_logits(inputs, token_ids) | |
| logits = [float(x) for x in scored.cpu().tolist()] | |
| note = {"thought_tokens": len(tokens), "closed": closed} | |
| if thinking.return_thought: | |
| note["text"] = self.engine.processor.tokenizer.decode(tokens) | |
| return result_from_logits(choices, logits, token_ids=token_ids), note | |
| def _warn_layout(trained, served): | |
| if trained != served: | |
| log.warning("serving prompt layout %r but the adapter was trained with %r", served, trained) | |
| def build_backend(kind, adapter=None, no_adapter=False, bundle=BUNDLE, rotations=1, max_input_tokens=4096, | |
| readout_codes=None, prompt_layout=None, fast=False, merge_lora=False, float32=False): | |
| """`auto` prefers MLX with the converted adapter and falls back to torch + the PEFT adapter. | |
| readout_codes None / prompt_layout None follow the adapter (its readout rows; its decision_readout.json layout).""" | |
| if kind == "auto": | |
| kind = "mlx" if MLX_ADAPTER.is_dir() else "torch" | |
| default = MLX_ADAPTER if kind == "mlx" else TORCH_ADAPTER | |
| if no_adapter: | |
| chosen = None | |
| elif adapter is not None: | |
| chosen = Path(adapter) | |
| else: | |
| chosen = default if default.is_dir() else None | |
| if chosen is None: | |
| log.warning("adapter %s is missing; serving the base model", default) | |
| if chosen is not None and not Path(chosen).is_dir(): | |
| raise ValueError(f"Adapter directory {chosen} does not exist") | |
| if kind == "mlx": | |
| return MLXBackend(bundle, chosen, rotations=rotations, max_input_tokens=max_input_tokens, | |
| readout_codes=readout_codes, prompt_layout=prompt_layout) | |
| if kind == "torch": | |
| return TorchBackend(bundle, chosen, rotations=rotations, max_input_tokens=max_input_tokens, | |
| readout_codes=readout_codes, prompt_layout=prompt_layout, fast=fast, merge_lora=merge_lora, | |
| float32=float32) | |
| raise ValueError(f"Unknown backend {kind!r}") | |
| # ----------------------------------------------------------------------------------- request body | |
| def decode_data_url(value, position): | |
| if not isinstance(value, str) or not value: | |
| raise PlaygroundError(422, "bad_image", f"images[{position}] must be a data URL string") | |
| match = DATA_URL.match(value.strip()) | |
| if match is None: | |
| raise PlaygroundError(422, "bad_image", f"images[{position}] is not a data: URL") | |
| if not match.group("b64"): | |
| raise PlaygroundError(422, "bad_image", f"images[{position}] must be base64-encoded (data:...;base64,...)") | |
| try: | |
| return base64.b64decode(match.group("payload"), validate=True) | |
| except (binascii.Error, ValueError): | |
| raise PlaygroundError(422, "bad_image", f"images[{position}] is not valid base64") | |
| IMAGE_DATA_URL = re.compile(r"data:image/[\w.+-]+;base64,[A-Za-z0-9+/=]+") | |
| def extract_state_images(payload): | |
| """Image JevBench-style requests carry the photo as a data:image URI inside the state instead of the `images` field. | |
| Pull every such URI out of the state (a string, a dict value, a list item or a messages[].content), in reading order, | |
| replace it with "[image N]" and return the data URLs. The state keeps its structure; nothing else is touched.""" | |
| found = [] | |
| def take(value): | |
| if not isinstance(value, str) or "data:image/" not in value: | |
| return value | |
| def repl(match): | |
| found.append(match.group(0)); return f"[image {len(found)}]" | |
| return IMAGE_DATA_URL.sub(repl, value) | |
| def walk(node): | |
| if isinstance(node, str): return take(node) | |
| if isinstance(node, list): return [walk(x) for x in node] | |
| if isinstance(node, dict): return {k: walk(v) for k, v in node.items()} | |
| return node | |
| if "state" in payload and payload["state"] is not None: | |
| payload["state"] = walk(payload["state"]) | |
| return found | |
| async def read_payload(http_request): | |
| """-> (payload dict without images, [image bytes]) for multipart or JSON encodings.""" | |
| content_type = (http_request.headers.get("content-type") or "").split(";")[0].strip().lower() | |
| if content_type == "multipart/form-data": | |
| try: | |
| form = await http_request.form() | |
| except Exception as exc: # malformed multipart body | |
| raise PlaygroundError(422, "bad_request", f"Malformed multipart body: {exc}") | |
| raw = form.get("request") | |
| if raw is None: | |
| raise PlaygroundError(422, "bad_request", "Missing the 'request' form field") | |
| if not isinstance(raw, str): | |
| raw = (await raw.read()).decode("utf-8", "replace") | |
| try: | |
| payload = json.loads(raw) | |
| except json.JSONDecodeError as exc: | |
| raise PlaygroundError(422, "bad_json", f"The 'request' field is not valid JSON: {exc}") | |
| blobs = [] | |
| for key in ("image", "images", "image[]"): | |
| for upload in form.getlist(key): | |
| if isinstance(upload, str): | |
| raise PlaygroundError(422, "bad_image", f"Form field {key!r} must be an uploaded file") | |
| blobs.append(await upload.read()) | |
| if isinstance(payload, dict): | |
| blobs += [decode_data_url(value, i) for i, value in enumerate(extract_state_images(payload))] | |
| elif content_type == "application/json": | |
| body = await http_request.body() | |
| try: | |
| payload = json.loads(body) | |
| except json.JSONDecodeError as exc: | |
| raise PlaygroundError(422, "bad_json", f"Request body is not valid JSON: {exc}") | |
| if not isinstance(payload, dict): | |
| raise PlaygroundError(422, "bad_request", "Request body must be a JSON object") | |
| images = payload.get("images", []) | |
| if not isinstance(images, list): | |
| raise PlaygroundError(422, "bad_image", "'images' must be a list of data URLs") | |
| images = list(images) + extract_state_images(payload) # data:image URIs embedded in the state count as images too | |
| blobs = [decode_data_url(value, i) for i, value in enumerate(images)] | |
| else: | |
| raise PlaygroundError(422, "bad_request", | |
| "Use multipart/form-data (field 'request' + files 'image') or application/json") | |
| if not isinstance(payload, dict): | |
| raise PlaygroundError(422, "bad_request", "The request must be a JSON object") | |
| payload = {k: v for k, v in payload.items() if k not in ("images", "model")} | |
| return payload, blobs | |
| def decode_images(blobs): | |
| """Zero images is a text-only request; one or two go to the image model. Three is a 422.""" | |
| if len(blobs) > 2: | |
| raise PlaygroundError(422, "bad_image", | |
| f"Provide at most two images (first = reference, second = target); got {len(blobs)}") | |
| loaded = [] | |
| for position, data in enumerate(blobs): | |
| if len(data) > MAX_BYTES: | |
| raise PlaygroundError(413, "image_too_large", f"Image {position} exceeds 20 MiB") | |
| try: | |
| loaded.append(load_image_bytes(data)) | |
| except ValueError as exc: | |
| raise PlaygroundError(_image_status(str(exc)), "bad_image", f"Image {position}: {exc}") | |
| except OSError as exc: | |
| raise PlaygroundError(422, "bad_image", f"Image {position} could not be decoded: {exc}") | |
| return loaded | |
| def compile_payload(payload, max_options=MAX_OPTIONS): | |
| """-> (internal Request, multi plan). max_options is the loaded readout's limit (254, or 255 with 256 codes).""" | |
| try: | |
| return to_request_with_plan(payload, request_id="playground", max_options=max_options) | |
| except ValidationError as exc: | |
| raise PlaygroundError(422, "invalid_request", "; ".join( | |
| f"{'.'.join(str(p) for p in e['loc'])}: {e['msg']}" for e in exc.errors()) or str(exc)) | |
| except (ValueError, TypeError) as exc: | |
| raise PlaygroundError(422, "invalid_request", str(exc)) | |
| # ------------------------------------------------------------------------------------------- app | |
| def create_app(backend, examples=None, static=STATIC, calibration=None, thinking=None): | |
| """`backend` needs .name, .model, .adapter, .load_seconds and .score(images, request); a backend with | |
| .supports_thinking also takes .score(images, request, thinking=ThinkingPolicy). `thinking` = the server's default policy | |
| (off unless the server was started with --thinking); a request's "thinking" object overrides it.""" | |
| thinking = thinking or ThinkingPolicy() | |
| app = FastAPI(title="imajev playground", docs_url="/docs", redoc_url=None) | |
| app.state.backend = backend | |
| app.state.examples = load_examples() if examples is None else examples | |
| app.state.lock = threading.Lock() | |
| async def playground_error(request, exc): | |
| return JSONResponse({"error": exc.error, "detail": exc.detail}, status_code=exc.status) | |
| async def validation_error(request, exc): | |
| return JSONResponse({"error": "invalid_request", "detail": str(exc)}, status_code=422) | |
| async def http_error(request, exc): | |
| return JSONResponse({"error": "http_error", "detail": exc.detail}, status_code=exc.status_code) | |
| def models(): | |
| body = {"model": backend.model, "adapter": backend.adapter, "backend": backend.name, | |
| "loaded": True, "load_seconds": round(backend.load_seconds, 3)} | |
| for extra in ("readout_codes", "prompt_layout", "max_options"): | |
| if hasattr(backend, extra): | |
| body[extra] = getattr(backend, extra) | |
| return body | |
| def examples_index(): | |
| return app.state.examples | |
| def example_image(example: int, index: int): | |
| items = app.state.examples | |
| if not 0 <= example < len(items) or not 0 <= index < len(items[example]["images"]): | |
| raise PlaygroundError(404, "not_found", "No such example image") | |
| path = ROOT / items[example]["images"][index]["path"] | |
| if not path.is_file(): | |
| raise PlaygroundError(404, "not_found", "Example image is missing from this checkout") | |
| media = mimetypes.guess_type(path.name)[0] or "application/octet-stream" | |
| return FileResponse(path, media_type=media, headers={"Cache-Control": "no-cache"}) | |
| async def systemone(http_request: HttpRequest): | |
| started = perf_counter() | |
| payload, blobs = await read_payload(http_request) | |
| loaded = decode_images(blobs) | |
| request, plan = compile_payload(payload, getattr(backend, "max_options", MAX_OPTIONS)) | |
| images = [image for image, _ in loaded] # [] is a text-only request; the model handles it | |
| try: | |
| policy = thinking.with_request(payload.get("thinking")) | |
| except (TypeError, ValueError) as exc: | |
| raise PlaygroundError(422, "invalid_request", str(exc)) | |
| if policy.active and not getattr(backend, "supports_thinking", False): | |
| raise PlaygroundError(422, "invalid_request", f"thinking is not available on the {backend.name} backend") | |
| def run(): | |
| with app.state.lock: # one GPU: strictly sequential | |
| return backend.score(images, request, thinking=policy) if policy.active else backend.score(images, request) | |
| try: | |
| results, usage = await run_in_threadpool(run) | |
| except PlaygroundError: | |
| raise | |
| except Exception as exc: | |
| log.exception("scoring failed") | |
| raise PlaygroundError(500, type(exc).__name__, str(exc)) | |
| if calibration is not None: | |
| photo_only = bool(images) and not request.state # images with no record: schema-1.2 files carry their own temperature | |
| results = [calibration.calibrate_result(result, field.type, len(result.scores) - 1, image=bool(images), photo_only=photo_only) | |
| for field, result in zip(request.fields, results)] | |
| body = to_response(request, results, model=backend.model, plan=plan) | |
| total_ms = round((perf_counter() - started) * 1000, 1) | |
| body["usage"] = {**usage, "total_ms": total_ms, | |
| "images": [{k: meta[k] for k in ("sha256", "width", "height")} for _, meta in loaded]} | |
| abstained = sum(1 for answer in body["answers"].values() if answer["abstained"]) | |
| log.info("%s images=%d questions=%d total_ms=%.1f abstained=%d", | |
| datetime.now(timezone.utc).isoformat(timespec="milliseconds"), | |
| len(images), len(request.fields), total_ms, abstained) | |
| return body | |
| static = Path(static) | |
| if static.is_dir(): | |
| app.mount("/", StaticFiles(directory=str(static), html=True), name="static") | |
| else: | |
| def placeholder(): | |
| return ("<!doctype html><meta charset=utf-8><title>imajev playground</title>" | |
| "<p>Playground UI not built yet — the API is at <code>POST /v1/systemone</code>.</p>") | |
| return app | |
| def main(argv=None): | |
| parser = argparse.ArgumentParser(prog="playground-server", description=__doc__) | |
| parser.add_argument("--backend", choices=("auto", "mlx", "torch"), default="auto") | |
| parser.add_argument("--adapter", help="override the adapter directory") | |
| parser.add_argument("--no-adapter", action="store_true", help="serve the base model") | |
| parser.add_argument("--model-bundle", default=str(BUNDLE)) | |
| parser.add_argument("--rotations", type=int, default=1, help="candidate orders averaged per question (MLX and torch)") | |
| parser.add_argument("--calibration", help="held-out temperature calibration artifact") | |
| parser.add_argument("--max-input-tokens", type=int, default=4096, | |
| help="refuse requests longer than this many processed tokens (training used <= 4096)") | |
| parser.add_argument("--model-name", help="public model name reported by the API and the UI (e.g. imajev-2b)") | |
| parser.add_argument("--readout-codes", type=int, choices=(255, 256), | |
| help="decision readout size; default = the adapter's. 256 extends a 255-code adapter by one " | |
| "LM-head-initialised code so choice questions may have 255 options") | |
| parser.add_argument("--prompt-layout", choices=("auto", "standard", "compact"), default="auto", | |
| help="prompt layout; auto = the layout recorded in the adapter (standard when absent)") | |
| parser.add_argument("--fast", action="store_true", | |
| help="torch only: tokenize once and, on CUDA, replay recorded graphs of the language model " | |
| "(captured at load, about a minute); same decisions as without it") | |
| parser.add_argument("--merge-lora", action="store_true", | |
| help="torch only: fold the LoRA into the weights at load (a little faster; bf16 rounding can move " | |
| "near-tie decisions)") | |
| parser.add_argument("--thinking", choices=("off", "auto", "always"), default="off", | |
| help="default thinking mode (torch only); a request's \"thinking\" object overrides it. auto thinks only " | |
| "when the single-pass answer is unsure (docs/think-if-unsure-plan.md)") | |
| parser.add_argument("--think-max-tokens", type=int, default=256, help="default thought budget in tokens") | |
| parser.add_argument("--think-threshold", type=float, default=0.7, | |
| help="auto: think when the top option's share of the non-unknown mass is below this") | |
| parser.add_argument("--think-unknown-threshold", type=float, default=None, | |
| help="auto: also think when the unknown mass is above this (default: not used)") | |
| parser.add_argument("--think-engine", choices=("hf", "vllm"), default="hf", | |
| help="who writes thoughts: transformers generate in this process (hf), or a vLLM OpenAI server running " | |
| "the same merged weights (vllm; see scripts/export_merged.py)") | |
| parser.add_argument("--think-url", default="http://127.0.0.1:8000", help="--think-engine vllm: the vLLM server") | |
| parser.add_argument("--think-model", default="imajev-4b", help="--think-engine vllm: its served model name") | |
| parser.add_argument("--float32", action="store_true", | |
| help="torch only: serve in float32 on CUDA too (a slow reference for checking bf16 serving paths)") | |
| parser.add_argument("--host", default="127.0.0.1") | |
| parser.add_argument("--port", type=int, default=8765) | |
| args = parser.parse_args(argv) | |
| logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) | |
| backend = build_backend(args.backend, args.adapter, args.no_adapter, Path(args.model_bundle), args.rotations, | |
| args.max_input_tokens, readout_codes=args.readout_codes, | |
| prompt_layout=None if args.prompt_layout == "auto" else args.prompt_layout, fast=args.fast, | |
| merge_lora=args.merge_lora, float32=args.float32) | |
| if args.model_name: | |
| backend.model = args.model_name | |
| log.info("backend=%s model=%s adapter=%s load_seconds=%.1f readout_codes=%s prompt_layout=%s", | |
| backend.name, backend.model, backend.adapter, backend.load_seconds, backend.readout_codes, backend.prompt_layout) | |
| calibration = None | |
| if args.calibration: | |
| from vision_decision.calibration import TemperatureCalibrator | |
| calibration = TemperatureCalibrator.load(args.calibration) | |
| import uvicorn | |
| thinking = ThinkingPolicy(mode=args.thinking, max_tokens=args.think_max_tokens, threshold=args.think_threshold, | |
| unknown_threshold=args.think_unknown_threshold) | |
| if thinking.active and not getattr(backend, "supports_thinking", False): | |
| parser.error(f"--thinking needs the torch backend (this server runs {backend.name})") | |
| if args.think_engine == "vllm": | |
| if not getattr(backend, "supports_thinking", False): | |
| parser.error("--think-engine vllm needs the torch backend") | |
| from thought_engine import VLLMThoughts | |
| backend.thoughts = VLLMThoughts(args.think_url, args.think_model, backend.engine.processor.tokenizer) | |
| uvicorn.run(create_app(backend, calibration=calibration, thinking=thinking), host=args.host, port=args.port, log_level="warning") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |