"""A small HTTP server for Clairic on MLX: POST /v1/systemone with a System One: Extract request. python server.py --port 8093 # serves the model in this folder python server.py --model mlx-community/clairic-flash-4bit # or a Hugging Face repository curl localhost:8093/v1/systemone -H 'content-type: application/json' -d '{ "state": "Invoice from Northwind Supply. Total: $1080.00.", "fields": {"vendor": {"type": "text"}, "total": {"type": "number", "decimals": 2}}}' The request body is the upstream format: `state`, `fields`, and optionally `images` (base64 data URIs), `documents` (PDF data URIs), `reference_date`, `options` and `model`. The model answers one request at a time (a request already fans out into many batched prompts), so concurrent requests wait their turn in arrival order. Set CLAIRIC_API_KEY to require `Authorization: Bearer `. """ from __future__ import annotations import argparse import asyncio import hmac import logging import os import time from pathlib import Path from typing import Any import uvicorn from fastapi import FastAPI, Header, HTTPException, Request from fastapi.responses import JSONResponse from pydantic import ValidationError from clairic_mlx import ClairicModel, load from clairic_runtime.errors import ClairInferenceError, ClairRequestError log = logging.getLogger("clairic") MAX_BODY_BYTES = 16 * 1024 * 1024 # embedded media is limited to 8 MiB decoded def create_app(model: ClairicModel, max_waiting: int) -> FastAPI: app = FastAPI(title="Clairic MLX") turn = asyncio.Lock() stats = {"requests": 0, "errors": 0, "seconds": 0.0, "input_tokens": 0, "output_tokens": 0, "waiting": 0} key = os.environ.get("CLAIRIC_API_KEY") def authorize(authorization: str | None) -> None: if key and not hmac.compare_digest(authorization or "", f"Bearer {key}"): raise HTTPException(status_code=401, detail="invalid API key") @app.get("/health") async def health() -> dict[str, Any]: return {"status": "ok", "model": model.model_id} @app.get("/v1/models") async def models(authorization: str | None = Header(default=None)) -> dict[str, Any]: authorize(authorization) return {"object": "list", "data": [{"id": model.model_id, "object": "model", "owned_by": "cloudflare"}]} @app.get("/stats") async def statistics(authorization: str | None = Header(default=None)) -> dict[str, Any]: authorize(authorization) return stats @app.post("/v1/systemone") async def systemone(request: Request, authorization: str | None = Header(default=None)) -> JSONResponse: authorize(authorization) body = await request.body() if len(body) > MAX_BODY_BYTES: raise HTTPException(status_code=413, detail="request body too large") try: payload = await request.json() except ValueError: raise HTTPException(status_code=400, detail="request body must be JSON") from None if not isinstance(payload, dict): raise HTTPException(status_code=400, detail="request body must be a JSON object") if stats["waiting"] >= max_waiting: raise HTTPException(status_code=429, detail="too many requests waiting") stats["waiting"] += 1 try: async with turn: begin = time.time() try: response = await model.predict_async(payload) except ValidationError as error: stats["errors"] += 1 detail = [{"loc": list(e["loc"]), "msg": e["msg"], "type": e["type"]} for e in error.errors()] return JSONResponse(status_code=422, content={"error": {"type": "invalid_request", "detail": detail}}) except ClairRequestError as error: stats["errors"] += 1 return JSONResponse(status_code=400, content={"error": {"type": "invalid_request", "message": str(error)}}) except ClairInferenceError as error: stats["errors"] += 1 log.exception("inference failed") return JSONResponse(status_code=500, content={"error": {"type": "inference_error", "message": str(error)}}) seconds = time.time() - begin finally: stats["waiting"] -= 1 usage = response["usage"] stats["requests"] += 1 stats["seconds"] += seconds stats["input_tokens"] += usage["input_tokens"] stats["output_tokens"] += usage["output_tokens"] log.info("%d fields, %d input tokens, %.2fs", len(response["answers"]), usage["input_tokens"], seconds) return JSONResponse(content=response, headers={"x-clairic-seconds": f"{seconds:.3f}"}) return app def main() -> None: ap = argparse.ArgumentParser(description="Serve Clairic on MLX") ap.add_argument("--model", default=str(Path(__file__).resolve().parent)) ap.add_argument("--host", default="127.0.0.1") ap.add_argument("--port", type=int, default=8093) ap.add_argument("--max-length", type=int, default=65_273, help="largest prompt in tokens") ap.add_argument("--batch-size", type=int, default=None) ap.add_argument("--token-budget", type=int, default=None) ap.add_argument("--max-waiting", type=int, default=64, help="requests allowed to wait for the model") args = ap.parse_args() logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s") options = {name: value for name, value in (("batch_size", args.batch_size), ("batch_token_budget", args.token_budget)) if value is not None} model = load(args.model, max_length=args.max_length, **options) log.info("loaded %s", model.model_id) uvicorn.run(create_app(model, args.max_waiting), host=args.host, port=args.port, log_level="warning") if __name__ == "__main__": main()