clairic-4bit / server.py
lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
6 kB
"""A small HTTP server for Clairic on MLX: POST /v1/systemone with a System One: Extract request.
python server.py --port 8093 # serves the model in this folder
python server.py --model mlx-community/clairic-flash-4bit # or a Hugging Face repository
curl localhost:8093/v1/systemone -H 'content-type: application/json' -d '{
"state": "Invoice from Northwind Supply. Total: $1080.00.",
"fields": {"vendor": {"type": "text"}, "total": {"type": "number", "decimals": 2}}}'
The request body is the upstream format: `state`, `fields`, and optionally `images` (base64 data URIs),
`documents` (PDF data URIs), `reference_date`, `options` and `model`. The model answers one request at a time
(a request already fans out into many batched prompts), so concurrent requests wait their turn in arrival order.
Set CLAIRIC_API_KEY to require `Authorization: Bearer <key>`.
"""
from __future__ import annotations
import argparse
import asyncio
import hmac
import logging
import os
import time
from pathlib import Path
from typing import Any
import uvicorn
from fastapi import FastAPI, Header, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import ValidationError
from clairic_mlx import ClairicModel, load
from clairic_runtime.errors import ClairInferenceError, ClairRequestError
log = logging.getLogger("clairic")
MAX_BODY_BYTES = 16 * 1024 * 1024 # embedded media is limited to 8 MiB decoded
def create_app(model: ClairicModel, max_waiting: int) -> FastAPI:
app = FastAPI(title="Clairic MLX")
turn = asyncio.Lock()
stats = {"requests": 0, "errors": 0, "seconds": 0.0, "input_tokens": 0, "output_tokens": 0, "waiting": 0}
key = os.environ.get("CLAIRIC_API_KEY")
def authorize(authorization: str | None) -> None:
if key and not hmac.compare_digest(authorization or "", f"Bearer {key}"):
raise HTTPException(status_code=401, detail="invalid API key")
@app.get("/health")
async def health() -> dict[str, Any]:
return {"status": "ok", "model": model.model_id}
@app.get("/v1/models")
async def models(authorization: str | None = Header(default=None)) -> dict[str, Any]:
authorize(authorization)
return {"object": "list", "data": [{"id": model.model_id, "object": "model", "owned_by": "cloudflare"}]}
@app.get("/stats")
async def statistics(authorization: str | None = Header(default=None)) -> dict[str, Any]:
authorize(authorization)
return stats
@app.post("/v1/systemone")
async def systemone(request: Request, authorization: str | None = Header(default=None)) -> JSONResponse:
authorize(authorization)
body = await request.body()
if len(body) > MAX_BODY_BYTES:
raise HTTPException(status_code=413, detail="request body too large")
try:
payload = await request.json()
except ValueError:
raise HTTPException(status_code=400, detail="request body must be JSON") from None
if not isinstance(payload, dict):
raise HTTPException(status_code=400, detail="request body must be a JSON object")
if stats["waiting"] >= max_waiting:
raise HTTPException(status_code=429, detail="too many requests waiting")
stats["waiting"] += 1
try:
async with turn:
begin = time.time()
try:
response = await model.predict_async(payload)
except ValidationError as error:
stats["errors"] += 1
detail = [{"loc": list(e["loc"]), "msg": e["msg"], "type": e["type"]} for e in error.errors()]
return JSONResponse(status_code=422, content={"error": {"type": "invalid_request", "detail": detail}})
except ClairRequestError as error:
stats["errors"] += 1
return JSONResponse(status_code=400, content={"error": {"type": "invalid_request", "message": str(error)}})
except ClairInferenceError as error:
stats["errors"] += 1
log.exception("inference failed")
return JSONResponse(status_code=500, content={"error": {"type": "inference_error", "message": str(error)}})
seconds = time.time() - begin
finally:
stats["waiting"] -= 1
usage = response["usage"]
stats["requests"] += 1
stats["seconds"] += seconds
stats["input_tokens"] += usage["input_tokens"]
stats["output_tokens"] += usage["output_tokens"]
log.info("%d fields, %d input tokens, %.2fs", len(response["answers"]), usage["input_tokens"], seconds)
return JSONResponse(content=response, headers={"x-clairic-seconds": f"{seconds:.3f}"})
return app
def main() -> None:
ap = argparse.ArgumentParser(description="Serve Clairic on MLX")
ap.add_argument("--model", default=str(Path(__file__).resolve().parent))
ap.add_argument("--host", default="127.0.0.1")
ap.add_argument("--port", type=int, default=8093)
ap.add_argument("--max-length", type=int, default=65_273, help="largest prompt in tokens")
ap.add_argument("--batch-size", type=int, default=None)
ap.add_argument("--token-budget", type=int, default=None)
ap.add_argument("--max-waiting", type=int, default=64, help="requests allowed to wait for the model")
args = ap.parse_args()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
options = {name: value for name, value in (("batch_size", args.batch_size), ("batch_token_budget", args.token_budget))
if value is not None}
model = load(args.model, max_length=args.max_length, **options)
log.info("loaded %s", model.model_id)
uvicorn.run(create_app(model, args.max_waiting), host=args.host, port=args.port, log_level="warning")
if __name__ == "__main__":
main()