File size: 5,998 Bytes
ae97c5d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
"""A small HTTP server for Clairic on MLX: POST /v1/systemone with a System One: Extract request.

    python server.py --port 8093                                    # serves the model in this folder
    python server.py --model mlx-community/clairic-flash-4bit        # or a Hugging Face repository

    curl localhost:8093/v1/systemone -H 'content-type: application/json' -d '{
      "state": "Invoice from Northwind Supply. Total: $1080.00.",
      "fields": {"vendor": {"type": "text"}, "total": {"type": "number", "decimals": 2}}}'

The request body is the upstream format: `state`, `fields`, and optionally `images` (base64 data URIs),
`documents` (PDF data URIs), `reference_date`, `options` and `model`. The model answers one request at a time
(a request already fans out into many batched prompts), so concurrent requests wait their turn in arrival order.
Set CLAIRIC_API_KEY to require `Authorization: Bearer <key>`.
"""

from __future__ import annotations

import argparse
import asyncio
import hmac
import logging
import os
import time
from pathlib import Path
from typing import Any

import uvicorn
from fastapi import FastAPI, Header, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import ValidationError

from clairic_mlx import ClairicModel, load
from clairic_runtime.errors import ClairInferenceError, ClairRequestError

log = logging.getLogger("clairic")
MAX_BODY_BYTES = 16 * 1024 * 1024  # embedded media is limited to 8 MiB decoded


def create_app(model: ClairicModel, max_waiting: int) -> FastAPI:
    app = FastAPI(title="Clairic MLX")
    turn = asyncio.Lock()
    stats = {"requests": 0, "errors": 0, "seconds": 0.0, "input_tokens": 0, "output_tokens": 0, "waiting": 0}
    key = os.environ.get("CLAIRIC_API_KEY")

    def authorize(authorization: str | None) -> None:
        if key and not hmac.compare_digest(authorization or "", f"Bearer {key}"):
            raise HTTPException(status_code=401, detail="invalid API key")

    @app.get("/health")
    async def health() -> dict[str, Any]:
        return {"status": "ok", "model": model.model_id}

    @app.get("/v1/models")
    async def models(authorization: str | None = Header(default=None)) -> dict[str, Any]:
        authorize(authorization)
        return {"object": "list", "data": [{"id": model.model_id, "object": "model", "owned_by": "cloudflare"}]}

    @app.get("/stats")
    async def statistics(authorization: str | None = Header(default=None)) -> dict[str, Any]:
        authorize(authorization)
        return stats

    @app.post("/v1/systemone")
    async def systemone(request: Request, authorization: str | None = Header(default=None)) -> JSONResponse:
        authorize(authorization)
        body = await request.body()
        if len(body) > MAX_BODY_BYTES:
            raise HTTPException(status_code=413, detail="request body too large")
        try:
            payload = await request.json()
        except ValueError:
            raise HTTPException(status_code=400, detail="request body must be JSON") from None
        if not isinstance(payload, dict):
            raise HTTPException(status_code=400, detail="request body must be a JSON object")
        if stats["waiting"] >= max_waiting:
            raise HTTPException(status_code=429, detail="too many requests waiting")
        stats["waiting"] += 1
        try:
            async with turn:
                begin = time.time()
                try:
                    response = await model.predict_async(payload)
                except ValidationError as error:
                    stats["errors"] += 1
                    detail = [{"loc": list(e["loc"]), "msg": e["msg"], "type": e["type"]} for e in error.errors()]
                    return JSONResponse(status_code=422, content={"error": {"type": "invalid_request", "detail": detail}})
                except ClairRequestError as error:
                    stats["errors"] += 1
                    return JSONResponse(status_code=400, content={"error": {"type": "invalid_request", "message": str(error)}})
                except ClairInferenceError as error:
                    stats["errors"] += 1
                    log.exception("inference failed")
                    return JSONResponse(status_code=500, content={"error": {"type": "inference_error", "message": str(error)}})
                seconds = time.time() - begin
        finally:
            stats["waiting"] -= 1
        usage = response["usage"]
        stats["requests"] += 1
        stats["seconds"] += seconds
        stats["input_tokens"] += usage["input_tokens"]
        stats["output_tokens"] += usage["output_tokens"]
        log.info("%d fields, %d input tokens, %.2fs", len(response["answers"]), usage["input_tokens"], seconds)
        return JSONResponse(content=response, headers={"x-clairic-seconds": f"{seconds:.3f}"})

    return app


def main() -> None:
    ap = argparse.ArgumentParser(description="Serve Clairic on MLX")
    ap.add_argument("--model", default=str(Path(__file__).resolve().parent))
    ap.add_argument("--host", default="127.0.0.1")
    ap.add_argument("--port", type=int, default=8093)
    ap.add_argument("--max-length", type=int, default=65_273, help="largest prompt in tokens")
    ap.add_argument("--batch-size", type=int, default=None)
    ap.add_argument("--token-budget", type=int, default=None)
    ap.add_argument("--max-waiting", type=int, default=64, help="requests allowed to wait for the model")
    args = ap.parse_args()
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
    options = {name: value for name, value in (("batch_size", args.batch_size), ("batch_token_budget", args.token_budget))
               if value is not None}
    model = load(args.model, max_length=args.max_length, **options)
    log.info("loaded %s", model.model_id)
    uvicorn.run(create_app(model, args.max_waiting), host=args.host, port=args.port, log_level="warning")


if __name__ == "__main__":
    main()