clef_mlx.py: add 'predict' CLI and 'serve' (local POST /v1/systemone server); card: CLI and server docs

README.md

@@ -83,6 +83,44 @@ model.predict({
 
 See the [original model card](https://huggingface.co/Cloudflare/clef) for the input format, question types, and benchmarks.
 
+## Try it from the command line
+
+`clef_mlx.py` runs on its own. From the downloaded repo it uses that repo by default; elsewhere pass
+`--model mlx-community/clef-8bit`.
+
+```bash
+cd "$(hf download mlx-community/clef-8bit --quiet)"
+python clef_mlx.py predict \
+  --state "Checkout has been failing for every customer for the last hour." \
+  --questions '{"urgent": {"type": "noul", "instructions": "Is this urgent?"},
+               "team": {"type": "choice", "criteria": {"billing": "Payments", "technical": "Outages and errors"}}}'
+```
+
+`--questions` takes JSON, a `.json` file, or `-` for stdin; `--request` takes a whole request body instead;
+`--image photo.jpg` attaches an image (repeatable). The output is the SystemOne response as JSON.
+
+## Run a local SystemOne server
+
+```bash
+python clef_mlx.py serve --port 8000          # http://127.0.0.1:8000, local only by default
+curl http://127.0.0.1:8000/v1/systemone -H "Content-Type: application/json" -d '{
+  "model": "clef",
+  "state": "Checkout has been failing for every customer for the last hour.",
+  "questions": {"urgent": {"type": "noul", "instructions": "Is this urgent?"}}
+}'
+```
+
+- `POST /v1/systemone` takes the same request body as the Jev/SystemOne API and returns the same response
+  (plus `usage.latency_ms`), so existing SystemOne clients can point at your laptop. `GET /health` and
+  `GET /v1/models` are also available.
+- Images go in `images` as data URLs, base64, or http(s) URLs. Videos aren't supported over HTTP.
+- Errors return JSON: `400` for invalid requests, `413` with "maximum context length" for inputs that don't fit.
+  Add `"truncate": false` to a request (or start with `--no-truncate`) to get the `413` instead of truncation.
+- Requests run one at a time on the GPU. There is no authentication: keep the default `127.0.0.1` binding
+  unless you put your own proxy in front.
+- For the [Decision Index](https://github.com/apolinario/decision-index), start with `--no-truncate` and use
+  its `http` engine: `python -m decision_index run --engine http --option base_url=http://127.0.0.1:8000`.
+
 ## Limitations
 
 - **Not a chat model.** Text generation tools (`mlx_lm.generate`, `mlx_vlm.generate`, LM Studio, Ollama) load

clef_mlx.py

@@ -8,6 +8,11 @@ Torch-free. The backbone is loaded with mlx-vlm (text + images/video) or mlx-lm
     model = clef_mlx.load("mlx-community/clef-flash-4bit")
     model.predict(record)        # {question_id: {option_id: probability}}
     model.systemone(request)     # Jev/SystemOne /v1/systemone response body
+
+Command line (from a downloaded repo, --model defaults to that repo):
+
+    python clef_mlx.py predict --state "..." --questions '{"q": {"type": "noul"}}'
+    python clef_mlx.py serve --port 8000     # POST /v1/systemone on 127.0.0.1
 """
 
 from __future__ import annotations
@@ -573,3 +578,200 @@ def load(path: str | Path, head_dtype=mx.bfloat16, backend: str = "auto") -> Cle
     head.set_dtype(head_dtype)
     mx.eval(head.parameters())
     return ClefMLX(backbone, tokenizer, head, processor)
+
+
+# --------------------------------------------------------------------------- CLI and server
+
+
+def _default_model() -> str | None:
+    """When run from inside a downloaded repo, default to that repo."""
+    here = Path(__file__).resolve().parent
+    return str(here) if (here / "joint_head.safetensors").exists() else None
+
+
+def _json_arg(value: str) -> Any:
+    """A JSON string, a path to a JSON file, or '-' for stdin."""
+    if value == "-":
+        return json.load(__import__("sys").stdin)
+    try:
+        candidate = Path(value).expanduser()
+        if candidate.is_file():
+            return json.loads(candidate.read_text())
+    except OSError:  # e.g. inline JSON too long to be a file name
+        pass
+    return json.loads(value)
+
+
+def _state_arg(value: str) -> Any:
+    """State text; JSON objects/arrays are parsed so they are rendered like the reference does."""
+    stripped = value.strip()
+    if stripped[:1] in "{[":
+        try:
+            return json.loads(stripped)
+        except json.JSONDecodeError:
+            pass
+    return value
+
+
+def _decode_image(item: Any):
+    """An image from a data URL, raw base64 or an http(s) URL (no local paths over HTTP)."""
+    import base64
+    import io
+    import urllib.request
+
+    from PIL import Image
+
+    if not isinstance(item, str) or not item:
+        raise ValueError("images must be strings: data URLs, base64, or http(s) URLs")
+    if item.startswith(("http://", "https://")):
+        with urllib.request.urlopen(item, timeout=30) as response:
+            data = response.read()
+    else:
+        if item.startswith("data:"):
+            item = item.split(",", 1)[-1]
+        try:
+            data = base64.b64decode(item, validate=True)
+        except ValueError:
+            raise ValueError("could not decode image: expected a data URL, base64, or http(s) URL") from None
+    return Image.open(io.BytesIO(data)).convert("RGB")
+
+
+def _cmd_predict(args) -> None:
+    import sys
+
+    if args.request:
+        request = _json_arg(args.request)
+    else:
+        if args.state is None or args.questions is None:
+            sys.exit("predict needs --state and --questions, or --request")
+        request = {"state": _state_arg(args.state), "questions": _json_arg(args.questions)}
+    request.setdefault("model", args.name or Path(args.model).name)
+    if args.image:
+        from PIL import Image
+
+        request["images"] = [Image.open(Path(p).expanduser()).convert("RGB") for p in args.image]
+    model = load(args.model)
+    try:
+        response = model.systemone(request, max_length=args.max_length, truncate=not args.no_truncate)
+    except ValueError as error:
+        sys.exit(f"error: {error}")
+    print(json.dumps(response, indent=2, ensure_ascii=False))
+
+
+def _cmd_serve(args) -> None:
+    import threading
+    import time
+    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+    model = load(args.model)
+    served_name = args.name or Path(args.model).name
+    model.systemone({"model": served_name, "state": "warmup",
+                     "questions": {"w": {"type": "noul", "instructions": "Is this a warmup?"}}})
+    lock = threading.Lock()  # one GPU: run requests one at a time
+    max_body = args.max_body_mb * 1024 * 1024
+
+    class Handler(BaseHTTPRequestHandler):
+        server_version = "clef-mlx"
+
+        def _send(self, status: int, body: dict) -> None:
+            data = json.dumps(body, ensure_ascii=False).encode()
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+
+        def _error(self, status: int, message: str) -> None:
+            self._send(status, {"error": {"message": message}})
+
+        def do_GET(self):
+            if self.path == "/health":
+                self._send(200, {"status": "ok", "model": served_name})
+            elif self.path == "/v1/models":
+                self._send(200, {"object": "list", "data": [{"id": served_name, "object": "model"}]})
+            else:
+                self._error(404, f"not found: {self.path}")
+
+        def do_POST(self):
+            if self.path != "/v1/systemone":
+                return self._error(404, f"not found: {self.path}")
+            length = int(self.headers.get("Content-Length") or 0)
+            if length > max_body:
+                return self._error(413, f"request body larger than {args.max_body_mb} MB")
+            try:
+                request = json.loads(self.rfile.read(length))
+                if not isinstance(request, dict):
+                    raise ValueError("request body must be a JSON object")
+                if request.get("videos"):
+                    raise ValueError("videos are not supported over HTTP; use clef_mlx.py from Python")
+                request.setdefault("model", served_name)
+                if request.get("images"):
+                    request["images"] = [_decode_image(i) for i in request["images"]]
+                truncate = request.pop("truncate", not args.no_truncate)
+            except (json.JSONDecodeError, ValueError, OSError) as error:
+                return self._error(400, str(error))
+            started = time.perf_counter()
+            try:
+                with lock:
+                    response = model.systemone(request, max_length=args.max_length, truncate=bool(truncate))
+            except ContextTooLong as error:
+                return self._error(413, f"maximum context length exceeded: {error}")
+            except (ValueError, KeyError, TypeError) as error:
+                return self._error(400, str(error))
+            except Exception as error:  # noqa: BLE001 - report, keep serving
+                return self._error(500, f"{type(error).__name__}: {error}")
+            response["usage"]["latency_ms"] = round((time.perf_counter() - started) * 1000, 1)
+            self._send(200, response)
+
+        def log_message(self, fmt, *fmt_args):
+            if not args.quiet:
+                super().log_message(fmt, *fmt_args)
+
+    server = ThreadingHTTPServer((args.host, args.port), Handler)
+    print(f"Serving {served_name} on http://{args.host}:{args.port}/v1/systemone "
+          f"(truncate={'off' if args.no_truncate else 'on'}, max_length={args.max_length})", flush=True)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        server.server_close()
+
+
+def main(argv: list[str] | None = None) -> None:
+    import argparse
+
+    default_model = _default_model()
+    parser = argparse.ArgumentParser(prog="clef_mlx.py", description="Run Clef locally with MLX.")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    def common(p):
+        p.add_argument("--model", default=default_model, required=default_model is None,
+                       help="Hub repo id or local dir (default: the repo this file is in)")
+        p.add_argument("--name", help="model name echoed in responses (default: the model dir/repo name)")
+        p.add_argument("--max-length", type=int, default=16384, help="prompt token limit (default 16384)")
+        p.add_argument("--no-truncate", action="store_true",
+                       help="refuse over-length inputs instead of truncating the state")
+
+    p = sub.add_parser("predict", help="answer one request and print the SystemOne response")
+    common(p)
+    p.add_argument("--state", help="state text (JSON objects/arrays are parsed)")
+    p.add_argument("--questions", help="questions as JSON, a .json file, or '-' for stdin")
+    p.add_argument("--request", help="a full /v1/systemone request body as JSON, a .json file, or '-'")
+    p.add_argument("--image", action="append", help="image file to attach (repeatable)")
+    p.set_defaults(func=_cmd_predict)
+
+    p = sub.add_parser("serve", help="serve POST /v1/systemone over HTTP")
+    common(p)
+    p.add_argument("--host", default="127.0.0.1", help="bind address (default 127.0.0.1, local only)")
+    p.add_argument("--port", type=int, default=8000)
+    p.add_argument("--max-body-mb", type=int, default=64)
+    p.add_argument("--quiet", action="store_true", help="no per-request access log")
+    p.set_defaults(func=_cmd_serve)
+
+    args = parser.parse_args(argv)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()