"""FastAPI inference service. uvicorn serve.main:app --host 0.0.0.0 --port 7860 Endpoints POST /predict multipart image upload -> JSON detections (or ?visualize=true -> PNG) GET /health 200 once the model is loaded GET /metrics Prometheus metrics GET /docs interactive OpenAPI docs """ from __future__ import annotations import asyncio import json import logging import os import platform import sys import time import uuid from contextlib import asynccontextmanager from datetime import datetime, timezone from pathlib import Path import cv2 import numpy as np import onnxruntime as ort from fastapi import FastAPI, File, HTTPException, Query, Request, UploadFile from fastapi.concurrency import run_in_threadpool from fastapi.responses import JSONResponse, Response from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Counter, Gauge, Histogram, generate_latest from pcb_infer import Detector, draw from serve.config import ALLOWED_CONTENT_TYPES, Settings from serve.weights import ensure_model, sha256_of LATENCY_BUCKETS = (0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0) def cpu_info() -> dict: """CPU the server runs on, as far as the container can see it (for honest benchmark labels).""" model = platform.processor() or "" try: for line in Path("/proc/cpuinfo").read_text().splitlines(): if line.startswith("model name"): model = line.split(":", 1)[1].strip() break except OSError: pass quota = None try: # cgroup v2 CPU limit, e.g. "200000 100000" -> 2.0 CPUs q, period = Path("/sys/fs/cgroup/cpu.max").read_text().split() quota = None if q == "max" else round(int(q) / int(period), 2) except (OSError, ValueError): pass return {"model": model or "unknown", "logical_cpus": os.cpu_count(), "cgroup_cpu_quota": quota} class JsonFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: payload = {"ts": round(record.created, 3), "level": record.levelname, "msg": record.getMessage()} payload.update(getattr(record, "fields", {})) if record.exc_info: payload["exc"] = self.formatException(record.exc_info) return json.dumps(payload) def _make_logger() -> logging.Logger: log = logging.getLogger("pcb_serve") if not log.handlers: handler = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) log.addHandler(handler) log.setLevel(logging.INFO) log.propagate = False return log class Metrics: def __init__(self): self.registry = CollectorRegistry() self.requests = Counter("http_requests_total", "HTTP requests", ["method", "route", "status"], registry=self.registry) self.errors = Counter("predict_errors_total", "Rejected or failed /predict requests", ["reason"], registry=self.registry) self.latency = Histogram("http_request_duration_seconds", "End-to-end server time per request", ["route"], buckets=LATENCY_BUCKETS, registry=self.registry) self.model_latency = Histogram("model_inference_seconds", "ONNX Runtime session.run time", buckets=LATENCY_BUCKETS, registry=self.registry) self.detections = Histogram("detections_per_image", "Boxes returned per image", buckets=(0, 1, 2, 5, 10, 20, 50), registry=self.registry) self.model_loaded = Gauge("model_loaded", "1 when the model is ready", registry=self.registry) def create_app(settings: Settings | None = None, load_model: bool = False) -> FastAPI: """Build the app. The model loads at startup (lifespan), or immediately with load_model=True, which is needed when the app is mounted inside another server whose startup it doesn't own.""" settings = settings or Settings() log = _make_logger() metrics = Metrics() # One id per server process. On Cloud Run every new instance gets a new id, so a client can tell # a cold start (X-Instance-Age-Ms smaller than its own round trip) from a warm request. instance_id = uuid.uuid4().hex[:12] started_mono = time.monotonic() started_utc = datetime.now(timezone.utc).isoformat(timespec="seconds") def _load(app: FastAPI) -> None: path = ensure_model(settings.model_path, settings.model_url, settings.model_sha256) detector = Detector(path, threads=settings.ort_threads) detector.predict(np.zeros((detector.imgsz, detector.imgsz, 3), np.uint8)) # warm-up run app.state.detector = detector app.state.model_sha256 = sha256_of(path) app.state.infer_slots = asyncio.Semaphore(settings.max_concurrent_inferences) metrics.model_loaded.set(1) log.info("model loaded", extra={"fields": {"event": "startup", "model": path.name, "imgsz": detector.imgsz, "ort": ort.__version__}}) @asynccontextmanager async def lifespan(app: FastAPI): if getattr(app.state, "detector", None) is None: _load(app) yield metrics.model_loaded.set(0) app = FastAPI(title="PCB defect detection API", version="1.0.0", lifespan=lifespan, description="YOLO11n, ONNX Runtime static INT8 (detection head FP32), CPU inference.") app.state.settings = settings app.state.metrics = metrics def reject(reason: str, status: int, message: str): metrics.errors.labels(reason).inc() raise HTTPException(status_code=status, detail=message) @app.middleware("http") async def observe(request: Request, call_next): request_id = request.headers.get("x-request-id") or uuid.uuid4().hex[:16] request.state.request_id = request_id request.state.log = {} t0 = time.perf_counter() try: response = await call_next(request) except Exception: log.exception("unhandled error", extra={"fields": {"request_id": request_id}}) metrics.errors.labels("internal").inc() response = JSONResponse({"detail": "Internal server error", "request_id": request_id}, 500) elapsed = time.perf_counter() - t0 route = getattr(request.scope.get("route"), "path", "unmatched") metrics.requests.labels(request.method, route, str(response.status_code)).inc() metrics.latency.labels(route).observe(elapsed) response.headers["X-Request-ID"] = request_id response.headers["X-Instance-Id"] = instance_id response.headers["X-Instance-Age-Ms"] = f"{(time.monotonic() - started_mono) * 1000:.0f}" if route != "/metrics": log.info("request", extra={"fields": { "event": "request", "request_id": request_id, "method": request.method, "path": route, "status": response.status_code, "latency_ms": round(elapsed * 1000, 1), **request.state.log}}) return response @app.get("/") async def root(): return {"service": "PCB defect detection API", "docs": "/docs", "health": "/health", "predict": "POST /predict (multipart field 'file')", "metrics": "/metrics"} @app.get("/health") async def health(request: Request): detector = getattr(request.app.state, "detector", None) if detector is None: return JSONResponse({"status": "loading"}, status_code=503) return {"status": "ok", "model": detector.model_path.name, "model_sha256": request.app.state.model_sha256, "imgsz": detector.imgsz, "onnxruntime": ort.__version__, "ort_threads": settings.ort_threads, "max_concurrent_inferences": settings.max_concurrent_inferences, "cpu": cpu_info(), "visualize_enabled": settings.enable_visualize, "instance": {"id": instance_id, "started_at_utc": started_utc, "uptime_s": round(time.monotonic() - started_mono, 1)}} @app.get("/metrics") async def prometheus_metrics(): return Response(generate_latest(metrics.registry), media_type=CONTENT_TYPE_LATEST) @app.post("/predict") async def predict( request: Request, file: UploadFile = File(..., description="PCB image (JPEG, PNG, BMP, WebP or TIFF)"), visualize: bool = Query(False, description="Return the annotated image as PNG instead of JSON"), conf: float | None = Query(None, gt=0, lt=1, description="Confidence threshold (default 0.25)"), ): detector: Detector = request.app.state.detector if visualize and not settings.enable_visualize: reject("visualize_disabled", 403, "visualize=true is disabled on this deployment; request JSON instead.") content_type = (file.content_type or "").split(";")[0].strip().lower() if content_type not in ALLOWED_CONTENT_TYPES: reject("unsupported_type", 415, f"Unsupported file type '{content_type or 'unknown'}'. Send one of: " f"{', '.join(sorted(ALLOWED_CONTENT_TYPES))}.") data = await file.read(settings.max_upload_bytes + 1) if len(data) > settings.max_upload_bytes: reject("file_too_large", 413, f"File is larger than {settings.max_upload_mb:g} MB.") if not data: reject("empty_file", 400, "The uploaded file is empty.") t0 = time.perf_counter() bgr = cv2.imdecode(np.frombuffer(data, np.uint8), cv2.IMREAD_COLOR) decode_ms = (time.perf_counter() - t0) * 1000 if bgr is None: reject("undecodable", 400, "Could not decode the image (corrupt file, or larger than the pixel limit).") h, w = bgr.shape[:2] if max(h, w) > settings.max_image_side or h * w > settings.max_image_pixels: reject("image_too_large", 413, f"Image is {w}x{h}; the limit is {settings.max_image_side} px per side and " f"{settings.max_image_pixels / 1e6:g} megapixels.") threshold = conf if conf is not None else settings.default_conf tq = time.perf_counter() async with request.app.state.infer_slots: queue_ms = (time.perf_counter() - tq) * 1000 dets, timings = await run_in_threadpool(detector.predict, bgr, threshold, settings.nms_iou) metrics.model_latency.observe(timings.model_ms / 1000) metrics.detections.observe(len(dets)) timing = {"decode": decode_ms, "queue": queue_ms, "preprocess": timings.preprocess_ms, "model": timings.model_ms, "postprocess": timings.postprocess_ms} request.state.log.update({"image_w": w, "image_h": h, "num_boxes": len(dets), **{f"{k}_ms": round(v, 1) for k, v in timing.items()}}) headers = { "Server-Timing": ", ".join(f"{k};dur={v:.1f}" for k, v in timing.items()), "X-Model-Ms": f"{timings.model_ms:.1f}", "X-Num-Boxes": str(len(dets)), } if visualize: ok, png = cv2.imencode(".png", draw(bgr, dets)) if not ok: reject("encode_failed", 500, "Could not encode the annotated image.") return Response(png.tobytes(), media_type="image/png", headers=headers) return JSONResponse({ "request_id": request.state.request_id, "model": detector.model_path.name, "image": {"width": w, "height": h}, "conf_threshold": threshold, "num_detections": len(dets), "detections": [d.to_dict() for d in dets], "timings_ms": {k: round(v, 1) for k, v in timing.items()}, }, headers=headers) if load_model: _load(app) return app app = create_app()