pcb-defect-api / serve /main.py
michaelandrew35's picture
Deploy 55ea6e0b from GitLab CI
a9cca77 verified
Raw History Blame Contribute Delete
11.9 kB
"""FastAPI inference service.
uvicorn serve.main:app --host 0.0.0.0 --port 7860
Endpoints
POST /predict multipart image upload -> JSON detections (or ?visualize=true -> PNG)
GET /health 200 once the model is loaded
GET /metrics Prometheus metrics
GET /docs interactive OpenAPI docs
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import platform
import sys
import time
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from pathlib import Path
import cv2
import numpy as np
import onnxruntime as ort
from fastapi import FastAPI, File, HTTPException, Query, Request, UploadFile
from fastapi.concurrency import run_in_threadpool
from fastapi.responses import JSONResponse, Response
from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Counter, Gauge, Histogram, generate_latest
from pcb_infer import Detector, draw
from serve.config import ALLOWED_CONTENT_TYPES, Settings
from serve.weights import ensure_model, sha256_of
LATENCY_BUCKETS = (0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0)
def cpu_info() -> dict:
"""CPU the server runs on, as far as the container can see it (for honest benchmark labels)."""
model = platform.processor() or ""
try:
for line in Path("/proc/cpuinfo").read_text().splitlines():
if line.startswith("model name"):
model = line.split(":", 1)[1].strip()
break
except OSError:
pass
quota = None
try: # cgroup v2 CPU limit, e.g. "200000 100000" -> 2.0 CPUs
q, period = Path("/sys/fs/cgroup/cpu.max").read_text().split()
quota = None if q == "max" else round(int(q) / int(period), 2)
except (OSError, ValueError):
pass
return {"model": model or "unknown", "logical_cpus": os.cpu_count(), "cgroup_cpu_quota": quota}
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
payload = {"ts": round(record.created, 3), "level": record.levelname, "msg": record.getMessage()}
payload.update(getattr(record, "fields", {}))
if record.exc_info:
payload["exc"] = self.formatException(record.exc_info)
return json.dumps(payload)
def _make_logger() -> logging.Logger:
log = logging.getLogger("pcb_serve")
if not log.handlers:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
log.addHandler(handler)
log.setLevel(logging.INFO)
log.propagate = False
return log
class Metrics:
def __init__(self):
self.registry = CollectorRegistry()
self.requests = Counter("http_requests_total", "HTTP requests", ["method", "route", "status"],
registry=self.registry)
self.errors = Counter("predict_errors_total", "Rejected or failed /predict requests", ["reason"],
registry=self.registry)
self.latency = Histogram("http_request_duration_seconds", "End-to-end server time per request",
["route"], buckets=LATENCY_BUCKETS, registry=self.registry)
self.model_latency = Histogram("model_inference_seconds", "ONNX Runtime session.run time",
buckets=LATENCY_BUCKETS, registry=self.registry)
self.detections = Histogram("detections_per_image", "Boxes returned per image",
buckets=(0, 1, 2, 5, 10, 20, 50), registry=self.registry)
self.model_loaded = Gauge("model_loaded", "1 when the model is ready", registry=self.registry)
def create_app(settings: Settings | None = None, load_model: bool = False) -> FastAPI:
"""Build the app. The model loads at startup (lifespan), or immediately with load_model=True,
which is needed when the app is mounted inside another server whose startup it doesn't own."""
settings = settings or Settings()
log = _make_logger()
metrics = Metrics()
# One id per server process. On Cloud Run every new instance gets a new id, so a client can tell
# a cold start (X-Instance-Age-Ms smaller than its own round trip) from a warm request.
instance_id = uuid.uuid4().hex[:12]
started_mono = time.monotonic()
started_utc = datetime.now(timezone.utc).isoformat(timespec="seconds")
def _load(app: FastAPI) -> None:
path = ensure_model(settings.model_path, settings.model_url, settings.model_sha256)
detector = Detector(path, threads=settings.ort_threads)
detector.predict(np.zeros((detector.imgsz, detector.imgsz, 3), np.uint8)) # warm-up run
app.state.detector = detector
app.state.model_sha256 = sha256_of(path)
app.state.infer_slots = asyncio.Semaphore(settings.max_concurrent_inferences)
metrics.model_loaded.set(1)
log.info("model loaded", extra={"fields": {"event": "startup", "model": path.name,
"imgsz": detector.imgsz, "ort": ort.__version__}})
@asynccontextmanager
async def lifespan(app: FastAPI):
if getattr(app.state, "detector", None) is None:
_load(app)
yield
metrics.model_loaded.set(0)
app = FastAPI(title="PCB defect detection API", version="1.0.0", lifespan=lifespan,
description="YOLO11n, ONNX Runtime static INT8 (detection head FP32), CPU inference.")
app.state.settings = settings
app.state.metrics = metrics
def reject(reason: str, status: int, message: str):
metrics.errors.labels(reason).inc()
raise HTTPException(status_code=status, detail=message)
@app.middleware("http")
async def observe(request: Request, call_next):
request_id = request.headers.get("x-request-id") or uuid.uuid4().hex[:16]
request.state.request_id = request_id
request.state.log = {}
t0 = time.perf_counter()
try:
response = await call_next(request)
except Exception:
log.exception("unhandled error", extra={"fields": {"request_id": request_id}})
metrics.errors.labels("internal").inc()
response = JSONResponse({"detail": "Internal server error", "request_id": request_id}, 500)
elapsed = time.perf_counter() - t0
route = getattr(request.scope.get("route"), "path", "unmatched")
metrics.requests.labels(request.method, route, str(response.status_code)).inc()
metrics.latency.labels(route).observe(elapsed)
response.headers["X-Request-ID"] = request_id
response.headers["X-Instance-Id"] = instance_id
response.headers["X-Instance-Age-Ms"] = f"{(time.monotonic() - started_mono) * 1000:.0f}"
if route != "/metrics":
log.info("request", extra={"fields": {
"event": "request", "request_id": request_id, "method": request.method, "path": route,
"status": response.status_code, "latency_ms": round(elapsed * 1000, 1), **request.state.log}})
return response
@app.get("/")
async def root():
return {"service": "PCB defect detection API", "docs": "/docs", "health": "/health",
"predict": "POST /predict (multipart field 'file')", "metrics": "/metrics"}
@app.get("/health")
async def health(request: Request):
detector = getattr(request.app.state, "detector", None)
if detector is None:
return JSONResponse({"status": "loading"}, status_code=503)
return {"status": "ok", "model": detector.model_path.name,
"model_sha256": request.app.state.model_sha256, "imgsz": detector.imgsz,
"onnxruntime": ort.__version__, "ort_threads": settings.ort_threads,
"max_concurrent_inferences": settings.max_concurrent_inferences, "cpu": cpu_info(),
"visualize_enabled": settings.enable_visualize,
"instance": {"id": instance_id, "started_at_utc": started_utc,
"uptime_s": round(time.monotonic() - started_mono, 1)}}
@app.get("/metrics")
async def prometheus_metrics():
return Response(generate_latest(metrics.registry), media_type=CONTENT_TYPE_LATEST)
@app.post("/predict")
async def predict(
request: Request,
file: UploadFile = File(..., description="PCB image (JPEG, PNG, BMP, WebP or TIFF)"),
visualize: bool = Query(False, description="Return the annotated image as PNG instead of JSON"),
conf: float | None = Query(None, gt=0, lt=1, description="Confidence threshold (default 0.25)"),
):
detector: Detector = request.app.state.detector
if visualize and not settings.enable_visualize:
reject("visualize_disabled", 403, "visualize=true is disabled on this deployment; request JSON instead.")
content_type = (file.content_type or "").split(";")[0].strip().lower()
if content_type not in ALLOWED_CONTENT_TYPES:
reject("unsupported_type", 415,
f"Unsupported file type '{content_type or 'unknown'}'. Send one of: "
f"{', '.join(sorted(ALLOWED_CONTENT_TYPES))}.")
data = await file.read(settings.max_upload_bytes + 1)
if len(data) > settings.max_upload_bytes:
reject("file_too_large", 413, f"File is larger than {settings.max_upload_mb:g} MB.")
if not data:
reject("empty_file", 400, "The uploaded file is empty.")
t0 = time.perf_counter()
bgr = cv2.imdecode(np.frombuffer(data, np.uint8), cv2.IMREAD_COLOR)
decode_ms = (time.perf_counter() - t0) * 1000
if bgr is None:
reject("undecodable", 400, "Could not decode the image (corrupt file, or larger than the pixel limit).")
h, w = bgr.shape[:2]
if max(h, w) > settings.max_image_side or h * w > settings.max_image_pixels:
reject("image_too_large", 413,
f"Image is {w}x{h}; the limit is {settings.max_image_side} px per side and "
f"{settings.max_image_pixels / 1e6:g} megapixels.")
threshold = conf if conf is not None else settings.default_conf
tq = time.perf_counter()
async with request.app.state.infer_slots:
queue_ms = (time.perf_counter() - tq) * 1000
dets, timings = await run_in_threadpool(detector.predict, bgr, threshold, settings.nms_iou)
metrics.model_latency.observe(timings.model_ms / 1000)
metrics.detections.observe(len(dets))
timing = {"decode": decode_ms, "queue": queue_ms, "preprocess": timings.preprocess_ms,
"model": timings.model_ms, "postprocess": timings.postprocess_ms}
request.state.log.update({"image_w": w, "image_h": h, "num_boxes": len(dets),
**{f"{k}_ms": round(v, 1) for k, v in timing.items()}})
headers = {
"Server-Timing": ", ".join(f"{k};dur={v:.1f}" for k, v in timing.items()),
"X-Model-Ms": f"{timings.model_ms:.1f}",
"X-Num-Boxes": str(len(dets)),
}
if visualize:
ok, png = cv2.imencode(".png", draw(bgr, dets))
if not ok:
reject("encode_failed", 500, "Could not encode the annotated image.")
return Response(png.tobytes(), media_type="image/png", headers=headers)
return JSONResponse({
"request_id": request.state.request_id,
"model": detector.model_path.name,
"image": {"width": w, "height": h},
"conf_threshold": threshold,
"num_detections": len(dets),
"detections": [d.to_dict() for d in dets],
"timings_ms": {k: round(v, 1) for k, v in timing.items()},
}, headers=headers)
if load_model:
_load(app)
return app
app = create_app()