"""Measure typed API processing, including validation, tensor preparation and decoding. The in-process HTTP client includes FastAPI routing/serialization but excludes socket/network latency. Explicitly synchronize MPS around timed requests. """ import argparse import copy import json import time from pathlib import Path import numpy as np import torch from tfm.api.server import PredictionEngine, create_app def profile(checkpoints, request, *, calibration=None, device="cpu", repetitions=50): from fastapi.testclient import TestClient engine = PredictionEngine(checkpoints, calibration, device=device) single = copy.deepcopy(request) single["actions"] = {next(iter(single["actions"])): next(iter(single["actions"].values()))} single["predict"] = single["predict"][:1] single["horizons_minutes"] = single["horizons_minutes"][:1] measurements = [] with TestClient(create_app(engine)) as client: for label, payload in (("single_query", single), ("batched_actions_horizons", request)): durations = [] for i in range(5 + repetitions): if device == "mps": torch.mps.synchronize() start = time.perf_counter_ns() response = client.post("/v1/predict", json=payload) if device == "mps": torch.mps.synchronize() duration = (time.perf_counter_ns() - start) / 1e6 response.raise_for_status() if i >= 5: durations.append(duration) measurements.append( { "case": label, "device": device, "p50_ms": float(np.percentile(durations, 50)), "p95_ms": float(np.percentile(durations, 95)), "repetitions": repetitions, "warmup": 5, "actions": len(payload["actions"]), "fields": len(payload["predict"]), "horizons": len(payload["horizons_minutes"]), "timing_scope": "In-process HTTP request: validation, tensor prep/transfers, model, decoding, serialization; excludes network sockets", } ) return {"version": "tfm-api-latency-v1", "measurements": measurements} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--checkpoint", required=True, action="append") parser.add_argument("--calibration") parser.add_argument("--request", required=True) parser.add_argument("--output", required=True) parser.add_argument("--device", choices=("cpu", "mps"), default="cpu") parser.add_argument("--repetitions", type=int, default=50) args = parser.parse_args() if args.repetitions < 1: raise ValueError("At least one latency repetition is required") results = profile( args.checkpoint, json.loads(Path(args.request).read_text()), calibration=args.calibration, device=args.device, repetitions=args.repetitions, ) Path(args.output).write_text(json.dumps(results, indent=2) + "\n") print(json.dumps(results)) if __name__ == "__main__": main()