File size: 3,252 Bytes
1baf215 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | """Measure typed API processing, including validation, tensor preparation and decoding.
The in-process HTTP client includes FastAPI routing/serialization but excludes
socket/network latency. Explicitly synchronize MPS around timed requests.
"""
import argparse
import copy
import json
import time
from pathlib import Path
import numpy as np
import torch
from tfm.api.server import PredictionEngine, create_app
def profile(checkpoints, request, *, calibration=None, device="cpu", repetitions=50):
from fastapi.testclient import TestClient
engine = PredictionEngine(checkpoints, calibration, device=device)
single = copy.deepcopy(request)
single["actions"] = {next(iter(single["actions"])): next(iter(single["actions"].values()))}
single["predict"] = single["predict"][:1]
single["horizons_minutes"] = single["horizons_minutes"][:1]
measurements = []
with TestClient(create_app(engine)) as client:
for label, payload in (("single_query", single), ("batched_actions_horizons", request)):
durations = []
for i in range(5 + repetitions):
if device == "mps":
torch.mps.synchronize()
start = time.perf_counter_ns()
response = client.post("/v1/predict", json=payload)
if device == "mps":
torch.mps.synchronize()
duration = (time.perf_counter_ns() - start) / 1e6
response.raise_for_status()
if i >= 5:
durations.append(duration)
measurements.append(
{
"case": label,
"device": device,
"p50_ms": float(np.percentile(durations, 50)),
"p95_ms": float(np.percentile(durations, 95)),
"repetitions": repetitions,
"warmup": 5,
"actions": len(payload["actions"]),
"fields": len(payload["predict"]),
"horizons": len(payload["horizons_minutes"]),
"timing_scope": "In-process HTTP request: validation, tensor prep/transfers, model, decoding, serialization; excludes network sockets",
}
)
return {"version": "tfm-api-latency-v1", "measurements": measurements}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--checkpoint", required=True, action="append")
parser.add_argument("--calibration")
parser.add_argument("--request", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--device", choices=("cpu", "mps"), default="cpu")
parser.add_argument("--repetitions", type=int, default=50)
args = parser.parse_args()
if args.repetitions < 1:
raise ValueError("At least one latency repetition is required")
results = profile(
args.checkpoint,
json.loads(Path(args.request).read_text()),
calibration=args.calibration,
device=args.device,
repetitions=args.repetitions,
)
Path(args.output).write_text(json.dumps(results, indent=2) + "\n")
print(json.dumps(results))
if __name__ == "__main__":
main()
|