File size: 3,252 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
"""Measure typed API processing, including validation, tensor preparation and decoding.

The in-process HTTP client includes FastAPI routing/serialization but excludes
socket/network latency. Explicitly synchronize MPS around timed requests.
"""

import argparse
import copy
import json
import time
from pathlib import Path

import numpy as np
import torch

from tfm.api.server import PredictionEngine, create_app


def profile(checkpoints, request, *, calibration=None, device="cpu", repetitions=50):
    from fastapi.testclient import TestClient

    engine = PredictionEngine(checkpoints, calibration, device=device)
    single = copy.deepcopy(request)
    single["actions"] = {next(iter(single["actions"])): next(iter(single["actions"].values()))}
    single["predict"] = single["predict"][:1]
    single["horizons_minutes"] = single["horizons_minutes"][:1]
    measurements = []
    with TestClient(create_app(engine)) as client:
        for label, payload in (("single_query", single), ("batched_actions_horizons", request)):
            durations = []
            for i in range(5 + repetitions):
                if device == "mps":
                    torch.mps.synchronize()
                start = time.perf_counter_ns()
                response = client.post("/v1/predict", json=payload)
                if device == "mps":
                    torch.mps.synchronize()
                duration = (time.perf_counter_ns() - start) / 1e6
                response.raise_for_status()
                if i >= 5:
                    durations.append(duration)
            measurements.append(
                {
                    "case": label,
                    "device": device,
                    "p50_ms": float(np.percentile(durations, 50)),
                    "p95_ms": float(np.percentile(durations, 95)),
                    "repetitions": repetitions,
                    "warmup": 5,
                    "actions": len(payload["actions"]),
                    "fields": len(payload["predict"]),
                    "horizons": len(payload["horizons_minutes"]),
                    "timing_scope": "In-process HTTP request: validation, tensor prep/transfers, model, decoding, serialization; excludes network sockets",
                }
            )
    return {"version": "tfm-api-latency-v1", "measurements": measurements}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--checkpoint", required=True, action="append")
    parser.add_argument("--calibration")
    parser.add_argument("--request", required=True)
    parser.add_argument("--output", required=True)
    parser.add_argument("--device", choices=("cpu", "mps"), default="cpu")
    parser.add_argument("--repetitions", type=int, default=50)
    args = parser.parse_args()
    if args.repetitions < 1:
        raise ValueError("At least one latency repetition is required")
    results = profile(
        args.checkpoint,
        json.loads(Path(args.request).read_text()),
        calibration=args.calibration,
        device=args.device,
        repetitions=args.repetitions,
    )
    Path(args.output).write_text(json.dumps(results, indent=2) + "\n")
    print(json.dumps(results))


if __name__ == "__main__":
    main()