# SPDX-License-Identifier: Apache-2.0 """Device test of the warm-up: the first real call after ``from_pretrained`` is fast. cd code && python -m pytest -s -q models/tests/test_api_warmup_device.py One chip, opened by the API. Real inputs: crops and mirror images of the demo photo (each call gets an image it has not seen). For each variant, call 1 (a new image) is compared with the steady latency of the same image (median of 5 later calls, each one after a call of the previous variant, as call 1 was): the ratio must be at most ``SP_FIRST_CALL_MAX`` (default 1.25, the target is 1.10; the margin is for a shared, loaded host) or within 0.3 ms. Variants: the default ``warmup_variants`` (1600x900 RGB array, 1920x1080 RGB array, 480x640 plane, JPEG file, ``max_keypoints=-1``, ``return_descriptors=False``, ``nms_radius=0``) and two variants that ``model.warmup(...)`` adds later (``nms_radius=3``, a 1024x768 image). Then the outputs of the warmed model are compared with a model made with ``warmup_variants="minimal"``: bit-identical. A fresh process gives the most exact numbers (``bench_first_call.py``): pytest has already imported Pillow and others before the model loads. """ from __future__ import annotations import os import statistics import time from pathlib import Path import numpy as np import pytest import torch from PIL import Image, ImageOps from tt_superpoint import SuperPoint CODE = Path(__file__).resolve().parents[2] SAMPLE = CODE / "sample_data" / "house_in_field_1080p.jpg" DEVICE_ID = int(os.environ.get("TT_DEVICE_ID", "0")) MAX_RATIO = float(os.environ.get("SP_FIRST_CALL_MAX", "1.25")) ABS_MS = 0.3 def _real_images(n): """n different real images (crops / mirror images of the demo photo), RGB PIL.""" with Image.open(SAMPLE) as im: base = im.convert("RGB") out = [] for i in range(n): x0, y0 = 40 * (i % 5), 30 * (i // 5) c = base.crop((x0, y0, x0 + 1400, y0 + 790)) out.append(ImageOps.mirror(c) if i % 2 else c) return out def _variants(tmp_path): imgs = _real_images(8) jpgs = [] for i, im in enumerate(imgs): p = tmp_path / f"real_{i}.jpg" im.resize((1600, 900), Image.BILINEAR).save(p, quality=92) jpgs.append(str(p)) rgb = lambda w, h: [np.asarray(im.resize((w, h), Image.BILINEAR)) for im in imgs] # noqa: E731 plane = [np.ascontiguousarray(a[..., 0]) for a in rgb(640, 480)] r1600 = rgb(1600, 900) # name -> (inputs, call kwargs, warm-up added later with model.warmup or None) return { "rgb_1600x900": (r1600, {}, None), "rgb_1920x1080": (rgb(1920, 1080), {}, None), "plane_480x640": (plane, {}, None), "jpeg_1600x900": (jpgs, {}, None), "all_keypoints": (r1600, {"max_keypoints": -1}, None), "no_descriptors": (r1600, {"return_descriptors": False}, None), "host_nms_r0": (r1600, {"nms_radius": 0}, None), "nms_r3_added": (r1600, {"nms_radius": 3}, {"nms_radius": 3}), "size_1024x768_added": (rgb(1024, 768), {}, {"size": (1024, 768)}), } def _ms(f): t0 = time.perf_counter() out = f() return (time.perf_counter() - t0) * 1e3, out def test_first_call_fast_and_outputs_unchanged(tmp_path): variants = _variants(tmp_path) t0 = time.perf_counter() model = SuperPoint.from_pretrained(device_id=DEVICE_ID) t_fp = time.perf_counter() - t0 print(f"\nfrom_pretrained (default warmup_variants): {t_fp:.2f} s, {model.config['warmup_s']}") rows, outs, fails, prev = [], {}, [], None try: for name, (inputs, kw, later) in variants.items(): if later is not None: t0 = time.perf_counter() spent = model.warmup(**later) t_w = time.perf_counter() - t0 assert spent, (name, "warm-up did nothing") assert model.warmup(**later) == {}, "warmup() must be idempotent" print(f"model.warmup({later}): {t_w * 1e3:.0f} ms") first, out = _ms(lambda: model(inputs[0], **kw)) again = [] for _ in range(5): if prev is not None: # the same switch as before call 1 (resize tables, bucket guess) model(prev[0], **prev[1]) again.append(_ms(lambda: model(inputs[0], **kw))[0]) steady = statistics.median(again) prev = (inputs[2], kw) # the last call of this variant (below) o2 = model(inputs[0], **kw) assert torch.equal(out.keypoints, o2.keypoints) and torch.equal(out.scores, o2.scores), name outs[name] = [out] + [model(x, **kw) for x in inputs[1:3]] ratio = first / steady rows.append((name, first, steady, ratio)) print(f"{name:20s} call 1 {first:7.2f} ms steady (same image) {steady:7.2f} ms ratio {ratio:.2f}") if not (ratio <= MAX_RATIO or first - steady <= ABS_MS): fails.append((name, round(first, 2), round(steady, 2))) finally: model.close() assert not fails, f"first call slower than {MAX_RATIO}x the steady latency: {fails}" # the warm-up does not change the outputs: same results with the minimal warm-up with SuperPoint.from_pretrained(device_id=DEVICE_ID, warmup_variants="minimal") as ref: for name, (inputs, kw, _) in variants.items(): for x, o in zip(inputs[:3], outs[name]): r = ref(x, **kw) assert torch.equal(r.keypoints, o.keypoints) and torch.equal(r.scores, o.scores), name assert (r.descriptors is None) == (o.descriptors is None), name if r.descriptors is not None: assert torch.equal(r.descriptors, o.descriptors), name print("outputs bit-identical to warmup_variants='minimal' for every variant") @pytest.mark.parametrize("spec", ["minimal"]) def test_warmup_spec_reported(spec): with SuperPoint.from_pretrained(device_id=DEVICE_ID, warmup_variants=spec) as model: assert model.config["warmup_variants"]["nms_radii"] == (4,) assert model.config["warmup_s"]["variants"] < 1.0