scorevision: push artifact

detect_crime_ops/scripts/ab_compare.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import statistics
+import time
+from pathlib import Path
+
+from ultralytics import YOLO
+
+
+def load_images(images_file: Path) -> list[str]:
+    out: list[str] = []
+    for line in images_file.read_text().splitlines():
+        s = line.strip()
+        if not s or s.startswith("#"):
+            continue
+        out.append(s)
+    return out
+
+
+def bench_model(model_path: Path, images: list[str], imgsz: int, device: str, conf: float, iou: float):
+    model = YOLO(str(model_path))
+    lat_ms = []
+    pred_counts = []
+    conf_means = []
+    for im in images:
+        t0 = time.perf_counter()
+        res = model.predict(source=im, imgsz=imgsz, device=device, conf=conf, iou=iou, verbose=False)
+        dt = (time.perf_counter() - t0) * 1000.0
+        lat_ms.append(dt)
+        boxes = res[0].boxes
+        n = 0 if boxes is None else len(boxes)
+        pred_counts.append(float(n))
+        if n > 0:
+            conf_means.append(float(boxes.conf.mean().item()))
+        else:
+            conf_means.append(0.0)
+    return {
+        "n_images": len(images),
+        "lat_p50": statistics.median(lat_ms) if lat_ms else 0.0,
+        "lat_p95": sorted(lat_ms)[max(0, int(0.95 * len(lat_ms)) - 1)] if lat_ms else 0.0,
+        "pred_avg": statistics.mean(pred_counts) if pred_counts else 0.0,
+        "conf_avg": statistics.mean(conf_means) if conf_means else 0.0,
+    }
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--a", required=True, help="candidate model")
+    ap.add_argument("--b", required=True, help="baseline model (e.g. king)")
+    ap.add_argument("--images-file", required=True)
+    ap.add_argument("--imgsz", type=int, default=1280)
+    ap.add_argument("--device", default="0")
+    ap.add_argument("--conf", type=float, default=0.05)
+    ap.add_argument("--iou", type=float, default=0.45)
+    args = ap.parse_args()
+
+    imgs = load_images(Path(args.images_file))
+    if not imgs:
+        print("[error] no images in images file")
+        return 1
+
+    a = bench_model(Path(args.a), imgs, args.imgsz, args.device, args.conf, args.iou)
+    b = bench_model(Path(args.b), imgs, args.imgsz, args.device, args.conf, args.iou)
+
+    print("=== A/B Inference Compare (same frames) ===")
+    print(f"A: {args.a}")
+    print(f"  n={a['n_images']} p50={a['lat_p50']:.1f}ms p95={a['lat_p95']:.1f}ms pred_avg={a['pred_avg']:.2f} conf_avg={a['conf_avg']:.3f}")
+    print(f"B: {args.b}")
+    print(f"  n={b['n_images']} p50={b['lat_p50']:.1f}ms p95={b['lat_p95']:.1f}ms pred_avg={b['pred_avg']:.2f} conf_avg={b['conf_avg']:.3f}")
+    print("--- delta (A - B) ---")
+    print(f"d_p50_ms    = {a['lat_p50'] - b['lat_p50']:+.1f}")
+    print(f"d_p95_ms    = {a['lat_p95'] - b['lat_p95']:+.1f}")
+    print(f"d_pred_avg  = {a['pred_avg'] - b['pred_avg']:+.2f}")
+    print(f"d_conf_avg  = {a['conf_avg'] - b['conf_avg']:+.3f}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())