scorevision: push artifact

training/eval_mine_vs_king.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""
+Val-split mAP: your exported ONNX vs current king ONNX (alfred8995/crime001).
+
+King ONNX uses semantic class ids in a different order than `scratch/data/data.yaml`.
+Remap king outputs to dataset indices before TP/FP matching:
+
+    king id  king name      -> yaml id / name
+       0     balaclava      -> 0 balaclava
+       1     hoodie         -> 4 hoodie
+       2     glove          -> 2 glove
+       3     wooden bat     -> 1 bat
+       4     spray can      -> 5 spray paint
+       5     graffiti       -> 3 graffiti
+
+Usage:
+  python3 eval_mine_vs_king.py --data /root/turbovision_crime/scratch/data/data.yaml \
+      --mine /root/turbovision_crime/scratch/crime_miner/weights.onnx \
+      --king /root/turbovision_crime/king_models/alfred8995_crime001/weights.onnx
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+from datetime import datetime, timezone
+from pathlib import Path
+
+import torch
+from ultralytics import YOLO
+from ultralytics.models.yolo.detect import DetectionValidator
+
+# king cls -> dataset cls (see module docstring)
+KING_TO_DATASET = torch.tensor([0, 4, 2, 1, 5, 3])
+
+
+class KingRemappedValidator(DetectionValidator):
+    def postprocess(self, preds):  # noqa: ANN001
+        outs = super().postprocess(preds)
+        for d in outs:
+            if d["cls"].numel() == 0:
+                continue
+            ci = d["cls"].long().clamp(0, 5).view(-1)
+            mapped = KING_TO_DATASET[ci.cpu()].to(device=d["cls"].device, dtype=d["cls"].dtype)
+            d["cls"] = mapped
+        return outs
+
+    def init_metrics(self, model):  # noqa: ANN001
+        super().init_metrics(model)
+        # Confusion-matrix / prints use canonical names from dataset
+        names = {
+            0: "balaclava",
+            1: "bat",
+            2: "glove",
+            3: "graffiti",
+            4: "hoodie",
+            5: "spray paint",
+        }
+        self.names = names
+        self.nc = len(names)
+        self.metrics.names = names
+        try:
+            from ultralytics.utils.metrics import ConfusionMatrix
+
+            self.confusion_matrix = ConfusionMatrix(names=names, save_matches=self.args.plots and self.args.visualize)
+        except Exception:
+            pass
+
+
+def run_val(weights: Path, data: Path, imgsz: int, batch: int, device: str, validator_cls: type):
+    model = YOLO(str(weights))
+    return model.val(
+        data=str(data),
+        imgsz=imgsz,
+        batch=batch,
+        device=device,
+        plots=False,
+        verbose=False,
+        validator=validator_cls,
+    )
+
+
+def maybe_log_ops_row(
+    csv_path: Path,
+    challenge_id: str,
+    scene: str,
+    mine: Path,
+    king: Path,
+    m_mine,
+    m_king,
+) -> None:
+    fields = [
+        "timestamp_utc",
+        "challenge_id",
+        "scene",
+        "lighting",
+        "camera_angle",
+        "motion_blur",
+        "crowd_level",
+        "my_model_id",
+        "my_score",
+        "king_model_id",
+        "king_score",
+        "delta_score",
+        "my_pred_count",
+        "king_pred_count",
+        "my_latency_p50_ms",
+        "my_latency_p95_ms",
+        "my_latency_p99_ms",
+        "dropped_frames",
+        "timeout_count",
+        "balaclava_hits",
+        "bat_hits",
+        "glove_hits",
+        "graffiti_hits",
+        "hoodie_hits",
+        "spray_paint_hits",
+        "tag_list",
+        "data_source_mix",
+        "notes",
+    ]
+    row = {
+        "timestamp_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "challenge_id": challenge_id,
+        "scene": scene,
+        "lighting": "",
+        "camera_angle": "",
+        "motion_blur": "",
+        "crowd_level": "",
+        "my_model_id": mine.name,
+        "my_score": f"{m_mine.box.map50:.6f}",
+        "king_model_id": king.name,
+        "king_score": f"{m_king.box.map50:.6f}",
+        "delta_score": f"{(m_mine.box.map50 - m_king.box.map50):.6f}",
+        "my_pred_count": "",
+        "king_pred_count": "",
+        "my_latency_p50_ms": f"{m_mine.speed.get('inference', 0.0):.3f}",
+        "my_latency_p95_ms": "",
+        "my_latency_p99_ms": "",
+        "dropped_frames": "",
+        "timeout_count": "",
+        "balaclava_hits": "",
+        "bat_hits": "",
+        "glove_hits": "",
+        "graffiti_hits": "",
+        "hoodie_hits": "",
+        "spray_paint_hits": "",
+        "tag_list": "offline_val_split",
+        "data_source_mix": "",
+        "notes": "auto-log from eval_mine_vs_king.py; scores are mAP50 on local val split",
+    }
+    csv_path.parent.mkdir(parents=True, exist_ok=True)
+    new_file = not csv_path.exists()
+    with csv_path.open("a", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fields)
+        if new_file:
+            writer.writeheader()
+        writer.writerow(row)
+    print(f"[ops-log] appended row -> {csv_path}", flush=True)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--data", required=True)
+    ap.add_argument("--mine", required=True)
+    ap.add_argument("--king", required=True)
+    ap.add_argument("--imgsz", type=int, default=1280)
+    ap.add_argument("--batch", type=int, default=8)
+    ap.add_argument("--device", default="0")
+    ap.add_argument("--log-csv", default="", help="optional path to detect_crime_ops challenge_log.csv")
+    ap.add_argument("--challenge-id", default="", help="optional challenge/eval id for ops logging")
+    ap.add_argument("--scene", default="offline_val_split", help="scene tag used when --log-csv is set")
+    args = ap.parse_args()
+
+    data = Path(args.data)
+    mine = Path(args.mine)
+    king = Path(args.king)
+
+    print("=== Yours (exported miner ONNX) ===", flush=True)
+    m_mine = run_val(mine, data, args.imgsz, args.batch, args.device, DetectionValidator)
+    print(f"images={m_mine.speed.get('images', '?')}  "
+          f"mAP50={m_mine.box.map50:.5f}  mAP50-95={m_mine.box.map:.5f}  "
+          f"P={m_mine.box.mp:.5f}  R={m_mine.box.mr:.5f}", flush=True)
+
+    print("\n=== King (alfred8995/crime001, cls remapped to data.yaml order) ===", flush=True)
+    m_king = run_val(king, data, args.imgsz, args.batch, args.device, KingRemappedValidator)
+    print(f"images={m_king.speed.get('images', '?')}  "
+          f"mAP50={m_king.box.map50:.5f}  mAP50-95={m_king.box.map:.5f}  "
+          f"P={m_king.box.mp:.5f}  R={m_king.box.mr:.5f}", flush=True)
+
+    print("\n=== Delta (yours − king) on same val split ===", flush=True)
+    print(f"d_mAP50     = {m_mine.box.map50 - m_king.box.map50:+.5f}", flush=True)
+    print(f"d_mAP50-95  = {m_mine.box.map - m_king.box.map:+.5f}", flush=True)
+    print("\nInterpretation:", flush=True)
+    print("- Same boxes (IoU/geometric inference) regardless of remap; remap only aligns class IDs for AP.", flush=True)
+    print("- This is YOUR silver-label val set, not the live Manako leaderboard.", flush=True)
+    if args.log_csv:
+        cid = args.challenge_id or f"offline_val_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}"
+        maybe_log_ops_row(Path(args.log_csv), cid, args.scene, mine, king, m_mine, m_king)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())