scorevision: push artifact

detect_crime_ops/scripts/summarize_challenges.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import csv
+from collections import Counter
+from pathlib import Path
+
+
+def _f(x: str) -> float:
+    try:
+        return float(x)
+    except Exception:
+        return 0.0
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--csv", required=True)
+    args = ap.parse_args()
+
+    p = Path(args.csv)
+    if not p.exists():
+        print(f"[error] missing {p}")
+        return 1
+
+    rows = list(csv.DictReader(p.open()))
+    if not rows:
+        print("[info] no rows yet")
+        return 0
+
+    n = len(rows)
+    my = [_f(r.get("my_score", "0")) for r in rows]
+    king = [_f(r.get("king_score", "0")) for r in rows]
+    delta = [m - k for m, k in zip(my, king)]
+    p95 = [_f(r.get("my_latency_p95_ms", "0")) for r in rows]
+    preds = [_f(r.get("my_pred_count", "0")) for r in rows]
+
+    tags = Counter()
+    for r in rows:
+        raw = r.get("tag_list", "")
+        for t in [x.strip() for x in raw.split(",") if x.strip()]:
+            tags[t] += 1
+
+    print("=== Detect-crime challenge summary ===")
+    print(f"rows={n}")
+    print(f"my_score:   avg={sum(my)/n:.4f} min={min(my):.4f} max={max(my):.4f}")
+    print(f"king_score: avg={sum(king)/n:.4f} min={min(king):.4f} max={max(king):.4f}")
+    print(f"delta:      avg={sum(delta)/n:+.4f} wins={sum(1 for d in delta if d > 0)}/{n}")
+    print(f"latency p95 avg={sum(p95)/n:.1f}ms")
+    print(f"pred_count  avg={sum(preds)/n:.2f}")
+    if tags:
+        print("top failure tags:")
+        for k, v in tags.most_common(8):
+            print(f"  - {k}: {v}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())