Add training-kit/monitor_ppocr_eval.py

training-kit/monitor_ppocr_eval.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Stop PP-OCR training when held-out normalized edit similarity stalls or diverges."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import signal
+import time
+from pathlib import Path
+
+
+PATTERN = re.compile(r"cur metric,.*norm_edit_dis: ([0-9.eE+-]+)")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--log", type=Path, required=True)
+    parser.add_argument("--pid-file", type=Path, required=True)
+    parser.add_argument("--min-evals", type=int, default=5)
+    parser.add_argument("--patience", type=int, default=5)
+    parser.add_argument("--min-delta", type=float, default=0.002)
+    parser.add_argument("--divergence-delta", type=float, default=0.03)
+    parser.add_argument("--divergence-patience", type=int, default=2)
+    parser.add_argument("--initial-best", type=float, default=-1.0)
+    args = parser.parse_args()
+
+    best = args.initial_best
+    stale = diverged = seen = 0
+    while True:
+        if not args.pid_file.exists():
+            time.sleep(2)
+            continue
+        pid = int(args.pid_file.read_text())
+        try:
+            os.kill(pid, 0)
+        except ProcessLookupError:
+            return
+
+        scores = [float(value) for value in PATTERN.findall(args.log.read_text(errors="ignore"))]
+        for score in scores[seen:]:
+            seen += 1
+            if score > best + args.min_delta:
+                best, stale = score, 0
+            else:
+                stale += 1
+            diverged = diverged + 1 if score < best - args.divergence_delta else 0
+            print(
+                f"eval={seen} score={score:.6f} best={best:.6f} "
+                f"stale={stale} diverged={diverged}",
+                flush=True,
+            )
+            if seen >= args.min_evals and (
+                stale >= args.patience or diverged >= args.divergence_patience
+            ):
+                os.kill(pid, signal.SIGTERM)
+                print("EARLY_STOP", flush=True)
+                return
+        time.sleep(3)
+
+
+if __name__ == "__main__":
+    main()