p4_stop_probe: pins at a813e5c7 + bounded-readback rehearsal stage / tee-prefix fix

kernels/p4_stop_probe.py

@@ -20,18 +20,18 @@
 # hundred steps, the save path's host/GPU copies while the card is 84 % full, and the eval forward pass.
 # Peak memory is asserted, not eyeballed. If it holds, the run adopts it as D-018 with a finer push cadence
 # as the bounded blast radius; if it does not, D-017 stands and this is the measurement that says so.
-import hashlib, json, os, shutil, signal, subprocess, sys, threading, time
+import hashlib, json, os, re, shutil, signal, subprocess, sys, threading, time
 
 os.chdir("/kaggle/working")
 sys.path.insert(0, "/kaggle/working")
-REV = "7570b7b264f82a536a7a0f8f3e45241dca10d883"
+REV = "a813e5c7fb7539a0586756c76b95f7520934f7a0"
 WANT = {
     "ounce100m_credentials.py": ("ounce100m_credentials.py",
                                  "6525f62f03f2d73650a1eb4f70fcb52d1194caad4ca88b2d8bd8fd54f88339b6"),
     "shard_dataset.py": ("train/shard_dataset.py",
                          "f35653bf4c8f2cfe7bb0c2c7835e308505fb5a84b4eb002d0f82f2cada768ca6"),
     "hubckpt.py": ("train/hubckpt.py",
-                   "568c31b300906cb8d78a59ef890baa9731b18e064e3d69b0eaea4b45c546f23e"),
+                   "a21de427b9c7586d3cc104e80d230a085d00907829fe1148d0b1ddbec9f4d20a"),
     "train_ounce100m.py": ("train/train_ounce100m.py",
                            "744259c7519e5a12c2d2bf4dca534fa39f7ddb7e6689e1a5a9e0925bfaa640aa"),
 }
@@ -70,6 +70,11 @@ TLOG = "/kaggle/working/tlogs"
 # watching stays live *and* each rank's traceback is on disk for diag() below.
 TORCHRUN = ["torchrun", "--nproc_per_node=2", "--redirects", "3", "--tee", "3", "--log-dir", TLOG]
 TPS = 262144                                        # the run's real step shape, in tokens
+# `--tee` prefixes every forwarded line with the worker name, and this file parses those lines: v5's
+# harvest() found no `RUN_JSON ` because the trainer's rank-0 output arrived as
+# "[default0]: RUN_JSON {...}", so every step/token/param check reported false on a leg that had actually
+# passed, and the failure looked like a trainer bug rather than a probe bug. Strip it once, on the way in.
+TEE = re.compile(r"^\[(?:default|rank|worker)\d*\]:\s*")
 if MODE == "smoke":
     STEPS, PUSH_EVERY, STOP1, VAL = 20, 10, 10, 200000
     T_LEG1, T_LEG2, T_FRESH = 1200, 900, 1500
@@ -119,7 +124,7 @@ def run(argv, label, timeout):
     keep, lines = [], []
     try:
         for line in p.stdout:
-            line = line.rstrip("\n")
+            line = TEE.sub("", line.rstrip("\n"), count=1)
             lines.append(line)
             if line.startswith(("CKPT ", "RUN_JSON ", "resume from", "auto-resume", "precision:",
                                 "params:", "mix:", "checkpoint hub target", "segment boundary",