p4_stop_probe: repin to b6210a29 (per-rank post-train marks + readable probe failures)

kernels/p4_stop_probe.py

@@ -24,7 +24,7 @@ import hashlib, json, os, re, shutil, signal, subprocess, sys, threading, time
 
 os.chdir("/kaggle/working")
 sys.path.insert(0, "/kaggle/working")
-REV = "5889e40c446ce96f549de105c9984c2ddd63588e"
+REV = "b6210a298619be7534b4a06e722bd31592f5d986"
 WANT = {
     "ounce100m_credentials.py": ("ounce100m_credentials.py",
                                  "6525f62f03f2d73650a1eb4f70fcb52d1194caad4ca88b2d8bd8fd54f88339b6"),
@@ -33,7 +33,7 @@ WANT = {
     "hubckpt.py": ("train/hubckpt.py",
                    "d4b50ed0928c678c94ce6764612a4655f2959c0a7b91d6fd588e8efddff298b2"),
     "train_ounce100m.py": ("train/train_ounce100m.py",
-                           "34d541785ac2aa94038b463d5595334b1937c427698fcf2e10134f34cbf91888"),
+                           "d795bb388dab42be5efedf133d0124265b3a0c60ddd8f5c375986a85de70b954"),
 }
 BASE = "https://huggingface.co/Cion-lab/ounce100m-code/resolve/" + REV
 for p, (rp, want) in sorted(WANT.items()):
@@ -128,10 +128,11 @@ def run(argv, label, timeout):
             lines.append(line)
             if line.startswith(("CKPT ", "RUN_JSON ", "resume from", "auto-resume", "precision:",
                                 "params:", "mix:", "checkpoint hub target", "segment boundary",
-                                "latest.json", "TRAIN DONE", "validation loss", "Traceback", "Error")):
+                                "validation skipped", "latest.json", "TRAIN DONE", "validation loss",
+                                "[rank ", "past the ", "peak_stats:", "Traceback", "Error")):
                 print("  KEY>", line[:300], flush=True)
             keep.append(line)
-            del keep[:-40]
+            del keep[:-400]
     finally:
         timer.cancel()
         hard.cancel()
@@ -140,6 +141,16 @@ def run(argv, label, timeout):
         print("   TIMEOUT after %d s" % timeout, flush=True)
         rc = -9
     if rc != 0:
+        # Probe v6 died on the early-stop leg and everything that could have said *where* was in `lines`
+        # and never printed: the last-40-line tail was all torchrun's summary, and the failing rank's own
+        # traceback came a hundred lines earlier. Grep the whole capture, then show the tail.
+        pat = ("[rank", "Traceback", "Error", "error", "Exception", "assert", "exitcode", "Signal",
+               "SystemExit", "refusing", "skipped", "CUDA", "NCCL", "out of memory", 'File "', "line ",
+               "raise ", "FileNotFoundError", "RuntimeError")
+        hits = [l for l in lines if any(q in l for q in pat)]
+        print("   FAILURE LINES (%d of %d):" % (len(hits), len(lines)), flush=True)
+        for l in hits[-45:]:
+            print("   !", l[:300], flush=True)
         print("   TAIL:\n" + "\n".join(keep)[-2500:], flush=True)
     print("%s_RC %s seconds %.1f elapsed %.0f" % (label, rc, time.time() - t0, time.time() - T0),
           flush=True)
@@ -179,6 +190,11 @@ def diag(tag):
                    'ls -R "%s" 2>&1 | head -40; '
                    'for f in $(find "%s" -type f 2>/dev/null | head -8); do echo "==== $f"; '
                    'tail -70 "$f"; done' % (TLOG, TLOG)], "DIAG_" + tag, 180)
+    # run() only echoes lines it recognises, which made v4's and v6's diag print *nothing* while the
+    # rank logs sat right there on disk. Say what was found, even if it is a traceback shape we do not
+    # have a filter word for.
+    print("   DIAG %s (%d chars, rc %s):\n%s" % (tag, len(out or ""), rc, (out or "")[-4000:]),
+          flush=True)
     return out