review round 3 (probe/model-card/eval): rank-visible eval failure + cross-rank peak memory + complete run_summary; card derived-and-asserted from config; harness install --no-deps and task ids confirmed; torchrun redirects 3 + tee 3

kernels/p4_stop_probe.py

@@ -60,19 +60,22 @@ ROOT, RUN = "/kaggle/working/mixroot", "/kaggle/working/run"
 # first run of this probe failed at, in 8.6 seconds, on a malformed torchrun command line (E-037). `soak`
 # is the 180-step memory question, and it is only worth 1.6 GPU-hours once the mechanics are known to work.
 MODE = os.environ.get("P4_PROBE_MODE", "soak")
+if MODE not in ("smoke", "soak"):
+    # A typo here would otherwise run the 1.6-hour soak when a 13-minute smoke was asked for.
+    raise SystemExit("P4_PROBE_MODE must be smoke or soak, got %r" % MODE)
 TLOG = "/kaggle/working/tlogs"
 # torchrun takes its first positional as the SCRIPT, not a command: passing sys.executable made
 # it compile the Python binary (E-037). --redirects is a bitmask into the --log-dir per rank:
 # 2 = redirect only stderr, so rank 1's traceback lands in a file while stdout (the loss curve,
 # which §5 requires monitoring on every wake) still streams live to this log. 3 would file both.
-TORCHRUN = ["torchrun", "--nproc_per_node=2", "--redirects", "2", "--log-dir", TLOG]
+TORCHRUN = ["torchrun", "--nproc_per_node=2", "--redirects", "3", "--tee", "3", "--log-dir", TLOG]
 TPS = 262144                                        # the run's real step shape, in tokens
 if MODE == "smoke":
     STEPS, PUSH_EVERY, STOP1, VAL = 20, 10, 10, 200000
-    T_LEG1, T_LEG2, T_FRESH = 1500, 1200, 2400
+    T_LEG1, T_LEG2, T_FRESH = 1200, 900, 1500
 else:
     STEPS, PUSH_EVERY, STOP1, VAL = 180, 60, 120, 2000000
-    T_LEG1, T_LEG2, T_FRESH = 4200, 3000, 2400
+    T_LEG1, T_LEG2, T_FRESH = 4800, 2700, 1500
 TOKENS = STEPS * TPS
 GATE_PEAK = (MODE != "smoke")                       # 20 steps says nothing about allocator drift
 T0 = time.time()
@@ -83,6 +86,8 @@ def run(argv, label, timeout):
     t0 = time.time()
     e = dict(os.environ); e["PYTHONPATH"] = "/kaggle/working"
     e["PYTHONUNBUFFERED"] = "1"   # the -u that torchrun cannot carry
+    e["NCCL_DEBUG"] = "WARN"        # a rank that dies in a collective says so here and nowhere else
+    e["TORCH_CPP_LOG_LEVEL"] = "WARNING"
     p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
                          env=e, bufsize=1, start_new_session=True)
     killed = []
@@ -94,9 +99,19 @@ def run(argv, label, timeout):
         except Exception:
             p.kill()
 
+    def _hard():
+        killed.append(True)
+        try:
+            os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+        except Exception:
+            p.kill()
+
     timer = threading.Timer(timeout, _kill)
     timer.daemon = True
     timer.start()
+    hard = threading.Timer(timeout + 90, _hard)
+    hard.daemon = True
+    hard.start()
     keep, lines = [], []
     try:
         for line in p.stdout:
@@ -110,6 +125,7 @@ def run(argv, label, timeout):
             del keep[:-40]
     finally:
         timer.cancel()
+        hard.cancel()
     rc = p.wait()
     if killed:
         print("   TIMEOUT after %d s" % timeout, flush=True)
@@ -148,9 +164,12 @@ def diag(tag):
     """torchrun's ChildFailedError said `error_file: <N/A>` and printed no traceback, which is useless for
     a rank-1-only failure. --redirects writes each rank's own stdout/stderr into the log dir, so print
     those on a failed leg: the exception is in there, and guessing at it costs a session."""
+    # v4's diag found no *.log at all, so list the tree as well as tailing it: if torchrun named the
+    # files something else, this says so instead of printing nothing and leaving me guessing.
     rc, out = run(["bash", "-c",
-                   'for f in $(find "%s" -name "*.log" 2>/dev/null | head -8); do echo "==== $f"; '
-                   'tail -60 "$f"; done' % TLOG], "DIAG_" + tag, 180)
+                   'ls -R "%s" 2>&1 | head -40; '
+                   'for f in $(find "%s" -type f 2>/dev/null | head -8); do echo "==== $f"; '
+                   'tail -70 "$f"; done' % (TLOG, TLOG)], "DIAG_" + tag, 180)
     return out
 
 
@@ -198,10 +217,16 @@ res = {
              "tokens_consumed": rj1.get("tokens_consumed"), "tok_per_s": rj1.get("tok_per_s"),
              "params": rj1.get("params"), "peak_gpu_gb": rj1.get("peak_gpu_gb"),
              "tok_per_s": rj1.get("tok_per_s"), "final_loss": rj1.get("final_loss"),
+             "peak_alloc_gb": rj1.get("peak_gpu_gb"), "peak_alloc_gb_max_rank":
+             rj1.get("peak_gpu_gb_max_rank"), "peak_reserved_gb_max_rank":
+             rj1.get("peak_reserved_gb_max_rank"), "val_error": rj1.get("val_error"),
              "pushed": pushed, "pointer": {k: ptr.get(k) for k in ("step", "path_in_repo")}},
     "leg2": {"final_step": rj2.get("final_step"), "segment_stop": rj2.get("segment_stop"),
              "tokens_consumed": rj2.get("tokens_consumed"), "val_ppl": rj2.get("val_ppl"),
-             "peak_gpu_gb": rj2.get("peak_gpu_gb"), "tok_per_s": rj2.get("tok_per_s"),
+             "peak_gpu_gb": rj2.get("peak_gpu_gb"), "peak_gpu_gb_max_rank":
+             rj2.get("peak_gpu_gb_max_rank"), "peak_reserved_gb_max_rank":
+             rj2.get("peak_reserved_gb_max_rank"), "val_error": rj2.get("val_error"),
+             "tok_per_s": rj2.get("tok_per_s"),
              "final_loss": rj2.get("final_loss"),
              "pointer_after": {k: ptr2.get(k) for k in ("step", "path_in_repo")},
              "has_final": any(k.startswith("final/") for k in listed2)},
@@ -229,8 +254,13 @@ res["checks"] = {
     # 13.6 GB of ~14.56 usable leaves the run room for a fragmentation spike and for the two CUDA contexts;
     # above that the answer to the user's question is "not on this hardware", not "keep going". Reported in
     # both modes, gated only in the soak: a 20-step peak is not evidence about 3,814 steps.
-    "peak_memory_below_13_6_gb": (not GATE_PEAK) or ((rj1.get("peak_gpu_gb") or 99) <= 13.6
-                                                     and (rj2.get("peak_gpu_gb") or 99) <= 13.6),
+    # rank 0's allocated peak is not the constraint: reserved bytes, the CUDA contexts and the *other*
+    # rank's peak are what an OOM is actually about, so the max across ranks is what gets gated.
+    "peak_memory_below_13_6_gb": (not GATE_PEAK) or (
+        (rj1.get("peak_reserved_gb_max_rank") or rj1.get("peak_gpu_gb_max_rank") or 99) <= 13.6
+        and (rj2.get("peak_reserved_gb_max_rank") or rj2.get("peak_gpu_gb_max_rank") or 99) <= 13.6),
+    "validation_actually_ran": (rj1.get("val_ppl") is not None and rj1.get("val_error") is None
+                                and rj2.get("val_ppl") is not None and rj2.get("val_error") is None),
     "no_nan_and_loss_moved": (rj1.get("final_loss") or 1e9) < 11.0
     and (rj2.get("final_loss") or 1e9) < 11.0,
 }