trainer: per-rank marks through the post-training sequence (v6's rank-1 death was unlocalisable because every line there is rank-0-only)

train/train_ounce100m.py

@@ -618,6 +618,12 @@ def main():
     consumed_this_run = max(0, step - start_step) * tokens_per_step
     hist = [h for h in tr.state.log_history if "loss" in h]
     final_loss = hist[-1]["loss"] if hist else None
+    # Every rank, with `print` and not `say`: probe v6 died on the leg that stopped early and the only
+    # evidence torchrun passed up was "rank 1 exitcode 1, error_file <N/A>", because the lines that would
+    # have localised it are rank-0-only. These four marks say which statement in the post-training
+    # sequence a rank never reached.
+    print(f"[rank {rank}] post-train: step={step}/{steps_planned} log_history_loss_entries={len(hist)} "
+          f"final_loss={final_loss} start_step={start_step}", flush=True)
     say(f"TRAIN DONE step={step}/{steps_planned} elapsed={elapsed/60:.1f} min "
         f"tokens_consumed={consumed:,} last_loss={final_loss}")
     # E-006, again: a run that produced NaN still reports a confident tokens/sec. Never let that pass.
@@ -633,6 +639,8 @@ def main():
     # raising -- which is exactly the signature. The report's PPL comes from the horizon leg, and the
     # mid-run legs keep their training loss, which is what §5 asks to watch on every wake.
     forced_stop = bool(hub_cb and hub_cb.stopped_at and step < steps_planned)
+    print(f"[rank {rank}] past the stop guard: forced_stop={forced_stop} "
+          f"stopped_at={getattr(hub_cb, 'stopped_at', None)}", flush=True)
     ppl = None
     val_err = None
     if forced_stop:
@@ -655,7 +663,9 @@ def main():
     # `if rank == 0:` would hang, and reading rank 0's allocator alone is what made the probe's memory
     # verdict systematically optimistic -- it hides the other card's peak, the two CUDA contexts, and the
     # reserved-but-unallocated blocks that are the real ceiling on a 14.56 GiB T4 (review finding B4).
+    print(f"[rank {rank}] peak_stats: entering (this is a collective)", flush=True)
     mem = peak_stats()
+    print(f"[rank {rank}] peak_stats: done, max_reserved={mem['max_reserved_gb']} GiB", flush=True)
     say("memory:", json.dumps(mem, sort_keys=True))
 
     # Everything below writes files, so it is rank 0's alone (E-029). Two ranks building the same export
@@ -705,6 +715,7 @@ def main():
                    "peak_gpu_gb_max_rank": mem["max_allocated_gb"],
                    "peak_reserved_gb_max_rank": mem["max_reserved_gb"]},
                   open(os.path.join(exp, "run_summary.json"), "w"), indent=1, sort_keys=True)
+    print(f"[rank {rank}] past the export and push block", flush=True)
     segment = forced_stop        # the same predicate, computed before the validation decision above
     if api is not None and not segment and rank == 0:
         r = hubckpt.push_and_prune(args.hub_repo, exp, "final", api, repo_type=args.hub_repo_type,