stage5 resume support

run_all.py

@@ -154,6 +154,16 @@ def stage5(args):
     (B) R = k+4 + untied stem block + dense per-iteration supervision with
     hold-after-k targets f^{min(t,k)}(x) - the full fix package."""
     lr = chosen_lr(args.hub_repo)
+    # resume support: a fresh container has no local checkpoints, so pull
+    # finished ones from the Hub; the skip-if-present check below then
+    # skips them and only unfinished variants train
+    if not glob.glob("out/rec1m4*.pt"):
+        from huggingface_hub import snapshot_download
+        snapshot_download(repo_id=args.hub_repo, repo_type="model",
+                          allow_patterns=["ckpts/rec1m4*.pt"],
+                          local_dir="out")
+        for f in glob.glob("out/ckpts/rec1m4*.pt"):
+            os.replace(f, os.path.join("out", os.path.basename(f)))
     variants = [
         ("rec1m4", {"arch": "rec", "depth": 1, "iters": "aligned4"}),
         ("rec1m4dense", {"arch": "rec", "depth": 1, "iters": "aligned4",