Upload folder using huggingface_hub

modal_train.py

@@ -76,8 +76,15 @@ def train_on_l4(hours: float = 20.0, ckpt_every: int = 250,
     os.makedirs(ckpt_dir, exist_ok=True)
 
     # ---- ensure data (generate in-container; fast HF egress) ----
-    if not os.path.exists(os.path.join(data_dir, "train.bin")):
-        print(f"[modal] train.bin missing; generating data locally (scale={scale}) ...")
+    # Only treat the dataset as complete if meta.json exists (a partial
+    # train.bin from a killed run must be regenerated).
+    meta_path = os.path.join(data_dir, "meta.json")
+    if not os.path.exists(meta_path):
+        stale = os.path.join(data_dir, "train.bin")
+        if os.path.exists(stale):
+            print("[modal] found partial train.bin without meta.json; discarding.")
+            os.remove(stale)
+        print(f"[modal] generating data locally (scale={scale}) ...")
         subprocess.run([sys.executable, "prep.py", "--out-dir", data_dir,
                         "--scale", str(scale)], check=True)