Upload train/train_ounce100m.py with huggingface_hub

train/train_ounce100m.py

@@ -322,8 +322,21 @@ def main():
         tk = os.environ.get("HF_TOKEN")
         ptr = hubckpt.latest_pointer(args.hub_repo, repo_type=args.hub_repo_type, token=tk)
         if ptr.get("not_found"):
-            # A 404 is a fact: this run has never checkpointed, so step 0 is the right place to start.
-            say(f"no latest.json on {args.hub_repo} -> starting from step 0")
+            # A 404 is normally a fact -- "this run has never checkpointed" -- and step 0 is then right.
+            # But a missing pointer beside existing checkpoints means the pointer upload failed after a
+            # good checkpoint, and starting from zero would silently retrain the corpus from the top,
+            # which §3.1 forbids and E-018 is about. So list the repo before believing the 404.
+            try:
+                listed = sorted(k for k in hubckpt.hub_listing(args.hub_repo, args.hub_repo_type,
+                                                               token=tk)
+                                if k.startswith("ckpt/checkpoint-"))
+            except Exception as e:
+                listed = [f"<listing failed: {type(e).__name__}>"]
+            if listed:
+                raise SystemExit(f"--resume auto found no latest.json on {args.hub_repo}, but the repo "
+                                 f"holds {len(listed)} checkpoint path(s) ({listed[:3]}). The pointer is "
+                                 "missing, not the run -- refusing to start from step 0.")
+            say(f"no latest.json and no checkpoints on {args.hub_repo} -> starting from step 0")
             args.resume = ""
         elif "error" in ptr:
             # Anything else (401, timeout, DNS) means the lookup failed, not that there is nothing there.
@@ -331,7 +344,13 @@ def main():
             # exactly the §3.1 violation this whole branch exists to prevent (E-018's shape).
             raise SystemExit(f"--resume auto could not read the hub pointer: {ptr['error']}")
         else:
-            dest = os.path.join(args.out, os.path.basename(ptr["path_in_repo"]))
+            # Both ranks need the checkpoint on disk, and this block runs before TrainingArguments has
+            # created a process group -- so there is no barrier to synchronise on. Downloading into a
+            # per-rank directory instead removes the race entirely (two writers, one path), at the cost of
+            # a second 1.7 GB pull that happens in parallel. Each rank then reads its own verified copy and
+            # derives the same start_sample/start_step from the same cursor.json bytes.
+            dest = os.path.join(args.out, f"resume_r{rank}",
+                                os.path.basename(ptr["path_in_repo"]))
             say(f"auto-resume: hub says step {ptr.get('step')} at {ptr['path_in_repo']}")
             got = hubckpt.download_checkpoint(args.hub_repo, ptr["path_in_repo"], dest, HfApi(),
                                               repo_type=args.hub_repo_type, token=tk)
@@ -374,7 +393,7 @@ def main():
     vstore = SD.PackedTokenStore(args.root, man, files=vs)
     n_val = min(max(1, args.val_tokens // args.seq_len), vstore.total_tokens // args.seq_len)
     val_ds = SD.make_dataset(vstore, args.seq_len, args.data_seed, shuffle=False, count=n_val)
-    need = steps_planned * seqs_per_step
+    need = (steps_planned - start_step) * seqs_per_step
     if len(train_ds) < need:
         # Reading past the end of the stream makes the Trainer begin epoch 2 at the top of the same
         # permutation: the run would then report the full token target while quietly re-training on the