train: shard_dataset -- static-review fixes: absolute-step cursor, hub_listing pagination, cursor required on resume, repo_type everywhere, rank-0 repo create

train/shard_dataset.py

@@ -121,11 +121,16 @@ def make_dataset(store, seq_len, seed, start_sample=0, count=None, shuffle=True)
     return _DS()
 
 
-def cursor_dict(seq_len, start_sample, seed, dataset_files_sha=None):
-    """The whole of the data-position state. Written into every checkpoint, checked on every resume."""
-    return {"seq_len": seq_len, "samples_consumed": start_sample,
-            "tokens_consumed": start_sample * seq_len, "shuffle_seed": seed,
-            "dataset_files_sha": dataset_files_sha}
+def cursor_dict(seq_len, samples_consumed, seed, dataset_files_sha=None, step=None):
+    """The whole of the data-position state. Written into every checkpoint, checked on every resume.
+
+    `step` is recorded alongside `samples_consumed` so the two can be cross-checked on resume: they are
+    redundant by construction (samples = step x sequences-per-step), and a cursor where they disagree is a
+    corrupted or hand-edited file, not a position to continue from.
+    """
+    return {"seq_len": seq_len, "samples_consumed": samples_consumed,
+            "tokens_consumed": samples_consumed * seq_len, "shuffle_seed": seed,
+            "step": step, "dataset_files_sha": dataset_files_sha}
 
 
 def load_cursor(dirpath):