perm_sha: identify the visit order by hashing the permutation, not just its seed (E-035 residual) -- cursor gains shuffle_perm_sha, asserted on resume; session repins the two files

train/shard_dataset.py

@@ -80,6 +80,22 @@ class PackedTokenStore:
         return out
 
 
+def perm_sha(n_samples, seed, base=None):
+    """Identity of the visit order itself, not just the seed that produced it.
+
+    `seed` pins the order only for a fixed numpy. Kaggle updates its notebook image underneath us, and a
+    session that resumed after such a bump would read a different permutation from the same seed -- some
+    windows twice, others never -- while every other guard (weights, cursor arithmetic, fingerprint,
+    batch size) still agreed. Hashing the full permutation is ~10 ms and makes that state impossible to
+    enter quietly. Computed before the resume slice, so two sessions at different offsets agree on it.
+    `base` lets a caller that already generated the permutation avoid generating it twice.
+    """
+    import hashlib
+    if base is None:
+        base = np.random.default_rng(np.random.PCG64(seed)).permutation(n_samples)
+    return hashlib.sha256(np.ascontiguousarray(base).tobytes()).hexdigest()[:16]
+
+
 def make_dataset(store, seq_len, seed, start_sample=0, count=None, shuffle=True):
     """A map-style torch Dataset of (input_ids, labels, attention_mask) with an exact resume cursor.
 
@@ -91,11 +107,14 @@ def make_dataset(store, seq_len, seed, start_sample=0, count=None, shuffle=True)
 
     n_samples = store.total_tokens // seq_len
     perm = None
+    order_sha = "unshuffled"
     if shuffle:
         # Seed must be identical on every resume or the two runs read different data orders. Stored in
-        # the checkpoint (cursor.json) and asserted by the trainer.
-        perm = np.random.default_rng(np.random.PCG64(seed)).permutation(n_samples)
-        perm = perm[start_sample:]
+        # the checkpoint (cursor.json) and asserted by the trainer -- alongside perm_sha(), which catches
+        # the case the seed alone cannot: the same seed on a different numpy.
+        base = np.random.default_rng(np.random.PCG64(seed)).permutation(n_samples)
+        order_sha = perm_sha(n_samples, seed, base=base)
+        perm = base[start_sample:]
     else:
         perm = np.arange(start_sample, n_samples, dtype=np.int64)
     if count is not None:
@@ -117,20 +136,25 @@ def make_dataset(store, seq_len, seed, start_sample=0, count=None, shuffle=True)
         # these names keeps it from dropping anything.
         column_names = ["input_ids", "labels", "attention_mask"]
 
-    _DS.perm = perm
-    return _DS()
+    ds = _DS()
+    # Instance attributes, not class attributes: `_DS.perm = perm` would let a second make_dataset() call
+    # in the same process overwrite the first one's order.
+    ds.perm = perm
+    ds.order_sha = order_sha
+    return ds
 
 
-def cursor_dict(seq_len, samples_consumed, seed, dataset_files_sha=None, step=None):
+def cursor_dict(seq_len, samples_consumed, seed, dataset_files_sha=None, step=None, order_sha=None):
     """The whole of the data-position state. Written into every checkpoint, checked on every resume.
 
     `step` is recorded alongside `samples_consumed` so the two can be cross-checked on resume: they are
     redundant by construction (samples = step x sequences-per-step), and a cursor where they disagree is a
-    corrupted or hand-edited file, not a position to continue from.
+    corrupted or hand-edited file, not a position to continue from. `shuffle_perm_sha` is the identity of
+    the visit order, which the seed alone does not pin (see perm_sha).
     """
     return {"seq_len": seq_len, "samples_consumed": samples_consumed,
             "tokens_consumed": samples_consumed * seq_len, "shuffle_seed": seed,
-            "step": step, "dataset_files_sha": dataset_files_sha}
+            "shuffle_perm_sha": order_sha, "step": step, "dataset_files_sha": dataset_files_sha}
 
 
 def load_cursor(dirpath):