phase4_prep: exercise the permutation-identity guard (perm_sha) through save_cursor/load_cursor

kernels/phase4_prep.py

@@ -1,11 +1,11 @@
 # Phase 4 rehearsal on a CPU instance: every check a session can fail, run where failing costs no quota.
 #
-# The first GPU session of the run bills ~11 hours before it is provably healthy, and E-031 is exactly the
-# shape of bug that only shows up mid-run (a dataset-length guard that compared the wrong quantity would
-# have refused the run's third leg). So this kernel does, on free CPU: fetches the pinned code and asserts
-# its sha256s, resolves credentials, reads the checkpoint pointer, downloads the PUBLISHED mix from the
-# Hub, hashes every shard against the manifest, rehearses the session planner for the whole schedule, and
-# proves resume-equivalence of the reader at the real checkpoint boundaries with the real seed.
+# The first GPU session of the run bills ~6.5 hours before it is provably healthy, and E-031/E-034 are both
+# exactly the shape of bug that only shows up mid-run or at the very first launch. So this kernel does, on
+# free CPU: fetches the pinned code and asserts its sha256s, resolves credentials, reads the checkpoint
+# pointer, downloads the PUBLISHED mix from the Hub, hashes every shard against the manifest, rehearses the
+# session planner across the whole schedule, and proves resume-equivalence of the reader at the boundaries
+# the run actually crosses.
 #
 # It ends with the same PLAN_JSON the GPU session will print, so the numbers can be compared before and
 # after rather than trusting that the code did not change in between.
@@ -173,16 +173,20 @@ assert legs[-1][1] == steps and [l[0] for l in legs[1:]] == [l[1] for l in legs[
 # dataset from load_cursor() rather than a hand-computed offset is the point -- the derivation from
 # step to samples_consumed is part of what is being tested.
 full = SD.make_dataset(store, SEQ, 20260919, start_sample=0)
+order = SD.perm_sha(n_samples, 20260919)
+assert full.order_sha == order, "two derivations of the order identity disagree"
 for boundary in sorted({S.CKPT_EVERY, legs[len(legs) // 2][0], steps - S.CKPT_EVERY}):
     d = "/tmp/cur_%d" % boundary
     os.makedirs(d, exist_ok=True)
     SD.save_cursor(d, SD.cursor_dict(SEQ, boundary * seqs_per_step, 20260919, fingerprint,
-                                     step=boundary))
+                                     step=boundary, order_sha=order))
     c = SD.load_cursor(d)
     assert c["step"] * seqs_per_step == c["samples_consumed"], c
     assert c["dataset_files_sha"] == fingerprint, c
+    assert c["shuffle_perm_sha"] == order, c
     s0 = c["samples_consumed"]
     resumed = SD.make_dataset(store, SEQ, c["shuffle_seed"], start_sample=s0)
+    assert resumed.order_sha == order, "a resume lost the visit order"
     assert len(resumed) == len(full) - s0, (len(resumed), len(full), s0)
     same = True
     for j in (0, 1, len(resumed) // 2, len(resumed) - 1):