phase4_prep: pins at a813e5c7 + bounded-readback rehearsal stage / tee-prefix fix

kernels/phase4_prep.py

@@ -13,18 +13,18 @@ import hashlib, json, os, subprocess, sys, time
 
 os.chdir("/kaggle/working")
 sys.path.insert(0, "/kaggle/working")
-REV = os.environ.get("OUNCE100M_REV") or "7570b7b264f82a536a7a0f8f3e45241dca10d883"
+REV = os.environ.get("OUNCE100M_REV") or "a813e5c7fb7539a0586756c76b95f7520934f7a0"
 WANT = {
   "ounce100m_credentials.py": ("ounce100m_credentials.py",
                                "6525f62f03f2d73650a1eb4f70fcb52d1194caad4ca88b2d8bd8fd54f88339b6"),
   "shard_dataset.py": ("train/shard_dataset.py",
                        "f35653bf4c8f2cfe7bb0c2c7835e308505fb5a84b4eb002d0f82f2cada768ca6"),
   "hubckpt.py": ("train/hubckpt.py",
-                 "568c31b300906cb8d78a59ef890baa9731b18e064e3d69b0eaea4b45c546f23e"),
+                 "a21de427b9c7586d3cc104e80d230a085d00907829fe1148d0b1ddbec9f4d20a"),
   "train_ounce100m.py": ("train/train_ounce100m.py",
                          "744259c7519e5a12c2d2bf4dca534fa39f7ddb7e6689e1a5a9e0925bfaa640aa"),
   "phase4_session.py": ("kernels/phase4_session.py",
-                        "28fb3bb477f956b5dab804d8d335acf85193900116ee8d0c520139dad495e112"),
+                        "925ef1f160dcd7ae52090d10fbb7e8abca1f52f3b087d36a6fc35fad8a5de6e1"),
 }
 BASE = "https://huggingface.co/Cion-lab/ounce100m-code/resolve/" + REV
 for p, (rp, want) in sorted(WANT.items()):
@@ -219,6 +219,87 @@ rc3, _ = run([sys.executable, "-c",
               "print('TRAINER_HELP_RC', p.returncode, 'lines', len((p.stdout or '').splitlines()))\n"
               "assert p.returncode == 0, (p.stdout or '')[-1500:] + (p.stderr or '')[-1500:]\n"],
              "CURSOR_AND_IMPORT", 900)
-print("VERDICT PHASE4_PREP rc_launcher", rc1, "rc_check", rc2, "rc_cursor", rc3,
+
+# Stage 4 -- checkpoint verification against real Hub bytes, on the theory that a gate which authorises
+# deleting the only local copy of a checkpoint should be tested where failing is free. Probe v5 measured
+# the old whole-object read-back at `push+verify=797.7s` for an 850 MB optimizer.pt (and 26 s for the same
+# cycle once the CDN was warm), so the read-back is now two bounded Range requests; this proves the
+# replacement still says yes to the 1.27 GB the probe really pushed and still says no three different ways.
+# PREP_VERIFY_REPO exists because the probe repos are scratch: point it at `Cion-lab/ounce100m-ckpt` plus a
+# `ckpt/checkpoint-N` once the run has one and this stage outlives them.
+VERIFY = """
+import json, os, sys, time
+sys.path.insert(0, "/kaggle/working")
+import hubckpt
+from huggingface_hub import snapshot_download
+REPO = os.environ.get("PREP_VERIFY_REPO") or "Cion-lab/ounce100m-ckpt-probe-0920-0747"
+SUB = os.environ.get("PREP_VERIFY_SUB") or "ckpt/checkpoint-10"
+d = "/kaggle/working/verify_subject"
+t0 = time.time()
+snapshot_download(repo_id=REPO, repo_type="dataset", allow_patterns=SUB + "/*", local_dir=d,
+                  max_workers=4)
+local = os.path.join(d, SUB)
+print("GOT", len(os.listdir(local)), "files from", REPO, SUB, "in", round(time.time() - t0, 1), "s",
+      flush=True)
+assert os.path.exists(os.path.join(local, "model.safetensors")), sorted(os.listdir(local))
+
+t0 = time.time()
+v = hubckpt.verify_checkpoint(REPO, local, SUB, repo_type="dataset")
+secs = round(time.time() - t0, 1)
+print("VERIFY_CLEAN ok", v["ok"], "readback_ok", v["readback_ok"], "seconds", secs,
+      json.dumps({k: v[k] for k in ("n_local", "n_hub", "missing", "mismatch", "extra", "readback")},
+                 default=str)[:600], flush=True)
+assert v["ok"] and v["readback_ok"], v
+assert secs < 180, f"verification took {secs}s -- the bounded read-back was supposed to make this cheap"
+rb = v["readback"]
+assert rb["path"] == "optimizer.pt" and rb["bytes"] < (10 << 20) and len(rb["ranges"]) == 2, rb
+
+# 1. A byte the Hub does not have: the local sha256 stops matching the stored lfs.oid.
+p = os.path.join(local, "optimizer.pt")
+with open(p, "r+b") as fh:
+    fh.seek(500_000_001)
+    b = fh.read(1)
+    fh.seek(500_000_001)
+    fh.write(bytes([b[0] ^ 0xFF]))
+v2 = hubckpt.verify_checkpoint(REPO, local, SUB, repo_type="dataset")
+print("VERIFY_TAMPERED ok", v2["ok"], "mismatch", json.dumps(v2["mismatch"])[:260], flush=True)
+assert not v2["ok"] and v2["mismatch"], "a flipped byte at 500 MB verified clean"
+with open(p, "r+b") as fh:
+    fh.seek(500_000_001)
+    fh.write(b)
+assert hubckpt.verify_checkpoint(REPO, local, SUB, repo_type="dataset")["ok"], "restoring did not help"
+
+# 2. A path that is not on the Hub at all: nothing listed means nothing verified.
+v3 = hubckpt.verify_checkpoint(REPO, local, "ckpt/checkpoint-99999999", repo_type="dataset")
+print("VERIFY_ABSENT ok", v3["ok"], "missing", len(v3["missing"]), "of", v3["n_local"], flush=True)
+assert not v3["ok"] and len(v3["missing"]) == v3["n_local"], v3["missing"][:3]
+
+# 3. A server that ignores Range and streams the whole object: the read-back must fail rather than pay
+#    for 850 MB, because a silent full download is the thing this change exists to prevent.
+real = hubckpt.urllib.request.urlopen
+class _Whole:
+    status = 200
+    def __enter__(self): return self
+    def __exit__(self, *a): return False
+    def read(self, n=-1): return b"x" * (len(self) if n is None or n < 0 else min(n, len(self)))
+    def __len__(self): return 1 << 20
+def _fake(req, *a, **k):
+    # Only the download path is faked: hub_listing uses the same urlopen, and breaking it here would
+    # report a "server ignored Range" test as a listing crash.
+    if "/resolve/" in getattr(req, "full_url", str(req)):
+        return _Whole()
+    return real(req, *a, **k)
+hubckpt.urllib.request.urlopen = _fake
+try:
+    v4 = hubckpt.verify_checkpoint(REPO, local, SUB, repo_type="dataset")
+finally:
+    hubckpt.urllib.request.urlopen = real
+print("VERIFY_NO_206 ok", v4["ok"], "readback", json.dumps(v4["readback"])[:260], flush=True)
+assert not v4["ok"] and "206" in str(v4["readback"]), v4["readback"]
+print("VERIFY_STAGE_OK all four paths behave", flush=True)
+"""
+open("prepverify.py", "w").write(VERIFY)
+rc4, _ = run([sys.executable, "prepverify.py"], "CHECK_VERIFY", 1800, env=dict(PLANENV))
+print("VERDICT PHASE4_PREP rc_launcher", rc1, "rc_check", rc2, "rc_cursor", rc3, "rc_verify", rc4,
       "seconds", round(time.time() - T0, 1), flush=True)
-raise SystemExit(0 if (rc1 == rc2 == rc3 == 0) else 4)
+raise SystemExit(0 if (rc1 == rc2 == rc3 == rc4 == 0) else 4)