trainer: an unverified checkpoint never rolls latest.json and never prunes (E-042)

train/train_ounce100m.py

@@ -166,13 +166,25 @@ class HubPush(TrainerCallback):
         # session would quietly re-read 100 M tokens.
         res = hubckpt.push_and_prune(self.repo, d, f"ckpt/checkpoint-{step}", self.api,
                                      repo_type=self.repo_type, token=self.token, prune=False)
+        # E-042: `prune=False` above means `push_and_prune`'s own guard never runs, so this path had to
+        # carry it. An unverified checkpoint must stop the session *before* the pointer moves: a pointer
+        # rolled onto bytes nobody checked, with the local copy then deleted, is a run that cannot be
+        # resumed and does not know it.
+        ver = res["verify"]
+        if not ver["ok"]:
+            raise RuntimeError(
+                f"checkpoint {step} did not verify on the Hub "
+                f"(missing={len(ver['missing'])} mismatched={len(ver['mismatch'])} "
+                f"extra={len(ver['extra'])} readback_ok={ver['readback_ok']}). Keeping the local copy in "
+                f"{d} and refusing to roll latest.json -- the next session would otherwise resume from "
+                "bytes that were never confirmed to exist.")
         ptr_body = {"path_in_repo": f"ckpt/checkpoint-{step}", "step": step,
                     "samples_consumed": samples, "tokens_consumed": samples * self.seq_len,
                     "fingerprint": self.fingerprint, "seed": self.seed}
         hubckpt.write_latest_pointer(self.repo, ptr_body, self.api, repo_type=self.repo_type,
                                      token=self.token)
         if self.prune:
-            hubckpt.prune_local(d, res, self.min_free_gb)
+            hubckpt.prune_verified(d, res, self.min_free_gb)
         loss = next((h["loss"] for h in reversed(state.log_history) if "loss" in h), None)
         print(f"CKPT {step} samples={samples:,} push+verify={round(time.time()-t0,1)}s "
               f"ok={res['verify']['ok']} pruned={res.get('pruned')} free={res['free_after_gb']}GB "