Upload train/hubckpt.py with huggingface_hub

train/hubckpt.py

@@ -131,10 +131,13 @@ def verify_checkpoint(repo, dirpath, path_in_repo, token=None, repo_type="datase
     rb = None
     rb_ok = False
     if readback_one and have:
-        # Download one file back and re-hash it: cheapest possible proof that the bytes are *readable*,
-        # not merely listed (§3.13 is explicit that listing is not verification). The token is sent when
-        # there is one so this also works if the checkpoint repo is ever made private.
-        pick = min(have, key=lambda k: (have[k][0] is None, have[k][0]))
+        # Read back the LARGEST file and re-hash it. In a Trainer checkpoint that is the fp16
+        # model.safetensors -- an LFS object -- and it is the only file whose retrievability the Hub's
+        # listing cannot establish, because the listing reports the oid the commit *claims*. Picking the
+        # cheapest file (cursor.json, a few dozen git-stored bytes) proved the git tree while leaving
+        # 1.7 GB of weights untested, and `ok=True` then authorised deleting the only local copy (E-031).
+        # The token is sent when there is one, so this also works if the repo is ever made private.
+        pick = max(have, key=lambda k: (have[k][0] or 0))
         tmp = os.path.join("/tmp", "hubverify_" + os.path.basename(pick))
         url = f"{HUB}/{repo_type}s/{repo}/resolve/main/{prefix}{pick}"
         try:
@@ -153,8 +156,10 @@ def verify_checkpoint(repo, dirpath, path_in_repo, token=None, repo_type="datase
             if os.path.exists(tmp):
                 os.remove(tmp)
     # ok must include the read-back: a repo whose files are listed but not downloadable is not a
-    # checkpoint, and pruning into that state loses the run.
-    ok = bool(want) and not missing and not mismatch and (rb_ok or not readback_one)
+    # checkpoint, and pruning into that state loses the run. `extra` is gated too: a file on the Hub that
+    # is not in this directory means the path being pushed to already held something else, which is a
+    # naming collision, not a clean checkpoint (E-031 -- the docstring here used to claim otherwise).
+    ok = bool(want) and not missing and not mismatch and not extra and (rb_ok or not readback_one)
     return {"ok": ok, "n_local": len(want), "n_hub": len(have), "missing": missing[:10],
             "mismatch": mismatch[:10], "extra": extra[:10], "readback": rb,
             "readback_ok": rb_ok}