Upload build/publish_mix.py with huggingface_hub

build/publish_mix.py

@@ -244,6 +244,7 @@ def main():
         print(f"  (no resumable listing: {type(e).__name__}; uploading everything)", flush=True)
 
     failures = []
+    todo = []
     for np, lp in to_upload:
         with open(lp, "rb") as fh:
             payload = fh.read()
@@ -251,20 +252,39 @@ def main():
         if remote_sha.get(np) == local_sha:
             print(f"  = {np} already on the Hub with this sha256, skipped", flush=True)
             continue
-        for attempt in (1, 2, 3):
-            try:
-                api.upload_file(path_or_fileobj=payload, path_in_repo=np, repo_id=a.repo,
-                                repo_type="dataset", commit_message=f"add {np} ({local_sha[:12]})")
-                print(f"  + {np} ({len(payload) / 1024**2:.1f} MB)", flush=True)
-                break
-            except Exception as e:
-                # One dropped upload must not discard 129 completed ones. Retry, then record and move on,
-                # so the final verification below can name every file that is still missing.
-                print(f"  ! {np} attempt {attempt}: {type(e).__name__}: {str(e)[:140]}", flush=True)
-                if attempt == 3:
-                    failures.append(np)
-                else:
-                    __import__("time").sleep(5 * attempt)
+        todo.append((np, lp, local_sha, payload))
+    print(f"  {len(todo)} of {len(to_upload)} files still need uploading", flush=True)
+
+    # ONE commit for the whole batch, not one commit per file. Gate 2f proved why: 139 sequential
+    # `upload_file` commits succeeded and then the Hub began rejecting every request -- including 30-byte
+    # files -- with a bare HfHubHTTPError, which is a commit-rate limit, not a network fault. A single
+    # upload_folder call is one commit and handles LFS itself.
+    if todo:
+        try:
+            api.upload_folder(repo_id=a.repo, repo_type="dataset", folder_path=a.root, path_in_repo="",
+                              allow_patterns=[t[0] for t in todo], max_workers=2,
+                              commit_message=f"publish mix-v1: {len(todo)} file(s)",
+                              commit_description="single batched commit; see audit.json for the "
+                                                 "contamination verdict bound to these bytes")
+            print(f"  + batched commit with {len(todo)} file(s)", flush=True)
+        except Exception as e:
+            print(f"  batched commit failed ({type(e).__name__}: {str(e)[:160]}); "
+                  "falling back to per-file uploads with retries", flush=True)
+            for np, lp, local_sha, payload in todo:
+                for attempt in (1, 2, 3):
+                    try:
+                        api.upload_file(path_or_fileobj=payload, path_in_repo=np, repo_id=a.repo,
+                                        repo_type="dataset",
+                                        commit_message=f"add {np} ({local_sha[:12]})")
+                        print(f"  + {np} ({len(payload) / 1024**2:.1f} MB)", flush=True)
+                        break
+                    except Exception as e2:
+                        print(f"  ! {np} attempt {attempt}: {type(e2).__name__}: {str(e2)[:140]}",
+                              flush=True)
+                        if attempt == 3:
+                            failures.append(np)
+                        else:
+                            __import__("time").sleep(20 * attempt)
     if failures:
         print(f"FATAL: {len(failures)} file(s) never landed: {failures[:8]} -- re-running publish_mix "
               "resumes from the ones that did", flush=True)