Spaces:
Sleeping
Sleeping
Commit ·
4815f65
1
Parent(s): 31a2ff2
Add per-file delay and enhanced rate limiting handling in upload process
Browse files
hf_bulk_upload_tool/upload_dataset.py
CHANGED
|
@@ -57,7 +57,8 @@ def parse_args() -> argparse.Namespace:
|
|
| 57 |
parser.add_argument("--create-repo-attempts", type=int, default=3, help="Number of retries for repo creation.")
|
| 58 |
parser.add_argument("--upload-attempts", type=int, default=3, help="Number of retries for folder and CSV uploads.")
|
| 59 |
parser.add_argument("--retry-backoff", type=float, default=5.0, help="Base delay in seconds between retries.")
|
| 60 |
-
parser.add_argument("--max-workers", type=int, default=
|
|
|
|
| 61 |
parser.add_argument("--resume", action="store_true", help="Check repository and skip files that are already uploaded (resume mode).")
|
| 62 |
parser.add_argument("--checkpoint-dir", default=None, help="Directory to store upload checkpoints. Defaults to ./hf_bulk_upload_tool/.checkpoints.")
|
| 63 |
parser.add_argument("--verify-remote", action="store_true", help="When resuming, verify remote file size via HTTP HEAD to avoid skipping mismatched files.")
|
|
@@ -102,7 +103,26 @@ def retry_call(label: str, attempts: int, backoff: float, func):
|
|
| 102 |
last_error = exc
|
| 103 |
if attempt >= attempts:
|
| 104 |
break
|
|
|
|
| 105 |
wait_seconds = backoff * attempt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
print(f"[{label}] attempt {attempt} failed: {exc}")
|
| 107 |
print(f"[{label}] retrying in {wait_seconds:.0f}s")
|
| 108 |
time.sleep(wait_seconds)
|
|
@@ -295,87 +315,45 @@ def main() -> int:
|
|
| 295 |
|
| 296 |
# If not resuming and hf offers upload_large_folder, use it for speed
|
| 297 |
if not args.resume and hasattr(api, "upload_large_folder"):
|
| 298 |
-
|
| 299 |
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
try:
|
| 305 |
-
total_bytes += (Path(root) / f).stat().st_size
|
| 306 |
-
except Exception:
|
| 307 |
-
continue
|
| 308 |
-
|
| 309 |
-
kwargs = {
|
| 310 |
-
"folder_path": str(segments_path),
|
| 311 |
-
"repo_id": repo_id,
|
| 312 |
-
"repo_type": "dataset",
|
| 313 |
-
"path_in_repo": args.segments_path_in_repo,
|
| 314 |
-
"commit_message": args.commit_message,
|
| 315 |
-
"num_workers": args.max_workers,
|
| 316 |
-
"print_report": False,
|
| 317 |
-
}
|
| 318 |
-
|
| 319 |
-
upload_exc: list[Exception] = []
|
| 320 |
-
|
| 321 |
-
def get_remote_repo_bytes() -> int | None:
|
| 322 |
-
if requests is None:
|
| 323 |
-
return None
|
| 324 |
-
url = f"https://huggingface.co/api/datasets/{repo_id}"
|
| 325 |
-
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
| 326 |
try:
|
| 327 |
-
|
| 328 |
-
r.raise_for_status()
|
| 329 |
-
data = r.json()
|
| 330 |
-
for key in ("size_in_bytes", "size", "totalBytes", "total_bytes"):
|
| 331 |
-
if key in data and isinstance(data[key], int):
|
| 332 |
-
return data[key]
|
| 333 |
-
for v in data.values():
|
| 334 |
-
if isinstance(v, dict):
|
| 335 |
-
for key in ("size_in_bytes", "size", "totalBytes"):
|
| 336 |
-
if key in v and isinstance(v[key], int):
|
| 337 |
-
return v[key]
|
| 338 |
except Exception:
|
| 339 |
-
return None
|
| 340 |
-
return None
|
| 341 |
-
|
| 342 |
-
def _run_upload():
|
| 343 |
-
try:
|
| 344 |
-
retry_call("upload_large_folder", args.upload_attempts, args.retry_backoff, lambda: api.upload_large_folder(**kwargs))
|
| 345 |
-
except Exception as e:
|
| 346 |
-
upload_exc.append(e)
|
| 347 |
-
|
| 348 |
-
uploader = threading.Thread(target=_run_upload, daemon=True)
|
| 349 |
-
uploader.start()
|
| 350 |
-
|
| 351 |
-
if tqdm is None:
|
| 352 |
-
# Fall back to synchronous call if tqdm not installed
|
| 353 |
-
uploader.join()
|
| 354 |
-
if upload_exc:
|
| 355 |
-
raise upload_exc[0]
|
| 356 |
-
return
|
| 357 |
-
|
| 358 |
-
pbar = tqdm(total=total_bytes, unit="B", unit_scale=True, desc="Uploading", leave=True)
|
| 359 |
-
last_remote = 0
|
| 360 |
-
poll_interval = 10.0
|
| 361 |
-
while uploader.is_alive():
|
| 362 |
-
remote_bytes = get_remote_repo_bytes()
|
| 363 |
-
if remote_bytes is None:
|
| 364 |
-
time.sleep(poll_interval)
|
| 365 |
continue
|
| 366 |
-
delta = max(0, remote_bytes - last_remote)
|
| 367 |
-
if delta > 0:
|
| 368 |
-
pbar.update(delta)
|
| 369 |
-
last_remote = remote_bytes
|
| 370 |
-
time.sleep(poll_interval)
|
| 371 |
-
|
| 372 |
-
final_remote = get_remote_repo_bytes() or last_remote
|
| 373 |
-
if final_remote > last_remote:
|
| 374 |
-
pbar.update(final_remote - last_remote)
|
| 375 |
-
pbar.close()
|
| 376 |
|
| 377 |
-
|
| 378 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 379 |
return
|
| 380 |
|
| 381 |
# Otherwise, perform per-file upload and skip existing files
|
|
@@ -425,6 +403,17 @@ def main() -> int:
|
|
| 425 |
|
| 426 |
print(f"Found {len(files)} files to upload; launching {args.max_workers} workers")
|
| 427 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 428 |
def upload_one(pair: Tuple[Path, str]):
|
| 429 |
local_path, repo_path = pair
|
| 430 |
start = time.time()
|
|
@@ -446,7 +435,27 @@ def main() -> int:
|
|
| 446 |
else:
|
| 447 |
elapsed = time.time() - start
|
| 448 |
append_progress(local_path, repo_path, 'uploaded', None, elapsed)
|
| 449 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
|
| 451 |
with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as ex:
|
| 452 |
futures = [ex.submit(upload_one, f) for f in files]
|
|
@@ -469,6 +478,11 @@ def main() -> int:
|
|
| 469 |
except Exception:
|
| 470 |
pass
|
| 471 |
save_checkpoint(already_uploaded, checkpoint_hashes)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
|
| 473 |
retry_call("upload_segments", args.upload_attempts, args.retry_backoff, upload_segments)
|
| 474 |
|
|
|
|
| 57 |
parser.add_argument("--create-repo-attempts", type=int, default=3, help="Number of retries for repo creation.")
|
| 58 |
parser.add_argument("--upload-attempts", type=int, default=3, help="Number of retries for folder and CSV uploads.")
|
| 59 |
parser.add_argument("--retry-backoff", type=float, default=5.0, help="Base delay in seconds between retries.")
|
| 60 |
+
parser.add_argument("--max-workers", type=int, default=1, help="Number of parallel worker threads for per-file uploads (fallback path).")
|
| 61 |
+
parser.add_argument("--per-file-delay", type=float, default=0.5, help="Seconds to wait between per-file uploads to avoid rate limits.")
|
| 62 |
parser.add_argument("--resume", action="store_true", help="Check repository and skip files that are already uploaded (resume mode).")
|
| 63 |
parser.add_argument("--checkpoint-dir", default=None, help="Directory to store upload checkpoints. Defaults to ./hf_bulk_upload_tool/.checkpoints.")
|
| 64 |
parser.add_argument("--verify-remote", action="store_true", help="When resuming, verify remote file size via HTTP HEAD to avoid skipping mismatched files.")
|
|
|
|
| 103 |
last_error = exc
|
| 104 |
if attempt >= attempts:
|
| 105 |
break
|
| 106 |
+
# Detect possible 429 / rate-limit signals and increase backoff
|
| 107 |
wait_seconds = backoff * attempt
|
| 108 |
+
try:
|
| 109 |
+
resp = getattr(exc, 'response', None)
|
| 110 |
+
if resp is not None:
|
| 111 |
+
status = getattr(resp, 'status_code', None)
|
| 112 |
+
if status == 429:
|
| 113 |
+
# obey Retry-After header when present
|
| 114 |
+
ra = resp.headers.get('Retry-After') if hasattr(resp, 'headers') else None
|
| 115 |
+
if ra:
|
| 116 |
+
try:
|
| 117 |
+
wait_seconds = max(wait_seconds, int(ra))
|
| 118 |
+
except Exception:
|
| 119 |
+
wait_seconds = max(wait_seconds, backoff * attempt * 4)
|
| 120 |
+
else:
|
| 121 |
+
wait_seconds = max(wait_seconds, backoff * attempt * 4)
|
| 122 |
+
elif '429' in str(exc) or 'Too Many Requests' in str(exc):
|
| 123 |
+
wait_seconds = max(wait_seconds, backoff * attempt * 4)
|
| 124 |
+
except Exception:
|
| 125 |
+
pass
|
| 126 |
print(f"[{label}] attempt {attempt} failed: {exc}")
|
| 127 |
print(f"[{label}] retrying in {wait_seconds:.0f}s")
|
| 128 |
time.sleep(wait_seconds)
|
|
|
|
| 315 |
|
| 316 |
# If not resuming and hf offers upload_large_folder, use it for speed
|
| 317 |
if not args.resume and hasattr(api, "upload_large_folder"):
|
| 318 |
+
print("Attempting HfApi.upload_large_folder for optimized large uploads (will fall back on failure)...")
|
| 319 |
|
| 320 |
+
# compute total bytes locally for ETA/progress
|
| 321 |
+
total_bytes = 0
|
| 322 |
+
for root, _, files in os.walk(segments_path):
|
| 323 |
+
for f in files:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
try:
|
| 325 |
+
total_bytes += (Path(root) / f).stat().st_size
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 326 |
except Exception:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
|
| 329 |
+
kwargs = {
|
| 330 |
+
"folder_path": str(segments_path),
|
| 331 |
+
"repo_id": repo_id,
|
| 332 |
+
"repo_type": "dataset",
|
| 333 |
+
"path_in_repo": args.segments_path_in_repo,
|
| 334 |
+
"commit_message": args.commit_message,
|
| 335 |
+
"num_workers": args.max_workers,
|
| 336 |
+
"print_report": False,
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
upload_exc: list[Exception] = []
|
| 340 |
+
|
| 341 |
+
def _run_upload():
|
| 342 |
+
try:
|
| 343 |
+
retry_call("upload_large_folder", args.upload_attempts, args.retry_backoff, lambda: api.upload_large_folder(**kwargs))
|
| 344 |
+
except Exception as e:
|
| 345 |
+
upload_exc.append(e)
|
| 346 |
+
|
| 347 |
+
uploader = threading.Thread(target=_run_upload, daemon=True)
|
| 348 |
+
uploader.start()
|
| 349 |
+
|
| 350 |
+
# wait for upload to finish but do not immediately fail on error; fallback to per-file
|
| 351 |
+
uploader.join()
|
| 352 |
+
if upload_exc:
|
| 353 |
+
print(f"upload_large_folder failed: {upload_exc[0]}; falling back to per-file upload with throttling.")
|
| 354 |
+
# continue to per-file upload path below
|
| 355 |
+
else:
|
| 356 |
+
print("upload_large_folder completed successfully.")
|
| 357 |
return
|
| 358 |
|
| 359 |
# Otherwise, perform per-file upload and skip existing files
|
|
|
|
| 403 |
|
| 404 |
print(f"Found {len(files)} files to upload; launching {args.max_workers} workers")
|
| 405 |
|
| 406 |
+
# aggregated progress bar for per-file uploads
|
| 407 |
+
pbar = None
|
| 408 |
+
pbar_lock = None
|
| 409 |
+
if tqdm is not None:
|
| 410 |
+
try:
|
| 411 |
+
pbar = tqdm(total=run_stats['bytes_planned'], unit="B", unit_scale=True, desc="Uploading", leave=True)
|
| 412 |
+
import threading as _threading
|
| 413 |
+
pbar_lock = _threading.Lock()
|
| 414 |
+
except Exception:
|
| 415 |
+
pbar = None
|
| 416 |
+
|
| 417 |
def upload_one(pair: Tuple[Path, str]):
|
| 418 |
local_path, repo_path = pair
|
| 419 |
start = time.time()
|
|
|
|
| 435 |
else:
|
| 436 |
elapsed = time.time() - start
|
| 437 |
append_progress(local_path, repo_path, 'uploaded', None, elapsed)
|
| 438 |
+
# update aggregated progress bar
|
| 439 |
+
if pbar is not None:
|
| 440 |
+
size = 0
|
| 441 |
+
try:
|
| 442 |
+
size = local_path.stat().st_size
|
| 443 |
+
except Exception:
|
| 444 |
+
size = 0
|
| 445 |
+
try:
|
| 446 |
+
if pbar_lock:
|
| 447 |
+
with pbar_lock:
|
| 448 |
+
pbar.update(size)
|
| 449 |
+
else:
|
| 450 |
+
pbar.update(size)
|
| 451 |
+
except Exception:
|
| 452 |
+
pass
|
| 453 |
+
# throttle to reduce HF rate pressure
|
| 454 |
+
try:
|
| 455 |
+
time.sleep(args.per_file_delay)
|
| 456 |
+
except Exception:
|
| 457 |
+
pass
|
| 458 |
+
return repo_path, local_path
|
| 459 |
|
| 460 |
with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as ex:
|
| 461 |
futures = [ex.submit(upload_one, f) for f in files]
|
|
|
|
| 478 |
except Exception:
|
| 479 |
pass
|
| 480 |
save_checkpoint(already_uploaded, checkpoint_hashes)
|
| 481 |
+
if pbar is not None:
|
| 482 |
+
try:
|
| 483 |
+
pbar.close()
|
| 484 |
+
except Exception:
|
| 485 |
+
pass
|
| 486 |
|
| 487 |
retry_call("upload_segments", args.upload_attempts, args.retry_backoff, upload_segments)
|
| 488 |
|
hf_bulk_upload_tool_v2/upload_dataset_v2.py
CHANGED
|
@@ -61,7 +61,25 @@ def retry(label: str, attempts: int, backoff_seconds: float, func):
|
|
| 61 |
last_error = exc
|
| 62 |
if attempt >= attempts:
|
| 63 |
break
|
|
|
|
| 64 |
wait_seconds = backoff_seconds * attempt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
print(f"[{label}] failed: {exc}")
|
| 66 |
print(f"[{label}] retrying in {wait_seconds:.0f}s")
|
| 67 |
time.sleep(wait_seconds)
|
|
|
|
| 61 |
last_error = exc
|
| 62 |
if attempt >= attempts:
|
| 63 |
break
|
| 64 |
+
# Increase backoff if we detect rate limiting (429)
|
| 65 |
wait_seconds = backoff_seconds * attempt
|
| 66 |
+
try:
|
| 67 |
+
resp = getattr(exc, 'response', None)
|
| 68 |
+
if resp is not None:
|
| 69 |
+
status = getattr(resp, 'status_code', None)
|
| 70 |
+
if status == 429:
|
| 71 |
+
ra = resp.headers.get('Retry-After') if hasattr(resp, 'headers') else None
|
| 72 |
+
if ra:
|
| 73 |
+
try:
|
| 74 |
+
wait_seconds = max(wait_seconds, int(ra))
|
| 75 |
+
except Exception:
|
| 76 |
+
wait_seconds = max(wait_seconds, backoff_seconds * attempt * 4)
|
| 77 |
+
else:
|
| 78 |
+
wait_seconds = max(wait_seconds, backoff_seconds * attempt * 4)
|
| 79 |
+
elif '429' in str(exc) or 'Too Many Requests' in str(exc):
|
| 80 |
+
wait_seconds = max(wait_seconds, backoff_seconds * attempt * 4)
|
| 81 |
+
except Exception:
|
| 82 |
+
pass
|
| 83 |
print(f"[{label}] failed: {exc}")
|
| 84 |
print(f"[{label}] retrying in {wait_seconds:.0f}s")
|
| 85 |
time.sleep(wait_seconds)
|