jrrribeiro commited on
Commit
4815f65
·
1 Parent(s): 31a2ff2

Add per-file delay and enhanced rate limiting handling in upload process

Browse files
hf_bulk_upload_tool/upload_dataset.py CHANGED
@@ -57,7 +57,8 @@ def parse_args() -> argparse.Namespace:
57
  parser.add_argument("--create-repo-attempts", type=int, default=3, help="Number of retries for repo creation.")
58
  parser.add_argument("--upload-attempts", type=int, default=3, help="Number of retries for folder and CSV uploads.")
59
  parser.add_argument("--retry-backoff", type=float, default=5.0, help="Base delay in seconds between retries.")
60
- parser.add_argument("--max-workers", type=int, default=4, help="Number of parallel worker threads for per-file uploads (fallback path).")
 
61
  parser.add_argument("--resume", action="store_true", help="Check repository and skip files that are already uploaded (resume mode).")
62
  parser.add_argument("--checkpoint-dir", default=None, help="Directory to store upload checkpoints. Defaults to ./hf_bulk_upload_tool/.checkpoints.")
63
  parser.add_argument("--verify-remote", action="store_true", help="When resuming, verify remote file size via HTTP HEAD to avoid skipping mismatched files.")
@@ -102,7 +103,26 @@ def retry_call(label: str, attempts: int, backoff: float, func):
102
  last_error = exc
103
  if attempt >= attempts:
104
  break
 
105
  wait_seconds = backoff * attempt
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  print(f"[{label}] attempt {attempt} failed: {exc}")
107
  print(f"[{label}] retrying in {wait_seconds:.0f}s")
108
  time.sleep(wait_seconds)
@@ -295,87 +315,45 @@ def main() -> int:
295
 
296
  # If not resuming and hf offers upload_large_folder, use it for speed
297
  if not args.resume and hasattr(api, "upload_large_folder"):
298
- print("Using HfApi.upload_large_folder for optimized large uploads...")
299
 
300
- # compute total bytes locally for ETA/progress
301
- total_bytes = 0
302
- for root, _, files in os.walk(segments_path):
303
- for f in files:
304
- try:
305
- total_bytes += (Path(root) / f).stat().st_size
306
- except Exception:
307
- continue
308
-
309
- kwargs = {
310
- "folder_path": str(segments_path),
311
- "repo_id": repo_id,
312
- "repo_type": "dataset",
313
- "path_in_repo": args.segments_path_in_repo,
314
- "commit_message": args.commit_message,
315
- "num_workers": args.max_workers,
316
- "print_report": False,
317
- }
318
-
319
- upload_exc: list[Exception] = []
320
-
321
- def get_remote_repo_bytes() -> int | None:
322
- if requests is None:
323
- return None
324
- url = f"https://huggingface.co/api/datasets/{repo_id}"
325
- headers = {"Authorization": f"Bearer {token}"} if token else {}
326
  try:
327
- r = requests.get(url, headers=headers, timeout=20)
328
- r.raise_for_status()
329
- data = r.json()
330
- for key in ("size_in_bytes", "size", "totalBytes", "total_bytes"):
331
- if key in data and isinstance(data[key], int):
332
- return data[key]
333
- for v in data.values():
334
- if isinstance(v, dict):
335
- for key in ("size_in_bytes", "size", "totalBytes"):
336
- if key in v and isinstance(v[key], int):
337
- return v[key]
338
  except Exception:
339
- return None
340
- return None
341
-
342
- def _run_upload():
343
- try:
344
- retry_call("upload_large_folder", args.upload_attempts, args.retry_backoff, lambda: api.upload_large_folder(**kwargs))
345
- except Exception as e:
346
- upload_exc.append(e)
347
-
348
- uploader = threading.Thread(target=_run_upload, daemon=True)
349
- uploader.start()
350
-
351
- if tqdm is None:
352
- # Fall back to synchronous call if tqdm not installed
353
- uploader.join()
354
- if upload_exc:
355
- raise upload_exc[0]
356
- return
357
-
358
- pbar = tqdm(total=total_bytes, unit="B", unit_scale=True, desc="Uploading", leave=True)
359
- last_remote = 0
360
- poll_interval = 10.0
361
- while uploader.is_alive():
362
- remote_bytes = get_remote_repo_bytes()
363
- if remote_bytes is None:
364
- time.sleep(poll_interval)
365
  continue
366
- delta = max(0, remote_bytes - last_remote)
367
- if delta > 0:
368
- pbar.update(delta)
369
- last_remote = remote_bytes
370
- time.sleep(poll_interval)
371
-
372
- final_remote = get_remote_repo_bytes() or last_remote
373
- if final_remote > last_remote:
374
- pbar.update(final_remote - last_remote)
375
- pbar.close()
376
 
377
- if upload_exc:
378
- raise upload_exc[0]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
379
  return
380
 
381
  # Otherwise, perform per-file upload and skip existing files
@@ -425,6 +403,17 @@ def main() -> int:
425
 
426
  print(f"Found {len(files)} files to upload; launching {args.max_workers} workers")
427
 
 
 
 
 
 
 
 
 
 
 
 
428
  def upload_one(pair: Tuple[Path, str]):
429
  local_path, repo_path = pair
430
  start = time.time()
@@ -446,7 +435,27 @@ def main() -> int:
446
  else:
447
  elapsed = time.time() - start
448
  append_progress(local_path, repo_path, 'uploaded', None, elapsed)
449
- return repo_path, local_path
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
450
 
451
  with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as ex:
452
  futures = [ex.submit(upload_one, f) for f in files]
@@ -469,6 +478,11 @@ def main() -> int:
469
  except Exception:
470
  pass
471
  save_checkpoint(already_uploaded, checkpoint_hashes)
 
 
 
 
 
472
 
473
  retry_call("upload_segments", args.upload_attempts, args.retry_backoff, upload_segments)
474
 
 
57
  parser.add_argument("--create-repo-attempts", type=int, default=3, help="Number of retries for repo creation.")
58
  parser.add_argument("--upload-attempts", type=int, default=3, help="Number of retries for folder and CSV uploads.")
59
  parser.add_argument("--retry-backoff", type=float, default=5.0, help="Base delay in seconds between retries.")
60
+ parser.add_argument("--max-workers", type=int, default=1, help="Number of parallel worker threads for per-file uploads (fallback path).")
61
+ parser.add_argument("--per-file-delay", type=float, default=0.5, help="Seconds to wait between per-file uploads to avoid rate limits.")
62
  parser.add_argument("--resume", action="store_true", help="Check repository and skip files that are already uploaded (resume mode).")
63
  parser.add_argument("--checkpoint-dir", default=None, help="Directory to store upload checkpoints. Defaults to ./hf_bulk_upload_tool/.checkpoints.")
64
  parser.add_argument("--verify-remote", action="store_true", help="When resuming, verify remote file size via HTTP HEAD to avoid skipping mismatched files.")
 
103
  last_error = exc
104
  if attempt >= attempts:
105
  break
106
+ # Detect possible 429 / rate-limit signals and increase backoff
107
  wait_seconds = backoff * attempt
108
+ try:
109
+ resp = getattr(exc, 'response', None)
110
+ if resp is not None:
111
+ status = getattr(resp, 'status_code', None)
112
+ if status == 429:
113
+ # obey Retry-After header when present
114
+ ra = resp.headers.get('Retry-After') if hasattr(resp, 'headers') else None
115
+ if ra:
116
+ try:
117
+ wait_seconds = max(wait_seconds, int(ra))
118
+ except Exception:
119
+ wait_seconds = max(wait_seconds, backoff * attempt * 4)
120
+ else:
121
+ wait_seconds = max(wait_seconds, backoff * attempt * 4)
122
+ elif '429' in str(exc) or 'Too Many Requests' in str(exc):
123
+ wait_seconds = max(wait_seconds, backoff * attempt * 4)
124
+ except Exception:
125
+ pass
126
  print(f"[{label}] attempt {attempt} failed: {exc}")
127
  print(f"[{label}] retrying in {wait_seconds:.0f}s")
128
  time.sleep(wait_seconds)
 
315
 
316
  # If not resuming and hf offers upload_large_folder, use it for speed
317
  if not args.resume and hasattr(api, "upload_large_folder"):
318
+ print("Attempting HfApi.upload_large_folder for optimized large uploads (will fall back on failure)...")
319
 
320
+ # compute total bytes locally for ETA/progress
321
+ total_bytes = 0
322
+ for root, _, files in os.walk(segments_path):
323
+ for f in files:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
324
  try:
325
+ total_bytes += (Path(root) / f).stat().st_size
 
 
 
 
 
 
 
 
 
 
326
  except Exception:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
327
  continue
 
 
 
 
 
 
 
 
 
 
328
 
329
+ kwargs = {
330
+ "folder_path": str(segments_path),
331
+ "repo_id": repo_id,
332
+ "repo_type": "dataset",
333
+ "path_in_repo": args.segments_path_in_repo,
334
+ "commit_message": args.commit_message,
335
+ "num_workers": args.max_workers,
336
+ "print_report": False,
337
+ }
338
+
339
+ upload_exc: list[Exception] = []
340
+
341
+ def _run_upload():
342
+ try:
343
+ retry_call("upload_large_folder", args.upload_attempts, args.retry_backoff, lambda: api.upload_large_folder(**kwargs))
344
+ except Exception as e:
345
+ upload_exc.append(e)
346
+
347
+ uploader = threading.Thread(target=_run_upload, daemon=True)
348
+ uploader.start()
349
+
350
+ # wait for upload to finish but do not immediately fail on error; fallback to per-file
351
+ uploader.join()
352
+ if upload_exc:
353
+ print(f"upload_large_folder failed: {upload_exc[0]}; falling back to per-file upload with throttling.")
354
+ # continue to per-file upload path below
355
+ else:
356
+ print("upload_large_folder completed successfully.")
357
  return
358
 
359
  # Otherwise, perform per-file upload and skip existing files
 
403
 
404
  print(f"Found {len(files)} files to upload; launching {args.max_workers} workers")
405
 
406
+ # aggregated progress bar for per-file uploads
407
+ pbar = None
408
+ pbar_lock = None
409
+ if tqdm is not None:
410
+ try:
411
+ pbar = tqdm(total=run_stats['bytes_planned'], unit="B", unit_scale=True, desc="Uploading", leave=True)
412
+ import threading as _threading
413
+ pbar_lock = _threading.Lock()
414
+ except Exception:
415
+ pbar = None
416
+
417
  def upload_one(pair: Tuple[Path, str]):
418
  local_path, repo_path = pair
419
  start = time.time()
 
435
  else:
436
  elapsed = time.time() - start
437
  append_progress(local_path, repo_path, 'uploaded', None, elapsed)
438
+ # update aggregated progress bar
439
+ if pbar is not None:
440
+ size = 0
441
+ try:
442
+ size = local_path.stat().st_size
443
+ except Exception:
444
+ size = 0
445
+ try:
446
+ if pbar_lock:
447
+ with pbar_lock:
448
+ pbar.update(size)
449
+ else:
450
+ pbar.update(size)
451
+ except Exception:
452
+ pass
453
+ # throttle to reduce HF rate pressure
454
+ try:
455
+ time.sleep(args.per_file_delay)
456
+ except Exception:
457
+ pass
458
+ return repo_path, local_path
459
 
460
  with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as ex:
461
  futures = [ex.submit(upload_one, f) for f in files]
 
478
  except Exception:
479
  pass
480
  save_checkpoint(already_uploaded, checkpoint_hashes)
481
+ if pbar is not None:
482
+ try:
483
+ pbar.close()
484
+ except Exception:
485
+ pass
486
 
487
  retry_call("upload_segments", args.upload_attempts, args.retry_backoff, upload_segments)
488
 
hf_bulk_upload_tool_v2/upload_dataset_v2.py CHANGED
@@ -61,7 +61,25 @@ def retry(label: str, attempts: int, backoff_seconds: float, func):
61
  last_error = exc
62
  if attempt >= attempts:
63
  break
 
64
  wait_seconds = backoff_seconds * attempt
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  print(f"[{label}] failed: {exc}")
66
  print(f"[{label}] retrying in {wait_seconds:.0f}s")
67
  time.sleep(wait_seconds)
 
61
  last_error = exc
62
  if attempt >= attempts:
63
  break
64
+ # Increase backoff if we detect rate limiting (429)
65
  wait_seconds = backoff_seconds * attempt
66
+ try:
67
+ resp = getattr(exc, 'response', None)
68
+ if resp is not None:
69
+ status = getattr(resp, 'status_code', None)
70
+ if status == 429:
71
+ ra = resp.headers.get('Retry-After') if hasattr(resp, 'headers') else None
72
+ if ra:
73
+ try:
74
+ wait_seconds = max(wait_seconds, int(ra))
75
+ except Exception:
76
+ wait_seconds = max(wait_seconds, backoff_seconds * attempt * 4)
77
+ else:
78
+ wait_seconds = max(wait_seconds, backoff_seconds * attempt * 4)
79
+ elif '429' in str(exc) or 'Too Many Requests' in str(exc):
80
+ wait_seconds = max(wait_seconds, backoff_seconds * attempt * 4)
81
+ except Exception:
82
+ pass
83
  print(f"[{label}] failed: {exc}")
84
  print(f"[{label}] retrying in {wait_seconds:.0f}s")
85
  time.sleep(wait_seconds)