jrrribeiro commited on
Commit
5ff126a
·
1 Parent(s): aed52e5

Add v2 of BirdNET HF Bulk Upload Tool with enhanced features and performance optimizations

Browse files
hf_bulk_upload_tool/README.md CHANGED
@@ -13,6 +13,7 @@ This folder contains a standalone script for uploading a large BirdNET segment t
13
 
14
  - `upload_dataset.py` - command line uploader
15
  - `requirements.txt` - minimal dependency list for this folder
 
16
 
17
  ## Install
18
 
@@ -70,6 +71,22 @@ The script uploads the segments folder under `audio/` by default, preserving the
70
 
71
  Optional CSV files are uploaded to `index/detections.csv` by default.
72
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  ## Notes for large uploads
74
 
75
  - Start with `--private` and one small test folder if you want to verify permissions first
 
13
 
14
  - `upload_dataset.py` - command line uploader
15
  - `requirements.txt` - minimal dependency list for this folder
16
+ - `run_upload.ps1` - Windows helper launcher
17
 
18
  ## Install
19
 
 
71
 
72
  Optional CSV files are uploaded to `index/detections.csv` by default.
73
 
74
+ ## Large uploads and performance
75
+
76
+ - The script will use `HfApi.upload_large_folder` when available in your `huggingface_hub` version for optimal throughput.
77
+ - If that API is not available, the script falls back to uploading files in parallel using `--max-workers` (default 4).
78
+ - For very large datasets, increase `--max-workers` and system timeouts via environment variables:
79
+
80
+ ```powershell
81
+ $env:HF_HUB_ETAG_TIMEOUT = '60'
82
+ $env:HF_HUB_DOWNLOAD_TIMEOUT = '300'
83
+ $env:HF_XET_HIGH_PERFORMANCE = '1'
84
+ ```
85
+
86
+ ## Resume-only mode
87
+
88
+ Use `--resume-only` when you want the script to inspect the repository, compare against local files, and write checkpoints/progress logs without uploading anything.
89
+
90
  ## Notes for large uploads
91
 
92
  - Start with `--private` and one small test folder if you want to verify permissions first
hf_bulk_upload_tool/run_upload.ps1 CHANGED
@@ -9,6 +9,12 @@ param(
9
  [switch]$Private,
10
  [switch]$Public,
11
  [switch]$DryRun,
 
 
 
 
 
 
12
  [int]$CreateRepoAttempts = 3,
13
  [int]$UploadAttempts = 3,
14
  [double]$RetryBackoff = 5
@@ -38,6 +44,12 @@ if ($Csv) { $argsList += @('--csv', $Csv) }
38
  if ($Private) { $argsList += '--private' }
39
  if ($Public) { $argsList += '--public' }
40
  if ($DryRun) { $argsList += '--dry-run' }
 
 
 
 
 
 
41
 
42
  $argsList += @('--create-repo-attempts', $CreateRepoAttempts.ToString())
43
  $argsList += @('--upload-attempts', $UploadAttempts.ToString())
 
9
  [switch]$Private,
10
  [switch]$Public,
11
  [switch]$DryRun,
12
+ [switch]$Resume,
13
+ [switch]$VerifyRemote,
14
+ [switch]$VerifyEtag,
15
+ [string]$ProgressLog,
16
+ [switch]$ResumeOnly,
17
+ [string]$CheckpointDir,
18
  [int]$CreateRepoAttempts = 3,
19
  [int]$UploadAttempts = 3,
20
  [double]$RetryBackoff = 5
 
44
  if ($Private) { $argsList += '--private' }
45
  if ($Public) { $argsList += '--public' }
46
  if ($DryRun) { $argsList += '--dry-run' }
47
+ if ($Resume) { $argsList += '--resume' }
48
+ if ($VerifyRemote) { $argsList += '--verify-remote' }
49
+ if ($CheckpointDir) { $argsList += @('--checkpoint-dir', $CheckpointDir) }
50
+ if ($VerifyEtag) { $argsList += '--verify-etag' }
51
+ if ($ProgressLog) { $argsList += @('--progress-log', $ProgressLog) }
52
+ if ($ResumeOnly) { $argsList += '--resume-only' }
53
 
54
  $argsList += @('--create-repo-attempts', $CreateRepoAttempts.ToString())
55
  $argsList += @('--upload-attempts', $UploadAttempts.ToString())
hf_bulk_upload_tool/upload_dataset.py CHANGED
@@ -3,10 +3,21 @@ from __future__ import annotations
3
 
4
  import argparse
5
  import getpass
 
6
  import os
7
  import sys
8
  import time
9
  from pathlib import Path
 
 
 
 
 
 
 
 
 
 
10
 
11
 
12
  def configure_hf_env() -> None:
@@ -41,6 +52,13 @@ def parse_args() -> argparse.Namespace:
41
  parser.add_argument("--create-repo-attempts", type=int, default=3, help="Number of retries for repo creation.")
42
  parser.add_argument("--upload-attempts", type=int, default=3, help="Number of retries for folder and CSV uploads.")
43
  parser.add_argument("--retry-backoff", type=float, default=5.0, help="Base delay in seconds between retries.")
 
 
 
 
 
 
 
44
  parser.add_argument("--dry-run", action="store_true", help="Print the planned actions without uploading anything.")
45
  return parser.parse_args()
46
 
@@ -121,38 +139,325 @@ def main() -> int:
121
 
122
  api = HfApi(token=token)
123
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  def create_repo() -> None:
125
  api.create_repo(repo_id=repo_id, repo_type="dataset", private=is_private, exist_ok=True)
126
 
127
  retry_call("create_repo", args.create_repo_attempts, args.retry_backoff, create_repo)
 
 
 
 
 
 
 
 
 
 
128
 
129
  def upload_segments() -> None:
130
- api.upload_folder(
131
- folder_path=str(segments_path),
132
- repo_id=repo_id,
133
- repo_type="dataset",
134
- path_in_repo=args.segments_path_in_repo,
135
- commit_message=args.commit_message,
136
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
 
138
  retry_call("upload_segments", args.upload_attempts, args.retry_backoff, upload_segments)
139
 
 
 
 
 
 
 
 
 
140
  if csv_path:
141
 
142
  def upload_csv() -> None:
143
- api.upload_file(
144
- path_or_fileobj=str(csv_path),
145
- path_in_repo=args.csv_path_in_repo,
146
- repo_id=repo_id,
147
- repo_type="dataset",
148
- commit_message=f"Upload CSV for {repo_id}",
149
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
 
151
  retry_call("upload_csv", args.upload_attempts, args.retry_backoff, upload_csv)
152
 
153
  print()
154
  print("Upload complete.")
155
  print(f"Dataset URL: https://huggingface.co/datasets/{repo_id}")
 
156
  return 0
157
 
158
 
 
3
 
4
  import argparse
5
  import getpass
6
+ import hashlib
7
  import os
8
  import sys
9
  import time
10
  from pathlib import Path
11
+ import concurrent.futures
12
+ from typing import List, Tuple
13
+ import json
14
+ import csv
15
+ from urllib.parse import quote
16
+
17
+ try:
18
+ import requests
19
+ except Exception:
20
+ requests = None
21
 
22
 
23
  def configure_hf_env() -> None:
 
52
  parser.add_argument("--create-repo-attempts", type=int, default=3, help="Number of retries for repo creation.")
53
  parser.add_argument("--upload-attempts", type=int, default=3, help="Number of retries for folder and CSV uploads.")
54
  parser.add_argument("--retry-backoff", type=float, default=5.0, help="Base delay in seconds between retries.")
55
+ parser.add_argument("--max-workers", type=int, default=4, help="Number of parallel worker threads for per-file uploads (fallback path).")
56
+ parser.add_argument("--resume", action="store_true", help="Check repository and skip files that are already uploaded (resume mode).")
57
+ parser.add_argument("--checkpoint-dir", default=None, help="Directory to store upload checkpoints. Defaults to ./hf_bulk_upload_tool/.checkpoints.")
58
+ parser.add_argument("--verify-remote", action="store_true", help="When resuming, verify remote file size via HTTP HEAD to avoid skipping mismatched files.")
59
+ parser.add_argument("--verify-etag", action="store_true", help="When resuming, prefer ETag comparison if available from the server.")
60
+ parser.add_argument("--progress-log", default=None, help="Path to CSV progress log. Defaults to <checkpoint-dir>/progress.csv")
61
+ parser.add_argument("--resume-only", action="store_true", help="Only compare local files with the repo and report what would be uploaded; do not upload anything.")
62
  parser.add_argument("--dry-run", action="store_true", help="Print the planned actions without uploading anything.")
63
  return parser.parse_args()
64
 
 
139
 
140
  api = HfApi(token=token)
141
 
142
+ def list_existing_files() -> set:
143
+ try:
144
+ print("Listing files in target repo to determine already-uploaded files...")
145
+ files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
146
+ return set(files or [])
147
+ except Exception as e:
148
+ print(f"Warning: could not list repo files: {e}")
149
+ return set()
150
+
151
+
152
+ # Checkpoint helpers
153
+ default_checkpoint_dir = Path(__file__).parent / ".checkpoints"
154
+ checkpoint_dir = Path(args.checkpoint_dir) if args.checkpoint_dir else default_checkpoint_dir
155
+ checkpoint_dir.mkdir(parents=True, exist_ok=True)
156
+ checkpoint_file = checkpoint_dir / f"{repo_id.replace('/', '__')}.json"
157
+
158
+ def load_checkpoint() -> tuple[set, dict[str, str]]:
159
+ if not checkpoint_file.exists():
160
+ return set(), {}
161
+ try:
162
+ with open(checkpoint_file, 'r', encoding='utf-8') as fh:
163
+ data = json.load(fh)
164
+ if isinstance(data, list):
165
+ return set(data), {}
166
+ uploaded = set(data.get('uploaded', []))
167
+ file_hashes = data.get('file_hashes', {})
168
+ if not isinstance(file_hashes, dict):
169
+ file_hashes = {}
170
+ return uploaded, {str(k): str(v) for k, v in file_hashes.items()}
171
+ except Exception as e:
172
+ print(f"Warning: failed to load checkpoint {checkpoint_file}: {e}")
173
+ return set(), {}
174
+
175
+ def save_checkpoint(uploaded_set: set, file_hashes: dict[str, str]) -> None:
176
+ try:
177
+ with open(checkpoint_file, 'w', encoding='utf-8') as fh:
178
+ json.dump({'uploaded': sorted(list(uploaded_set)), 'file_hashes': file_hashes}, fh, indent=2)
179
+ except Exception as e:
180
+ print(f"Warning: failed to save checkpoint {checkpoint_file}: {e}")
181
+
182
+ def compute_sha256(file_path: Path) -> str:
183
+ digest = hashlib.sha256()
184
+ with open(file_path, 'rb') as fh:
185
+ for chunk in iter(lambda: fh.read(1024 * 1024), b''):
186
+ digest.update(chunk)
187
+ return digest.hexdigest()
188
+
189
+ def remote_file_info(repo_path: str) -> Tuple[int | None, str | None]:
190
+ """Return (size, etag) for a repo file via HTTP HEAD, or (None, None)."""
191
+ if requests is None:
192
+ return None, None
193
+ quoted = '/'.join(quote(p) for p in repo_path.split('/'))
194
+ url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{quoted}"
195
+ try:
196
+ r = requests.head(url, allow_redirects=True, timeout=10)
197
+ if r.status_code >= 400:
198
+ return None, None
199
+ cl = r.headers.get('content-length')
200
+ etag = r.headers.get('etag')
201
+ size = int(cl) if cl is not None else None
202
+ return size, etag
203
+ except Exception:
204
+ return None, None
205
+
206
+ # Progress logging
207
+ progress_log_path = Path(args.progress_log) if args.progress_log else (checkpoint_dir / 'progress.csv')
208
+ def append_progress(local_path: Path | None, repo_path: str, status: str, msg: str | None = None, elapsed: float | None = None) -> None:
209
+ header = ['timestamp','local_path','repo_path','status','message','elapsed_seconds']
210
+ row = [time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), str(local_path) if local_path else '', repo_path, status, (msg or '').replace('\n',' '), f"{elapsed:.2f}" if elapsed is not None else '']
211
+ write_header = not progress_log_path.exists()
212
+ try:
213
+ with open(progress_log_path, 'a', newline='', encoding='utf-8') as fh:
214
+ writer = csv.writer(fh)
215
+ if write_header:
216
+ writer.writerow(header)
217
+ writer.writerow(row)
218
+ except Exception as e:
219
+ print(f"Warning: failed to write progress log {progress_log_path}: {e}")
220
+
221
+ run_stats = {
222
+ 'planned': 0,
223
+ 'skipped': 0,
224
+ 'uploaded': 0,
225
+ 'failed': 0,
226
+ 'bytes_planned': 0,
227
+ 'bytes_uploaded': 0,
228
+ 'started_at': time.time(),
229
+ }
230
+
231
+ def record_summary() -> None:
232
+ elapsed = time.time() - run_stats['started_at']
233
+ summary_message = (
234
+ f"planned={run_stats['planned']} skipped={run_stats['skipped']} "
235
+ f"uploaded={run_stats['uploaded']} failed={run_stats['failed']} "
236
+ f"bytes_planned={run_stats['bytes_planned']} bytes_uploaded={run_stats['bytes_uploaded']}"
237
+ )
238
+ append_progress(None, '', 'summary', summary_message, elapsed)
239
+ try:
240
+ summary_json = checkpoint_dir / f"{repo_id.replace('/', '__')}.summary.json"
241
+ with open(summary_json, 'w', encoding='utf-8') as fh:
242
+ json.dump({
243
+ 'repo_id': repo_id,
244
+ 'planned': run_stats['planned'],
245
+ 'skipped': run_stats['skipped'],
246
+ 'uploaded': run_stats['uploaded'],
247
+ 'failed': run_stats['failed'],
248
+ 'bytes_planned': run_stats['bytes_planned'],
249
+ 'bytes_uploaded': run_stats['bytes_uploaded'],
250
+ 'elapsed_seconds': round(elapsed, 2),
251
+ }, fh, indent=2)
252
+ except Exception as e:
253
+ print(f"Warning: failed to write summary json: {e}")
254
+
255
+ def remote_file_size(repo_path: str) -> int | None:
256
+ if requests is None:
257
+ return None
258
+ # Construct raw file URL: https://huggingface.co/datasets/{repo_id}/resolve/main/{path}
259
+ # Path components must be URL-quoted
260
+ quoted = '/'.join(quote(p) for p in repo_path.split('/'))
261
+ url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{quoted}"
262
+ try:
263
+ r = requests.head(url, allow_redirects=True, timeout=10)
264
+ if r.status_code >= 400:
265
+ return None
266
+ cl = r.headers.get('content-length')
267
+ if cl is None:
268
+ return None
269
+ return int(cl)
270
+ except Exception:
271
+ return None
272
+
273
  def create_repo() -> None:
274
  api.create_repo(repo_id=repo_id, repo_type="dataset", private=is_private, exist_ok=True)
275
 
276
  retry_call("create_repo", args.create_repo_attempts, args.retry_backoff, create_repo)
277
+ # Prepare set of existing files if resume requested
278
+ existing_files: set = set()
279
+ checkpoint_uploaded: set = set()
280
+ checkpoint_hashes: dict[str, str] = {}
281
+ if args.resume:
282
+ existing_files = list_existing_files()
283
+ checkpoint_uploaded, checkpoint_hashes = load_checkpoint()
284
+
285
+ # union of already-known uploaded files
286
+ already_uploaded = existing_files.union(checkpoint_uploaded)
287
 
288
  def upload_segments() -> None:
289
+ # existing_files is computed above when resume is requested
290
+
291
+ # If not resuming and hf offers upload_large_folder, use it for speed
292
+ if not args.resume and hasattr(api, "upload_large_folder"):
293
+ print("Using HfApi.upload_large_folder for optimized large uploads...")
294
+ api.upload_large_folder(
295
+ folder_path=str(segments_path),
296
+ repo_id=repo_id,
297
+ repo_type="dataset",
298
+ path_in_repo=args.segments_path_in_repo,
299
+ commit_message=args.commit_message,
300
+ )
301
+ return
302
+
303
+ # Otherwise, perform per-file upload and skip existing files
304
+ print("Performing per-file upload (skipping already uploaded files if any)...")
305
+
306
+ files: List[Tuple[Path, str]] = []
307
+ for p in segments_path.rglob("*"):
308
+ if p.is_file():
309
+ rel = p.relative_to(segments_path).as_posix()
310
+ repo_path = f"{args.segments_path_in_repo}/{rel}"
311
+ run_stats['bytes_planned'] += p.stat().st_size
312
+ skip = False
313
+ if repo_path in already_uploaded:
314
+ local_hash = None
315
+ checkpoint_hash = checkpoint_hashes.get(repo_path)
316
+ if checkpoint_hash:
317
+ local_hash = compute_sha256(p)
318
+ if local_hash != checkpoint_hash:
319
+ skip = False
320
+ print(f"Hash mismatch for {repo_path}; will re-upload.")
321
+ else:
322
+ skip = True
323
+ elif args.verify_remote and requests is not None:
324
+ local_size = p.stat().st_size
325
+ remote_size, remote_etag = remote_file_info(repo_path)
326
+ if args.verify_etag and remote_etag:
327
+ # We do not compute a remote-matching hash locally here; rely on size and the checkpoint hash path.
328
+ skip = (remote_size is not None and local_size == remote_size)
329
+ else:
330
+ skip = (remote_size is not None and local_size == remote_size)
331
+ else:
332
+ skip = True
333
+
334
+ if skip:
335
+ run_stats['skipped'] += 1
336
+ append_progress(p, repo_path, 'skipped', 'already uploaded')
337
+ continue
338
+ files.append((p, repo_path))
339
+
340
+ run_stats['planned'] = len(files)
341
+
342
+ if args.resume_only:
343
+ print("Resume-only mode: no upload will be performed.")
344
+ for local_path, repo_path in files:
345
+ append_progress(local_path, repo_path, 'planned', 'resume-only')
346
+ return
347
+
348
+ print(f"Found {len(files)} files to upload; launching {args.max_workers} workers")
349
+
350
+ def upload_one(pair: Tuple[Path, str]):
351
+ local_path, repo_path = pair
352
+ start = time.time()
353
+ try:
354
+ def do_upload():
355
+ api.upload_file(
356
+ path_or_fileobj=str(local_path),
357
+ path_in_repo=repo_path,
358
+ repo_id=repo_id,
359
+ repo_type="dataset",
360
+ commit_message=args.commit_message,
361
+ )
362
+
363
+ retry_call(f"upload_file:{repo_path}", args.upload_attempts, args.retry_backoff, do_upload)
364
+ except Exception as e:
365
+ elapsed = time.time() - start
366
+ append_progress(local_path, repo_path, 'failed', str(e), elapsed)
367
+ raise
368
+ else:
369
+ elapsed = time.time() - start
370
+ append_progress(local_path, repo_path, 'uploaded', None, elapsed)
371
+ return repo_path, local_path
372
+
373
+ with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as ex:
374
+ futures = [ex.submit(upload_one, f) for f in files]
375
+ for fut in concurrent.futures.as_completed(futures):
376
+ try:
377
+ repo_path, local_path = fut.result()
378
+ except Exception as e:
379
+ print(f"Error uploading file: {e}")
380
+ continue
381
+ # mark as uploaded in checkpoint
382
+ if repo_path:
383
+ already_uploaded.add(repo_path)
384
+ try:
385
+ checkpoint_hashes[repo_path] = compute_sha256(local_path)
386
+ except Exception as e:
387
+ print(f"Warning: failed to hash {repo_path} after upload: {e}")
388
+ run_stats['uploaded'] += 1
389
+ try:
390
+ run_stats['bytes_uploaded'] += Path(local_path).stat().st_size
391
+ except Exception:
392
+ pass
393
+ save_checkpoint(already_uploaded, checkpoint_hashes)
394
 
395
  retry_call("upload_segments", args.upload_attempts, args.retry_backoff, upload_segments)
396
 
397
+ if args.resume_only:
398
+ print("Resume-only mode completed; no uploads were performed.")
399
+ record_summary()
400
+ print()
401
+ print("Upload complete.")
402
+ print(f"Dataset URL: https://huggingface.co/datasets/{repo_id}")
403
+ return 0
404
+
405
  if csv_path:
406
 
407
  def upload_csv() -> None:
408
+ skip_csv = False
409
+ if args.resume and args.csv_path_in_repo in already_uploaded:
410
+ if args.verify_remote and requests is not None:
411
+ local_size = csv_path.stat().st_size
412
+ remote_size, remote_etag = remote_file_info(args.csv_path_in_repo)
413
+ if args.verify_etag and remote_etag:
414
+ # no local etag available -> fall back to size check
415
+ if remote_size is not None and local_size == remote_size:
416
+ skip_csv = True
417
+ else:
418
+ if remote_size is not None and local_size == remote_size:
419
+ skip_csv = True
420
+
421
+ if skip_csv:
422
+ print(f"CSV already exists at {args.csv_path_in_repo}; skipping upload.")
423
+ run_stats['skipped'] += 1
424
+ append_progress(csv_path, args.csv_path_in_repo, 'skipped', 'already uploaded')
425
+ return
426
+
427
+ start = time.time()
428
+ try:
429
+ api.upload_file(
430
+ path_or_fileobj=str(csv_path),
431
+ path_in_repo=args.csv_path_in_repo,
432
+ repo_id=repo_id,
433
+ repo_type="dataset",
434
+ commit_message=f"Upload CSV for {repo_id}",
435
+ )
436
+ except Exception as e:
437
+ elapsed = time.time() - start
438
+ run_stats['failed'] += 1
439
+ append_progress(csv_path, args.csv_path_in_repo, 'failed', str(e), elapsed)
440
+ raise
441
+ else:
442
+ elapsed = time.time() - start
443
+ run_stats['uploaded'] += 1
444
+ try:
445
+ run_stats['bytes_uploaded'] += csv_path.stat().st_size
446
+ except Exception:
447
+ pass
448
+ try:
449
+ checkpoint_hashes[args.csv_path_in_repo] = compute_sha256(csv_path)
450
+ save_checkpoint(already_uploaded.union({args.csv_path_in_repo}), checkpoint_hashes)
451
+ except Exception as e:
452
+ print(f"Warning: failed to update CSV checkpoint hash: {e}")
453
+ append_progress(csv_path, args.csv_path_in_repo, 'uploaded', None, elapsed)
454
 
455
  retry_call("upload_csv", args.upload_attempts, args.retry_backoff, upload_csv)
456
 
457
  print()
458
  print("Upload complete.")
459
  print(f"Dataset URL: https://huggingface.co/datasets/{repo_id}")
460
+ record_summary()
461
  return 0
462
 
463
 
hf_bulk_upload_tool_v2/README.md ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # BirdNET HF Bulk Upload Tool v2
2
+
3
+ This version is a clean HF-first uploader for large BirdNET segment folders.
4
+
5
+ ## What it does
6
+
7
+ - Creates the dataset repo if needed
8
+ - Uploads the segment tree with `HfApi.upload_large_folder(...)`
9
+ - Uploads an optional detections CSV to `index/detections.csv`
10
+ - Uses the same local species subfolder tree as the source folder
11
+
12
+ ## Files
13
+
14
+ - `upload_dataset_v2.py` - standalone uploader
15
+ - `run_upload.ps1` - Windows launcher
16
+
17
+ ## Run
18
+
19
+ ### PowerShell launcher
20
+
21
+ ```powershell
22
+ .\hf_bulk_upload_tool_v2\run_upload.ps1 `
23
+ -RepoId "jrrribeiro/PPBIO_Audio_Library" `
24
+ -Segments "C:\Users\jonat\OneDrive - MSFT\Doutorado PPGEtno\BirdNET Segments" `
25
+ -Csv "C:\Users\jonat\OneDrive - MSFT\Doutorado PPGEtno\BirdNET Analyzer\00_ALL_DETECTIONS.csv" `
26
+ -Private
27
+ ```
28
+
29
+ Add `-HfToken` if you prefer passing the token explicitly. Otherwise set `HF_TOKEN` in the environment or let the script prompt.
30
+
31
+ ### Python directly
32
+
33
+ ```powershell
34
+ python .\hf_bulk_upload_tool_v2\upload_dataset_v2.py `
35
+ --repo-id "jrrribeiro/PPBIO_Audio_Library" `
36
+ --segments "C:\Users\jonat\OneDrive - MSFT\Doutorado PPGEtno\BirdNET Segments" `
37
+ --csv "C:\Users\jonat\OneDrive - MSFT\Doutorado PPGEtno\BirdNET Analyzer\00_ALL_DETECTIONS.csv" `
38
+ --private
39
+ ```
40
+
41
+ ## Notes
42
+
43
+ - This version uses the HF optimized large-folder API first.
44
+ - If you need a subset upload, use `--allow-patterns` or `--ignore-patterns`.
hf_bulk_upload_tool_v2/requirements.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ huggingface_hub==0.28.1
hf_bulk_upload_tool_v2/run_upload.ps1 ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ param(
2
+ [Parameter(Mandatory = $true)]
3
+ [string]$RepoId,
4
+ [Parameter(Mandatory = $true)]
5
+ [string]$Segments,
6
+ [string]$Csv,
7
+ [string]$HfToken,
8
+ [switch]$Private,
9
+ [switch]$Public,
10
+ [switch]$DryRun,
11
+ [int]$MaxWorkers = 4,
12
+ [string]$AllowPatterns,
13
+ [string]$IgnorePatterns
14
+ )
15
+
16
+ $ErrorActionPreference = 'Stop'
17
+
18
+ $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
19
+ $pythonExe = Join-Path (Split-Path -Parent $scriptDir) '.venv\Scripts\python.exe'
20
+ $scriptPath = Join-Path $scriptDir 'upload_dataset_v2.py'
21
+
22
+ if (-not (Test-Path $pythonExe)) {
23
+ throw "Python executable not found at: $pythonExe"
24
+ }
25
+
26
+ $argsList = @($scriptPath, '--repo-id', $RepoId, '--segments', $Segments)
27
+
28
+ if ($Csv) { $argsList += @('--csv', $Csv) }
29
+ if ($HfToken) { $argsList += @('--hf-token', $HfToken) }
30
+ if ($Private) { $argsList += '--private' }
31
+ if ($Public) { $argsList += '--public' }
32
+ if ($DryRun) { $argsList += '--dry-run' }
33
+ if ($MaxWorkers) { $argsList += @('--max-workers', $MaxWorkers.ToString()) }
34
+ if ($AllowPatterns) { $argsList += @('--allow-patterns', $AllowPatterns) }
35
+ if ($IgnorePatterns) { $argsList += @('--ignore-patterns', $IgnorePatterns) }
36
+
37
+ & $pythonExe $argsList
hf_bulk_upload_tool_v2/upload_dataset_v2.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import getpass
6
+ import os
7
+ import shutil
8
+ import subprocess
9
+ import tempfile
10
+ import time
11
+ from pathlib import Path
12
+
13
+ from huggingface_hub import HfApi
14
+
15
+
16
+ def configure_hf_env() -> None:
17
+ os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "20")
18
+ os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "120")
19
+ os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1")
20
+
21
+
22
+ configure_hf_env()
23
+
24
+
25
+ def parse_args() -> argparse.Namespace:
26
+ parser = argparse.ArgumentParser(
27
+ description="Create a HF dataset repo and upload a large BirdNET segment folder plus optional CSV."
28
+ )
29
+ parser.add_argument("--repo-id", required=True, help="Dataset repo id, for example username/dataset-name.")
30
+ parser.add_argument("--segments", required=True, help="Path to the local BirdNET segments folder.")
31
+ parser.add_argument("--csv", help="Optional CSV file to upload as index/detections.csv.")
32
+ parser.add_argument("--hf-token", help="Hugging Face write token. If omitted, the script uses HF_TOKEN or prompts.")
33
+ parser.add_argument("--private", action="store_true", help="Create the dataset as private. This is the default.")
34
+ parser.add_argument("--public", action="store_true", help="Create the dataset as public.")
35
+ parser.add_argument("--max-workers", type=int, default=4, help="Worker threads for upload_large_folder.")
36
+ parser.add_argument("--commit-message", default="Upload BirdNET data", help="Commit message for the CSV upload fallback.")
37
+ parser.add_argument("--dry-run", action="store_true", help="Show what would happen without uploading anything.")
38
+ parser.add_argument("--ignore-patterns", default=None, help="Optional ignore patterns passed to upload_large_folder.")
39
+ parser.add_argument("--allow-patterns", default=None, help="Optional allow patterns passed to upload_large_folder.")
40
+ return parser.parse_args()
41
+
42
+
43
+ def get_token(args: argparse.Namespace) -> str:
44
+ token = args.hf_token or os.environ.get("HF_TOKEN")
45
+ if token:
46
+ return token.strip()
47
+ return getpass.getpass("Hugging Face token: ").strip()
48
+
49
+
50
+ def retry(label: str, attempts: int, backoff_seconds: float, func):
51
+ last_error: Exception | None = None
52
+ for attempt in range(1, attempts + 1):
53
+ try:
54
+ print(f"[{label}] attempt {attempt}/{attempts}")
55
+ return func()
56
+ except Exception as exc: # noqa: BLE001
57
+ last_error = exc
58
+ if attempt >= attempts:
59
+ break
60
+ wait_seconds = backoff_seconds * attempt
61
+ print(f"[{label}] failed: {exc}")
62
+ print(f"[{label}] retrying in {wait_seconds:.0f}s")
63
+ time.sleep(wait_seconds)
64
+ raise SystemExit(f"{label} failed after {attempts} attempts: {last_error}")
65
+
66
+
67
+ def create_audio_junction(staging_root: Path, segments_path: Path) -> Path:
68
+ audio_link = staging_root / "audio"
69
+ if audio_link.exists():
70
+ if audio_link.is_symlink() or audio_link.is_dir():
71
+ shutil.rmtree(audio_link)
72
+ else:
73
+ audio_link.unlink()
74
+
75
+ # Prefer a directory junction on Windows so we do not duplicate the dataset on disk.
76
+ command = ["cmd", "/c", "mklink", "/J", str(audio_link), str(segments_path)]
77
+ result = subprocess.run(command, capture_output=True, text=True)
78
+ if result.returncode != 0:
79
+ raise SystemExit(f"Failed to create audio junction: {result.stderr.strip() or result.stdout.strip()}")
80
+ return audio_link
81
+
82
+
83
+ def main() -> int:
84
+ args = parse_args()
85
+
86
+ if args.private and args.public:
87
+ raise SystemExit("Choose either --private or --public, not both.")
88
+
89
+ segments_path = Path(args.segments).expanduser().resolve()
90
+ if not segments_path.is_dir():
91
+ raise SystemExit(f"Segments path does not exist or is not a directory: {segments_path}")
92
+
93
+ csv_path = None
94
+ if args.csv:
95
+ csv_path = Path(args.csv).expanduser().resolve()
96
+ if not csv_path.is_file():
97
+ raise SystemExit(f"CSV path does not exist or is not a file: {csv_path}")
98
+
99
+ is_private = True if args.private or not args.public else False
100
+
101
+ print(f"Repository: {args.repo_id}")
102
+ print(f"Segments: {segments_path}")
103
+ if csv_path:
104
+ print(f"CSV: {csv_path}")
105
+ print(f"Visibility: {'private' if is_private else 'public'}")
106
+
107
+ if args.dry_run:
108
+ print("Dry run selected. No network calls will be made.")
109
+ print("Dataset layout will be staged as audio/* plus index/detections.csv.")
110
+ return 0
111
+
112
+ token = get_token(args)
113
+ api = HfApi(token=token)
114
+
115
+ def create_repo() -> None:
116
+ api.create_repo(repo_id=args.repo_id, repo_type="dataset", private=is_private, exist_ok=True)
117
+
118
+ retry("create_repo", 3, 5.0, create_repo)
119
+
120
+ allow_patterns = args.allow_patterns
121
+ ignore_patterns = args.ignore_patterns
122
+
123
+ def upload_segments() -> None:
124
+ print("Uploading large folder with upload_large_folder (HF optimized path)...")
125
+ with tempfile.TemporaryDirectory(prefix="birdnet_upload_stage_") as temp_dir:
126
+ staging_root = Path(temp_dir)
127
+ create_audio_junction(staging_root, segments_path)
128
+ kwargs = {
129
+ "repo_id": args.repo_id,
130
+ "folder_path": str(staging_root),
131
+ "repo_type": "dataset",
132
+ "private": is_private,
133
+ "num_workers": args.max_workers,
134
+ "print_report": True,
135
+ }
136
+ if allow_patterns:
137
+ kwargs["allow_patterns"] = allow_patterns
138
+ if ignore_patterns:
139
+ kwargs["ignore_patterns"] = ignore_patterns
140
+ api.upload_large_folder(**kwargs)
141
+
142
+ retry("upload_segments", 3, 5.0, upload_segments)
143
+
144
+ if csv_path:
145
+ def upload_csv() -> None:
146
+ api.upload_file(
147
+ repo_id=args.repo_id,
148
+ repo_type="dataset",
149
+ path_or_fileobj=str(csv_path),
150
+ path_in_repo="index/detections.csv",
151
+ commit_message=args.commit_message,
152
+ )
153
+
154
+ retry("upload_csv", 3, 5.0, upload_csv)
155
+
156
+ print()
157
+ print("Upload complete.")
158
+ print(f"Dataset URL: https://huggingface.co/datasets/{args.repo_id}")
159
+ return 0
160
+
161
+
162
+ if __name__ == "__main__":
163
+ raise SystemExit(main())