jrrribeiro commited on
Commit
eb46582
·
1 Parent(s): 60bb2f0

feat(cli): add sync-audio with retry and resume state

Browse files
Files changed (1) hide show
  1. cli/hf_dataset_cli.py +183 -0
cli/hf_dataset_cli.py CHANGED
@@ -3,6 +3,7 @@ import hashlib
3
  import json
4
  import math
5
  import tempfile
 
6
  from dataclasses import asdict, dataclass
7
  from datetime import UTC, datetime
8
  from io import BytesIO
@@ -36,6 +37,7 @@ REQUIRED_DETECTIONS_COLUMNS = (
36
  "start_time",
37
  "end_time",
38
  )
 
39
 
40
 
41
  @dataclass(slots=True)
@@ -203,6 +205,164 @@ def build_manifest_payload(
203
  }
204
 
205
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
206
  def build_and_upload_index(
207
  api: HfApi,
208
  project_slug: str,
@@ -324,6 +484,15 @@ def build_parser() -> argparse.ArgumentParser:
324
  build_index.add_argument("--detections-file", required=True)
325
  build_index.add_argument("--shard-size", type=int, default=10000)
326
 
 
 
 
 
 
 
 
 
 
327
  verify_project = sub.add_parser("verify-project", help="Verify project integrity")
328
  verify_project.add_argument("--project-slug", required=True)
329
  verify_project.add_argument("--dataset-repo", required=True)
@@ -357,6 +526,20 @@ def main() -> int:
357
  print(json.dumps(result, indent=2))
358
  return 0
359
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
360
  if args.command == "verify-project":
361
  result = verify_project(
362
  api=api,
 
3
  import json
4
  import math
5
  import tempfile
6
+ import time
7
  from dataclasses import asdict, dataclass
8
  from datetime import UTC, datetime
9
  from io import BytesIO
 
37
  "start_time",
38
  "end_time",
39
  )
40
+ SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
41
 
42
 
43
  @dataclass(slots=True)
 
205
  }
206
 
207
 
208
+ def discover_audio_files(local_audio_dir: str) -> list[Path]:
209
+ audio_root = Path(local_audio_dir)
210
+ if not audio_root.exists() or not audio_root.is_dir():
211
+ raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
212
+
213
+ files = [
214
+ path
215
+ for path in audio_root.rglob("*")
216
+ if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
217
+ ]
218
+ return sorted(files)
219
+
220
+
221
+ def _chunk_items(items: list[Path], chunk_size: int) -> list[list[Path]]:
222
+ if chunk_size <= 0:
223
+ raise ValueError("chunk_size must be greater than zero")
224
+ return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
225
+
226
+
227
+ def _load_resume_state(state_file: Path) -> dict[str, Any]:
228
+ if not state_file.exists():
229
+ return {"uploaded": [], "failed": []}
230
+
231
+ payload = json.loads(state_file.read_text(encoding="utf-8"))
232
+ uploaded = payload.get("uploaded", [])
233
+ failed = payload.get("failed", [])
234
+ return {
235
+ "uploaded": uploaded if isinstance(uploaded, list) else [],
236
+ "failed": failed if isinstance(failed, list) else [],
237
+ }
238
+
239
+
240
+ def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
241
+ state_file.parent.mkdir(parents=True, exist_ok=True)
242
+ payload = {
243
+ "updated_at": _utcnow_iso(),
244
+ "uploaded": sorted(uploaded),
245
+ "failed": sorted(failed),
246
+ }
247
+ state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
248
+
249
+
250
+ def _upload_audio_with_retry(
251
+ api: HfApi,
252
+ dataset_repo: str,
253
+ local_path: Path,
254
+ path_in_repo: str,
255
+ max_retries: int,
256
+ retry_backoff_seconds: float,
257
+ ) -> None:
258
+ attempts = 0
259
+ while True:
260
+ attempts += 1
261
+ try:
262
+ api.upload_file(
263
+ path_or_fileobj=str(local_path),
264
+ path_in_repo=path_in_repo,
265
+ repo_id=dataset_repo,
266
+ repo_type="dataset",
267
+ )
268
+ return
269
+ except Exception:
270
+ if attempts > max_retries:
271
+ raise
272
+ time.sleep(retry_backoff_seconds * attempts)
273
+
274
+
275
+ def sync_audio_batches(
276
+ api: HfApi,
277
+ project_slug: str,
278
+ dataset_repo: str,
279
+ local_audio_dir: str,
280
+ batch_size: int,
281
+ max_retries: int,
282
+ retry_backoff_seconds: float,
283
+ resume_state_file: str,
284
+ ) -> dict[str, Any]:
285
+ ensure_project_dataset_structure(
286
+ api=api,
287
+ project_slug=project_slug,
288
+ dataset_repo=dataset_repo,
289
+ create_private_repo=False,
290
+ )
291
+
292
+ all_files = discover_audio_files(local_audio_dir=local_audio_dir)
293
+ state_path = Path(resume_state_file)
294
+ state_payload = _load_resume_state(state_file=state_path)
295
+ uploaded_state = set(str(item) for item in state_payload["uploaded"])
296
+
297
+ remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
298
+
299
+ pending: list[tuple[Path, str]] = []
300
+ for path in all_files:
301
+ rel = path.relative_to(Path(local_audio_dir)).as_posix()
302
+ repo_path = f"audio/{rel}"
303
+
304
+ if repo_path in remote_files:
305
+ uploaded_state.add(repo_path)
306
+ continue
307
+
308
+ if repo_path in uploaded_state:
309
+ continue
310
+
311
+ pending.append((path, repo_path))
312
+
313
+ pending_paths = [item[0] for item in pending]
314
+ by_local_to_repo = {local: repo for local, repo in pending}
315
+ batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
316
+
317
+ uploaded_now: set[str] = set()
318
+ failed: set[str] = set()
319
+
320
+ for batch_index, batch in enumerate(batches, start=1):
321
+ batch_uploaded = 0
322
+ batch_failed = 0
323
+
324
+ for local_path in batch:
325
+ repo_path = by_local_to_repo[local_path]
326
+ try:
327
+ _upload_audio_with_retry(
328
+ api=api,
329
+ dataset_repo=dataset_repo,
330
+ local_path=local_path,
331
+ path_in_repo=repo_path,
332
+ max_retries=max_retries,
333
+ retry_backoff_seconds=retry_backoff_seconds,
334
+ )
335
+ uploaded_now.add(repo_path)
336
+ uploaded_state.add(repo_path)
337
+ batch_uploaded += 1
338
+ except Exception:
339
+ failed.add(repo_path)
340
+ batch_failed += 1
341
+
342
+ print(
343
+ json.dumps(
344
+ {
345
+ "event": "sync-audio-batch",
346
+ "batch_index": batch_index,
347
+ "batch_size": len(batch),
348
+ "uploaded": batch_uploaded,
349
+ "failed": batch_failed,
350
+ }
351
+ )
352
+ )
353
+ _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
354
+
355
+ return {
356
+ "project_slug": project_slug,
357
+ "dataset_repo": dataset_repo,
358
+ "total_local_audio_files": len(all_files),
359
+ "pending_uploads": len(pending),
360
+ "uploaded_now": len(uploaded_now),
361
+ "failed": len(failed),
362
+ "resume_state_file": str(state_path),
363
+ }
364
+
365
+
366
  def build_and_upload_index(
367
  api: HfApi,
368
  project_slug: str,
 
484
  build_index.add_argument("--detections-file", required=True)
485
  build_index.add_argument("--shard-size", type=int, default=10000)
486
 
487
+ sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
488
+ sync_audio.add_argument("--project-slug", required=True)
489
+ sync_audio.add_argument("--dataset-repo", required=True)
490
+ sync_audio.add_argument("--local-audio-dir", required=True)
491
+ sync_audio.add_argument("--batch-size", type=int, default=100)
492
+ sync_audio.add_argument("--max-retries", type=int, default=3)
493
+ sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
494
+ sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
495
+
496
  verify_project = sub.add_parser("verify-project", help="Verify project integrity")
497
  verify_project.add_argument("--project-slug", required=True)
498
  verify_project.add_argument("--dataset-repo", required=True)
 
526
  print(json.dumps(result, indent=2))
527
  return 0
528
 
529
+ if args.command == "sync-audio":
530
+ result = sync_audio_batches(
531
+ api=api,
532
+ project_slug=args.project_slug,
533
+ dataset_repo=args.dataset_repo,
534
+ local_audio_dir=args.local_audio_dir,
535
+ batch_size=args.batch_size,
536
+ max_retries=args.max_retries,
537
+ retry_backoff_seconds=args.retry_backoff_seconds,
538
+ resume_state_file=args.resume_state_file,
539
+ )
540
+ print(json.dumps(result, indent=2))
541
+ return 0 if result["failed"] == 0 else 1
542
+
543
  if args.command == "verify-project":
544
  result = verify_project(
545
  api=api,