jrrribeiro commited on
Commit
b572423
·
1 Parent(s): 090ffd1

feat(cli): implement ingest-segments execute mode with HF upload/indexing

Browse files

- Add full execute path for ingest-segments (without --dry-run)
- Upload matched segment audio to audio/segments in resumable batches
- Reuse retry/backoff and resume state for robust uploads
- Build and upload index shards from normalized matched rows
- Update manifest.json with detection/audio counters and shard metadata
- Upload ingestion audit report under audit/ingestion-runs
- Extend parser options (batch-size, shard-size, retries, resume-state)
- Keep dry-run behavior backward compatible

cli/hf_dataset_cli.py CHANGED
@@ -241,7 +241,7 @@ def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
241
  return records
242
 
243
 
244
- def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
245
  frame = load_birdnet_detections_csv(detections_csv=detections_csv)
246
  segment_records = discover_segment_records(segments_root=segments_root)
247
 
@@ -328,6 +328,26 @@ def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments
328
  }
329
  )
330
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
331
  unique_audio = {row["audio_id"] for row in matched_rows}
332
  result = {
333
  "mode": "dry-run",
@@ -335,7 +355,7 @@ def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments
335
  "detections_csv": detections_csv,
336
  "segments_root": segments_root,
337
  "csv_rows_total": int(len(frame)),
338
- "segments_found_total": len(segment_records),
339
  "matched_rows": len(matched_rows),
340
  "unmatched_rows": len(unmatched_rows),
341
  "ambiguous_rows": len(ambiguous_rows),
@@ -422,7 +442,7 @@ def discover_audio_files(local_audio_dir: str) -> list[Path]:
422
  return sorted(files)
423
 
424
 
425
- def _chunk_items(items: list[Path], chunk_size: int) -> list[list[Path]]:
426
  if chunk_size <= 0:
427
  raise ValueError("chunk_size must be greater than zero")
428
  return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
@@ -476,6 +496,154 @@ def _upload_audio_with_retry(
476
  time.sleep(retry_backoff_seconds * attempts)
477
 
478
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
479
  def sync_audio_batches(
480
  api: HfApi,
481
  project_slug: str,
@@ -706,6 +874,11 @@ def build_parser() -> argparse.ArgumentParser:
706
  ingest_segments.add_argument("--dataset-repo", required=True)
707
  ingest_segments.add_argument("--detections-csv", required=True)
708
  ingest_segments.add_argument("--segments-root", required=True)
 
 
 
 
 
709
  ingest_segments.add_argument("--dry-run", action="store_true")
710
  ingest_segments.add_argument("--report-file", default="")
711
 
@@ -762,20 +935,31 @@ def main() -> int:
762
  return 0 if result["ok"] else 1
763
 
764
  if args.command == "ingest-segments":
765
- if not args.dry_run:
766
- parser.error("ingest-segments currently supports only --dry-run in this phase")
767
-
768
- result = run_ingest_segments_dry_run(
769
- project_slug=args.project_slug,
770
- detections_csv=args.detections_csv,
771
- segments_root=args.segments_root,
772
- )
 
 
 
 
 
 
 
 
 
 
 
773
 
774
  if args.report_file:
775
  Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
776
 
777
  print(json.dumps(result, indent=2))
778
- return 0
779
 
780
  parser.error("Unknown command")
781
  return 2
 
241
  return records
242
 
243
 
244
+ def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
245
  frame = load_birdnet_detections_csv(detections_csv=detections_csv)
246
  segment_records = discover_segment_records(segments_root=segments_root)
247
 
 
328
  }
329
  )
330
 
331
+ return {
332
+ "frame": frame,
333
+ "segment_records_total": len(segment_records),
334
+ "matched_rows": matched_rows,
335
+ "unmatched_rows": unmatched_rows,
336
+ "ambiguous_rows": ambiguous_rows,
337
+ }
338
+
339
+
340
+ def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
341
+ match_result = _match_birdnet_rows(
342
+ project_slug=project_slug,
343
+ detections_csv=detections_csv,
344
+ segments_root=segments_root,
345
+ )
346
+ frame = match_result["frame"]
347
+ matched_rows = match_result["matched_rows"]
348
+ unmatched_rows = match_result["unmatched_rows"]
349
+ ambiguous_rows = match_result["ambiguous_rows"]
350
+
351
  unique_audio = {row["audio_id"] for row in matched_rows}
352
  result = {
353
  "mode": "dry-run",
 
355
  "detections_csv": detections_csv,
356
  "segments_root": segments_root,
357
  "csv_rows_total": int(len(frame)),
358
+ "segments_found_total": int(match_result["segment_records_total"]),
359
  "matched_rows": len(matched_rows),
360
  "unmatched_rows": len(unmatched_rows),
361
  "ambiguous_rows": len(ambiguous_rows),
 
442
  return sorted(files)
443
 
444
 
445
+ def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
446
  if chunk_size <= 0:
447
  raise ValueError("chunk_size must be greater than zero")
448
  return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
 
496
  time.sleep(retry_backoff_seconds * attempts)
497
 
498
 
499
+ def ingest_segments_to_hf(
500
+ api: HfApi,
501
+ project_slug: str,
502
+ dataset_repo: str,
503
+ detections_csv: str,
504
+ segments_root: str,
505
+ batch_size: int,
506
+ shard_size: int,
507
+ max_retries: int,
508
+ retry_backoff_seconds: float,
509
+ resume_state_file: str,
510
+ ) -> dict[str, Any]:
511
+ ensure_project_dataset_structure(
512
+ api=api,
513
+ project_slug=project_slug,
514
+ dataset_repo=dataset_repo,
515
+ create_private_repo=False,
516
+ )
517
+
518
+ match_result = _match_birdnet_rows(
519
+ project_slug=project_slug,
520
+ detections_csv=detections_csv,
521
+ segments_root=segments_root,
522
+ )
523
+ frame = match_result["frame"]
524
+ matched_rows = match_result["matched_rows"]
525
+ unmatched_rows = match_result["unmatched_rows"]
526
+ ambiguous_rows = match_result["ambiguous_rows"]
527
+
528
+ # Deduplicate by remote path so retries/resume work on a stable key.
529
+ path_to_local: dict[str, Path] = {}
530
+ for row in matched_rows:
531
+ repo_path = f"audio/segments/{row['segment_relpath']}"
532
+ if repo_path in path_to_local:
533
+ continue
534
+ path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
535
+
536
+ state_path = Path(resume_state_file)
537
+ state_payload = _load_resume_state(state_file=state_path)
538
+ uploaded_state = set(str(item) for item in state_payload["uploaded"])
539
+ failed_state = set(str(item) for item in state_payload["failed"])
540
+
541
+ remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
542
+
543
+ pending: list[tuple[Path, str]] = []
544
+ skipped_existing = 0
545
+ for repo_path, local_path in path_to_local.items():
546
+ if repo_path in remote_files:
547
+ uploaded_state.add(repo_path)
548
+ skipped_existing += 1
549
+ continue
550
+ if repo_path in uploaded_state:
551
+ continue
552
+ pending.append((local_path, repo_path))
553
+
554
+ uploaded_now: set[str] = set()
555
+ failed_now: set[str] = set()
556
+
557
+ for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
558
+ batch_uploaded = 0
559
+ batch_failed = 0
560
+ for local_path, repo_path in batch:
561
+ try:
562
+ _upload_audio_with_retry(
563
+ api=api,
564
+ dataset_repo=dataset_repo,
565
+ local_path=local_path,
566
+ path_in_repo=repo_path,
567
+ max_retries=max_retries,
568
+ retry_backoff_seconds=retry_backoff_seconds,
569
+ )
570
+ uploaded_state.add(repo_path)
571
+ failed_state.discard(repo_path)
572
+ uploaded_now.add(repo_path)
573
+ batch_uploaded += 1
574
+ except Exception:
575
+ failed_state.add(repo_path)
576
+ failed_now.add(repo_path)
577
+ batch_failed += 1
578
+
579
+ print(
580
+ json.dumps(
581
+ {
582
+ "event": "ingest-segments-audio-batch",
583
+ "batch_index": batch_index,
584
+ "batch_size": len(batch),
585
+ "uploaded": batch_uploaded,
586
+ "failed": batch_failed,
587
+ }
588
+ )
589
+ )
590
+ _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
591
+
592
+ index_frame = pd.DataFrame(matched_rows)
593
+ if not index_frame.empty:
594
+ index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
595
+
596
+ with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
597
+ temp_dir = Path(temp_dir_str)
598
+ shard_dir = temp_dir / "index" / "shards"
599
+ shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
600
+
601
+ for item in shard_metadata:
602
+ local_path = temp_dir / item.path
603
+ api.upload_file(
604
+ path_or_fileobj=str(local_path),
605
+ path_in_repo=item.path,
606
+ repo_id=dataset_repo,
607
+ repo_type="dataset",
608
+ )
609
+
610
+ manifest = build_manifest_payload(
611
+ project_slug=project_slug,
612
+ dataset_repo=dataset_repo,
613
+ frame=index_frame,
614
+ shard_size=shard_size,
615
+ shard_metadata=shard_metadata,
616
+ )
617
+ _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
618
+
619
+ run_report = {
620
+ "mode": "execute",
621
+ "project_slug": project_slug,
622
+ "dataset_repo": dataset_repo,
623
+ "detections_csv": detections_csv,
624
+ "segments_root": segments_root,
625
+ "csv_rows_total": int(len(frame)),
626
+ "segments_found_total": int(match_result["segment_records_total"]),
627
+ "matched_rows": len(matched_rows),
628
+ "unmatched_rows": len(unmatched_rows),
629
+ "ambiguous_rows": len(ambiguous_rows),
630
+ "unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
631
+ "pending_audio_uploads": len(pending),
632
+ "uploaded_audio_now": len(uploaded_now),
633
+ "uploaded_audio_skipped_existing": skipped_existing,
634
+ "failed_uploads": len(failed_now),
635
+ "resume_state_file": str(state_path),
636
+ "index_rows_written": len(index_frame),
637
+ "shards_written": len(shard_metadata),
638
+ "sample_unmatched": unmatched_rows[:20],
639
+ "sample_ambiguous": ambiguous_rows[:20],
640
+ }
641
+
642
+ audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
643
+ _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
644
+ return run_report
645
+
646
+
647
  def sync_audio_batches(
648
  api: HfApi,
649
  project_slug: str,
 
874
  ingest_segments.add_argument("--dataset-repo", required=True)
875
  ingest_segments.add_argument("--detections-csv", required=True)
876
  ingest_segments.add_argument("--segments-root", required=True)
877
+ ingest_segments.add_argument("--batch-size", type=int, default=200)
878
+ ingest_segments.add_argument("--shard-size", type=int, default=10000)
879
+ ingest_segments.add_argument("--max-retries", type=int, default=3)
880
+ ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
881
+ ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
882
  ingest_segments.add_argument("--dry-run", action="store_true")
883
  ingest_segments.add_argument("--report-file", default="")
884
 
 
935
  return 0 if result["ok"] else 1
936
 
937
  if args.command == "ingest-segments":
938
+ if args.dry_run:
939
+ result = run_ingest_segments_dry_run(
940
+ project_slug=args.project_slug,
941
+ detections_csv=args.detections_csv,
942
+ segments_root=args.segments_root,
943
+ )
944
+ else:
945
+ result = ingest_segments_to_hf(
946
+ api=api,
947
+ project_slug=args.project_slug,
948
+ dataset_repo=args.dataset_repo,
949
+ detections_csv=args.detections_csv,
950
+ segments_root=args.segments_root,
951
+ batch_size=args.batch_size,
952
+ shard_size=args.shard_size,
953
+ max_retries=args.max_retries,
954
+ retry_backoff_seconds=args.retry_backoff_seconds,
955
+ resume_state_file=args.resume_state_file,
956
+ )
957
 
958
  if args.report_file:
959
  Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
960
 
961
  print(json.dumps(result, indent=2))
962
+ return 0 if result.get("failed_uploads", 0) == 0 else 1
963
 
964
  parser.error("Unknown command")
965
  return 2
tests/unit/test_hf_ingest_segments_cli.py CHANGED
@@ -4,6 +4,7 @@ import pandas as pd
4
 
5
  from cli.hf_dataset_cli import (
6
  _build_detection_key,
 
7
  parse_segment_filename,
8
  run_ingest_segments_dry_run,
9
  )
@@ -99,3 +100,28 @@ def test_run_ingest_segments_dry_run_matches_rows(tmp_path: Path):
99
  assert result["segments_found_total"] == 2
100
  assert result["matched_rows"] == 1
101
  assert result["unmatched_rows"] == 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
 
5
  from cli.hf_dataset_cli import (
6
  _build_detection_key,
7
+ build_parser,
8
  parse_segment_filename,
9
  run_ingest_segments_dry_run,
10
  )
 
100
  assert result["segments_found_total"] == 2
101
  assert result["matched_rows"] == 1
102
  assert result["unmatched_rows"] == 1
103
+
104
+
105
+ def test_build_parser_ingest_segments_defaults():
106
+ parser = build_parser()
107
+ args = parser.parse_args(
108
+ [
109
+ "ingest-segments",
110
+ "--project-slug",
111
+ "ppbio-aiuaba",
112
+ "--dataset-repo",
113
+ "owner/repo",
114
+ "--detections-csv",
115
+ "detections.csv",
116
+ "--segments-root",
117
+ "segments",
118
+ ]
119
+ )
120
+
121
+ assert args.command == "ingest-segments"
122
+ assert args.batch_size == 200
123
+ assert args.shard_size == 10000
124
+ assert args.max_retries == 3
125
+ assert args.retry_backoff_seconds == 1.0
126
+ assert args.resume_state_file == ".ingest-segments-state.json"
127
+ assert args.dry_run is False