Spaces:
Sleeping
Sleeping
Commit ·
b572423
1
Parent(s): 090ffd1
feat(cli): implement ingest-segments execute mode with HF upload/indexing
Browse files- Add full execute path for ingest-segments (without --dry-run)
- Upload matched segment audio to audio/segments in resumable batches
- Reuse retry/backoff and resume state for robust uploads
- Build and upload index shards from normalized matched rows
- Update manifest.json with detection/audio counters and shard metadata
- Upload ingestion audit report under audit/ingestion-runs
- Extend parser options (batch-size, shard-size, retries, resume-state)
- Keep dry-run behavior backward compatible
- cli/hf_dataset_cli.py +196 -12
- tests/unit/test_hf_ingest_segments_cli.py +26 -0
cli/hf_dataset_cli.py
CHANGED
|
@@ -241,7 +241,7 @@ def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
|
|
| 241 |
return records
|
| 242 |
|
| 243 |
|
| 244 |
-
def
|
| 245 |
frame = load_birdnet_detections_csv(detections_csv=detections_csv)
|
| 246 |
segment_records = discover_segment_records(segments_root=segments_root)
|
| 247 |
|
|
@@ -328,6 +328,26 @@ def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments
|
|
| 328 |
}
|
| 329 |
)
|
| 330 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
unique_audio = {row["audio_id"] for row in matched_rows}
|
| 332 |
result = {
|
| 333 |
"mode": "dry-run",
|
|
@@ -335,7 +355,7 @@ def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments
|
|
| 335 |
"detections_csv": detections_csv,
|
| 336 |
"segments_root": segments_root,
|
| 337 |
"csv_rows_total": int(len(frame)),
|
| 338 |
-
"segments_found_total":
|
| 339 |
"matched_rows": len(matched_rows),
|
| 340 |
"unmatched_rows": len(unmatched_rows),
|
| 341 |
"ambiguous_rows": len(ambiguous_rows),
|
|
@@ -422,7 +442,7 @@ def discover_audio_files(local_audio_dir: str) -> list[Path]:
|
|
| 422 |
return sorted(files)
|
| 423 |
|
| 424 |
|
| 425 |
-
def _chunk_items(items: list[
|
| 426 |
if chunk_size <= 0:
|
| 427 |
raise ValueError("chunk_size must be greater than zero")
|
| 428 |
return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
|
|
@@ -476,6 +496,154 @@ def _upload_audio_with_retry(
|
|
| 476 |
time.sleep(retry_backoff_seconds * attempts)
|
| 477 |
|
| 478 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 479 |
def sync_audio_batches(
|
| 480 |
api: HfApi,
|
| 481 |
project_slug: str,
|
|
@@ -706,6 +874,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
| 706 |
ingest_segments.add_argument("--dataset-repo", required=True)
|
| 707 |
ingest_segments.add_argument("--detections-csv", required=True)
|
| 708 |
ingest_segments.add_argument("--segments-root", required=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 709 |
ingest_segments.add_argument("--dry-run", action="store_true")
|
| 710 |
ingest_segments.add_argument("--report-file", default="")
|
| 711 |
|
|
@@ -762,20 +935,31 @@ def main() -> int:
|
|
| 762 |
return 0 if result["ok"] else 1
|
| 763 |
|
| 764 |
if args.command == "ingest-segments":
|
| 765 |
-
if
|
| 766 |
-
|
| 767 |
-
|
| 768 |
-
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
|
| 772 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 773 |
|
| 774 |
if args.report_file:
|
| 775 |
Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
| 776 |
|
| 777 |
print(json.dumps(result, indent=2))
|
| 778 |
-
return 0
|
| 779 |
|
| 780 |
parser.error("Unknown command")
|
| 781 |
return 2
|
|
|
|
| 241 |
return records
|
| 242 |
|
| 243 |
|
| 244 |
+
def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 245 |
frame = load_birdnet_detections_csv(detections_csv=detections_csv)
|
| 246 |
segment_records = discover_segment_records(segments_root=segments_root)
|
| 247 |
|
|
|
|
| 328 |
}
|
| 329 |
)
|
| 330 |
|
| 331 |
+
return {
|
| 332 |
+
"frame": frame,
|
| 333 |
+
"segment_records_total": len(segment_records),
|
| 334 |
+
"matched_rows": matched_rows,
|
| 335 |
+
"unmatched_rows": unmatched_rows,
|
| 336 |
+
"ambiguous_rows": ambiguous_rows,
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 341 |
+
match_result = _match_birdnet_rows(
|
| 342 |
+
project_slug=project_slug,
|
| 343 |
+
detections_csv=detections_csv,
|
| 344 |
+
segments_root=segments_root,
|
| 345 |
+
)
|
| 346 |
+
frame = match_result["frame"]
|
| 347 |
+
matched_rows = match_result["matched_rows"]
|
| 348 |
+
unmatched_rows = match_result["unmatched_rows"]
|
| 349 |
+
ambiguous_rows = match_result["ambiguous_rows"]
|
| 350 |
+
|
| 351 |
unique_audio = {row["audio_id"] for row in matched_rows}
|
| 352 |
result = {
|
| 353 |
"mode": "dry-run",
|
|
|
|
| 355 |
"detections_csv": detections_csv,
|
| 356 |
"segments_root": segments_root,
|
| 357 |
"csv_rows_total": int(len(frame)),
|
| 358 |
+
"segments_found_total": int(match_result["segment_records_total"]),
|
| 359 |
"matched_rows": len(matched_rows),
|
| 360 |
"unmatched_rows": len(unmatched_rows),
|
| 361 |
"ambiguous_rows": len(ambiguous_rows),
|
|
|
|
| 442 |
return sorted(files)
|
| 443 |
|
| 444 |
|
| 445 |
+
def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
|
| 446 |
if chunk_size <= 0:
|
| 447 |
raise ValueError("chunk_size must be greater than zero")
|
| 448 |
return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
|
|
|
|
| 496 |
time.sleep(retry_backoff_seconds * attempts)
|
| 497 |
|
| 498 |
|
| 499 |
+
def ingest_segments_to_hf(
|
| 500 |
+
api: HfApi,
|
| 501 |
+
project_slug: str,
|
| 502 |
+
dataset_repo: str,
|
| 503 |
+
detections_csv: str,
|
| 504 |
+
segments_root: str,
|
| 505 |
+
batch_size: int,
|
| 506 |
+
shard_size: int,
|
| 507 |
+
max_retries: int,
|
| 508 |
+
retry_backoff_seconds: float,
|
| 509 |
+
resume_state_file: str,
|
| 510 |
+
) -> dict[str, Any]:
|
| 511 |
+
ensure_project_dataset_structure(
|
| 512 |
+
api=api,
|
| 513 |
+
project_slug=project_slug,
|
| 514 |
+
dataset_repo=dataset_repo,
|
| 515 |
+
create_private_repo=False,
|
| 516 |
+
)
|
| 517 |
+
|
| 518 |
+
match_result = _match_birdnet_rows(
|
| 519 |
+
project_slug=project_slug,
|
| 520 |
+
detections_csv=detections_csv,
|
| 521 |
+
segments_root=segments_root,
|
| 522 |
+
)
|
| 523 |
+
frame = match_result["frame"]
|
| 524 |
+
matched_rows = match_result["matched_rows"]
|
| 525 |
+
unmatched_rows = match_result["unmatched_rows"]
|
| 526 |
+
ambiguous_rows = match_result["ambiguous_rows"]
|
| 527 |
+
|
| 528 |
+
# Deduplicate by remote path so retries/resume work on a stable key.
|
| 529 |
+
path_to_local: dict[str, Path] = {}
|
| 530 |
+
for row in matched_rows:
|
| 531 |
+
repo_path = f"audio/segments/{row['segment_relpath']}"
|
| 532 |
+
if repo_path in path_to_local:
|
| 533 |
+
continue
|
| 534 |
+
path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
|
| 535 |
+
|
| 536 |
+
state_path = Path(resume_state_file)
|
| 537 |
+
state_payload = _load_resume_state(state_file=state_path)
|
| 538 |
+
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 539 |
+
failed_state = set(str(item) for item in state_payload["failed"])
|
| 540 |
+
|
| 541 |
+
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 542 |
+
|
| 543 |
+
pending: list[tuple[Path, str]] = []
|
| 544 |
+
skipped_existing = 0
|
| 545 |
+
for repo_path, local_path in path_to_local.items():
|
| 546 |
+
if repo_path in remote_files:
|
| 547 |
+
uploaded_state.add(repo_path)
|
| 548 |
+
skipped_existing += 1
|
| 549 |
+
continue
|
| 550 |
+
if repo_path in uploaded_state:
|
| 551 |
+
continue
|
| 552 |
+
pending.append((local_path, repo_path))
|
| 553 |
+
|
| 554 |
+
uploaded_now: set[str] = set()
|
| 555 |
+
failed_now: set[str] = set()
|
| 556 |
+
|
| 557 |
+
for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
|
| 558 |
+
batch_uploaded = 0
|
| 559 |
+
batch_failed = 0
|
| 560 |
+
for local_path, repo_path in batch:
|
| 561 |
+
try:
|
| 562 |
+
_upload_audio_with_retry(
|
| 563 |
+
api=api,
|
| 564 |
+
dataset_repo=dataset_repo,
|
| 565 |
+
local_path=local_path,
|
| 566 |
+
path_in_repo=repo_path,
|
| 567 |
+
max_retries=max_retries,
|
| 568 |
+
retry_backoff_seconds=retry_backoff_seconds,
|
| 569 |
+
)
|
| 570 |
+
uploaded_state.add(repo_path)
|
| 571 |
+
failed_state.discard(repo_path)
|
| 572 |
+
uploaded_now.add(repo_path)
|
| 573 |
+
batch_uploaded += 1
|
| 574 |
+
except Exception:
|
| 575 |
+
failed_state.add(repo_path)
|
| 576 |
+
failed_now.add(repo_path)
|
| 577 |
+
batch_failed += 1
|
| 578 |
+
|
| 579 |
+
print(
|
| 580 |
+
json.dumps(
|
| 581 |
+
{
|
| 582 |
+
"event": "ingest-segments-audio-batch",
|
| 583 |
+
"batch_index": batch_index,
|
| 584 |
+
"batch_size": len(batch),
|
| 585 |
+
"uploaded": batch_uploaded,
|
| 586 |
+
"failed": batch_failed,
|
| 587 |
+
}
|
| 588 |
+
)
|
| 589 |
+
)
|
| 590 |
+
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
|
| 591 |
+
|
| 592 |
+
index_frame = pd.DataFrame(matched_rows)
|
| 593 |
+
if not index_frame.empty:
|
| 594 |
+
index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
|
| 595 |
+
|
| 596 |
+
with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
|
| 597 |
+
temp_dir = Path(temp_dir_str)
|
| 598 |
+
shard_dir = temp_dir / "index" / "shards"
|
| 599 |
+
shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
|
| 600 |
+
|
| 601 |
+
for item in shard_metadata:
|
| 602 |
+
local_path = temp_dir / item.path
|
| 603 |
+
api.upload_file(
|
| 604 |
+
path_or_fileobj=str(local_path),
|
| 605 |
+
path_in_repo=item.path,
|
| 606 |
+
repo_id=dataset_repo,
|
| 607 |
+
repo_type="dataset",
|
| 608 |
+
)
|
| 609 |
+
|
| 610 |
+
manifest = build_manifest_payload(
|
| 611 |
+
project_slug=project_slug,
|
| 612 |
+
dataset_repo=dataset_repo,
|
| 613 |
+
frame=index_frame,
|
| 614 |
+
shard_size=shard_size,
|
| 615 |
+
shard_metadata=shard_metadata,
|
| 616 |
+
)
|
| 617 |
+
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 618 |
+
|
| 619 |
+
run_report = {
|
| 620 |
+
"mode": "execute",
|
| 621 |
+
"project_slug": project_slug,
|
| 622 |
+
"dataset_repo": dataset_repo,
|
| 623 |
+
"detections_csv": detections_csv,
|
| 624 |
+
"segments_root": segments_root,
|
| 625 |
+
"csv_rows_total": int(len(frame)),
|
| 626 |
+
"segments_found_total": int(match_result["segment_records_total"]),
|
| 627 |
+
"matched_rows": len(matched_rows),
|
| 628 |
+
"unmatched_rows": len(unmatched_rows),
|
| 629 |
+
"ambiguous_rows": len(ambiguous_rows),
|
| 630 |
+
"unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
|
| 631 |
+
"pending_audio_uploads": len(pending),
|
| 632 |
+
"uploaded_audio_now": len(uploaded_now),
|
| 633 |
+
"uploaded_audio_skipped_existing": skipped_existing,
|
| 634 |
+
"failed_uploads": len(failed_now),
|
| 635 |
+
"resume_state_file": str(state_path),
|
| 636 |
+
"index_rows_written": len(index_frame),
|
| 637 |
+
"shards_written": len(shard_metadata),
|
| 638 |
+
"sample_unmatched": unmatched_rows[:20],
|
| 639 |
+
"sample_ambiguous": ambiguous_rows[:20],
|
| 640 |
+
}
|
| 641 |
+
|
| 642 |
+
audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
|
| 643 |
+
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
|
| 644 |
+
return run_report
|
| 645 |
+
|
| 646 |
+
|
| 647 |
def sync_audio_batches(
|
| 648 |
api: HfApi,
|
| 649 |
project_slug: str,
|
|
|
|
| 874 |
ingest_segments.add_argument("--dataset-repo", required=True)
|
| 875 |
ingest_segments.add_argument("--detections-csv", required=True)
|
| 876 |
ingest_segments.add_argument("--segments-root", required=True)
|
| 877 |
+
ingest_segments.add_argument("--batch-size", type=int, default=200)
|
| 878 |
+
ingest_segments.add_argument("--shard-size", type=int, default=10000)
|
| 879 |
+
ingest_segments.add_argument("--max-retries", type=int, default=3)
|
| 880 |
+
ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 881 |
+
ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
|
| 882 |
ingest_segments.add_argument("--dry-run", action="store_true")
|
| 883 |
ingest_segments.add_argument("--report-file", default="")
|
| 884 |
|
|
|
|
| 935 |
return 0 if result["ok"] else 1
|
| 936 |
|
| 937 |
if args.command == "ingest-segments":
|
| 938 |
+
if args.dry_run:
|
| 939 |
+
result = run_ingest_segments_dry_run(
|
| 940 |
+
project_slug=args.project_slug,
|
| 941 |
+
detections_csv=args.detections_csv,
|
| 942 |
+
segments_root=args.segments_root,
|
| 943 |
+
)
|
| 944 |
+
else:
|
| 945 |
+
result = ingest_segments_to_hf(
|
| 946 |
+
api=api,
|
| 947 |
+
project_slug=args.project_slug,
|
| 948 |
+
dataset_repo=args.dataset_repo,
|
| 949 |
+
detections_csv=args.detections_csv,
|
| 950 |
+
segments_root=args.segments_root,
|
| 951 |
+
batch_size=args.batch_size,
|
| 952 |
+
shard_size=args.shard_size,
|
| 953 |
+
max_retries=args.max_retries,
|
| 954 |
+
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 955 |
+
resume_state_file=args.resume_state_file,
|
| 956 |
+
)
|
| 957 |
|
| 958 |
if args.report_file:
|
| 959 |
Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
| 960 |
|
| 961 |
print(json.dumps(result, indent=2))
|
| 962 |
+
return 0 if result.get("failed_uploads", 0) == 0 else 1
|
| 963 |
|
| 964 |
parser.error("Unknown command")
|
| 965 |
return 2
|
tests/unit/test_hf_ingest_segments_cli.py
CHANGED
|
@@ -4,6 +4,7 @@ import pandas as pd
|
|
| 4 |
|
| 5 |
from cli.hf_dataset_cli import (
|
| 6 |
_build_detection_key,
|
|
|
|
| 7 |
parse_segment_filename,
|
| 8 |
run_ingest_segments_dry_run,
|
| 9 |
)
|
|
@@ -99,3 +100,28 @@ def test_run_ingest_segments_dry_run_matches_rows(tmp_path: Path):
|
|
| 99 |
assert result["segments_found_total"] == 2
|
| 100 |
assert result["matched_rows"] == 1
|
| 101 |
assert result["unmatched_rows"] == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
from cli.hf_dataset_cli import (
|
| 6 |
_build_detection_key,
|
| 7 |
+
build_parser,
|
| 8 |
parse_segment_filename,
|
| 9 |
run_ingest_segments_dry_run,
|
| 10 |
)
|
|
|
|
| 100 |
assert result["segments_found_total"] == 2
|
| 101 |
assert result["matched_rows"] == 1
|
| 102 |
assert result["unmatched_rows"] == 1
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def test_build_parser_ingest_segments_defaults():
|
| 106 |
+
parser = build_parser()
|
| 107 |
+
args = parser.parse_args(
|
| 108 |
+
[
|
| 109 |
+
"ingest-segments",
|
| 110 |
+
"--project-slug",
|
| 111 |
+
"ppbio-aiuaba",
|
| 112 |
+
"--dataset-repo",
|
| 113 |
+
"owner/repo",
|
| 114 |
+
"--detections-csv",
|
| 115 |
+
"detections.csv",
|
| 116 |
+
"--segments-root",
|
| 117 |
+
"segments",
|
| 118 |
+
]
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
assert args.command == "ingest-segments"
|
| 122 |
+
assert args.batch_size == 200
|
| 123 |
+
assert args.shard_size == 10000
|
| 124 |
+
assert args.max_retries == 3
|
| 125 |
+
assert args.retry_backoff_seconds == 1.0
|
| 126 |
+
assert args.resume_state_file == ".ingest-segments-state.json"
|
| 127 |
+
assert args.dry_run is False
|