jrrribeiro commited on
Commit
090ffd1
·
1 Parent(s): afa898b

feat(cli): add ingest-segments dry-run matching for BirdNET CSV + segments tree

Browse files

- Add new CLI command: ingest-segments
- Implement phase-1 dry-run pipeline (no upload yet)
- Parse BirdNET segment filenames and match CSV rows deterministically
- Generate stable detection_key hash and normalized ingest-ready records
- Return quality report metrics (matched/unmatched/ambiguous)
- Optional JSON report output via --report-file
- Add unit tests for parser, key generation, and dry-run matching

cli/hf_dataset_cli.py CHANGED
@@ -2,6 +2,7 @@ import argparse
2
  import hashlib
3
  import json
4
  import math
 
5
  import tempfile
6
  import time
7
  from dataclasses import asdict, dataclass
@@ -38,6 +39,16 @@ REQUIRED_DETECTIONS_COLUMNS = (
38
  "end_time",
39
  )
40
  SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
 
 
 
 
 
 
 
 
 
 
41
 
42
 
43
  @dataclass(slots=True)
@@ -48,6 +59,17 @@ class ShardMetadata:
48
  size_bytes: int
49
 
50
 
 
 
 
 
 
 
 
 
 
 
 
51
  def _utcnow_iso() -> str:
52
  return datetime.now(UTC).replace(microsecond=0).isoformat()
53
 
@@ -142,6 +164,188 @@ def load_detections_table(detections_file: str) -> pd.DataFrame:
142
  return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
143
 
144
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
145
  def _file_sha256(file_path: Path) -> str:
146
  hasher = hashlib.sha256()
147
  with file_path.open("rb") as handle:
@@ -497,6 +701,14 @@ def build_parser() -> argparse.ArgumentParser:
497
  verify_project.add_argument("--project-slug", required=True)
498
  verify_project.add_argument("--dataset-repo", required=True)
499
 
 
 
 
 
 
 
 
 
500
  return parser
501
 
502
 
@@ -549,6 +761,22 @@ def main() -> int:
549
  print(json.dumps(result, indent=2))
550
  return 0 if result["ok"] else 1
551
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
552
  parser.error("Unknown command")
553
  return 2
554
 
 
2
  import hashlib
3
  import json
4
  import math
5
+ import re
6
  import tempfile
7
  import time
8
  from dataclasses import asdict, dataclass
 
39
  "end_time",
40
  )
41
  SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
42
+ REQUIRED_BIRDNET_COLUMNS = (
43
+ "source_file",
44
+ "scientific_name",
45
+ "confidence",
46
+ "start_time",
47
+ "end_time",
48
+ )
49
+ SEGMENT_FILENAME_PATTERN = re.compile(
50
+ r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
51
+ )
52
 
53
 
54
  @dataclass(slots=True)
 
59
  size_bytes: int
60
 
61
 
62
+ @dataclass(slots=True)
63
+ class SegmentFileRecord:
64
+ scientific_name: str
65
+ source_stem: str
66
+ start_time: float
67
+ end_time: float
68
+ confidence_pct: int
69
+ absolute_path: Path
70
+ relative_path: str
71
+
72
+
73
  def _utcnow_iso() -> str:
74
  return datetime.now(UTC).replace(microsecond=0).isoformat()
75
 
 
164
  return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
165
 
166
 
167
+ def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
168
+ input_path = Path(detections_csv)
169
+ if not input_path.exists():
170
+ raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
171
+
172
+ frame = pd.read_csv(input_path)
173
+ missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
174
+ if missing_columns:
175
+ raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
176
+
177
+ if frame.empty:
178
+ return frame
179
+
180
+ normalized = frame.copy()
181
+ normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
182
+ normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
183
+ normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
184
+ normalized["species_key"] = normalized["scientific_name"].str.casefold()
185
+ normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
186
+ normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
187
+ normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
188
+
189
+ normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
190
+ return normalized
191
+
192
+
193
+ def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
194
+ payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
195
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
196
+
197
+
198
+ def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
199
+ stem = Path(file_name).stem
200
+ match = SEGMENT_FILENAME_PATTERN.match(stem)
201
+ if not match:
202
+ return None
203
+
204
+ source_stem = match.group("source_stem")
205
+ start_time = float(match.group("start"))
206
+ end_time = float(match.group("end"))
207
+ confidence_pct = int(match.group("confidence_pct"))
208
+ return source_stem, start_time, end_time, confidence_pct
209
+
210
+
211
+ def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
212
+ root = Path(segments_root)
213
+ if not root.exists() or not root.is_dir():
214
+ raise FileNotFoundError(f"Segments root not found: {segments_root}")
215
+
216
+ records: list[SegmentFileRecord] = []
217
+ for absolute_path in root.rglob("*"):
218
+ if not absolute_path.is_file():
219
+ continue
220
+ if absolute_path.suffix.lower() != ".wav":
221
+ continue
222
+
223
+ rel = absolute_path.relative_to(root)
224
+ scientific_name = rel.parts[0] if rel.parts else ""
225
+ parsed = parse_segment_filename(absolute_path.name)
226
+ if not parsed:
227
+ continue
228
+ source_stem, start_time, end_time, confidence_pct = parsed
229
+ records.append(
230
+ SegmentFileRecord(
231
+ scientific_name=scientific_name,
232
+ source_stem=source_stem,
233
+ start_time=start_time,
234
+ end_time=end_time,
235
+ confidence_pct=confidence_pct,
236
+ absolute_path=absolute_path,
237
+ relative_path=rel.as_posix(),
238
+ )
239
+ )
240
+
241
+ return records
242
+
243
+
244
+ def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
245
+ frame = load_birdnet_detections_csv(detections_csv=detections_csv)
246
+ segment_records = discover_segment_records(segments_root=segments_root)
247
+
248
+ by_species: dict[str, list[SegmentFileRecord]] = {}
249
+ for record in segment_records:
250
+ by_species.setdefault(record.scientific_name.casefold(), []).append(record)
251
+
252
+ matched_rows: list[dict[str, Any]] = []
253
+ unmatched_rows: list[dict[str, Any]] = []
254
+ ambiguous_rows: list[dict[str, Any]] = []
255
+
256
+ for row in frame.to_dict(orient="records"):
257
+ species_key = str(row.get("species_key", "")).casefold()
258
+ source_stem = str(row.get("source_stem", "")).strip()
259
+ start_time = float(row["start_time"])
260
+ end_time = float(row["end_time"])
261
+ confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
262
+
263
+ species_candidates = by_species.get(species_key, [])
264
+ exact_matches = [
265
+ record
266
+ for record in species_candidates
267
+ if record.source_stem == source_stem
268
+ and abs(record.start_time - start_time) < 1e-6
269
+ and abs(record.end_time - end_time) < 1e-6
270
+ ]
271
+
272
+ if not exact_matches:
273
+ unmatched_rows.append(
274
+ {
275
+ "source_file": row.get("source_file", ""),
276
+ "scientific_name": row.get("scientific_name", ""),
277
+ "start_time": start_time,
278
+ "end_time": end_time,
279
+ }
280
+ )
281
+ continue
282
+
283
+ selected = exact_matches[0]
284
+ if len(exact_matches) > 1:
285
+ expected_pct = int(round(confidence * 100))
286
+ exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
287
+ selected = exact_matches[0]
288
+ ambiguous_rows.append(
289
+ {
290
+ "source_file": row.get("source_file", ""),
291
+ "scientific_name": row.get("scientific_name", ""),
292
+ "start_time": start_time,
293
+ "end_time": end_time,
294
+ "candidates": len(exact_matches),
295
+ }
296
+ )
297
+
298
+ matched_rows.append(
299
+ {
300
+ "project_slug": project_slug,
301
+ "detection_key": _build_detection_key(
302
+ project_slug=project_slug,
303
+ source_file=str(row.get("source_file", "")),
304
+ scientific_name=str(row.get("scientific_name", "")),
305
+ start_time=start_time,
306
+ end_time=end_time,
307
+ ),
308
+ "audio_id": source_stem,
309
+ "segment_filename": selected.absolute_path.name,
310
+ "segment_relpath": selected.relative_path,
311
+ "scientific_name": row.get("scientific_name", ""),
312
+ "common_name": row.get("common_name", ""),
313
+ "confidence": confidence,
314
+ "start_time": start_time,
315
+ "end_time": end_time,
316
+ "exact_start": row.get("exact_start", start_time),
317
+ "exact_end": row.get("exact_end", end_time),
318
+ "locality": row.get("locality", ""),
319
+ "point": row.get("point", ""),
320
+ "date_folder": row.get("date_folder", ""),
321
+ "min_freq": row.get("min_freq", None),
322
+ "max_freq": row.get("max_freq", None),
323
+ "box_source": row.get("box_source", ""),
324
+ "label": row.get("label", ""),
325
+ "source_file": row.get("source_file", ""),
326
+ "schema_version": SCHEMA_VERSION,
327
+ "ingested_at": _utcnow_iso(),
328
+ }
329
+ )
330
+
331
+ unique_audio = {row["audio_id"] for row in matched_rows}
332
+ result = {
333
+ "mode": "dry-run",
334
+ "project_slug": project_slug,
335
+ "detections_csv": detections_csv,
336
+ "segments_root": segments_root,
337
+ "csv_rows_total": int(len(frame)),
338
+ "segments_found_total": len(segment_records),
339
+ "matched_rows": len(matched_rows),
340
+ "unmatched_rows": len(unmatched_rows),
341
+ "ambiguous_rows": len(ambiguous_rows),
342
+ "unique_audio_ids": len(unique_audio),
343
+ "sample_unmatched": unmatched_rows[:20],
344
+ "sample_ambiguous": ambiguous_rows[:20],
345
+ }
346
+ return result
347
+
348
+
349
  def _file_sha256(file_path: Path) -> str:
350
  hasher = hashlib.sha256()
351
  with file_path.open("rb") as handle:
 
701
  verify_project.add_argument("--project-slug", required=True)
702
  verify_project.add_argument("--dataset-repo", required=True)
703
 
704
+ ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
705
+ ingest_segments.add_argument("--project-slug", required=True)
706
+ ingest_segments.add_argument("--dataset-repo", required=True)
707
+ ingest_segments.add_argument("--detections-csv", required=True)
708
+ ingest_segments.add_argument("--segments-root", required=True)
709
+ ingest_segments.add_argument("--dry-run", action="store_true")
710
+ ingest_segments.add_argument("--report-file", default="")
711
+
712
  return parser
713
 
714
 
 
761
  print(json.dumps(result, indent=2))
762
  return 0 if result["ok"] else 1
763
 
764
+ if args.command == "ingest-segments":
765
+ if not args.dry_run:
766
+ parser.error("ingest-segments currently supports only --dry-run in this phase")
767
+
768
+ result = run_ingest_segments_dry_run(
769
+ project_slug=args.project_slug,
770
+ detections_csv=args.detections_csv,
771
+ segments_root=args.segments_root,
772
+ )
773
+
774
+ if args.report_file:
775
+ Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
776
+
777
+ print(json.dumps(result, indent=2))
778
+ return 0
779
+
780
  parser.error("Unknown command")
781
  return 2
782
 
tests/unit/test_hf_ingest_segments_cli.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+
3
+ import pandas as pd
4
+
5
+ from cli.hf_dataset_cli import (
6
+ _build_detection_key,
7
+ parse_segment_filename,
8
+ run_ingest_segments_dry_run,
9
+ )
10
+
11
+
12
+ def test_parse_segment_filename_extracts_components():
13
+ parsed = parse_segment_filename("Aiuab_LN-0500_U16_20260121_153400_42.0-45.0s_98%.wav")
14
+ assert parsed is not None
15
+ source_stem, start_time, end_time, confidence_pct = parsed
16
+ assert source_stem == "Aiuab_LN-0500_U16_20260121_153400"
17
+ assert start_time == 42.0
18
+ assert end_time == 45.0
19
+ assert confidence_pct == 98
20
+
21
+
22
+ def test_detection_key_is_stable_and_long_enough():
23
+ key1 = _build_detection_key(
24
+ project_slug="ppbio-aiuaba",
25
+ source_file="Aiuab_LN-0500_U16_20260121_153400.wav",
26
+ scientific_name="Cyanocorax cyanopogon",
27
+ start_time=42.0,
28
+ end_time=45.0,
29
+ )
30
+ key2 = _build_detection_key(
31
+ project_slug="ppbio-aiuaba",
32
+ source_file="Aiuab_LN-0500_U16_20260121_153400.wav",
33
+ scientific_name="Cyanocorax cyanopogon",
34
+ start_time=42.0,
35
+ end_time=45.0,
36
+ )
37
+ assert key1 == key2
38
+ assert len(key1) >= 16
39
+
40
+
41
+ def test_run_ingest_segments_dry_run_matches_rows(tmp_path: Path):
42
+ segments_root = tmp_path / "segments"
43
+ species_dir = segments_root / "Cyanocorax cyanopogon"
44
+ species_dir.mkdir(parents=True, exist_ok=True)
45
+
46
+ (species_dir / "Aiuab_LN-0500_U16_20260121_153400_42.0-45.0s_98%.wav").write_bytes(b"dummy")
47
+ (species_dir / "Aiuab_LN-0500_U16_20260121_153400_45.0-48.0s_88%.wav").write_bytes(b"dummy")
48
+
49
+ frame = pd.DataFrame(
50
+ [
51
+ {
52
+ "locality": "PPBIO Aiuaba",
53
+ "point": "Aiuab_LN-0500_U16",
54
+ "date_folder": "20260121",
55
+ "source_file": "Aiuab_LN-0500_U16_20260121_153400.wav",
56
+ "scientific_name": "Cyanocorax cyanopogon",
57
+ "common_name": "White-naped Jay",
58
+ "confidence": 0.98,
59
+ "start_time": 42.0,
60
+ "end_time": 45.0,
61
+ "exact_start": 42.0,
62
+ "exact_end": 45.0,
63
+ "min_freq": 0,
64
+ "max_freq": 15000,
65
+ "box_source": "BirdNET_Original",
66
+ "label": "Cyanocorax cyanopogon_White-naped Jay",
67
+ },
68
+ {
69
+ "locality": "PPBIO Aiuaba",
70
+ "point": "Aiuab_LN-0500_U16",
71
+ "date_folder": "20260121",
72
+ "source_file": "Aiuab_LN-0500_U16_20260121_153400.wav",
73
+ "scientific_name": "Cyanocorax cyanopogon",
74
+ "common_name": "White-naped Jay",
75
+ "confidence": 0.50,
76
+ "start_time": 0.0,
77
+ "end_time": 3.0,
78
+ "exact_start": 0.0,
79
+ "exact_end": 3.0,
80
+ "min_freq": 0,
81
+ "max_freq": 15000,
82
+ "box_source": "BirdNET_Original",
83
+ "label": "Cyanocorax cyanopogon_White-naped Jay",
84
+ },
85
+ ]
86
+ )
87
+
88
+ csv_path = tmp_path / "detections.csv"
89
+ frame.to_csv(csv_path, index=False)
90
+
91
+ result = run_ingest_segments_dry_run(
92
+ project_slug="ppbio-aiuaba",
93
+ detections_csv=str(csv_path),
94
+ segments_root=str(segments_root),
95
+ )
96
+
97
+ assert result["mode"] == "dry-run"
98
+ assert result["csv_rows_total"] == 2
99
+ assert result["segments_found_total"] == 2
100
+ assert result["matched_rows"] == 1
101
+ assert result["unmatched_rows"] == 1