jrrribeiro commited on
Commit
cd7d464
·
1 Parent(s): 722e080

fix: support audiofolder datasets without detections metadata

Browse files
src/ui/app_factory.py CHANGED
@@ -2,6 +2,7 @@ import gradio as gr
2
  import csv
3
  import hashlib
4
  import json
 
5
  import tempfile
6
  import wave
7
  from datetime import date, datetime
@@ -264,9 +265,13 @@ def _load_dataset_detections_for_project(project: Project) -> tuple[list[Detecti
264
  selected_file = sorted(metadata_candidates, key=lambda value: (len(value), value))[0]
265
 
266
  if not selected_file:
 
 
 
267
  return [], (
268
  f"⚠️ Dataset {dataset_repo} has no detection metadata file for project {project.project_slug}. "
269
- "Expected names like detections.jsonl / segments.csv (top-level, metadata/, or <project_slug>/)."
 
270
  )
271
 
272
  try:
@@ -308,6 +313,9 @@ def _load_dataset_detections_for_project(project: Project) -> tuple[list[Detecti
308
  return [], f"⚠️ Failed to parse detection metadata {selected_file} from {dataset_repo}: {exc}"
309
 
310
  if not parsed:
 
 
 
311
  return [], (
312
  f"⚠️ Metadata file {selected_file} from {dataset_repo} has no valid detections for project {project.project_slug}."
313
  )
@@ -315,6 +323,70 @@ def _load_dataset_detections_for_project(project: Project) -> tuple[list[Detecti
315
  return parsed, ""
316
 
317
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
318
  def _seed_service_for_projects(
319
  project_slugs: list[str],
320
  seed_file_path: str | None = None,
 
2
  import csv
3
  import hashlib
4
  import json
5
+ import re
6
  import tempfile
7
  import wave
8
  from datetime import date, datetime
 
265
  selected_file = sorted(metadata_candidates, key=lambda value: (len(value), value))[0]
266
 
267
  if not selected_file:
268
+ parsed_from_paths = _build_detections_from_audio_paths(project, repo_files)
269
+ if parsed_from_paths:
270
+ return parsed_from_paths, ""
271
  return [], (
272
  f"⚠️ Dataset {dataset_repo} has no detection metadata file for project {project.project_slug}. "
273
+ "Expected names like detections.jsonl / segments.csv (top-level, metadata/, or <project_slug>/), "
274
+ "or audio files under audio/segments/<species>/..."
275
  )
276
 
277
  try:
 
313
  return [], f"⚠️ Failed to parse detection metadata {selected_file} from {dataset_repo}: {exc}"
314
 
315
  if not parsed:
316
+ parsed_from_paths = _build_detections_from_audio_paths(project, repo_files)
317
+ if parsed_from_paths:
318
+ return parsed_from_paths, ""
319
  return [], (
320
  f"⚠️ Metadata file {selected_file} from {dataset_repo} has no valid detections for project {project.project_slug}."
321
  )
 
323
  return parsed, ""
324
 
325
 
326
+ def _parse_segment_filename_hint(filename: str) -> tuple[float, float, float]:
327
+ # Common uploader pattern: ..._12.0-15.0s_85%.wav
328
+ segment_match = re.search(r"_(\d+(?:\.\d+)?)\-(\d+(?:\.\d+)?)s_(\d+(?:\.\d+)?)%", filename)
329
+ if segment_match:
330
+ start_time = float(segment_match.group(1))
331
+ end_time = float(segment_match.group(2))
332
+ confidence = float(segment_match.group(3)) / 100.0
333
+ return start_time, end_time, max(0.0, min(1.0, confidence))
334
+
335
+ # Fallback pattern without confidence: ..._12.0-15.0s
336
+ basic_match = re.search(r"_(\d+(?:\.\d+)?)\-(\d+(?:\.\d+)?)s", filename)
337
+ if basic_match:
338
+ return float(basic_match.group(1)), float(basic_match.group(2)), 0.5
339
+
340
+ return 0.0, 1.0, 0.5
341
+
342
+
343
+ def _build_detections_from_audio_paths(project: Project, repo_files: list[str]) -> list[Detection]:
344
+ detections: list[Detection] = []
345
+ seen_keys: set[str] = set()
346
+
347
+ for file_path in repo_files:
348
+ normalized = str(file_path).replace("\\", "/").strip()
349
+ lower = normalized.lower()
350
+ if not lower.startswith("audio/"):
351
+ continue
352
+ if not lower.endswith((".wav", ".mp3", ".flac", ".ogg", ".m4a")):
353
+ continue
354
+
355
+ relative_audio_id = normalized[len("audio/") :]
356
+ parts = relative_audio_id.split("/")
357
+ if len(parts) < 2:
358
+ continue
359
+
360
+ if parts[0].lower() == "segments" and len(parts) >= 3:
361
+ scientific_name = parts[1].replace("_", " ").strip() or "Unknown species"
362
+ else:
363
+ scientific_name = parts[-2].replace("_", " ").strip() or "Unknown species"
364
+
365
+ filename = parts[-1]
366
+ start_time, end_time, confidence = _parse_segment_filename_hint(filename)
367
+ stable = f"{project.project_slug}|{relative_audio_id}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
368
+ detection_key = hashlib.sha1(stable.encode("utf-8")).hexdigest()[:16]
369
+ if detection_key in seen_keys:
370
+ continue
371
+
372
+ try:
373
+ detections.append(
374
+ Detection(
375
+ detection_key=detection_key,
376
+ audio_id=relative_audio_id,
377
+ scientific_name=scientific_name,
378
+ confidence=confidence,
379
+ start_time=start_time,
380
+ end_time=end_time,
381
+ )
382
+ )
383
+ seen_keys.add(detection_key)
384
+ except Exception:
385
+ continue
386
+
387
+ return detections
388
+
389
+
390
  def _seed_service_for_projects(
391
  project_slugs: list[str],
392
  seed_file_path: str | None = None,
tests/unit/test_app_factory_audio_helpers.py CHANGED
@@ -939,3 +939,48 @@ def test_build_detection_repository_prefers_dataset_rows(monkeypatch: pytest.Mon
939
  assert warning == ""
940
  assert len(page.items) == 1
941
  assert page.items[0].scientific_name == "Dataset Species"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
939
  assert warning == ""
940
  assert len(page.items) == 1
941
  assert page.items[0].scientific_name == "Dataset Species"
942
+
943
+
944
+ def test_load_dataset_detections_for_project_falls_back_to_audiofolder_paths(monkeypatch: pytest.MonkeyPatch) -> None:
945
+ from src.ui import app_factory as module
946
+
947
+ class FakeHfApi:
948
+ def __init__(self, token: str | None = None) -> None:
949
+ _ = token
950
+
951
+ def list_repo_files(self, repo_id: str, repo_type: str = "dataset") -> list[str]:
952
+ _ = repo_id
953
+ _ = repo_type
954
+ return [
955
+ "audio/segments/Accipiter_striatus/Catim_20250221_060600_0.0-3.0s_85%.wav",
956
+ "audio/segments/Aegolius_harrisii/Aiuab_20260123_182900_9.0-12.0s_68%.wav",
957
+ ]
958
+
959
+ monkeypatch.setattr(module, "HfApi", FakeHfApi)
960
+
961
+ project = Project(
962
+ project_slug="teste7",
963
+ name="Teste 7",
964
+ dataset_repo_id="jrrribeiro/teste7",
965
+ active=True,
966
+ )
967
+
968
+ detections, warning = module._load_dataset_detections_for_project(project)
969
+
970
+ assert warning == ""
971
+ assert len(detections) == 2
972
+ assert {item.scientific_name for item in detections} == {"Accipiter striatus", "Aegolius harrisii"}
973
+ assert {item.audio_id for item in detections} == {
974
+ "segments/Accipiter_striatus/Catim_20250221_060600_0.0-3.0s_85%.wav",
975
+ "segments/Aegolius_harrisii/Aiuab_20260123_182900_9.0-12.0s_68%.wav",
976
+ }
977
+
978
+
979
+ def test_parse_segment_filename_hint_reads_time_and_confidence() -> None:
980
+ from src.ui import app_factory as module
981
+
982
+ start, end, conf = module._parse_segment_filename_hint("any_12.0-15.0s_85%.wav")
983
+
984
+ assert start == 12.0
985
+ assert end == 15.0
986
+ assert conf == 0.85