Spaces:
Sleeping
Sleeping
Commit ·
cd7d464
1
Parent(s): 722e080
fix: support audiofolder datasets without detections metadata
Browse files
src/ui/app_factory.py
CHANGED
|
@@ -2,6 +2,7 @@ import gradio as gr
|
|
| 2 |
import csv
|
| 3 |
import hashlib
|
| 4 |
import json
|
|
|
|
| 5 |
import tempfile
|
| 6 |
import wave
|
| 7 |
from datetime import date, datetime
|
|
@@ -264,9 +265,13 @@ def _load_dataset_detections_for_project(project: Project) -> tuple[list[Detecti
|
|
| 264 |
selected_file = sorted(metadata_candidates, key=lambda value: (len(value), value))[0]
|
| 265 |
|
| 266 |
if not selected_file:
|
|
|
|
|
|
|
|
|
|
| 267 |
return [], (
|
| 268 |
f"⚠️ Dataset {dataset_repo} has no detection metadata file for project {project.project_slug}. "
|
| 269 |
-
"Expected names like detections.jsonl / segments.csv (top-level, metadata/, or <project_slug>/)
|
|
|
|
| 270 |
)
|
| 271 |
|
| 272 |
try:
|
|
@@ -308,6 +313,9 @@ def _load_dataset_detections_for_project(project: Project) -> tuple[list[Detecti
|
|
| 308 |
return [], f"⚠️ Failed to parse detection metadata {selected_file} from {dataset_repo}: {exc}"
|
| 309 |
|
| 310 |
if not parsed:
|
|
|
|
|
|
|
|
|
|
| 311 |
return [], (
|
| 312 |
f"⚠️ Metadata file {selected_file} from {dataset_repo} has no valid detections for project {project.project_slug}."
|
| 313 |
)
|
|
@@ -315,6 +323,70 @@ def _load_dataset_detections_for_project(project: Project) -> tuple[list[Detecti
|
|
| 315 |
return parsed, ""
|
| 316 |
|
| 317 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
def _seed_service_for_projects(
|
| 319 |
project_slugs: list[str],
|
| 320 |
seed_file_path: str | None = None,
|
|
|
|
| 2 |
import csv
|
| 3 |
import hashlib
|
| 4 |
import json
|
| 5 |
+
import re
|
| 6 |
import tempfile
|
| 7 |
import wave
|
| 8 |
from datetime import date, datetime
|
|
|
|
| 265 |
selected_file = sorted(metadata_candidates, key=lambda value: (len(value), value))[0]
|
| 266 |
|
| 267 |
if not selected_file:
|
| 268 |
+
parsed_from_paths = _build_detections_from_audio_paths(project, repo_files)
|
| 269 |
+
if parsed_from_paths:
|
| 270 |
+
return parsed_from_paths, ""
|
| 271 |
return [], (
|
| 272 |
f"⚠️ Dataset {dataset_repo} has no detection metadata file for project {project.project_slug}. "
|
| 273 |
+
"Expected names like detections.jsonl / segments.csv (top-level, metadata/, or <project_slug>/), "
|
| 274 |
+
"or audio files under audio/segments/<species>/..."
|
| 275 |
)
|
| 276 |
|
| 277 |
try:
|
|
|
|
| 313 |
return [], f"⚠️ Failed to parse detection metadata {selected_file} from {dataset_repo}: {exc}"
|
| 314 |
|
| 315 |
if not parsed:
|
| 316 |
+
parsed_from_paths = _build_detections_from_audio_paths(project, repo_files)
|
| 317 |
+
if parsed_from_paths:
|
| 318 |
+
return parsed_from_paths, ""
|
| 319 |
return [], (
|
| 320 |
f"⚠️ Metadata file {selected_file} from {dataset_repo} has no valid detections for project {project.project_slug}."
|
| 321 |
)
|
|
|
|
| 323 |
return parsed, ""
|
| 324 |
|
| 325 |
|
| 326 |
+
def _parse_segment_filename_hint(filename: str) -> tuple[float, float, float]:
|
| 327 |
+
# Common uploader pattern: ..._12.0-15.0s_85%.wav
|
| 328 |
+
segment_match = re.search(r"_(\d+(?:\.\d+)?)\-(\d+(?:\.\d+)?)s_(\d+(?:\.\d+)?)%", filename)
|
| 329 |
+
if segment_match:
|
| 330 |
+
start_time = float(segment_match.group(1))
|
| 331 |
+
end_time = float(segment_match.group(2))
|
| 332 |
+
confidence = float(segment_match.group(3)) / 100.0
|
| 333 |
+
return start_time, end_time, max(0.0, min(1.0, confidence))
|
| 334 |
+
|
| 335 |
+
# Fallback pattern without confidence: ..._12.0-15.0s
|
| 336 |
+
basic_match = re.search(r"_(\d+(?:\.\d+)?)\-(\d+(?:\.\d+)?)s", filename)
|
| 337 |
+
if basic_match:
|
| 338 |
+
return float(basic_match.group(1)), float(basic_match.group(2)), 0.5
|
| 339 |
+
|
| 340 |
+
return 0.0, 1.0, 0.5
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def _build_detections_from_audio_paths(project: Project, repo_files: list[str]) -> list[Detection]:
|
| 344 |
+
detections: list[Detection] = []
|
| 345 |
+
seen_keys: set[str] = set()
|
| 346 |
+
|
| 347 |
+
for file_path in repo_files:
|
| 348 |
+
normalized = str(file_path).replace("\\", "/").strip()
|
| 349 |
+
lower = normalized.lower()
|
| 350 |
+
if not lower.startswith("audio/"):
|
| 351 |
+
continue
|
| 352 |
+
if not lower.endswith((".wav", ".mp3", ".flac", ".ogg", ".m4a")):
|
| 353 |
+
continue
|
| 354 |
+
|
| 355 |
+
relative_audio_id = normalized[len("audio/") :]
|
| 356 |
+
parts = relative_audio_id.split("/")
|
| 357 |
+
if len(parts) < 2:
|
| 358 |
+
continue
|
| 359 |
+
|
| 360 |
+
if parts[0].lower() == "segments" and len(parts) >= 3:
|
| 361 |
+
scientific_name = parts[1].replace("_", " ").strip() or "Unknown species"
|
| 362 |
+
else:
|
| 363 |
+
scientific_name = parts[-2].replace("_", " ").strip() or "Unknown species"
|
| 364 |
+
|
| 365 |
+
filename = parts[-1]
|
| 366 |
+
start_time, end_time, confidence = _parse_segment_filename_hint(filename)
|
| 367 |
+
stable = f"{project.project_slug}|{relative_audio_id}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
|
| 368 |
+
detection_key = hashlib.sha1(stable.encode("utf-8")).hexdigest()[:16]
|
| 369 |
+
if detection_key in seen_keys:
|
| 370 |
+
continue
|
| 371 |
+
|
| 372 |
+
try:
|
| 373 |
+
detections.append(
|
| 374 |
+
Detection(
|
| 375 |
+
detection_key=detection_key,
|
| 376 |
+
audio_id=relative_audio_id,
|
| 377 |
+
scientific_name=scientific_name,
|
| 378 |
+
confidence=confidence,
|
| 379 |
+
start_time=start_time,
|
| 380 |
+
end_time=end_time,
|
| 381 |
+
)
|
| 382 |
+
)
|
| 383 |
+
seen_keys.add(detection_key)
|
| 384 |
+
except Exception:
|
| 385 |
+
continue
|
| 386 |
+
|
| 387 |
+
return detections
|
| 388 |
+
|
| 389 |
+
|
| 390 |
def _seed_service_for_projects(
|
| 391 |
project_slugs: list[str],
|
| 392 |
seed_file_path: str | None = None,
|
tests/unit/test_app_factory_audio_helpers.py
CHANGED
|
@@ -939,3 +939,48 @@ def test_build_detection_repository_prefers_dataset_rows(monkeypatch: pytest.Mon
|
|
| 939 |
assert warning == ""
|
| 940 |
assert len(page.items) == 1
|
| 941 |
assert page.items[0].scientific_name == "Dataset Species"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 939 |
assert warning == ""
|
| 940 |
assert len(page.items) == 1
|
| 941 |
assert page.items[0].scientific_name == "Dataset Species"
|
| 942 |
+
|
| 943 |
+
|
| 944 |
+
def test_load_dataset_detections_for_project_falls_back_to_audiofolder_paths(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 945 |
+
from src.ui import app_factory as module
|
| 946 |
+
|
| 947 |
+
class FakeHfApi:
|
| 948 |
+
def __init__(self, token: str | None = None) -> None:
|
| 949 |
+
_ = token
|
| 950 |
+
|
| 951 |
+
def list_repo_files(self, repo_id: str, repo_type: str = "dataset") -> list[str]:
|
| 952 |
+
_ = repo_id
|
| 953 |
+
_ = repo_type
|
| 954 |
+
return [
|
| 955 |
+
"audio/segments/Accipiter_striatus/Catim_20250221_060600_0.0-3.0s_85%.wav",
|
| 956 |
+
"audio/segments/Aegolius_harrisii/Aiuab_20260123_182900_9.0-12.0s_68%.wav",
|
| 957 |
+
]
|
| 958 |
+
|
| 959 |
+
monkeypatch.setattr(module, "HfApi", FakeHfApi)
|
| 960 |
+
|
| 961 |
+
project = Project(
|
| 962 |
+
project_slug="teste7",
|
| 963 |
+
name="Teste 7",
|
| 964 |
+
dataset_repo_id="jrrribeiro/teste7",
|
| 965 |
+
active=True,
|
| 966 |
+
)
|
| 967 |
+
|
| 968 |
+
detections, warning = module._load_dataset_detections_for_project(project)
|
| 969 |
+
|
| 970 |
+
assert warning == ""
|
| 971 |
+
assert len(detections) == 2
|
| 972 |
+
assert {item.scientific_name for item in detections} == {"Accipiter striatus", "Aegolius harrisii"}
|
| 973 |
+
assert {item.audio_id for item in detections} == {
|
| 974 |
+
"segments/Accipiter_striatus/Catim_20250221_060600_0.0-3.0s_85%.wav",
|
| 975 |
+
"segments/Aegolius_harrisii/Aiuab_20260123_182900_9.0-12.0s_68%.wav",
|
| 976 |
+
}
|
| 977 |
+
|
| 978 |
+
|
| 979 |
+
def test_parse_segment_filename_hint_reads_time_and_confidence() -> None:
|
| 980 |
+
from src.ui import app_factory as module
|
| 981 |
+
|
| 982 |
+
start, end, conf = module._parse_segment_filename_hint("any_12.0-15.0s_85%.wav")
|
| 983 |
+
|
| 984 |
+
assert start == 12.0
|
| 985 |
+
assert end == 15.0
|
| 986 |
+
assert conf == 0.85
|