jrrribeiro commited on
Commit
722e080
·
1 Parent(s): e37709f

fix: load validation detections from selected project dataset

Browse files
src/ui/app_factory.py CHANGED
@@ -1,4 +1,5 @@
1
  import gradio as gr
 
2
  import hashlib
3
  import json
4
  import tempfile
@@ -8,6 +9,7 @@ from pathlib import Path
8
  from typing import Protocol
9
 
10
  import numpy as np
 
11
  from PIL import Image
12
 
13
  from src.config.runtime_config import RuntimeConfig
@@ -73,31 +75,296 @@ class _QueueServiceProtocol(Protocol):
73
 
74
 
75
  def _seed_service() -> DetectionQueueService:
76
- return _seed_service_for_projects(["demo-project"])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
 
78
 
79
  def _seed_service_for_projects(
80
  project_slugs: list[str],
81
  seed_file_path: str | None = None,
82
- ) -> DetectionQueueService:
 
 
83
  repo = InMemoryDetectionRepository()
84
  detected_by_project = _load_seed_detections(seed_file_path)
 
85
 
86
  for project_slug in project_slugs:
 
 
 
 
 
 
 
87
  seeded_items = detected_by_project.get(project_slug, [])
88
- items = seeded_items or _default_demo_detections(project_slug)
 
 
 
 
 
 
 
89
  items = sorted(items, key=lambda item: item.detection_key)
90
  repo.seed(project_slug, items)
91
 
92
- return DetectionQueueService(repo)
93
 
94
 
95
  def _build_detection_repository(
96
  project_slugs: list[str],
97
  seed_file_path: str | None,
 
 
98
  ) -> tuple[DetectionQueueService, str]:
99
  warning = _validate_seed_file(seed_file_path)
100
- return _seed_service_for_projects(project_slugs, seed_file_path=seed_file_path), warning
 
 
 
 
 
 
 
 
 
101
 
102
 
103
  def _validate_seed_file(seed_file_path: str | None) -> str:
@@ -1976,9 +2243,23 @@ def create_app() -> gr.Blocks:
1976
  user_access_file_path=str(user_access_file_path),
1977
  invites_file_path=str(invites_file_path),
1978
  )
 
 
 
 
 
 
 
 
 
 
 
 
1979
  queue_service, seed_warning = _build_detection_repository(
1980
  [project["project_slug"] for project in admin_manager.list_projects()],
1981
  seed_file_path=runtime_config.detection_seed_path,
 
 
1982
  )
1983
  service_ref: dict[str, DetectionQueueService] = {"queue": queue_service}
1984
  audio_service = AudioFetchService(EphemeralCacheManager(ttl_seconds=300, max_files=128))
@@ -2015,6 +2296,14 @@ def create_app() -> gr.Blocks:
2015
  def _project_slugs() -> list[str]:
2016
  return [p["project_slug"] for p in admin_manager.list_projects()]
2017
 
 
 
 
 
 
 
 
 
2018
  def _persist_admin_state() -> tuple[bool, str]:
2019
  try:
2020
  _persist_bootstrap_state(
@@ -2147,6 +2436,8 @@ def create_app() -> gr.Blocks:
2147
  refreshed_service, refreshed_warning = _build_detection_repository(
2148
  _project_slugs(),
2149
  seed_file_path=runtime_config.detection_seed_path,
 
 
2150
  )
2151
  service_ref["queue"] = refreshed_service
2152
  refreshed_session = auth_service.refresh_session_authorizations(session.session_id) or session
@@ -2226,6 +2517,16 @@ def create_app() -> gr.Blocks:
2226
  if not persisted:
2227
  message = f"{message} | ⚠️ Persistence failed: {persist_error}"
2228
 
 
 
 
 
 
 
 
 
 
 
2229
  return message, gr.update(value=""), _project_rows()
2230
 
2231
  token_update_btn.click(
 
1
  import gradio as gr
2
+ import csv
3
  import hashlib
4
  import json
5
  import tempfile
 
9
  from typing import Protocol
10
 
11
  import numpy as np
12
+ from huggingface_hub import HfApi, hf_hub_download
13
  from PIL import Image
14
 
15
  from src.config.runtime_config import RuntimeConfig
 
75
 
76
 
77
  def _seed_service() -> DetectionQueueService:
78
+ return _seed_service_for_projects(["demo-project"])[0]
79
+
80
+
81
+ def _candidate_metadata_files(project_slug: str) -> list[str]:
82
+ return [
83
+ f"{project_slug}/detections.jsonl",
84
+ f"{project_slug}/detections.json",
85
+ f"{project_slug}/detections.csv",
86
+ f"{project_slug}/segments.jsonl",
87
+ f"{project_slug}/segments.json",
88
+ f"{project_slug}/segments.csv",
89
+ "detections.jsonl",
90
+ "detections.json",
91
+ "detections.csv",
92
+ "segments.jsonl",
93
+ "segments.json",
94
+ "segments.csv",
95
+ "metadata/detections.jsonl",
96
+ "metadata/detections.json",
97
+ "metadata/detections.csv",
98
+ "metadata/segments.jsonl",
99
+ "metadata/segments.json",
100
+ "metadata/segments.csv",
101
+ "validation/detections.jsonl",
102
+ "validation/detections.json",
103
+ "validation/detections.csv",
104
+ ]
105
+
106
+
107
+ def _pick_row_value(raw: dict[str, object], keys: list[str]) -> str:
108
+ for key in keys:
109
+ value = raw.get(key)
110
+ if value is None:
111
+ continue
112
+ text = str(value).strip()
113
+ if text:
114
+ return text
115
+ return ""
116
+
117
+
118
+ def _to_float(value: object, default: float) -> float:
119
+ try:
120
+ return float(value)
121
+ except Exception:
122
+ return default
123
+
124
+
125
+ def _normalize_audio_id(audio_value: str) -> str:
126
+ normalized = audio_value.strip().replace("\\", "/")
127
+ if normalized.startswith("audio/"):
128
+ normalized = normalized[len("audio/") :]
129
+ return normalized
130
+
131
+
132
+ def _build_detection_from_row(raw: dict[str, object], row_index: int, project_slug: str) -> Detection | None:
133
+ row_project = _pick_row_value(raw, ["project_slug", "project", "project_id"])
134
+ if row_project and row_project != project_slug:
135
+ return None
136
+
137
+ audio_id = _normalize_audio_id(
138
+ _pick_row_value(
139
+ raw,
140
+ [
141
+ "audio_id",
142
+ "audio_file",
143
+ "audio_path",
144
+ "segment_path",
145
+ "file",
146
+ "filepath",
147
+ "path",
148
+ "filename",
149
+ ],
150
+ )
151
+ )
152
+ if not audio_id:
153
+ return None
154
+
155
+ scientific_name = _pick_row_value(
156
+ raw,
157
+ [
158
+ "scientific_name",
159
+ "species",
160
+ "species_name",
161
+ "predicted_species",
162
+ "label",
163
+ "taxon",
164
+ ],
165
+ )
166
+ if not scientific_name:
167
+ scientific_name = "Unknown species"
168
+
169
+ confidence = _to_float(
170
+ raw.get("confidence", raw.get("score", raw.get("probability", raw.get("prediction_confidence", 1.0)))),
171
+ 1.0,
172
+ )
173
+ confidence = max(0.0, min(1.0, confidence))
174
+
175
+ start_time = _to_float(
176
+ raw.get("start_time", raw.get("start", raw.get("begin", raw.get("offset", raw.get("segment_start", 0.0))))),
177
+ 0.0,
178
+ )
179
+ end_time = _to_float(
180
+ raw.get("end_time", raw.get("end", raw.get("stop", raw.get("segment_end", 0.0)))),
181
+ 0.0,
182
+ )
183
+ if end_time <= 0.0:
184
+ duration = _to_float(raw.get("duration", 0.0), 0.0)
185
+ if duration > 0.0:
186
+ end_time = start_time + duration
187
+ if end_time <= start_time:
188
+ end_time = start_time + 1.0
189
+
190
+ detection_key = _pick_row_value(raw, ["detection_key", "segment_id", "id", "uid", "key"])
191
+ if not detection_key:
192
+ stable = f"{project_slug}|{audio_id}|{scientific_name}|{start_time:.3f}|{end_time:.3f}|{row_index}"
193
+ detection_key = hashlib.sha1(stable.encode("utf-8")).hexdigest()[:16]
194
+
195
+ try:
196
+ return Detection(
197
+ detection_key=detection_key,
198
+ audio_id=audio_id,
199
+ scientific_name=scientific_name,
200
+ confidence=confidence,
201
+ start_time=start_time,
202
+ end_time=end_time,
203
+ )
204
+ except Exception:
205
+ return None
206
+
207
+
208
+ def _parse_detection_metadata_payload(payload: object, project_slug: str) -> list[Detection]:
209
+ rows: list[dict[str, object]] = []
210
+ if isinstance(payload, list):
211
+ rows = [item for item in payload if isinstance(item, dict)]
212
+ elif isinstance(payload, dict):
213
+ project_rows = payload.get(project_slug)
214
+ if isinstance(project_rows, list):
215
+ rows = [item for item in project_rows if isinstance(item, dict)]
216
+ else:
217
+ for key in ["detections", "segments", "items", "rows"]:
218
+ candidate = payload.get(key)
219
+ if isinstance(candidate, list):
220
+ rows = [item for item in candidate if isinstance(item, dict)]
221
+ break
222
+
223
+ parsed: list[Detection] = []
224
+ seen_keys: set[str] = set()
225
+ for index, row in enumerate(rows):
226
+ detection = _build_detection_from_row(row, index, project_slug)
227
+ if detection is None:
228
+ continue
229
+ if detection.detection_key in seen_keys:
230
+ continue
231
+ seen_keys.add(detection.detection_key)
232
+ parsed.append(detection)
233
+ return parsed
234
+
235
+
236
+ def _load_dataset_detections_for_project(project: Project) -> tuple[list[Detection], str]:
237
+ dataset_repo = project.dataset_repo_id.strip()
238
+ if not dataset_repo:
239
+ return [], ""
240
+
241
+ token = (project.dataset_token or "").strip() or None
242
+ try:
243
+ api = HfApi(token=token)
244
+ repo_files = api.list_repo_files(repo_id=dataset_repo, repo_type="dataset")
245
+ except Exception as exc:
246
+ return [], f"⚠️ Could not list files for dataset {dataset_repo}: {exc}"
247
+
248
+ if not repo_files:
249
+ return [], f"⚠️ Dataset {dataset_repo} has no files."
250
+
251
+ preferred = _candidate_metadata_files(project.project_slug)
252
+ selected_file = next((name for name in preferred if name in repo_files), "")
253
+ if not selected_file:
254
+ metadata_candidates = []
255
+ for name in repo_files:
256
+ lowered = name.lower()
257
+ if lowered.startswith("audio/"):
258
+ continue
259
+ if not lowered.endswith((".json", ".jsonl", ".csv")):
260
+ continue
261
+ if "detection" in lowered or "segment" in lowered:
262
+ metadata_candidates.append(name)
263
+ if metadata_candidates:
264
+ selected_file = sorted(metadata_candidates, key=lambda value: (len(value), value))[0]
265
+
266
+ if not selected_file:
267
+ return [], (
268
+ f"⚠️ Dataset {dataset_repo} has no detection metadata file for project {project.project_slug}. "
269
+ "Expected names like detections.jsonl / segments.csv (top-level, metadata/, or <project_slug>/)."
270
+ )
271
+
272
+ try:
273
+ if token:
274
+ downloaded_path = hf_hub_download(
275
+ repo_id=dataset_repo,
276
+ repo_type="dataset",
277
+ filename=selected_file,
278
+ token=token,
279
+ )
280
+ else:
281
+ downloaded_path = hf_hub_download(
282
+ repo_id=dataset_repo,
283
+ repo_type="dataset",
284
+ filename=selected_file,
285
+ )
286
+ except Exception as exc:
287
+ return [], f"⚠️ Failed to download {selected_file} from {dataset_repo}: {exc}"
288
+
289
+ metadata_path = Path(downloaded_path)
290
+ try:
291
+ if metadata_path.suffix.lower() == ".jsonl":
292
+ rows = []
293
+ for line in metadata_path.read_text(encoding="utf-8").splitlines():
294
+ text = line.strip()
295
+ if not text:
296
+ continue
297
+ value = json.loads(text)
298
+ if isinstance(value, dict):
299
+ rows.append(value)
300
+ parsed = _parse_detection_metadata_payload(rows, project.project_slug)
301
+ elif metadata_path.suffix.lower() == ".csv":
302
+ with metadata_path.open("r", encoding="utf-8", newline="") as file_handle:
303
+ parsed = _parse_detection_metadata_payload(list(csv.DictReader(file_handle)), project.project_slug)
304
+ else:
305
+ payload = json.loads(metadata_path.read_text(encoding="utf-8"))
306
+ parsed = _parse_detection_metadata_payload(payload, project.project_slug)
307
+ except Exception as exc:
308
+ return [], f"⚠️ Failed to parse detection metadata {selected_file} from {dataset_repo}: {exc}"
309
+
310
+ if not parsed:
311
+ return [], (
312
+ f"⚠️ Metadata file {selected_file} from {dataset_repo} has no valid detections for project {project.project_slug}."
313
+ )
314
+
315
+ return parsed, ""
316
 
317
 
318
  def _seed_service_for_projects(
319
  project_slugs: list[str],
320
  seed_file_path: str | None = None,
321
+ project_map: dict[str, Project] | None = None,
322
+ allow_demo_defaults: bool = True,
323
+ ) -> tuple[DetectionQueueService, list[str]]:
324
  repo = InMemoryDetectionRepository()
325
  detected_by_project = _load_seed_detections(seed_file_path)
326
+ warnings: list[str] = []
327
 
328
  for project_slug in project_slugs:
329
+ project = (project_map or {}).get(project_slug)
330
+ dataset_items: list[Detection] = []
331
+ if project is not None and project.active:
332
+ dataset_items, dataset_warning = _load_dataset_detections_for_project(project)
333
+ if dataset_warning:
334
+ warnings.append(dataset_warning)
335
+
336
  seeded_items = detected_by_project.get(project_slug, [])
337
+ if dataset_items:
338
+ items = dataset_items
339
+ elif seeded_items:
340
+ items = seeded_items
341
+ elif allow_demo_defaults:
342
+ items = _default_demo_detections(project_slug)
343
+ else:
344
+ items = []
345
  items = sorted(items, key=lambda item: item.detection_key)
346
  repo.seed(project_slug, items)
347
 
348
+ return DetectionQueueService(repo), warnings
349
 
350
 
351
  def _build_detection_repository(
352
  project_slugs: list[str],
353
  seed_file_path: str | None,
354
+ project_map: dict[str, Project] | None = None,
355
+ allow_demo_defaults: bool = True,
356
  ) -> tuple[DetectionQueueService, str]:
357
  warning = _validate_seed_file(seed_file_path)
358
+ service, dataset_warnings = _seed_service_for_projects(
359
+ project_slugs,
360
+ seed_file_path=seed_file_path,
361
+ project_map=project_map,
362
+ allow_demo_defaults=allow_demo_defaults,
363
+ )
364
+
365
+ warnings = [item for item in [warning, *dataset_warnings] if item.strip()]
366
+ joined_warning = "\n\n".join(dict.fromkeys(warnings))
367
+ return service, joined_warning
368
 
369
 
370
  def _validate_seed_file(seed_file_path: str | None) -> str:
 
2243
  user_access_file_path=str(user_access_file_path),
2244
  invites_file_path=str(invites_file_path),
2245
  )
2246
+
2247
+ def _current_project_map() -> dict[str, Project]:
2248
+ project_map: dict[str, Project] = {}
2249
+ for row in admin_manager.list_projects():
2250
+ slug = str(row.get("project_slug", "")).strip()
2251
+ if not slug:
2252
+ continue
2253
+ project = admin_manager.get_project(slug)
2254
+ if project is not None:
2255
+ project_map[slug] = project
2256
+ return project_map
2257
+
2258
  queue_service, seed_warning = _build_detection_repository(
2259
  [project["project_slug"] for project in admin_manager.list_projects()],
2260
  seed_file_path=runtime_config.detection_seed_path,
2261
+ project_map=_current_project_map(),
2262
+ allow_demo_defaults=runtime_config.enable_demo_bootstrap,
2263
  )
2264
  service_ref: dict[str, DetectionQueueService] = {"queue": queue_service}
2265
  audio_service = AudioFetchService(EphemeralCacheManager(ttl_seconds=300, max_files=128))
 
2296
  def _project_slugs() -> list[str]:
2297
  return [p["project_slug"] for p in admin_manager.list_projects()]
2298
 
2299
+ def _project_map() -> dict[str, Project]:
2300
+ result: dict[str, Project] = {}
2301
+ for slug in _project_slugs():
2302
+ project = admin_manager.get_project(slug)
2303
+ if project is not None:
2304
+ result[slug] = project
2305
+ return result
2306
+
2307
  def _persist_admin_state() -> tuple[bool, str]:
2308
  try:
2309
  _persist_bootstrap_state(
 
2436
  refreshed_service, refreshed_warning = _build_detection_repository(
2437
  _project_slugs(),
2438
  seed_file_path=runtime_config.detection_seed_path,
2439
+ project_map=_project_map(),
2440
+ allow_demo_defaults=runtime_config.enable_demo_bootstrap,
2441
  )
2442
  service_ref["queue"] = refreshed_service
2443
  refreshed_session = auth_service.refresh_session_authorizations(session.session_id) or session
 
2517
  if not persisted:
2518
  message = f"{message} | ⚠️ Persistence failed: {persist_error}"
2519
 
2520
+ refreshed_service, refreshed_warning = _build_detection_repository(
2521
+ _project_slugs(),
2522
+ seed_file_path=runtime_config.detection_seed_path,
2523
+ project_map=_project_map(),
2524
+ allow_demo_defaults=runtime_config.enable_demo_bootstrap,
2525
+ )
2526
+ service_ref["queue"] = refreshed_service
2527
+ if refreshed_warning:
2528
+ message = f"{message} | {refreshed_warning}"
2529
+
2530
  return message, gr.update(value=""), _project_rows()
2531
 
2532
  token_update_btn.click(
tests/unit/test_app_factory_audio_helpers.py CHANGED
@@ -29,6 +29,7 @@ from src.ui.app_factory import (
29
  )
30
  from src.auth.auth_service import AuthService
31
  from src.config.runtime_config import RuntimeConfig
 
32
  from src.ui.admin_panel import AdminPanelManager
33
 
34
 
@@ -830,3 +831,111 @@ def test_bootstrap_auth_and_projects_warns_when_not_configured(tmp_path: Path) -
830
  warning = _bootstrap_auth_and_projects(auth_service, admin_manager, runtime_config)
831
 
832
  assert "Production bootstrap incomplete" in warning
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  )
30
  from src.auth.auth_service import AuthService
31
  from src.config.runtime_config import RuntimeConfig
32
+ from src.domain.models import Detection, Project
33
  from src.ui.admin_panel import AdminPanelManager
34
 
35
 
 
831
  warning = _bootstrap_auth_and_projects(auth_service, admin_manager, runtime_config)
832
 
833
  assert "Production bootstrap incomplete" in warning
834
+
835
+
836
+ def test_load_dataset_detections_for_project_reads_jsonl(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
837
+ from src.ui import app_factory as module
838
+
839
+ class FakeHfApi:
840
+ def __init__(self, token: str | None = None) -> None:
841
+ _ = token
842
+
843
+ def list_repo_files(self, repo_id: str, repo_type: str = "dataset") -> list[str]:
844
+ _ = repo_id
845
+ _ = repo_type
846
+ return ["detections.jsonl"]
847
+
848
+ metadata_file = tmp_path / "detections.jsonl"
849
+ metadata_file.write_text(
850
+ "\n".join(
851
+ [
852
+ json.dumps(
853
+ {
854
+ "project_slug": "project-a",
855
+ "audio_id": "audio_0001",
856
+ "scientific_name": "Species A",
857
+ "confidence": 0.92,
858
+ "start_time": 1.0,
859
+ "end_time": 2.0,
860
+ }
861
+ ),
862
+ json.dumps(
863
+ {
864
+ "project_slug": "project-a",
865
+ "audio_id": "audio_0002",
866
+ "scientific_name": "Species B",
867
+ "confidence": 0.81,
868
+ "start_time": 2.0,
869
+ "end_time": 3.0,
870
+ }
871
+ ),
872
+ json.dumps(
873
+ {
874
+ "project_slug": "project-other",
875
+ "audio_id": "audio_9999",
876
+ "scientific_name": "Other",
877
+ "confidence": 0.5,
878
+ "start_time": 0.0,
879
+ "end_time": 1.0,
880
+ }
881
+ ),
882
+ ]
883
+ ),
884
+ encoding="utf-8",
885
+ )
886
+
887
+ monkeypatch.setattr(module, "HfApi", FakeHfApi)
888
+ monkeypatch.setattr(module, "hf_hub_download", lambda **kwargs: str(metadata_file))
889
+
890
+ project = Project(
891
+ project_slug="project-a",
892
+ name="Project A",
893
+ dataset_repo_id="org/project-a",
894
+ active=True,
895
+ )
896
+ detections, warning = module._load_dataset_detections_for_project(project)
897
+
898
+ assert warning == ""
899
+ assert len(detections) == 2
900
+ assert {item.scientific_name for item in detections} == {"Species A", "Species B"}
901
+
902
+
903
+ def test_build_detection_repository_prefers_dataset_rows(monkeypatch: pytest.MonkeyPatch) -> None:
904
+ from src.ui import app_factory as module
905
+
906
+ project = Project(
907
+ project_slug="project-a",
908
+ name="Project A",
909
+ dataset_repo_id="org/project-a",
910
+ active=True,
911
+ )
912
+
913
+ monkeypatch.setattr(
914
+ module,
915
+ "_load_dataset_detections_for_project",
916
+ lambda project_obj: (
917
+ [
918
+ Detection(
919
+ detection_key="0000000000002222",
920
+ audio_id="audio_dataset_1",
921
+ scientific_name="Dataset Species",
922
+ confidence=0.95,
923
+ start_time=0.0,
924
+ end_time=1.0,
925
+ )
926
+ ],
927
+ "",
928
+ ),
929
+ )
930
+
931
+ queue, warning = _build_detection_repository(
932
+ ["project-a"],
933
+ seed_file_path=None,
934
+ project_map={"project-a": project},
935
+ allow_demo_defaults=False,
936
+ )
937
+ page = queue.get_page(project_slug="project-a", page=1, page_size=10)
938
+
939
+ assert warning == ""
940
+ assert len(page.items) == 1
941
+ assert page.items[0].scientific_name == "Dataset Species"