Spaces:
Sleeping
Sleeping
Sync app from GitHub Actions
Browse files- requirements.txt +1 -0
- src/domain/models.py +1 -0
- src/services/detection_queue_service.py +24 -0
- src/ui/app_factory.py +215 -16
- tests/unit/test_app_factory_audio_helpers.py +122 -0
- tests/unit/test_detection_queue_service.py +10 -0
requirements.txt
CHANGED
|
@@ -2,6 +2,7 @@ gradio==5.23.1
|
|
| 2 |
huggingface_hub>=0.32.0
|
| 3 |
hf_xet>=1.0.0
|
| 4 |
numpy>=2.0.0
|
|
|
|
| 5 |
pandas>=2.2.0
|
| 6 |
Pillow>=10.4.0
|
| 7 |
pyarrow>=16.0.0
|
|
|
|
| 2 |
huggingface_hub>=0.32.0
|
| 3 |
hf_xet>=1.0.0
|
| 4 |
numpy>=2.0.0
|
| 5 |
+
openpyxl>=3.1.0
|
| 6 |
pandas>=2.2.0
|
| 7 |
Pillow>=10.4.0
|
| 8 |
pyarrow>=16.0.0
|
src/domain/models.py
CHANGED
|
@@ -34,6 +34,7 @@ class Detection(BaseModel):
|
|
| 34 |
confidence: float = Field(ge=0.0, le=1.0)
|
| 35 |
start_time: float = Field(ge=0.0)
|
| 36 |
end_time: float = Field(gt=0.0)
|
|
|
|
| 37 |
|
| 38 |
|
| 39 |
class Validation(BaseModel):
|
|
|
|
| 34 |
confidence: float = Field(ge=0.0, le=1.0)
|
| 35 |
start_time: float = Field(ge=0.0)
|
| 36 |
end_time: float = Field(gt=0.0)
|
| 37 |
+
source_metadata: dict[str, object] = Field(default_factory=dict)
|
| 38 |
|
| 39 |
|
| 40 |
class Validation(BaseModel):
|
src/services/detection_queue_service.py
CHANGED
|
@@ -20,6 +20,30 @@ class DetectionQueueService:
|
|
| 20 |
def __init__(self, repository: DetectionRepository) -> None:
|
| 21 |
self._repository = repository
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
def get_page(
|
| 24 |
self,
|
| 25 |
project_slug: str,
|
|
|
|
| 20 |
def __init__(self, repository: DetectionRepository) -> None:
|
| 21 |
self._repository = repository
|
| 22 |
|
| 23 |
+
def list_all_detections(
|
| 24 |
+
self,
|
| 25 |
+
project_slug: str,
|
| 26 |
+
scientific_name: str | None = None,
|
| 27 |
+
min_confidence: float | None = None,
|
| 28 |
+
max_confidence: float | None = None,
|
| 29 |
+
) -> list[Detection]:
|
| 30 |
+
total_items = self._repository.count_detections(
|
| 31 |
+
project_slug=project_slug,
|
| 32 |
+
scientific_name=scientific_name,
|
| 33 |
+
min_confidence=min_confidence,
|
| 34 |
+
max_confidence=max_confidence,
|
| 35 |
+
)
|
| 36 |
+
if total_items == 0:
|
| 37 |
+
return []
|
| 38 |
+
return self._repository.list_detections(
|
| 39 |
+
project_slug=project_slug,
|
| 40 |
+
page=1,
|
| 41 |
+
page_size=total_items,
|
| 42 |
+
scientific_name=scientific_name,
|
| 43 |
+
min_confidence=min_confidence,
|
| 44 |
+
max_confidence=max_confidence,
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
def get_page(
|
| 48 |
self,
|
| 49 |
project_slug: str,
|
src/ui/app_factory.py
CHANGED
|
@@ -6,6 +6,7 @@ import os
|
|
| 6 |
import re
|
| 7 |
import tempfile
|
| 8 |
import wave
|
|
|
|
| 9 |
from dataclasses import replace
|
| 10 |
from datetime import date, datetime
|
| 11 |
from pathlib import Path
|
|
@@ -212,6 +213,7 @@ def _build_detection_from_row(raw: dict[str, object], row_index: int, project_sl
|
|
| 212 |
confidence=confidence,
|
| 213 |
start_time=start_time,
|
| 214 |
end_time=end_time,
|
|
|
|
| 215 |
)
|
| 216 |
except Exception:
|
| 217 |
return None
|
|
@@ -2233,6 +2235,158 @@ def _batch_validate_conflicts(
|
|
| 2233 |
return status, "", None, refreshed_rows, refreshed_page
|
| 2234 |
|
| 2235 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2236 |
def create_app() -> gr.Blocks:
|
| 2237 |
"""Build the BirdNET Validator app with multi-project auth integration.
|
| 2238 |
|
|
@@ -4285,26 +4439,61 @@ def create_app() -> gr.Blocks:
|
|
| 4285 |
report_recent_prev_btn = gr.Button("Previous recent page")
|
| 4286 |
report_recent_next_btn = gr.Button("Next recent page")
|
| 4287 |
report_status = gr.Markdown("")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4288 |
|
| 4289 |
def _list_project_detections(project_slug: str) -> list[Detection]:
|
| 4290 |
if not project_slug:
|
| 4291 |
return []
|
| 4292 |
-
|
| 4293 |
-
|
| 4294 |
-
|
| 4295 |
-
|
| 4296 |
-
|
| 4297 |
-
|
| 4298 |
-
|
| 4299 |
-
|
| 4300 |
-
|
| 4301 |
-
|
| 4302 |
-
|
| 4303 |
-
|
| 4304 |
-
|
| 4305 |
-
|
| 4306 |
-
|
| 4307 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4308 |
|
| 4309 |
def _render_report_dashboard(project_slug: str, validator_page: int = 1, recent_page: int = 1):
|
| 4310 |
slug = (project_slug or "").strip()
|
|
@@ -4475,6 +4664,16 @@ def create_app() -> gr.Blocks:
|
|
| 4475 |
inputs=[report_project_selector, report_validator_page, report_recent_page],
|
| 4476 |
outputs=[report_kpis, report_coverage_bars, report_validator_table, report_recent_table, report_validator_page, report_recent_page, report_status],
|
| 4477 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4478 |
|
| 4479 |
# ===== TAB 6: Settings =====
|
| 4480 |
with gr.Tab("Settings", id="settings_tab"):
|
|
|
|
| 6 |
import re
|
| 7 |
import tempfile
|
| 8 |
import wave
|
| 9 |
+
from collections.abc import Mapping
|
| 10 |
from dataclasses import replace
|
| 11 |
from datetime import date, datetime
|
| 12 |
from pathlib import Path
|
|
|
|
| 213 |
confidence=confidence,
|
| 214 |
start_time=start_time,
|
| 215 |
end_time=end_time,
|
| 216 |
+
source_metadata=dict(raw),
|
| 217 |
)
|
| 218 |
except Exception:
|
| 219 |
return None
|
|
|
|
| 2235 |
return status, "", None, refreshed_rows, refreshed_page
|
| 2236 |
|
| 2237 |
|
| 2238 |
+
_VALIDATION_EXPORT_BASE_COLUMNS = [
|
| 2239 |
+
"project_slug",
|
| 2240 |
+
"dataset_repo_id",
|
| 2241 |
+
"detection_key",
|
| 2242 |
+
"audio_id",
|
| 2243 |
+
"detection_scientific_name",
|
| 2244 |
+
"detection_confidence",
|
| 2245 |
+
"detection_start_time",
|
| 2246 |
+
"detection_end_time",
|
| 2247 |
+
]
|
| 2248 |
+
|
| 2249 |
+
_VALIDATION_EXPORT_STATE_COLUMNS = [
|
| 2250 |
+
"validation_status",
|
| 2251 |
+
"validation_corrected_species",
|
| 2252 |
+
"validation_effective_species",
|
| 2253 |
+
"validation_notes",
|
| 2254 |
+
"validation_validator",
|
| 2255 |
+
"validation_updated_at",
|
| 2256 |
+
"validation_version",
|
| 2257 |
+
"validation_reviewed",
|
| 2258 |
+
]
|
| 2259 |
+
|
| 2260 |
+
_XLSX_EXPORT_MAX_DATA_ROWS = 1_048_575
|
| 2261 |
+
|
| 2262 |
+
|
| 2263 |
+
def _export_cell_value(value: object) -> object:
|
| 2264 |
+
if value is None:
|
| 2265 |
+
return ""
|
| 2266 |
+
if isinstance(value, np.generic):
|
| 2267 |
+
value = value.item()
|
| 2268 |
+
if isinstance(value, float) and np.isnan(value):
|
| 2269 |
+
return ""
|
| 2270 |
+
if isinstance(value, Mapping):
|
| 2271 |
+
return json.dumps(dict(value), ensure_ascii=True, sort_keys=True)
|
| 2272 |
+
if isinstance(value, (list, tuple, set)):
|
| 2273 |
+
return json.dumps(list(value), ensure_ascii=True)
|
| 2274 |
+
return value
|
| 2275 |
+
|
| 2276 |
+
|
| 2277 |
+
def _source_export_column(raw_key: object) -> str:
|
| 2278 |
+
key = str(raw_key or "").strip()
|
| 2279 |
+
return f"source_{key}" if key else ""
|
| 2280 |
+
|
| 2281 |
+
|
| 2282 |
+
def _build_validation_export_rows(
|
| 2283 |
+
detections: list[Detection],
|
| 2284 |
+
snapshot: dict[str, dict[str, object]],
|
| 2285 |
+
*,
|
| 2286 |
+
project_slug: str,
|
| 2287 |
+
dataset_repo_id: str,
|
| 2288 |
+
) -> tuple[list[str], list[dict[str, object]]]:
|
| 2289 |
+
"""Join complete project detections with their current validation state."""
|
| 2290 |
+
source_columns: list[str] = []
|
| 2291 |
+
for detection in detections:
|
| 2292 |
+
for raw_key in detection.source_metadata:
|
| 2293 |
+
column = _source_export_column(raw_key)
|
| 2294 |
+
if column and column not in source_columns:
|
| 2295 |
+
source_columns.append(column)
|
| 2296 |
+
|
| 2297 |
+
columns = [*_VALIDATION_EXPORT_BASE_COLUMNS, *source_columns, *_VALIDATION_EXPORT_STATE_COLUMNS]
|
| 2298 |
+
rows: list[dict[str, object]] = []
|
| 2299 |
+
sorted_detections = sorted(
|
| 2300 |
+
detections,
|
| 2301 |
+
key=lambda item: (
|
| 2302 |
+
item.scientific_name.lower(),
|
| 2303 |
+
item.audio_id.lower(),
|
| 2304 |
+
item.start_time,
|
| 2305 |
+
item.end_time,
|
| 2306 |
+
item.detection_key,
|
| 2307 |
+
),
|
| 2308 |
+
)
|
| 2309 |
+
for detection in sorted_detections:
|
| 2310 |
+
state = snapshot.get(detection.detection_key, {})
|
| 2311 |
+
validation_status = str(state.get("status") or "pending").strip() or "pending"
|
| 2312 |
+
corrected_species = str(state.get("corrected_species") or "").strip()
|
| 2313 |
+
row: dict[str, object] = {
|
| 2314 |
+
"project_slug": project_slug,
|
| 2315 |
+
"dataset_repo_id": dataset_repo_id,
|
| 2316 |
+
"detection_key": detection.detection_key,
|
| 2317 |
+
"audio_id": detection.audio_id,
|
| 2318 |
+
"detection_scientific_name": detection.scientific_name,
|
| 2319 |
+
"detection_confidence": detection.confidence,
|
| 2320 |
+
"detection_start_time": detection.start_time,
|
| 2321 |
+
"detection_end_time": detection.end_time,
|
| 2322 |
+
"validation_status": validation_status,
|
| 2323 |
+
"validation_corrected_species": corrected_species,
|
| 2324 |
+
"validation_effective_species": corrected_species or detection.scientific_name,
|
| 2325 |
+
"validation_notes": str(state.get("notes") or ""),
|
| 2326 |
+
"validation_validator": str(state.get("validator") or ""),
|
| 2327 |
+
"validation_updated_at": str(state.get("updated_at") or ""),
|
| 2328 |
+
"validation_version": int(state.get("version") or 0),
|
| 2329 |
+
"validation_reviewed": validation_status.lower() != "pending",
|
| 2330 |
+
}
|
| 2331 |
+
for raw_key, raw_value in detection.source_metadata.items():
|
| 2332 |
+
column = _source_export_column(raw_key)
|
| 2333 |
+
if column:
|
| 2334 |
+
row[column] = _export_cell_value(raw_value)
|
| 2335 |
+
rows.append({column: _export_cell_value(row.get(column, "")) for column in columns})
|
| 2336 |
+
return columns, rows
|
| 2337 |
+
|
| 2338 |
+
|
| 2339 |
+
def _write_validation_export(
|
| 2340 |
+
detections: list[Detection],
|
| 2341 |
+
snapshot: dict[str, dict[str, object]],
|
| 2342 |
+
*,
|
| 2343 |
+
project_slug: str,
|
| 2344 |
+
dataset_repo_id: str,
|
| 2345 |
+
file_format: str,
|
| 2346 |
+
) -> Path:
|
| 2347 |
+
columns, rows = _build_validation_export_rows(
|
| 2348 |
+
detections,
|
| 2349 |
+
snapshot,
|
| 2350 |
+
project_slug=project_slug,
|
| 2351 |
+
dataset_repo_id=dataset_repo_id,
|
| 2352 |
+
)
|
| 2353 |
+
normalized_format = file_format.lower().strip()
|
| 2354 |
+
if normalized_format not in {"csv", "xlsx"}:
|
| 2355 |
+
raise ValueError("Export format must be csv or xlsx")
|
| 2356 |
+
|
| 2357 |
+
safe_slug = re.sub(r"[^A-Za-z0-9_.-]+", "-", project_slug).strip("-") or "project"
|
| 2358 |
+
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
| 2359 |
+
output_dir = Path(tempfile.mkdtemp(prefix="birdnet-validation-export-"))
|
| 2360 |
+
output_path = output_dir / f"{safe_slug}-validation-data-{timestamp}.{normalized_format}"
|
| 2361 |
+
|
| 2362 |
+
if normalized_format == "csv":
|
| 2363 |
+
with output_path.open("w", encoding="utf-8", newline="") as file_handle:
|
| 2364 |
+
writer = csv.DictWriter(file_handle, fieldnames=columns)
|
| 2365 |
+
writer.writeheader()
|
| 2366 |
+
writer.writerows(rows)
|
| 2367 |
+
return output_path
|
| 2368 |
+
|
| 2369 |
+
import pandas as pd # type: ignore[import-not-found]
|
| 2370 |
+
|
| 2371 |
+
with pd.ExcelWriter(output_path, engine="openpyxl") as writer:
|
| 2372 |
+
if not rows:
|
| 2373 |
+
pd.DataFrame(columns=columns).to_excel(
|
| 2374 |
+
writer,
|
| 2375 |
+
sheet_name="validation_data",
|
| 2376 |
+
index=False,
|
| 2377 |
+
freeze_panes=(1, 0),
|
| 2378 |
+
)
|
| 2379 |
+
for chunk_index, start in enumerate(range(0, len(rows), _XLSX_EXPORT_MAX_DATA_ROWS), start=1):
|
| 2380 |
+
sheet_name = "validation_data" if chunk_index == 1 else f"validation_data_{chunk_index}"
|
| 2381 |
+
pd.DataFrame(rows[start : start + _XLSX_EXPORT_MAX_DATA_ROWS], columns=columns).to_excel(
|
| 2382 |
+
writer,
|
| 2383 |
+
sheet_name=sheet_name,
|
| 2384 |
+
index=False,
|
| 2385 |
+
freeze_panes=(1, 0),
|
| 2386 |
+
)
|
| 2387 |
+
return output_path
|
| 2388 |
+
|
| 2389 |
+
|
| 2390 |
def create_app() -> gr.Blocks:
|
| 2391 |
"""Build the BirdNET Validator app with multi-project auth integration.
|
| 2392 |
|
|
|
|
| 4439 |
report_recent_prev_btn = gr.Button("Previous recent page")
|
| 4440 |
report_recent_next_btn = gr.Button("Next recent page")
|
| 4441 |
report_status = gr.Markdown("")
|
| 4442 |
+
with gr.Group(elem_classes=["bn-report-panel"]):
|
| 4443 |
+
gr.HTML(
|
| 4444 |
+
section_header_html(
|
| 4445 |
+
"Download",
|
| 4446 |
+
"Complete validation dataset",
|
| 4447 |
+
"Export every detection with the validation fields already filled in for the selected project.",
|
| 4448 |
+
class_name="bn-panel-soft",
|
| 4449 |
+
)
|
| 4450 |
+
)
|
| 4451 |
+
with gr.Row():
|
| 4452 |
+
report_export_csv_btn = gr.Button("Download CSV", variant="primary")
|
| 4453 |
+
report_export_xlsx_btn = gr.Button("Download XLSX")
|
| 4454 |
+
report_export_status = gr.Markdown("")
|
| 4455 |
+
report_export_file = gr.File(
|
| 4456 |
+
label="Prepared file",
|
| 4457 |
+
interactive=False,
|
| 4458 |
+
)
|
| 4459 |
|
| 4460 |
def _list_project_detections(project_slug: str) -> list[Detection]:
|
| 4461 |
if not project_slug:
|
| 4462 |
return []
|
| 4463 |
+
return service_ref["queue"].list_all_detections(project_slug=project_slug)
|
| 4464 |
+
|
| 4465 |
+
def _prepare_report_export(project_slug: str, file_format: str, session):
|
| 4466 |
+
slug = (project_slug or "").strip()
|
| 4467 |
+
if session is None:
|
| 4468 |
+
return None, "Login before exporting project data."
|
| 4469 |
+
if not slug or slug not in session.authorized_projects:
|
| 4470 |
+
return None, "Choose an authorized project before exporting."
|
| 4471 |
+
|
| 4472 |
+
project = _project_map().get(slug)
|
| 4473 |
+
items = _list_project_detections(slug)
|
| 4474 |
+
if not items:
|
| 4475 |
+
return None, f"Project '{slug}' has no detections loaded for export."
|
| 4476 |
+
|
| 4477 |
+
snapshot = validation_repository.load_current_snapshot(project_slug=slug)
|
| 4478 |
+
output_path = _write_validation_export(
|
| 4479 |
+
items,
|
| 4480 |
+
snapshot,
|
| 4481 |
+
project_slug=slug,
|
| 4482 |
+
dataset_repo_id=(project.dataset_repo_id if project is not None else ""),
|
| 4483 |
+
file_format=file_format,
|
| 4484 |
+
)
|
| 4485 |
+
reviewed = sum(
|
| 4486 |
+
1
|
| 4487 |
+
for detection in items
|
| 4488 |
+
if str(snapshot.get(detection.detection_key, {}).get("status") or "pending").lower() != "pending"
|
| 4489 |
+
)
|
| 4490 |
+
return (
|
| 4491 |
+
str(output_path),
|
| 4492 |
+
(
|
| 4493 |
+
f"Prepared **{file_format.upper()}** with **{len(items)}** detections and "
|
| 4494 |
+
f"**{reviewed}** current validations for project **{slug}**."
|
| 4495 |
+
),
|
| 4496 |
+
)
|
| 4497 |
|
| 4498 |
def _render_report_dashboard(project_slug: str, validator_page: int = 1, recent_page: int = 1):
|
| 4499 |
slug = (project_slug or "").strip()
|
|
|
|
| 4664 |
inputs=[report_project_selector, report_validator_page, report_recent_page],
|
| 4665 |
outputs=[report_kpis, report_coverage_bars, report_validator_table, report_recent_table, report_validator_page, report_recent_page, report_status],
|
| 4666 |
)
|
| 4667 |
+
report_export_csv_btn.click(
|
| 4668 |
+
fn=lambda project_slug, session: _prepare_report_export(project_slug, "csv", session),
|
| 4669 |
+
inputs=[report_project_selector, session_state],
|
| 4670 |
+
outputs=[report_export_file, report_export_status],
|
| 4671 |
+
)
|
| 4672 |
+
report_export_xlsx_btn.click(
|
| 4673 |
+
fn=lambda project_slug, session: _prepare_report_export(project_slug, "xlsx", session),
|
| 4674 |
+
inputs=[report_project_selector, session_state],
|
| 4675 |
+
outputs=[report_export_file, report_export_status],
|
| 4676 |
+
)
|
| 4677 |
|
| 4678 |
# ===== TAB 6: Settings =====
|
| 4679 |
with gr.Tab("Settings", id="settings_tab"):
|
tests/unit/test_app_factory_audio_helpers.py
CHANGED
|
@@ -7,6 +7,7 @@ import pandas as pd
|
|
| 7 |
import pytest
|
| 8 |
|
| 9 |
from src.ui.app_factory import (
|
|
|
|
| 10 |
_build_validation_report,
|
| 11 |
_cleanup_selected_audio,
|
| 12 |
_advance_to_next_row_with_title,
|
|
@@ -31,6 +32,7 @@ from src.ui.app_factory import (
|
|
| 31 |
_load_user_access_from_file,
|
| 32 |
_bootstrap_auth_and_projects,
|
| 33 |
_resolve_project_fetch_token,
|
|
|
|
| 34 |
)
|
| 35 |
from src.auth.auth_service import AuthService
|
| 36 |
from src.config.runtime_config import RuntimeConfig
|
|
@@ -295,6 +297,126 @@ def test_build_validation_report() -> None:
|
|
| 295 |
assert "negative=1" in report
|
| 296 |
|
| 297 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
def test_page_to_table_includes_validation_status() -> None:
|
| 299 |
queue = FakeQueueService()
|
| 300 |
rows, status, page = _page_to_table(
|
|
|
|
| 7 |
import pytest
|
| 8 |
|
| 9 |
from src.ui.app_factory import (
|
| 10 |
+
_build_validation_export_rows,
|
| 11 |
_build_validation_report,
|
| 12 |
_cleanup_selected_audio,
|
| 13 |
_advance_to_next_row_with_title,
|
|
|
|
| 32 |
_load_user_access_from_file,
|
| 33 |
_bootstrap_auth_and_projects,
|
| 34 |
_resolve_project_fetch_token,
|
| 35 |
+
_write_validation_export,
|
| 36 |
)
|
| 37 |
from src.auth.auth_service import AuthService
|
| 38 |
from src.config.runtime_config import RuntimeConfig
|
|
|
|
| 297 |
assert "negative=1" in report
|
| 298 |
|
| 299 |
|
| 300 |
+
def test_build_validation_export_rows_keep_detections_metadata_and_current_validation() -> None:
|
| 301 |
+
columns, rows = _build_validation_export_rows(
|
| 302 |
+
[
|
| 303 |
+
Detection(
|
| 304 |
+
detection_key="0000000000000002",
|
| 305 |
+
audio_id="Species B/second.wav",
|
| 306 |
+
scientific_name="Species B",
|
| 307 |
+
confidence=0.92,
|
| 308 |
+
start_time=6.0,
|
| 309 |
+
end_time=9.0,
|
| 310 |
+
source_metadata={"Common Name": "Bird B", "Latitude": -3.1},
|
| 311 |
+
),
|
| 312 |
+
Detection(
|
| 313 |
+
detection_key="0000000000000001",
|
| 314 |
+
audio_id="Species A/first.wav",
|
| 315 |
+
scientific_name="Species A",
|
| 316 |
+
confidence=0.87,
|
| 317 |
+
start_time=0.0,
|
| 318 |
+
end_time=3.0,
|
| 319 |
+
source_metadata={"Common Name": "Bird A", "Latitude": -2.9},
|
| 320 |
+
),
|
| 321 |
+
],
|
| 322 |
+
{
|
| 323 |
+
"0000000000000001": {
|
| 324 |
+
"status": "positive",
|
| 325 |
+
"corrected_species": "Corrected species",
|
| 326 |
+
"notes": "Checked twice",
|
| 327 |
+
"validator": "scientist",
|
| 328 |
+
"updated_at": "2026-05-22T12:00:00+00:00",
|
| 329 |
+
"version": 2,
|
| 330 |
+
}
|
| 331 |
+
},
|
| 332 |
+
project_slug="analysis-project",
|
| 333 |
+
dataset_repo_id="birds/analysis-project",
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
assert columns[:8] == [
|
| 337 |
+
"project_slug",
|
| 338 |
+
"dataset_repo_id",
|
| 339 |
+
"detection_key",
|
| 340 |
+
"audio_id",
|
| 341 |
+
"detection_scientific_name",
|
| 342 |
+
"detection_confidence",
|
| 343 |
+
"detection_start_time",
|
| 344 |
+
"detection_end_time",
|
| 345 |
+
]
|
| 346 |
+
assert columns[8:10] == ["source_Common Name", "source_Latitude"]
|
| 347 |
+
assert rows[0]["detection_key"] == "0000000000000001"
|
| 348 |
+
assert rows[0]["validation_status"] == "positive"
|
| 349 |
+
assert rows[0]["validation_effective_species"] == "Corrected species"
|
| 350 |
+
assert rows[0]["validation_version"] == 2
|
| 351 |
+
assert rows[1]["validation_status"] == "pending"
|
| 352 |
+
assert rows[1]["validation_effective_species"] == "Species B"
|
| 353 |
+
assert rows[1]["validation_reviewed"] is False
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
def test_write_validation_export_creates_csv_and_xlsx(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 357 |
+
monkeypatch.setattr("src.ui.app_factory.tempfile.mkdtemp", lambda prefix: str(tmp_path))
|
| 358 |
+
detections = [
|
| 359 |
+
Detection(
|
| 360 |
+
detection_key="0000000000000001",
|
| 361 |
+
audio_id="Species A/first.wav",
|
| 362 |
+
scientific_name="Species A",
|
| 363 |
+
confidence=0.87,
|
| 364 |
+
start_time=0.0,
|
| 365 |
+
end_time=3.0,
|
| 366 |
+
source_metadata={"source_file": "first.wav"},
|
| 367 |
+
)
|
| 368 |
+
]
|
| 369 |
+
snapshot = {"0000000000000001": {"status": "negative", "validator": "reviewer", "version": 1}}
|
| 370 |
+
|
| 371 |
+
csv_path = _write_validation_export(
|
| 372 |
+
detections,
|
| 373 |
+
snapshot,
|
| 374 |
+
project_slug="analysis-project",
|
| 375 |
+
dataset_repo_id="birds/analysis-project",
|
| 376 |
+
file_format="csv",
|
| 377 |
+
)
|
| 378 |
+
xlsx_path = _write_validation_export(
|
| 379 |
+
detections,
|
| 380 |
+
snapshot,
|
| 381 |
+
project_slug="analysis-project",
|
| 382 |
+
dataset_repo_id="birds/analysis-project",
|
| 383 |
+
file_format="xlsx",
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
csv_frame = pd.read_csv(csv_path)
|
| 387 |
+
xlsx_frame = pd.read_excel(xlsx_path)
|
| 388 |
+
assert csv_frame.loc[0, "validation_status"] == "negative"
|
| 389 |
+
assert csv_frame.loc[0, "source_source_file"] == "first.wav"
|
| 390 |
+
assert xlsx_frame.loc[0, "validation_validator"] == "reviewer"
|
| 391 |
+
assert xlsx_frame.loc[0, "dataset_repo_id"] == "birds/analysis-project"
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def test_write_validation_export_splits_large_xlsx_into_sheets(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 395 |
+
monkeypatch.setattr("src.ui.app_factory.tempfile.mkdtemp", lambda prefix: str(tmp_path))
|
| 396 |
+
monkeypatch.setattr("src.ui.app_factory._XLSX_EXPORT_MAX_DATA_ROWS", 1)
|
| 397 |
+
detections = [
|
| 398 |
+
Detection(
|
| 399 |
+
detection_key=f"000000000000000{index}",
|
| 400 |
+
audio_id=f"Species A/{index}.wav",
|
| 401 |
+
scientific_name="Species A",
|
| 402 |
+
confidence=0.87,
|
| 403 |
+
start_time=0.0,
|
| 404 |
+
end_time=3.0,
|
| 405 |
+
)
|
| 406 |
+
for index in (1, 2)
|
| 407 |
+
]
|
| 408 |
+
|
| 409 |
+
path = _write_validation_export(
|
| 410 |
+
detections,
|
| 411 |
+
{},
|
| 412 |
+
project_slug="large-analysis-project",
|
| 413 |
+
dataset_repo_id="birds/large-analysis-project",
|
| 414 |
+
file_format="xlsx",
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
assert pd.ExcelFile(path).sheet_names == ["validation_data", "validation_data_2"]
|
| 418 |
+
|
| 419 |
+
|
| 420 |
def test_page_to_table_includes_validation_status() -> None:
|
| 421 |
queue = FakeQueueService()
|
| 422 |
rows, status, page = _page_to_table(
|
tests/unit/test_detection_queue_service.py
CHANGED
|
@@ -86,6 +86,16 @@ def test_get_page_with_filters() -> None:
|
|
| 86 |
assert page.items[0].detection_key == "0000000000000001"
|
| 87 |
|
| 88 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
def test_get_page_requires_positive_page_and_page_size() -> None:
|
| 90 |
repo = InMemoryDetectionRepository()
|
| 91 |
repo.seed("ppbio-rabeca", _sample())
|
|
|
|
| 86 |
assert page.items[0].detection_key == "0000000000000001"
|
| 87 |
|
| 88 |
|
| 89 |
+
def test_list_all_detections_returns_filtered_project_items_without_paging() -> None:
|
| 90 |
+
repo = InMemoryDetectionRepository()
|
| 91 |
+
repo.seed("ppbio-rabeca", _sample())
|
| 92 |
+
service = DetectionQueueService(repo)
|
| 93 |
+
|
| 94 |
+
items = service.list_all_detections(project_slug="ppbio-rabeca", scientific_name="Species A")
|
| 95 |
+
|
| 96 |
+
assert [item.detection_key for item in items] == ["0000000000000001", "0000000000000003"]
|
| 97 |
+
|
| 98 |
+
|
| 99 |
def test_get_page_requires_positive_page_and_page_size() -> None:
|
| 100 |
repo = InMemoryDetectionRepository()
|
| 101 |
repo.seed("ppbio-rabeca", _sample())
|