jrrribeiro commited on
Commit
c85843b
·
1 Parent(s): f5bcf37

refactor: focus repository on validator app only

Browse files
Files changed (45) hide show
  1. .ingest-dry-run-report.json +0 -14
  2. BirdNET Uploader App/README.md +0 -84
  3. BirdNET Uploader App/app.py +0 -9
  4. BirdNET Uploader App/cli/hf_dataset_cli.py +0 -969
  5. BirdNET Uploader App/pyproject.toml +0 -9
  6. BirdNET Uploader App/requirements.txt +0 -7
  7. BirdNET Uploader App/src/__init__.py +0 -0
  8. BirdNET Uploader App/src/ui/__init__.py +0 -0
  9. BirdNET Uploader App/src/ui/upload_app_factory.py +0 -177
  10. BirdNET Validator App/README.md +0 -83
  11. BirdNET Validator App/app.py +0 -6
  12. BirdNET Validator App/pyproject.toml +0 -9
  13. BirdNET Validator App/requirements.txt +0 -7
  14. BirdNET Validator App/src/__init__.py +0 -0
  15. BirdNET Validator App/src/auth/__init__.py +0 -0
  16. BirdNET Validator App/src/auth/auth_service.py +0 -200
  17. BirdNET Validator App/src/cache/__init__.py +0 -0
  18. BirdNET Validator App/src/cache/ephemeral_cache_manager.py +0 -82
  19. BirdNET Validator App/src/domain/__init__.py +0 -0
  20. BirdNET Validator App/src/domain/models.py +0 -46
  21. BirdNET Validator App/src/repositories/__init__.py +0 -0
  22. BirdNET Validator App/src/repositories/append_only_validation_repository.py +0 -89
  23. BirdNET Validator App/src/repositories/contracts.py +0 -35
  24. BirdNET Validator App/src/repositories/in_memory_detection_repository.py +0 -73
  25. BirdNET Validator App/src/repositories/in_memory_validation_repository.py +0 -25
  26. BirdNET Validator App/src/services/__init__.py +0 -0
  27. BirdNET Validator App/src/services/audio_fetch_service.py +0 -54
  28. BirdNET Validator App/src/services/detection_queue_service.py +0 -64
  29. BirdNET Validator App/src/services/validation_service.py +0 -39
  30. BirdNET Validator App/src/telemetry/__init__.py +0 -0
  31. BirdNET Validator App/src/ui/__init__.py +0 -0
  32. BirdNET Validator App/src/ui/admin_panel.py +0 -281
  33. BirdNET Validator App/src/ui/app_factory.py +0 -1542
  34. BirdNET Validator App/src/ui/login_page.py +0 -66
  35. BirdNET Validator App/src/ui/project_selector.py +0 -77
  36. BirdNET Validator App/src/ui/upload_app_factory.py +0 -177
  37. README.md +64 -82
  38. app_uploader.py +0 -9
  39. cli/hf_dataset_cli.py +0 -969
  40. docs/HF_SEGMENTS_INGESTION_PLAN.md +0 -194
  41. pyproject.toml +1 -1
  42. requirements.txt +0 -3
  43. src/ui/upload_app_factory.py +0 -177
  44. tests/unit/test_hf_dataset_cli.py +0 -231
  45. tests/unit/test_hf_ingest_segments_cli.py +0 -127
.ingest-dry-run-report.json DELETED
@@ -1,14 +0,0 @@
1
- {
2
- "mode": "dry-run",
3
- "project_slug": "ppbio-aiuaba",
4
- "detections_csv": "c:\\Users\\jonat\\OneDrive - MSFT\\Doutorado PPGEtno\\BirdNET Analyzer\\00_ALL_DETECTIONS.csv",
5
- "segments_root": "c:\\Users\\jonat\\OneDrive - MSFT\\Doutorado PPGEtno\\BirdNET Segments",
6
- "csv_rows_total": 146561,
7
- "segments_found_total": 146561,
8
- "matched_rows": 146561,
9
- "unmatched_rows": 0,
10
- "ambiguous_rows": 0,
11
- "unique_audio_ids": 31265,
12
- "sample_unmatched": [],
13
- "sample_ambiguous": []
14
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Uploader App/README.md DELETED
@@ -1,84 +0,0 @@
1
- # BirdNET Uploader App
2
-
3
- Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
4
- - Multi-projeto
5
- - Audio sob demanda
6
- - Persistencia de validacoes
7
- - Escalabilidade para datasets grandes
8
-
9
- ## Desenvolvimento local
10
- 1. Criar ambiente virtual Python 3.11+
11
- 2. Instalar dependencias:
12
- pip install -r requirements.txt
13
- 3. Rodar app:
14
- python app.py
15
-
16
- ## Objetivo desta pasta
17
- Contem apenas os arquivos necessarios para o app de upload/ingestao.
18
- Use esta pasta para criar/publicar o repositorio do uploader de forma independente.
19
-
20
- ## Fluxo recomendado (separado)
21
- 1. Uploader app (porta 7862 por padrao):
22
- - Rodar dry-run com CSV + pasta de segmentos
23
- - Rodar upload real para owner/repo do projeto
24
- 2. Validator app (porta 7860):
25
- - Validar deteccoes do projeto com dados ja publicados
26
-
27
- Essa separacao evita impacto de performance na validacao durante uploads grandes.
28
-
29
- ## CLI de dataset (Sprint 1)
30
- Comandos principais:
31
-
32
- 1. Criar estrutura do projeto no dataset Hugging Face:
33
- python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
34
-
35
- 2. Gerar manifesto e shards iniciais de indice:
36
- python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
37
-
38
- 3. Fazer upload de audio em lotes com retry e resume:
39
- python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
40
-
41
- 4. Verificar consistencia basica do projeto:
42
- python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
43
-
44
- 5. Ingestao BirdNET por projeto (dry-run):
45
- python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
46
-
47
- 6. Ingestao BirdNET por projeto (upload real):
48
- python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
49
-
50
- Estrutura base criada por projeto/dataset:
51
- - audio/
52
- - index/
53
- - index/shards/
54
- - validations/
55
- - audit/
56
- - manifest.json
57
-
58
- ## Estrutura
59
- - src/domain: modelos de dominio
60
- - src/repositories: contratos de persistencia
61
- - src/services: servicos de aplicacao
62
- - src/auth: autenticacao e autorizacao
63
- - src/ui: montagem da interface
64
- - src/cache: cache efemero local
65
- - cli: comandos de ingestao/publicacao
66
- - tests: testes unitarios e integracao
67
-
68
- ## Estado atual de implementacao
69
- - Sprint 1: CLI create-project, sync-audio, build-index, verify-project
70
- - Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
71
- - Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
72
- - Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
73
- - Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
74
- - Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
75
- - Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
76
- - Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
77
- - Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
78
- - Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
79
- - Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
80
- - Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
81
- - Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
82
- - Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
83
- - Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
84
- - Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Uploader App/app.py DELETED
@@ -1,9 +0,0 @@
1
- import os
2
-
3
- from src.ui.upload_app_factory import build_upload_app
4
-
5
-
6
- if __name__ == "__main__":
7
- app = build_upload_app()
8
- port = int(os.getenv("BIRDNET_UPLOADER_PORT", "7862"))
9
- app.launch(server_name="0.0.0.0", server_port=port)
 
 
 
 
 
 
 
 
 
 
BirdNET Uploader App/cli/hf_dataset_cli.py DELETED
@@ -1,969 +0,0 @@
1
- import argparse
2
- import hashlib
3
- import json
4
- import math
5
- import re
6
- import tempfile
7
- import time
8
- from dataclasses import asdict, dataclass
9
- from datetime import UTC, datetime
10
- from io import BytesIO
11
- from pathlib import Path
12
- from typing import Any
13
-
14
- import pandas as pd
15
- from huggingface_hub import HfApi, hf_hub_download
16
-
17
-
18
- SCHEMA_VERSION = "1.0.0"
19
- REQUIRED_PROJECT_PREFIXES = (
20
- "audio/",
21
- "index/",
22
- "index/shards/",
23
- "validations/",
24
- "audit/",
25
- )
26
- REQUIRED_PLACEHOLDER_FILES = (
27
- "audio/.gitkeep",
28
- "index/.gitkeep",
29
- "index/shards/.gitkeep",
30
- "validations/.gitkeep",
31
- "audit/.gitkeep",
32
- )
33
- REQUIRED_DETECTIONS_COLUMNS = (
34
- "detection_key",
35
- "audio_id",
36
- "scientific_name",
37
- "confidence",
38
- "start_time",
39
- "end_time",
40
- )
41
- SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
42
- REQUIRED_BIRDNET_COLUMNS = (
43
- "source_file",
44
- "scientific_name",
45
- "confidence",
46
- "start_time",
47
- "end_time",
48
- )
49
- SEGMENT_FILENAME_PATTERN = re.compile(
50
- r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
51
- )
52
-
53
-
54
- @dataclass(slots=True)
55
- class ShardMetadata:
56
- path: str
57
- rows: int
58
- sha256: str
59
- size_bytes: int
60
-
61
-
62
- @dataclass(slots=True)
63
- class SegmentFileRecord:
64
- scientific_name: str
65
- source_stem: str
66
- start_time: float
67
- end_time: float
68
- confidence_pct: int
69
- absolute_path: Path
70
- relative_path: str
71
-
72
-
73
- def _utcnow_iso() -> str:
74
- return datetime.now(UTC).replace(microsecond=0).isoformat()
75
-
76
-
77
- def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
78
- api.upload_file(
79
- path_or_fileobj=BytesIO(content.encode("utf-8")),
80
- path_in_repo=path_in_repo,
81
- repo_id=dataset_repo,
82
- repo_type="dataset",
83
- )
84
-
85
-
86
- def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
87
- api.upload_file(
88
- path_or_fileobj=BytesIO(b""),
89
- path_in_repo=path_in_repo,
90
- repo_id=dataset_repo,
91
- repo_type="dataset",
92
- )
93
-
94
-
95
- def ensure_project_dataset_structure(
96
- api: HfApi,
97
- project_slug: str,
98
- dataset_repo: str,
99
- create_private_repo: bool,
100
- ) -> dict[str, Any]:
101
- api.create_repo(
102
- repo_id=dataset_repo,
103
- repo_type="dataset",
104
- private=create_private_repo,
105
- exist_ok=True,
106
- )
107
-
108
- files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
109
- created_paths: list[str] = []
110
-
111
- for placeholder in REQUIRED_PLACEHOLDER_FILES:
112
- if placeholder not in files:
113
- _upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
114
- created_paths.append(placeholder)
115
-
116
- manifest_path = "manifest.json"
117
- if manifest_path not in files:
118
- manifest = {
119
- "schema_version": SCHEMA_VERSION,
120
- "project_slug": project_slug,
121
- "dataset_repo_id": dataset_repo,
122
- "created_at": _utcnow_iso(),
123
- "updated_at": _utcnow_iso(),
124
- "index": {
125
- "total_detections": 0,
126
- "total_audio_files": 0,
127
- "shard_size": 0,
128
- "shards": [],
129
- },
130
- }
131
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
132
- created_paths.append(manifest_path)
133
-
134
- return {
135
- "dataset_repo": dataset_repo,
136
- "project_slug": project_slug,
137
- "created_paths": created_paths,
138
- }
139
-
140
-
141
- def load_detections_table(detections_file: str) -> pd.DataFrame:
142
- input_path = Path(detections_file)
143
- if not input_path.exists():
144
- raise FileNotFoundError(f"Detections file not found: {detections_file}")
145
-
146
- suffix = input_path.suffix.lower()
147
- if suffix == ".csv":
148
- frame = pd.read_csv(input_path)
149
- elif suffix in {".jsonl", ".ndjson"}:
150
- frame = pd.read_json(input_path, lines=True)
151
- elif suffix == ".parquet":
152
- frame = pd.read_parquet(input_path)
153
- else:
154
- raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
155
-
156
- missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
157
- if missing_columns:
158
- raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
159
-
160
- if frame.empty:
161
- return frame
162
-
163
- deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
164
- return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
165
-
166
-
167
- def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
168
- input_path = Path(detections_csv)
169
- if not input_path.exists():
170
- raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
171
-
172
- frame = pd.read_csv(input_path)
173
- missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
174
- if missing_columns:
175
- raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
176
-
177
- if frame.empty:
178
- return frame
179
-
180
- normalized = frame.copy()
181
- normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
182
- normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
183
- normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
184
- normalized["species_key"] = normalized["scientific_name"].str.casefold()
185
- normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
186
- normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
187
- normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
188
-
189
- normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
190
- return normalized
191
-
192
-
193
- def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
194
- payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
195
- return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
196
-
197
-
198
- def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
199
- stem = Path(file_name).stem
200
- match = SEGMENT_FILENAME_PATTERN.match(stem)
201
- if not match:
202
- return None
203
-
204
- source_stem = match.group("source_stem")
205
- start_time = float(match.group("start"))
206
- end_time = float(match.group("end"))
207
- confidence_pct = int(match.group("confidence_pct"))
208
- return source_stem, start_time, end_time, confidence_pct
209
-
210
-
211
- def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
212
- root = Path(segments_root)
213
- if not root.exists() or not root.is_dir():
214
- raise FileNotFoundError(f"Segments root not found: {segments_root}")
215
-
216
- records: list[SegmentFileRecord] = []
217
- for absolute_path in root.rglob("*"):
218
- if not absolute_path.is_file():
219
- continue
220
- if absolute_path.suffix.lower() != ".wav":
221
- continue
222
-
223
- rel = absolute_path.relative_to(root)
224
- scientific_name = rel.parts[0] if rel.parts else ""
225
- parsed = parse_segment_filename(absolute_path.name)
226
- if not parsed:
227
- continue
228
- source_stem, start_time, end_time, confidence_pct = parsed
229
- records.append(
230
- SegmentFileRecord(
231
- scientific_name=scientific_name,
232
- source_stem=source_stem,
233
- start_time=start_time,
234
- end_time=end_time,
235
- confidence_pct=confidence_pct,
236
- absolute_path=absolute_path,
237
- relative_path=rel.as_posix(),
238
- )
239
- )
240
-
241
- return records
242
-
243
-
244
- def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
245
- frame = load_birdnet_detections_csv(detections_csv=detections_csv)
246
- segment_records = discover_segment_records(segments_root=segments_root)
247
-
248
- by_species: dict[str, list[SegmentFileRecord]] = {}
249
- for record in segment_records:
250
- by_species.setdefault(record.scientific_name.casefold(), []).append(record)
251
-
252
- matched_rows: list[dict[str, Any]] = []
253
- unmatched_rows: list[dict[str, Any]] = []
254
- ambiguous_rows: list[dict[str, Any]] = []
255
-
256
- for row in frame.to_dict(orient="records"):
257
- species_key = str(row.get("species_key", "")).casefold()
258
- source_stem = str(row.get("source_stem", "")).strip()
259
- start_time = float(row["start_time"])
260
- end_time = float(row["end_time"])
261
- confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
262
-
263
- species_candidates = by_species.get(species_key, [])
264
- exact_matches = [
265
- record
266
- for record in species_candidates
267
- if record.source_stem == source_stem
268
- and abs(record.start_time - start_time) < 1e-6
269
- and abs(record.end_time - end_time) < 1e-6
270
- ]
271
-
272
- if not exact_matches:
273
- unmatched_rows.append(
274
- {
275
- "source_file": row.get("source_file", ""),
276
- "scientific_name": row.get("scientific_name", ""),
277
- "start_time": start_time,
278
- "end_time": end_time,
279
- }
280
- )
281
- continue
282
-
283
- selected = exact_matches[0]
284
- if len(exact_matches) > 1:
285
- expected_pct = int(round(confidence * 100))
286
- exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
287
- selected = exact_matches[0]
288
- ambiguous_rows.append(
289
- {
290
- "source_file": row.get("source_file", ""),
291
- "scientific_name": row.get("scientific_name", ""),
292
- "start_time": start_time,
293
- "end_time": end_time,
294
- "candidates": len(exact_matches),
295
- }
296
- )
297
-
298
- matched_rows.append(
299
- {
300
- "project_slug": project_slug,
301
- "detection_key": _build_detection_key(
302
- project_slug=project_slug,
303
- source_file=str(row.get("source_file", "")),
304
- scientific_name=str(row.get("scientific_name", "")),
305
- start_time=start_time,
306
- end_time=end_time,
307
- ),
308
- "audio_id": source_stem,
309
- "segment_filename": selected.absolute_path.name,
310
- "segment_relpath": selected.relative_path,
311
- "scientific_name": row.get("scientific_name", ""),
312
- "common_name": row.get("common_name", ""),
313
- "confidence": confidence,
314
- "start_time": start_time,
315
- "end_time": end_time,
316
- "exact_start": row.get("exact_start", start_time),
317
- "exact_end": row.get("exact_end", end_time),
318
- "locality": row.get("locality", ""),
319
- "point": row.get("point", ""),
320
- "date_folder": row.get("date_folder", ""),
321
- "min_freq": row.get("min_freq", None),
322
- "max_freq": row.get("max_freq", None),
323
- "box_source": row.get("box_source", ""),
324
- "label": row.get("label", ""),
325
- "source_file": row.get("source_file", ""),
326
- "schema_version": SCHEMA_VERSION,
327
- "ingested_at": _utcnow_iso(),
328
- }
329
- )
330
-
331
- return {
332
- "frame": frame,
333
- "segment_records_total": len(segment_records),
334
- "matched_rows": matched_rows,
335
- "unmatched_rows": unmatched_rows,
336
- "ambiguous_rows": ambiguous_rows,
337
- }
338
-
339
-
340
- def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
341
- match_result = _match_birdnet_rows(
342
- project_slug=project_slug,
343
- detections_csv=detections_csv,
344
- segments_root=segments_root,
345
- )
346
- frame = match_result["frame"]
347
- matched_rows = match_result["matched_rows"]
348
- unmatched_rows = match_result["unmatched_rows"]
349
- ambiguous_rows = match_result["ambiguous_rows"]
350
-
351
- unique_audio = {row["audio_id"] for row in matched_rows}
352
- result = {
353
- "mode": "dry-run",
354
- "project_slug": project_slug,
355
- "detections_csv": detections_csv,
356
- "segments_root": segments_root,
357
- "csv_rows_total": int(len(frame)),
358
- "segments_found_total": int(match_result["segment_records_total"]),
359
- "matched_rows": len(matched_rows),
360
- "unmatched_rows": len(unmatched_rows),
361
- "ambiguous_rows": len(ambiguous_rows),
362
- "unique_audio_ids": len(unique_audio),
363
- "sample_unmatched": unmatched_rows[:20],
364
- "sample_ambiguous": ambiguous_rows[:20],
365
- }
366
- return result
367
-
368
-
369
- def _file_sha256(file_path: Path) -> str:
370
- hasher = hashlib.sha256()
371
- with file_path.open("rb") as handle:
372
- for chunk in iter(lambda: handle.read(65536), b""):
373
- hasher.update(chunk)
374
- return hasher.hexdigest()
375
-
376
-
377
- def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
378
- if shard_size <= 0:
379
- raise ValueError("shard_size must be greater than zero")
380
-
381
- output_dir.mkdir(parents=True, exist_ok=True)
382
- shards: list[ShardMetadata] = []
383
-
384
- if frame.empty:
385
- return shards
386
-
387
- total_rows = len(frame)
388
- total_shards = math.ceil(total_rows / shard_size)
389
-
390
- for index in range(total_shards):
391
- start = index * shard_size
392
- end = min(start + shard_size, total_rows)
393
- shard_frame = frame.iloc[start:end]
394
- shard_filename = f"shard-{index:05d}.parquet"
395
- local_path = output_dir / shard_filename
396
- shard_frame.to_parquet(local_path, index=False)
397
-
398
- shards.append(
399
- ShardMetadata(
400
- path=f"index/shards/{shard_filename}",
401
- rows=len(shard_frame),
402
- sha256=_file_sha256(local_path),
403
- size_bytes=local_path.stat().st_size,
404
- )
405
- )
406
-
407
- return shards
408
-
409
-
410
- def build_manifest_payload(
411
- project_slug: str,
412
- dataset_repo: str,
413
- frame: pd.DataFrame,
414
- shard_size: int,
415
- shard_metadata: list[ShardMetadata],
416
- ) -> dict[str, Any]:
417
- now = _utcnow_iso()
418
- return {
419
- "schema_version": SCHEMA_VERSION,
420
- "project_slug": project_slug,
421
- "dataset_repo_id": dataset_repo,
422
- "updated_at": now,
423
- "index": {
424
- "total_detections": len(frame),
425
- "total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
426
- "shard_size": shard_size,
427
- "shards": [asdict(item) for item in shard_metadata],
428
- },
429
- }
430
-
431
-
432
- def discover_audio_files(local_audio_dir: str) -> list[Path]:
433
- audio_root = Path(local_audio_dir)
434
- if not audio_root.exists() or not audio_root.is_dir():
435
- raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
436
-
437
- files = [
438
- path
439
- for path in audio_root.rglob("*")
440
- if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
441
- ]
442
- return sorted(files)
443
-
444
-
445
- def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
446
- if chunk_size <= 0:
447
- raise ValueError("chunk_size must be greater than zero")
448
- return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
449
-
450
-
451
- def _load_resume_state(state_file: Path) -> dict[str, Any]:
452
- if not state_file.exists():
453
- return {"uploaded": [], "failed": []}
454
-
455
- payload = json.loads(state_file.read_text(encoding="utf-8"))
456
- uploaded = payload.get("uploaded", [])
457
- failed = payload.get("failed", [])
458
- return {
459
- "uploaded": uploaded if isinstance(uploaded, list) else [],
460
- "failed": failed if isinstance(failed, list) else [],
461
- }
462
-
463
-
464
- def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
465
- state_file.parent.mkdir(parents=True, exist_ok=True)
466
- payload = {
467
- "updated_at": _utcnow_iso(),
468
- "uploaded": sorted(uploaded),
469
- "failed": sorted(failed),
470
- }
471
- state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
472
-
473
-
474
- def _upload_audio_with_retry(
475
- api: HfApi,
476
- dataset_repo: str,
477
- local_path: Path,
478
- path_in_repo: str,
479
- max_retries: int,
480
- retry_backoff_seconds: float,
481
- ) -> None:
482
- attempts = 0
483
- while True:
484
- attempts += 1
485
- try:
486
- api.upload_file(
487
- path_or_fileobj=str(local_path),
488
- path_in_repo=path_in_repo,
489
- repo_id=dataset_repo,
490
- repo_type="dataset",
491
- )
492
- return
493
- except Exception:
494
- if attempts > max_retries:
495
- raise
496
- time.sleep(retry_backoff_seconds * attempts)
497
-
498
-
499
- def ingest_segments_to_hf(
500
- api: HfApi,
501
- project_slug: str,
502
- dataset_repo: str,
503
- detections_csv: str,
504
- segments_root: str,
505
- batch_size: int,
506
- shard_size: int,
507
- max_retries: int,
508
- retry_backoff_seconds: float,
509
- resume_state_file: str,
510
- ) -> dict[str, Any]:
511
- ensure_project_dataset_structure(
512
- api=api,
513
- project_slug=project_slug,
514
- dataset_repo=dataset_repo,
515
- create_private_repo=False,
516
- )
517
-
518
- match_result = _match_birdnet_rows(
519
- project_slug=project_slug,
520
- detections_csv=detections_csv,
521
- segments_root=segments_root,
522
- )
523
- frame = match_result["frame"]
524
- matched_rows = match_result["matched_rows"]
525
- unmatched_rows = match_result["unmatched_rows"]
526
- ambiguous_rows = match_result["ambiguous_rows"]
527
-
528
- # Deduplicate by remote path so retries/resume work on a stable key.
529
- path_to_local: dict[str, Path] = {}
530
- for row in matched_rows:
531
- repo_path = f"audio/segments/{row['segment_relpath']}"
532
- if repo_path in path_to_local:
533
- continue
534
- path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
535
-
536
- state_path = Path(resume_state_file)
537
- state_payload = _load_resume_state(state_file=state_path)
538
- uploaded_state = set(str(item) for item in state_payload["uploaded"])
539
- failed_state = set(str(item) for item in state_payload["failed"])
540
-
541
- remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
542
-
543
- pending: list[tuple[Path, str]] = []
544
- skipped_existing = 0
545
- for repo_path, local_path in path_to_local.items():
546
- if repo_path in remote_files:
547
- uploaded_state.add(repo_path)
548
- skipped_existing += 1
549
- continue
550
- if repo_path in uploaded_state:
551
- continue
552
- pending.append((local_path, repo_path))
553
-
554
- uploaded_now: set[str] = set()
555
- failed_now: set[str] = set()
556
-
557
- for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
558
- batch_uploaded = 0
559
- batch_failed = 0
560
- for local_path, repo_path in batch:
561
- try:
562
- _upload_audio_with_retry(
563
- api=api,
564
- dataset_repo=dataset_repo,
565
- local_path=local_path,
566
- path_in_repo=repo_path,
567
- max_retries=max_retries,
568
- retry_backoff_seconds=retry_backoff_seconds,
569
- )
570
- uploaded_state.add(repo_path)
571
- failed_state.discard(repo_path)
572
- uploaded_now.add(repo_path)
573
- batch_uploaded += 1
574
- except Exception:
575
- failed_state.add(repo_path)
576
- failed_now.add(repo_path)
577
- batch_failed += 1
578
-
579
- print(
580
- json.dumps(
581
- {
582
- "event": "ingest-segments-audio-batch",
583
- "batch_index": batch_index,
584
- "batch_size": len(batch),
585
- "uploaded": batch_uploaded,
586
- "failed": batch_failed,
587
- }
588
- )
589
- )
590
- _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
591
-
592
- index_frame = pd.DataFrame(matched_rows)
593
- if not index_frame.empty:
594
- index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
595
-
596
- with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
597
- temp_dir = Path(temp_dir_str)
598
- shard_dir = temp_dir / "index" / "shards"
599
- shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
600
-
601
- for item in shard_metadata:
602
- local_path = temp_dir / item.path
603
- api.upload_file(
604
- path_or_fileobj=str(local_path),
605
- path_in_repo=item.path,
606
- repo_id=dataset_repo,
607
- repo_type="dataset",
608
- )
609
-
610
- manifest = build_manifest_payload(
611
- project_slug=project_slug,
612
- dataset_repo=dataset_repo,
613
- frame=index_frame,
614
- shard_size=shard_size,
615
- shard_metadata=shard_metadata,
616
- )
617
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
618
-
619
- run_report = {
620
- "mode": "execute",
621
- "project_slug": project_slug,
622
- "dataset_repo": dataset_repo,
623
- "detections_csv": detections_csv,
624
- "segments_root": segments_root,
625
- "csv_rows_total": int(len(frame)),
626
- "segments_found_total": int(match_result["segment_records_total"]),
627
- "matched_rows": len(matched_rows),
628
- "unmatched_rows": len(unmatched_rows),
629
- "ambiguous_rows": len(ambiguous_rows),
630
- "unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
631
- "pending_audio_uploads": len(pending),
632
- "uploaded_audio_now": len(uploaded_now),
633
- "uploaded_audio_skipped_existing": skipped_existing,
634
- "failed_uploads": len(failed_now),
635
- "resume_state_file": str(state_path),
636
- "index_rows_written": len(index_frame),
637
- "shards_written": len(shard_metadata),
638
- "sample_unmatched": unmatched_rows[:20],
639
- "sample_ambiguous": ambiguous_rows[:20],
640
- }
641
-
642
- audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
643
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
644
- return run_report
645
-
646
-
647
- def sync_audio_batches(
648
- api: HfApi,
649
- project_slug: str,
650
- dataset_repo: str,
651
- local_audio_dir: str,
652
- batch_size: int,
653
- max_retries: int,
654
- retry_backoff_seconds: float,
655
- resume_state_file: str,
656
- ) -> dict[str, Any]:
657
- ensure_project_dataset_structure(
658
- api=api,
659
- project_slug=project_slug,
660
- dataset_repo=dataset_repo,
661
- create_private_repo=False,
662
- )
663
-
664
- all_files = discover_audio_files(local_audio_dir=local_audio_dir)
665
- state_path = Path(resume_state_file)
666
- state_payload = _load_resume_state(state_file=state_path)
667
- uploaded_state = set(str(item) for item in state_payload["uploaded"])
668
-
669
- remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
670
-
671
- pending: list[tuple[Path, str]] = []
672
- for path in all_files:
673
- rel = path.relative_to(Path(local_audio_dir)).as_posix()
674
- repo_path = f"audio/{rel}"
675
-
676
- if repo_path in remote_files:
677
- uploaded_state.add(repo_path)
678
- continue
679
-
680
- if repo_path in uploaded_state:
681
- continue
682
-
683
- pending.append((path, repo_path))
684
-
685
- pending_paths = [item[0] for item in pending]
686
- by_local_to_repo = {local: repo for local, repo in pending}
687
- batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
688
-
689
- uploaded_now: set[str] = set()
690
- failed: set[str] = set()
691
-
692
- for batch_index, batch in enumerate(batches, start=1):
693
- batch_uploaded = 0
694
- batch_failed = 0
695
-
696
- for local_path in batch:
697
- repo_path = by_local_to_repo[local_path]
698
- try:
699
- _upload_audio_with_retry(
700
- api=api,
701
- dataset_repo=dataset_repo,
702
- local_path=local_path,
703
- path_in_repo=repo_path,
704
- max_retries=max_retries,
705
- retry_backoff_seconds=retry_backoff_seconds,
706
- )
707
- uploaded_now.add(repo_path)
708
- uploaded_state.add(repo_path)
709
- batch_uploaded += 1
710
- except Exception:
711
- failed.add(repo_path)
712
- batch_failed += 1
713
-
714
- print(
715
- json.dumps(
716
- {
717
- "event": "sync-audio-batch",
718
- "batch_index": batch_index,
719
- "batch_size": len(batch),
720
- "uploaded": batch_uploaded,
721
- "failed": batch_failed,
722
- }
723
- )
724
- )
725
- _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
726
-
727
- return {
728
- "project_slug": project_slug,
729
- "dataset_repo": dataset_repo,
730
- "total_local_audio_files": len(all_files),
731
- "pending_uploads": len(pending),
732
- "uploaded_now": len(uploaded_now),
733
- "failed": len(failed),
734
- "resume_state_file": str(state_path),
735
- }
736
-
737
-
738
- def build_and_upload_index(
739
- api: HfApi,
740
- project_slug: str,
741
- dataset_repo: str,
742
- detections_file: str,
743
- shard_size: int,
744
- ) -> dict[str, Any]:
745
- ensure_project_dataset_structure(
746
- api=api,
747
- project_slug=project_slug,
748
- dataset_repo=dataset_repo,
749
- create_private_repo=False,
750
- )
751
-
752
- frame = load_detections_table(detections_file=detections_file)
753
-
754
- with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
755
- temp_dir = Path(temp_dir_str)
756
- shard_dir = temp_dir / "index" / "shards"
757
- metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
758
-
759
- for item in metadata:
760
- local_path = temp_dir / item.path
761
- api.upload_file(
762
- path_or_fileobj=str(local_path),
763
- path_in_repo=item.path,
764
- repo_id=dataset_repo,
765
- repo_type="dataset",
766
- )
767
-
768
- manifest = build_manifest_payload(
769
- project_slug=project_slug,
770
- dataset_repo=dataset_repo,
771
- frame=frame,
772
- shard_size=shard_size,
773
- shard_metadata=metadata,
774
- )
775
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
776
-
777
- return {
778
- "project_slug": project_slug,
779
- "dataset_repo": dataset_repo,
780
- "total_detections": manifest["index"]["total_detections"],
781
- "total_audio_files": manifest["index"]["total_audio_files"],
782
- "total_shards": len(metadata),
783
- }
784
-
785
-
786
- def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
787
- errors: list[str] = []
788
-
789
- for prefix in REQUIRED_PROJECT_PREFIXES:
790
- if not any(path.startswith(prefix) for path in repo_files):
791
- errors.append(f"Missing prefix in dataset repo: {prefix}")
792
-
793
- if manifest_payload.get("schema_version") != SCHEMA_VERSION:
794
- errors.append(
795
- f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
796
- )
797
-
798
- if manifest_payload.get("project_slug") != project_slug:
799
- errors.append(
800
- f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
801
- )
802
-
803
- index_section = manifest_payload.get("index", {})
804
- shards = index_section.get("shards", [])
805
- if not isinstance(shards, list):
806
- errors.append("Manifest index.shards must be a list")
807
- return errors
808
-
809
- for shard in shards:
810
- shard_path = shard.get("path")
811
- if not shard_path or not isinstance(shard_path, str):
812
- errors.append("Shard entry missing path")
813
- continue
814
- if shard_path not in repo_files:
815
- errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
816
-
817
- return errors
818
-
819
-
820
- def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
821
- repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
822
- if "manifest.json" not in repo_files:
823
- return {
824
- "ok": False,
825
- "errors": ["manifest.json not found in dataset repository"],
826
- }
827
-
828
- manifest_local = hf_hub_download(
829
- repo_id=dataset_repo,
830
- repo_type="dataset",
831
- filename="manifest.json",
832
- )
833
- manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
834
- errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
835
-
836
- return {
837
- "ok": len(errors) == 0,
838
- "errors": errors,
839
- "total_files": len(repo_files),
840
- "total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
841
- }
842
-
843
-
844
- def build_parser() -> argparse.ArgumentParser:
845
- parser = argparse.ArgumentParser(description="HF dataset project CLI")
846
- sub = parser.add_subparsers(dest="command", required=True)
847
-
848
- create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
849
- create_project.add_argument("--project-slug", required=True)
850
- create_project.add_argument("--dataset-repo", required=True)
851
- create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
852
-
853
- build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
854
- build_index.add_argument("--project-slug", required=True)
855
- build_index.add_argument("--dataset-repo", required=True)
856
- build_index.add_argument("--detections-file", required=True)
857
- build_index.add_argument("--shard-size", type=int, default=10000)
858
-
859
- sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
860
- sync_audio.add_argument("--project-slug", required=True)
861
- sync_audio.add_argument("--dataset-repo", required=True)
862
- sync_audio.add_argument("--local-audio-dir", required=True)
863
- sync_audio.add_argument("--batch-size", type=int, default=100)
864
- sync_audio.add_argument("--max-retries", type=int, default=3)
865
- sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
866
- sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
867
-
868
- verify_project = sub.add_parser("verify-project", help="Verify project integrity")
869
- verify_project.add_argument("--project-slug", required=True)
870
- verify_project.add_argument("--dataset-repo", required=True)
871
-
872
- ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
873
- ingest_segments.add_argument("--project-slug", required=True)
874
- ingest_segments.add_argument("--dataset-repo", required=True)
875
- ingest_segments.add_argument("--detections-csv", required=True)
876
- ingest_segments.add_argument("--segments-root", required=True)
877
- ingest_segments.add_argument("--batch-size", type=int, default=200)
878
- ingest_segments.add_argument("--shard-size", type=int, default=10000)
879
- ingest_segments.add_argument("--max-retries", type=int, default=3)
880
- ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
881
- ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
882
- ingest_segments.add_argument("--dry-run", action="store_true")
883
- ingest_segments.add_argument("--report-file", default="")
884
-
885
- return parser
886
-
887
-
888
- def main() -> int:
889
- parser = build_parser()
890
- args = parser.parse_args()
891
- api = HfApi()
892
-
893
- if args.command == "create-project":
894
- result = ensure_project_dataset_structure(
895
- api=api,
896
- project_slug=args.project_slug,
897
- dataset_repo=args.dataset_repo,
898
- create_private_repo=bool(args.private),
899
- )
900
- print(json.dumps(result, indent=2))
901
- return 0
902
-
903
- if args.command == "build-index":
904
- result = build_and_upload_index(
905
- api=api,
906
- project_slug=args.project_slug,
907
- dataset_repo=args.dataset_repo,
908
- detections_file=args.detections_file,
909
- shard_size=args.shard_size,
910
- )
911
- print(json.dumps(result, indent=2))
912
- return 0
913
-
914
- if args.command == "sync-audio":
915
- result = sync_audio_batches(
916
- api=api,
917
- project_slug=args.project_slug,
918
- dataset_repo=args.dataset_repo,
919
- local_audio_dir=args.local_audio_dir,
920
- batch_size=args.batch_size,
921
- max_retries=args.max_retries,
922
- retry_backoff_seconds=args.retry_backoff_seconds,
923
- resume_state_file=args.resume_state_file,
924
- )
925
- print(json.dumps(result, indent=2))
926
- return 0 if result["failed"] == 0 else 1
927
-
928
- if args.command == "verify-project":
929
- result = verify_project(
930
- api=api,
931
- project_slug=args.project_slug,
932
- dataset_repo=args.dataset_repo,
933
- )
934
- print(json.dumps(result, indent=2))
935
- return 0 if result["ok"] else 1
936
-
937
- if args.command == "ingest-segments":
938
- if args.dry_run:
939
- result = run_ingest_segments_dry_run(
940
- project_slug=args.project_slug,
941
- detections_csv=args.detections_csv,
942
- segments_root=args.segments_root,
943
- )
944
- else:
945
- result = ingest_segments_to_hf(
946
- api=api,
947
- project_slug=args.project_slug,
948
- dataset_repo=args.dataset_repo,
949
- detections_csv=args.detections_csv,
950
- segments_root=args.segments_root,
951
- batch_size=args.batch_size,
952
- shard_size=args.shard_size,
953
- max_retries=args.max_retries,
954
- retry_backoff_seconds=args.retry_backoff_seconds,
955
- resume_state_file=args.resume_state_file,
956
- )
957
-
958
- if args.report_file:
959
- Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
960
-
961
- print(json.dumps(result, indent=2))
962
- return 0 if result.get("failed_uploads", 0) == 0 else 1
963
-
964
- parser.error("Unknown command")
965
- return 2
966
-
967
-
968
- if __name__ == "__main__":
969
- raise SystemExit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Uploader App/pyproject.toml DELETED
@@ -1,9 +0,0 @@
1
- [project]
2
- name = "BirdNET-validator-App"
3
- version = "0.1.0"
4
- description = "BirdNET detection validator for Hugging Face Spaces"
5
- readme = "README.md"
6
- requires-python = ">=3.11"
7
-
8
- [tool.pytest.ini_options]
9
- testpaths = ["tests"]
 
 
 
 
 
 
 
 
 
 
BirdNET Uploader App/requirements.txt DELETED
@@ -1,7 +0,0 @@
1
- gradio>=5.0.0
2
- huggingface_hub>=0.30.0
3
- pydantic>=2.8.0
4
- pandas>=2.2.0
5
- pyarrow>=17.0.0
6
- soundfile>=0.12.1
7
- pytest>=8.3.0
 
 
 
 
 
 
 
 
BirdNET Uploader App/src/__init__.py DELETED
File without changes
BirdNET Uploader App/src/ui/__init__.py DELETED
File without changes
BirdNET Uploader App/src/ui/upload_app_factory.py DELETED
@@ -1,177 +0,0 @@
1
- import json
2
- from pathlib import Path
3
- from typing import Any
4
-
5
- import gradio as gr
6
- from huggingface_hub import HfApi
7
-
8
- from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
9
-
10
-
11
- def _as_pretty_json(payload: dict[str, Any]) -> str:
12
- return json.dumps(payload, indent=2, ensure_ascii=False)
13
-
14
-
15
- def _build_api(token: str) -> HfApi:
16
- clean_token = (token or "").strip()
17
- if clean_token:
18
- return HfApi(token=clean_token)
19
- return HfApi()
20
-
21
-
22
- def build_upload_app() -> gr.Blocks:
23
- with gr.Blocks(title="BirdNET Segments Uploader") as demo:
24
- gr.Markdown("# BirdNET Segments Uploader")
25
- gr.Markdown(
26
- "Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
27
- "Recomendado: executar dry-run antes do upload real."
28
- )
29
-
30
- with gr.Row():
31
- project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
32
- dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
33
-
34
- detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
35
- segments_root = gr.Textbox(
36
- label="Pasta raiz de segmentos",
37
- placeholder=r"ex: C:\dados\BirdNET Segments",
38
- )
39
-
40
- with gr.Accordion("Configuracao avancada", open=False):
41
- hf_token = gr.Textbox(
42
- label="HF Token (opcional)",
43
- type="password",
44
- placeholder="Se vazio, usa sessao ja autenticada no ambiente",
45
- )
46
- with gr.Row():
47
- batch_size = gr.Number(label="Batch size", value=200, precision=0)
48
- shard_size = gr.Number(label="Shard size", value=10000, precision=0)
49
- with gr.Row():
50
- max_retries = gr.Number(label="Max retries", value=3, precision=0)
51
- retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
52
- resume_state_file = gr.Textbox(
53
- label="Resume state file",
54
- value=".ingest-segments-state.json",
55
- )
56
- report_file = gr.Textbox(
57
- label="Report file local (opcional)",
58
- placeholder="ex: .ingest-run-report.json",
59
- )
60
-
61
- with gr.Row():
62
- dry_run_button = gr.Button("Dry-run", variant="secondary")
63
- upload_button = gr.Button("Upload real", variant="primary")
64
-
65
- status = gr.Markdown(value="Pronto")
66
- result_json = gr.Code(label="Resultado JSON", language="json")
67
-
68
- def run_dry_run(
69
- project: str,
70
- repo: str,
71
- csv_path: str | None,
72
- segments_path: str,
73
- report_path: str,
74
- ) -> tuple[str, str]:
75
- project = (project or "").strip()
76
- if not project:
77
- return "❌ Informe o project slug", "{}"
78
-
79
- if not csv_path:
80
- return "❌ Selecione o CSV de deteccoes", "{}"
81
-
82
- if not segments_path or not Path(segments_path).exists():
83
- return "❌ Pasta de segmentos invalida", "{}"
84
-
85
- try:
86
- result = run_ingest_segments_dry_run(
87
- project_slug=project,
88
- detections_csv=csv_path,
89
- segments_root=segments_path,
90
- )
91
- if report_path.strip():
92
- Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
93
- summary = (
94
- "✅ Dry-run finalizado | "
95
- f"Matched: {result['matched_rows']} | "
96
- f"Unmatched: {result['unmatched_rows']} | "
97
- f"Ambiguous: {result['ambiguous_rows']}"
98
- )
99
- return summary, _as_pretty_json(result)
100
- except Exception as exc:
101
- return f"❌ Dry-run falhou: {exc}", "{}"
102
-
103
- def run_upload(
104
- project: str,
105
- repo: str,
106
- csv_path: str | None,
107
- segments_path: str,
108
- token: str,
109
- batch: float,
110
- shard: float,
111
- retries: float,
112
- backoff: float,
113
- resume_file: str,
114
- report_path: str,
115
- ) -> tuple[str, str]:
116
- project = (project or "").strip()
117
- repo = (repo or "").strip()
118
- if not project:
119
- return "❌ Informe o project slug", "{}"
120
- if not repo or "/" not in repo:
121
- return "❌ Informe dataset repo no formato owner/repo", "{}"
122
- if not csv_path:
123
- return "❌ Selecione o CSV de deteccoes", "{}"
124
- if not segments_path or not Path(segments_path).exists():
125
- return "❌ Pasta de segmentos invalida", "{}"
126
-
127
- try:
128
- api = _build_api(token)
129
- result = ingest_segments_to_hf(
130
- api=api,
131
- project_slug=project,
132
- dataset_repo=repo,
133
- detections_csv=csv_path,
134
- segments_root=segments_path,
135
- batch_size=int(batch),
136
- shard_size=int(shard),
137
- max_retries=int(retries),
138
- retry_backoff_seconds=float(backoff),
139
- resume_state_file=resume_file,
140
- )
141
- if report_path.strip():
142
- Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
143
- summary = (
144
- "✅ Upload finalizado | "
145
- f"Uploaded now: {result['uploaded_audio_now']} | "
146
- f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
147
- f"Failed uploads: {result['failed_uploads']}"
148
- )
149
- return summary, _as_pretty_json(result)
150
- except Exception as exc:
151
- return f"❌ Upload falhou: {exc}", "{}"
152
-
153
- dry_run_button.click(
154
- fn=run_dry_run,
155
- inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
156
- outputs=[status, result_json],
157
- )
158
-
159
- upload_button.click(
160
- fn=run_upload,
161
- inputs=[
162
- project_slug,
163
- dataset_repo,
164
- detections_csv,
165
- segments_root,
166
- hf_token,
167
- batch_size,
168
- shard_size,
169
- max_retries,
170
- retry_backoff_seconds,
171
- resume_state_file,
172
- report_file,
173
- ],
174
- outputs=[status, result_json],
175
- )
176
-
177
- return demo
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/README.md DELETED
@@ -1,83 +0,0 @@
1
- # BirdNET Validator App
2
-
3
- Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
4
- - Multi-projeto
5
- - Audio sob demanda
6
- - Persistencia de validacoes
7
- - Escalabilidade para datasets grandes
8
-
9
- ## Desenvolvimento local
10
- 1. Criar ambiente virtual Python 3.11+
11
- 2. Instalar dependencias:
12
- pip install -r requirements.txt
13
- 3. Rodar app:
14
- python app.py
15
-
16
- ## Objetivo desta pasta
17
- Contem os arquivos do app de validacao para evolucao/publicacao independente.
18
-
19
- ## Fluxo recomendado (separado)
20
- 1. Uploader app (porta 7862 por padrao):
21
- - Rodar dry-run com CSV + pasta de segmentos
22
- - Rodar upload real para owner/repo do projeto
23
- 2. Validator app (porta 7860):
24
- - Validar deteccoes do projeto com dados ja publicados
25
-
26
- Essa separacao evita impacto de performance na validacao durante uploads grandes.
27
-
28
- ## CLI de dataset (Sprint 1)
29
- Comandos principais:
30
-
31
- 1. Criar estrutura do projeto no dataset Hugging Face:
32
- python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
33
-
34
- 2. Gerar manifesto e shards iniciais de indice:
35
- python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
36
-
37
- 3. Fazer upload de audio em lotes com retry e resume:
38
- python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
39
-
40
- 4. Verificar consistencia basica do projeto:
41
- python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
42
-
43
- 5. Ingestao BirdNET por projeto (dry-run):
44
- python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
45
-
46
- 6. Ingestao BirdNET por projeto (upload real):
47
- python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
48
-
49
- Estrutura base criada por projeto/dataset:
50
- - audio/
51
- - index/
52
- - index/shards/
53
- - validations/
54
- - audit/
55
- - manifest.json
56
-
57
- ## Estrutura
58
- - src/domain: modelos de dominio
59
- - src/repositories: contratos de persistencia
60
- - src/services: servicos de aplicacao
61
- - src/auth: autenticacao e autorizacao
62
- - src/ui: montagem da interface
63
- - src/cache: cache efemero local
64
- - cli: comandos de ingestao/publicacao
65
- - tests: testes unitarios e integracao
66
-
67
- ## Estado atual de implementacao
68
- - Sprint 1: CLI create-project, sync-audio, build-index, verify-project
69
- - Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
70
- - Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
71
- - Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
72
- - Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
73
- - Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
74
- - Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
75
- - Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
76
- - Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
77
- - Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
78
- - Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
79
- - Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
80
- - Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
81
- - Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
82
- - Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
83
- - Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/app.py DELETED
@@ -1,6 +0,0 @@
1
- from src.ui.app_factory import create_app
2
-
3
-
4
- if __name__ == "__main__":
5
- app = create_app()
6
- app.launch(server_name="0.0.0.0", server_port=7860)
 
 
 
 
 
 
 
BirdNET Validator App/pyproject.toml DELETED
@@ -1,9 +0,0 @@
1
- [project]
2
- name = "BirdNET-validator-App"
3
- version = "0.1.0"
4
- description = "BirdNET detection validator for Hugging Face Spaces"
5
- readme = "README.md"
6
- requires-python = ">=3.11"
7
-
8
- [tool.pytest.ini_options]
9
- testpaths = ["tests"]
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/requirements.txt DELETED
@@ -1,7 +0,0 @@
1
- gradio>=5.0.0
2
- huggingface_hub>=0.30.0
3
- pydantic>=2.8.0
4
- pandas>=2.2.0
5
- pyarrow>=17.0.0
6
- soundfile>=0.12.1
7
- pytest>=8.3.0
 
 
 
 
 
 
 
 
BirdNET Validator App/src/__init__.py DELETED
File without changes
BirdNET Validator App/src/auth/__init__.py DELETED
File without changes
BirdNET Validator App/src/auth/auth_service.py DELETED
@@ -1,200 +0,0 @@
1
- """AuthService: User authentication, session management, and project ACL."""
2
-
3
- from dataclasses import dataclass, field
4
- from datetime import datetime, timedelta
5
- from typing import Dict, List, Optional
6
- import uuid
7
-
8
- from src.domain.models import Role, User
9
-
10
-
11
- @dataclass
12
- class UserProjectAccess:
13
- """Maps a user to projects with specific roles."""
14
-
15
- username: str
16
- project_slugs: Dict[str, Role] # Maps project_slug -> role (admin or validator)
17
- is_active: bool = True
18
-
19
-
20
- @dataclass
21
- class Session:
22
- """Represents an authenticated user session."""
23
-
24
- session_id: str
25
- username: str
26
- role: Role
27
- authorized_projects: List[str] # Projects this user can access
28
- created_at: datetime
29
- last_activity: datetime
30
- expires_at: datetime
31
-
32
- def is_expired(self) -> bool:
33
- """Check if session has expired."""
34
- return datetime.utcnow() > self.expires_at
35
-
36
- def update_activity(self, ttl_minutes: int = 120) -> None:
37
- """Update last activity timestamp and extend expiration."""
38
- self.last_activity = datetime.utcnow()
39
- self.expires_at = datetime.utcnow() + timedelta(minutes=ttl_minutes)
40
-
41
-
42
- class AuthService:
43
- """Manages user authentication, sessions, and project access control."""
44
-
45
- def __init__(self, session_ttl_minutes: int = 120):
46
- """Initialize AuthService with session TTL.
47
-
48
- Args:
49
- session_ttl_minutes: Session time-to-live in minutes (default: 2 hours)
50
- """
51
- self.session_ttl_minutes = session_ttl_minutes
52
- self._sessions: Dict[str, Session] = {} # session_id -> Session
53
- self._user_access: Dict[str, UserProjectAccess] = {} # username -> UserProjectAccess
54
-
55
- def register_user_project_access(
56
- self, username: str, project_access: Dict[str, Role]
57
- ) -> None:
58
- """Register or update a user's access to projects.
59
-
60
- Args:
61
- username: User name
62
- project_access: Dict mapping project_slug to Role (admin or validator)
63
- """
64
- self._user_access[username] = UserProjectAccess(
65
- username=username,
66
- project_slugs=project_access,
67
- is_active=True,
68
- )
69
-
70
- def login(self, username: str) -> Optional[Session]:
71
- """Authenticate a user and create a new session.
72
-
73
- Args:
74
- username: User name to authenticate
75
-
76
- Returns:
77
- Session if user exists and is active, None otherwise
78
- """
79
- if username not in self._user_access:
80
- return None
81
-
82
- access = self._user_access[username]
83
- if not access.is_active:
84
- return None
85
-
86
- # Determine highest role across all projects (admin > validator)
87
- role = Role.validator
88
- for proj_role in access.project_slugs.values():
89
- if proj_role == Role.admin:
90
- role = Role.admin
91
- break
92
-
93
- session_id = str(uuid.uuid4())
94
- now = datetime.utcnow()
95
- session = Session(
96
- session_id=session_id,
97
- username=username,
98
- role=role,
99
- authorized_projects=list(access.project_slugs.keys()),
100
- created_at=now,
101
- last_activity=now,
102
- expires_at=now + timedelta(minutes=self.session_ttl_minutes),
103
- )
104
-
105
- self._sessions[session_id] = session
106
- return session
107
-
108
- def get_session(self, session_id: str) -> Optional[Session]:
109
- """Retrieve an active session by ID.
110
-
111
- Args:
112
- session_id: Session ID
113
-
114
- Returns:
115
- Session if valid and not expired, None otherwise
116
- """
117
- session = self._sessions.get(session_id)
118
- if session is None:
119
- return None
120
-
121
- if session.is_expired():
122
- self._sessions.pop(session_id, None)
123
- return None
124
-
125
- session.update_activity(self.session_ttl_minutes)
126
- return session
127
-
128
- def logout(self, session_id: str) -> None:
129
- """Delete a session.
130
-
131
- Args:
132
- session_id: Session ID to invalidate
133
- """
134
- self._sessions.pop(session_id, None)
135
-
136
- def is_user_authorized_for_project(self, username: str, project_slug: str) -> bool:
137
- """Check if a user has access to a specific project.
138
-
139
- Args:
140
- username: User name
141
- project_slug: Project slug
142
-
143
- Returns:
144
- True if user has access (admin or validator), False otherwise
145
- """
146
- access = self._user_access.get(username)
147
- if access is None or not access.is_active:
148
- return False
149
-
150
- return project_slug in access.project_slugs
151
-
152
- def get_user_role_for_project(self, username: str, project_slug: str) -> Optional[Role]:
153
- """Get the specific role a user has for a project.
154
-
155
- Args:
156
- username: User name
157
- project_slug: Project slug
158
-
159
- Returns:
160
- Role if user has access, None otherwise
161
- """
162
- access = self._user_access.get(username)
163
- if access is None or not access.is_active:
164
- return None
165
-
166
- return access.project_slugs.get(project_slug)
167
-
168
- def list_user_projects(self, username: str) -> List[str]:
169
- """List all projects a user has access to.
170
-
171
- Args:
172
- username: User name
173
-
174
- Returns:
175
- List of project slugs
176
- """
177
- access = self._user_access.get(username)
178
- if access is None or not access.is_active:
179
- return []
180
-
181
- return list(access.project_slugs.keys())
182
-
183
- def cleanup_expired_sessions(self) -> None:
184
- """Remove all expired sessions (maintenance cleanup)."""
185
- now = datetime.utcnow()
186
- expired = [
187
- sid for sid, session in self._sessions.items() if session.is_expired()
188
- ]
189
- for sid in expired:
190
- self._sessions.pop(sid)
191
-
192
- def set_user_active(self, username: str, active: bool) -> None:
193
- """Enable or disable a user account.
194
-
195
- Args:
196
- username: User name
197
- active: Whether the user should be active
198
- """
199
- if username in self._user_access:
200
- self._user_access[username].is_active = active
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/cache/__init__.py DELETED
File without changes
BirdNET Validator App/src/cache/ephemeral_cache_manager.py DELETED
@@ -1,82 +0,0 @@
1
- import hashlib
2
- import tempfile
3
- import time
4
- from dataclasses import dataclass
5
- from pathlib import Path
6
-
7
-
8
- @dataclass(slots=True)
9
- class CacheEntry:
10
- key: str
11
- path: Path
12
- created_at: float
13
- expires_at: float
14
- size_bytes: int
15
-
16
-
17
- class EphemeralCacheManager:
18
- def __init__(
19
- self,
20
- cache_dir: str | None = None,
21
- ttl_seconds: int = 300,
22
- max_files: int = 256,
23
- ) -> None:
24
- self._base_dir = Path(cache_dir) if cache_dir else Path(tempfile.gettempdir()) / "birdnet-validator-cache"
25
- self._base_dir.mkdir(parents=True, exist_ok=True)
26
- self._ttl_seconds = ttl_seconds
27
- self._max_files = max_files
28
- self._entries: dict[str, CacheEntry] = {}
29
-
30
- def get(self, key: str) -> Path | None:
31
- self.cleanup_expired()
32
- entry = self._entries.get(key)
33
- if not entry:
34
- return None
35
- if not entry.path.exists():
36
- self._entries.pop(key, None)
37
- return None
38
- return entry.path
39
-
40
- def put_bytes(self, key: str, data: bytes, suffix: str = ".bin") -> Path:
41
- self.cleanup_expired()
42
- self._enforce_capacity()
43
-
44
- digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
45
- file_path = self._base_dir / f"{digest}{suffix}"
46
- file_path.write_bytes(data)
47
-
48
- now = time.time()
49
- self._entries[key] = CacheEntry(
50
- key=key,
51
- path=file_path,
52
- created_at=now,
53
- expires_at=now + self._ttl_seconds,
54
- size_bytes=len(data),
55
- )
56
- return file_path
57
-
58
- def cleanup_key(self, key: str) -> None:
59
- entry = self._entries.pop(key, None)
60
- if not entry:
61
- return
62
- if entry.path.exists():
63
- entry.path.unlink()
64
-
65
- def cleanup_expired(self) -> None:
66
- now = time.time()
67
- expired_keys = [key for key, entry in self._entries.items() if entry.expires_at <= now or not entry.path.exists()]
68
- for key in expired_keys:
69
- self.cleanup_key(key)
70
-
71
- def clear(self) -> None:
72
- for key in list(self._entries.keys()):
73
- self.cleanup_key(key)
74
-
75
- def _enforce_capacity(self) -> None:
76
- if len(self._entries) < self._max_files:
77
- return
78
-
79
- oldest_keys = sorted(self._entries.keys(), key=lambda key: self._entries[key].created_at)
80
- overflow = (len(self._entries) - self._max_files) + 1
81
- for key in oldest_keys[:overflow]:
82
- self.cleanup_key(key)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/domain/__init__.py DELETED
File without changes
BirdNET Validator App/src/domain/models.py DELETED
@@ -1,46 +0,0 @@
1
- from enum import Enum
2
- from typing import Optional
3
-
4
- from pydantic import BaseModel, Field
5
-
6
-
7
- class Role(str, Enum):
8
- admin = "admin"
9
- validator = "validator"
10
-
11
-
12
- class User(BaseModel):
13
- username: str = Field(min_length=1)
14
- role: Role
15
- active: bool = True
16
-
17
-
18
- class Project(BaseModel):
19
- project_slug: str = Field(min_length=3)
20
- name: str = Field(min_length=1)
21
- dataset_repo_id: str = Field(min_length=3)
22
- active: bool = True
23
-
24
-
25
- class Detection(BaseModel):
26
- detection_key: str = Field(min_length=16)
27
- audio_id: str = Field(min_length=1)
28
- scientific_name: str = Field(min_length=1)
29
- confidence: float = Field(ge=0.0, le=1.0)
30
- start_time: float = Field(ge=0.0)
31
- end_time: float = Field(gt=0.0)
32
-
33
-
34
- class Validation(BaseModel):
35
- detection_key: str = Field(min_length=16)
36
- status: str = Field(min_length=1)
37
- corrected_species: Optional[str] = None
38
- notes: str = ""
39
- validator: str = Field(min_length=1)
40
-
41
-
42
- class IndexManifest(BaseModel):
43
- schema_version: str = "1.0.0"
44
- project_slug: str
45
- total_detections: int = Field(ge=0)
46
- total_audio_files: int = Field(ge=0)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/repositories/__init__.py DELETED
File without changes
BirdNET Validator App/src/repositories/append_only_validation_repository.py DELETED
@@ -1,89 +0,0 @@
1
- import json
2
- from datetime import UTC, datetime
3
- from pathlib import Path
4
- from uuid import uuid4
5
-
6
- from src.domain.models import Validation
7
-
8
-
9
- class OptimisticLockError(Exception):
10
- def __init__(self, detection_key: str, expected_version: int, current_version: int) -> None:
11
- self.detection_key = detection_key
12
- self.expected_version = expected_version
13
- self.current_version = current_version
14
- super().__init__(
15
- f"Optimistic lock conflict for detection_key={detection_key}: "
16
- f"expected_version={expected_version}, current_version={current_version}"
17
- )
18
-
19
-
20
- class AppendOnlyValidationRepository:
21
- def __init__(self, base_dir: str) -> None:
22
- self._base_dir = Path(base_dir)
23
-
24
- def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
25
- project_dir = self._base_dir / project_slug / "validations"
26
- project_dir.mkdir(parents=True, exist_ok=True)
27
-
28
- current_file = project_dir / "current.json"
29
- if current_file.exists():
30
- current_payload = json.loads(current_file.read_text(encoding="utf-8"))
31
- else:
32
- current_payload = {}
33
-
34
- current_item = current_payload.get(item.detection_key, {})
35
- current_version = int(current_item.get("version", 0))
36
- expected = expected_version if expected_version is not None else current_version
37
- if expected != current_version:
38
- raise OptimisticLockError(
39
- detection_key=item.detection_key,
40
- expected_version=expected,
41
- current_version=current_version,
42
- )
43
-
44
- new_version = current_version + 1
45
-
46
- event_date = datetime.now(UTC).strftime("%Y%m%d")
47
- events_file = project_dir / f"events-{event_date}.jsonl"
48
- event = {
49
- "event_id": str(uuid4()),
50
- "timestamp": datetime.now(UTC).isoformat(),
51
- "project_slug": project_slug,
52
- "expected_version": expected,
53
- "previous_version": current_version,
54
- "new_version": new_version,
55
- **item.model_dump(),
56
- }
57
- with events_file.open("a", encoding="utf-8") as handle:
58
- handle.write(json.dumps(event, ensure_ascii=True) + "\n")
59
-
60
- current_payload[item.detection_key] = {
61
- "status": item.status,
62
- "corrected_species": item.corrected_species,
63
- "notes": item.notes,
64
- "validator": item.validator,
65
- "updated_at": event["timestamp"],
66
- "version": new_version,
67
- }
68
- current_file.write_text(json.dumps(current_payload, indent=2), encoding="utf-8")
69
- return new_version
70
-
71
- def list_events(self, project_slug: str) -> list[dict[str, object]]:
72
- project_dir = self._base_dir / project_slug / "validations"
73
- if not project_dir.exists():
74
- return []
75
-
76
- events: list[dict[str, object]] = []
77
- for path in sorted(project_dir.glob("events-*.jsonl")):
78
- with path.open("r", encoding="utf-8") as handle:
79
- for line in handle:
80
- payload = line.strip()
81
- if payload:
82
- events.append(json.loads(payload))
83
- return events
84
-
85
- def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]:
86
- current_file = self._base_dir / project_slug / "validations" / "current.json"
87
- if not current_file.exists():
88
- return {}
89
- return json.loads(current_file.read_text(encoding="utf-8"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/repositories/contracts.py DELETED
@@ -1,35 +0,0 @@
1
- from typing import Protocol
2
-
3
- from src.domain.models import Detection, Project, User, Validation
4
-
5
-
6
- class ProjectRepository(Protocol):
7
- def get_project(self, project_slug: str) -> Project: ...
8
-
9
-
10
- class DetectionRepository(Protocol):
11
- def list_detections(
12
- self,
13
- project_slug: str,
14
- page: int,
15
- page_size: int,
16
- scientific_name: str | None = None,
17
- min_confidence: float | None = None,
18
- max_confidence: float | None = None,
19
- ) -> list[Detection]: ...
20
-
21
- def count_detections(
22
- self,
23
- project_slug: str,
24
- scientific_name: str | None = None,
25
- min_confidence: float | None = None,
26
- max_confidence: float | None = None,
27
- ) -> int: ...
28
-
29
-
30
- class ValidationRepository(Protocol):
31
- def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int: ...
32
-
33
-
34
- class AuthRepository(Protocol):
35
- def authenticate(self, username: str) -> User: ...
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/repositories/in_memory_detection_repository.py DELETED
@@ -1,73 +0,0 @@
1
- from collections import defaultdict
2
- from typing import DefaultDict
3
-
4
- from src.domain.models import Detection
5
-
6
-
7
- class InMemoryDetectionRepository:
8
- def __init__(self) -> None:
9
- self._project_items: DefaultDict[str, list[Detection]] = defaultdict(list)
10
-
11
- def seed(self, project_slug: str, detections: list[Detection]) -> None:
12
- self._project_items[project_slug] = list(detections)
13
-
14
- def list_detections(
15
- self,
16
- project_slug: str,
17
- page: int,
18
- page_size: int,
19
- scientific_name: str | None = None,
20
- min_confidence: float | None = None,
21
- max_confidence: float | None = None,
22
- ) -> list[Detection]:
23
- if page < 1:
24
- raise ValueError("page must be >= 1")
25
- if page_size < 1:
26
- raise ValueError("page_size must be >= 1")
27
-
28
- filtered = self._apply_filters(
29
- project_slug=project_slug,
30
- scientific_name=scientific_name,
31
- min_confidence=min_confidence,
32
- max_confidence=max_confidence,
33
- )
34
-
35
- start = (page - 1) * page_size
36
- end = start + page_size
37
- return filtered[start:end]
38
-
39
- def count_detections(
40
- self,
41
- project_slug: str,
42
- scientific_name: str | None = None,
43
- min_confidence: float | None = None,
44
- max_confidence: float | None = None,
45
- ) -> int:
46
- return len(
47
- self._apply_filters(
48
- project_slug=project_slug,
49
- scientific_name=scientific_name,
50
- min_confidence=min_confidence,
51
- max_confidence=max_confidence,
52
- )
53
- )
54
-
55
- def _apply_filters(
56
- self,
57
- project_slug: str,
58
- scientific_name: str | None,
59
- min_confidence: float | None,
60
- max_confidence: float | None,
61
- ) -> list[Detection]:
62
- items = self._project_items.get(project_slug, [])
63
-
64
- def match(item: Detection) -> bool:
65
- if scientific_name and item.scientific_name != scientific_name:
66
- return False
67
- if min_confidence is not None and item.confidence < min_confidence:
68
- return False
69
- if max_confidence is not None and item.confidence > max_confidence:
70
- return False
71
- return True
72
-
73
- return [item for item in items if match(item)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/repositories/in_memory_validation_repository.py DELETED
@@ -1,25 +0,0 @@
1
- from collections import defaultdict
2
- from typing import DefaultDict
3
-
4
- from src.domain.models import Validation
5
- from src.repositories.append_only_validation_repository import OptimisticLockError
6
-
7
-
8
- class InMemoryValidationRepository:
9
- def __init__(self) -> None:
10
- self._by_project: DefaultDict[str, list[Validation]] = defaultdict(list)
11
- self._versions: DefaultDict[str, dict[str, int]] = defaultdict(dict)
12
-
13
- def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
14
- current_version = int(self._versions[project_slug].get(item.detection_key, 0))
15
- expected = expected_version if expected_version is not None else current_version
16
- if expected != current_version:
17
- raise OptimisticLockError(item.detection_key, expected, current_version)
18
-
19
- self._by_project[project_slug].append(item)
20
- new_version = current_version + 1
21
- self._versions[project_slug][item.detection_key] = new_version
22
- return new_version
23
-
24
- def list_validations(self, project_slug: str) -> list[Validation]:
25
- return list(self._by_project.get(project_slug, []))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/services/__init__.py DELETED
File without changes
BirdNET Validator App/src/services/audio_fetch_service.py DELETED
@@ -1,54 +0,0 @@
1
- from dataclasses import dataclass
2
- from pathlib import Path
3
-
4
- from huggingface_hub import hf_hub_download
5
-
6
- from src.cache.ephemeral_cache_manager import EphemeralCacheManager
7
-
8
-
9
- SUPPORTED_AUDIO_EXTENSIONS = (".wav", ".mp3", ".flac", ".ogg", ".m4a")
10
-
11
-
12
- @dataclass(slots=True)
13
- class AudioFetchResult:
14
- cache_key: str
15
- local_path: str
16
- source: str
17
-
18
-
19
- class AudioFetchService:
20
- def __init__(self, cache_manager: EphemeralCacheManager) -> None:
21
- self._cache = cache_manager
22
-
23
- def fetch(self, dataset_repo: str, audio_id: str) -> AudioFetchResult:
24
- cache_key = f"{dataset_repo}:{audio_id}"
25
- cached_path = self._cache.get(cache_key)
26
- if cached_path:
27
- return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="cache")
28
-
29
- target_filename, downloaded_path = self._resolve_remote_filename(dataset_repo=dataset_repo, audio_id=audio_id)
30
- if downloaded_path is None:
31
- downloaded = hf_hub_download(repo_id=dataset_repo, repo_type="dataset", filename=target_filename)
32
- downloaded_path = Path(downloaded)
33
-
34
- suffix = downloaded_path.suffix if downloaded_path.suffix else ".bin"
35
- local_path = self._cache.put_bytes(cache_key, downloaded_path.read_bytes(), suffix=suffix)
36
- return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="remote")
37
-
38
- def cleanup_after_validation(self, cache_key: str) -> None:
39
- self._cache.cleanup_key(cache_key)
40
-
41
- def _resolve_remote_filename(self, dataset_repo: str, audio_id: str) -> tuple[str, Path | None]:
42
- audio_path = Path(audio_id)
43
- if audio_path.suffix:
44
- return f"audio/{audio_id}", None
45
-
46
- for extension in SUPPORTED_AUDIO_EXTENSIONS:
47
- candidate = f"audio/{audio_id}{extension}"
48
- try:
49
- downloaded = hf_hub_download(repo_id=dataset_repo, repo_type="dataset", filename=candidate)
50
- return candidate, Path(downloaded)
51
- except Exception:
52
- continue
53
-
54
- raise FileNotFoundError(f"Unable to locate audio file for audio_id: {audio_id}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/services/detection_queue_service.py DELETED
@@ -1,64 +0,0 @@
1
- from dataclasses import dataclass
2
-
3
- from src.domain.models import Detection
4
- from src.repositories.contracts import DetectionRepository
5
-
6
-
7
- @dataclass(slots=True)
8
- class DetectionPage:
9
- project_slug: str
10
- page: int
11
- page_size: int
12
- total_items: int
13
- total_pages: int
14
- has_next: bool
15
- has_previous: bool
16
- items: list[Detection]
17
-
18
-
19
- class DetectionQueueService:
20
- def __init__(self, repository: DetectionRepository) -> None:
21
- self._repository = repository
22
-
23
- def get_page(
24
- self,
25
- project_slug: str,
26
- page: int,
27
- page_size: int,
28
- scientific_name: str | None = None,
29
- min_confidence: float | None = None,
30
- max_confidence: float | None = None,
31
- ) -> DetectionPage:
32
- if page < 1:
33
- raise ValueError("page must be >= 1")
34
- if page_size < 1:
35
- raise ValueError("page_size must be >= 1")
36
-
37
- total_items = self._repository.count_detections(
38
- project_slug=project_slug,
39
- scientific_name=scientific_name,
40
- min_confidence=min_confidence,
41
- max_confidence=max_confidence,
42
- )
43
- total_pages = max(1, ((total_items - 1) // page_size) + 1) if total_items else 1
44
- page = min(page, total_pages)
45
-
46
- items = self._repository.list_detections(
47
- project_slug=project_slug,
48
- page=page,
49
- page_size=page_size,
50
- scientific_name=scientific_name,
51
- min_confidence=min_confidence,
52
- max_confidence=max_confidence,
53
- )
54
-
55
- return DetectionPage(
56
- project_slug=project_slug,
57
- page=page,
58
- page_size=page_size,
59
- total_items=total_items,
60
- total_pages=total_pages,
61
- has_next=page < total_pages,
62
- has_previous=page > 1,
63
- items=items,
64
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/services/validation_service.py DELETED
@@ -1,39 +0,0 @@
1
- from dataclasses import dataclass
2
-
3
- from src.domain.models import Validation
4
- from src.repositories.contracts import ValidationRepository
5
-
6
-
7
- @dataclass(slots=True)
8
- class ValidationWriteResult:
9
- item: Validation
10
- new_version: int
11
-
12
-
13
- class ValidationService:
14
- def __init__(self, repository: ValidationRepository) -> None:
15
- self._repository = repository
16
-
17
- def validate_detection(
18
- self,
19
- project_slug: str,
20
- detection_key: str,
21
- status: str,
22
- validator: str,
23
- notes: str = "",
24
- corrected_species: str | None = None,
25
- expected_version: int | None = None,
26
- ) -> ValidationWriteResult:
27
- item = Validation(
28
- detection_key=detection_key,
29
- status=status,
30
- corrected_species=corrected_species,
31
- notes=notes,
32
- validator=validator,
33
- )
34
- new_version = self._repository.save_validation(
35
- project_slug=project_slug,
36
- item=item,
37
- expected_version=expected_version,
38
- )
39
- return ValidationWriteResult(item=item, new_version=new_version)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/telemetry/__init__.py DELETED
File without changes
BirdNET Validator App/src/ui/__init__.py DELETED
File without changes
BirdNET Validator App/src/ui/admin_panel.py DELETED
@@ -1,281 +0,0 @@
1
- """Gradio admin panel for managing projects and user assignments."""
2
-
3
- from typing import List, Tuple
4
-
5
- import gradio as gr
6
-
7
- from src.auth.auth_service import AuthService, Session
8
- from src.domain.models import Project, Role
9
-
10
-
11
- class AdminPanelManager:
12
- """Management backend for the admin panel."""
13
-
14
- def __init__(self, auth_service: AuthService):
15
- """Initialize admin panel manager.
16
-
17
- Args:
18
- auth_service: AuthService instance
19
- """
20
- self.auth_service = auth_service
21
- self._projects: dict[str, Project] = {} # project_slug -> Project
22
- self._user_project_assignments: dict[str, dict[str, str]] = {} # username -> {project_slug -> role}
23
-
24
- def register_project(self, project: Project) -> bool:
25
- """Register a new project (idempotent).
26
-
27
- Args:
28
- project: Project to register
29
-
30
- Returns:
31
- True if new, False if already existed
32
- """
33
- if project.project_slug in self._projects:
34
- return False
35
-
36
- self._projects[project.project_slug] = project
37
- return True
38
-
39
- def list_projects(self) -> List[dict]:
40
- """List all projects as dictionaries for Gradio display.
41
-
42
- Returns:
43
- List of project dicts with slug, name, repo_id, active status
44
- """
45
- return [
46
- {
47
- "project_slug": p.project_slug,
48
- "name": p.name,
49
- "dataset_repo_id": p.dataset_repo_id,
50
- "active": p.active,
51
- }
52
- for p in self._projects.values()
53
- ]
54
-
55
- def get_project(self, project_slug: str) -> Project | None:
56
- """Get project by slug.
57
-
58
- Args:
59
- project_slug: Project slug
60
-
61
- Returns:
62
- Project if found, None otherwise
63
- """
64
- return self._projects.get(project_slug)
65
-
66
- def list_users_for_project(self, project_slug: str) -> List[dict]:
67
- """List all users assigned to a project.
68
-
69
- Args:
70
- project_slug: Project slug
71
-
72
- Returns:
73
- List of dicts with username and role
74
- """
75
- result = []
76
- for username in self.auth_service._user_access.keys():
77
- role = self.auth_service.get_user_role_for_project(username, project_slug)
78
- if role is not None:
79
- result.append({"username": username, "role": role.value})
80
-
81
- return result
82
-
83
- def assign_user_to_project(
84
- self, username: str, project_slug: str, role: str
85
- ) -> Tuple[bool, str]:
86
- """Assign a user to a project with a role.
87
-
88
- Args:
89
- username: Username
90
- project_slug: Project slug
91
- role: "admin" or "validator"
92
-
93
- Returns:
94
- Tuple of (success, message)
95
- """
96
- if project_slug not in self._projects:
97
- return False, f"Project '{project_slug}' not found"
98
-
99
- if role not in ["admin", "validator"]:
100
- return False, f"Invalid role: {role}"
101
-
102
- # Get or create user's project access
103
- if username not in self.auth_service._user_access:
104
- self.auth_service.register_user_project_access(username, {})
105
-
106
- access = self.auth_service._user_access[username]
107
- access.project_slugs[project_slug] = Role(role)
108
-
109
- return True, f"✅ Assigned {username} to {project_slug} as {role}"
110
-
111
- def remove_user_from_project(self, username: str, project_slug: str) -> Tuple[bool, str]:
112
- """Remove a user's access to a project.
113
-
114
- Args:
115
- username: Username
116
- project_slug: Project slug
117
-
118
- Returns:
119
- Tuple of (success, message)
120
- """
121
- if username not in self.auth_service._user_access:
122
- return False, f"User '{username}' not found"
123
-
124
- access = self.auth_service._user_access[username]
125
- if project_slug not in access.project_slugs:
126
- return False, f"User '{username}' is not assigned to project '{project_slug}'"
127
-
128
- del access.project_slugs[project_slug]
129
- return True, f"✅ Removed {username} from {project_slug}"
130
-
131
- def toggleproject_active(self, project_slug: str, active: bool) -> Tuple[bool, str]:
132
- """Enable or disable a project.
133
-
134
- Args:
135
- project_slug: Project slug
136
- active: Whether to activate or deactivate
137
-
138
- Returns:
139
- Tuple of (success, message)
140
- """
141
- if project_slug not in self._projects:
142
- return False, f"Project '{project_slug}' not found"
143
-
144
- self._projects[project_slug].active = active
145
- status = "activated" if active else "deactivated"
146
- return True, f"✅ Project {project_slug} {status}"
147
-
148
-
149
- def create_admin_panel(admin_manager: AdminPanelManager, current_session: Session) -> gr.Blocks:
150
- """Create Gradio admin panel UI.
151
-
152
- Args:
153
- admin_manager: AdminPanelManager instance
154
- current_session: Current user's session
155
-
156
- Returns:
157
- Gradio Blocks with admin panel tabs
158
- """
159
- if current_session.role != Role.admin:
160
- with gr.Blocks() as restricted:
161
- gr.Markdown("❌ **Access Denied**\n\nOnly administrators can access this panel.")
162
- return restricted
163
-
164
- with gr.Blocks(title="BirdNET Admin Panel") as admin_block:
165
- gr.Markdown("# Admin Panel")
166
- gr.Markdown(f"Logged in as: **{current_session.username}** (Admin)")
167
-
168
- with gr.Tabs():
169
- # Projects Tab
170
- with gr.Tab("Projects"):
171
- gr.Markdown("## Manage Projects")
172
-
173
- with gr.Row():
174
- with gr.Column(scale=2):
175
- project_slug_input = gr.Textbox(
176
- label="Project Slug",
177
- placeholder="e.g., kenya-2024",
178
- lines=1,
179
- )
180
- project_name_input = gr.Textbox(
181
- label="Project Name",
182
- placeholder="e.g., Kenya Survey 2024",
183
- lines=1,
184
- )
185
-
186
- with gr.Column(scale=1):
187
- repo_id_input = gr.Textbox(
188
- label="HF Dataset Repo ID",
189
- placeholder="e.g., org/dataset-name",
190
- lines=1,
191
- )
192
-
193
- project_message = gr.Markdown()
194
-
195
- # TODO: Implement create project button
196
- # create_project_button = gr.Button("Create Project", variant="primary")
197
-
198
- # Projects list
199
- with gr.Row():
200
- refresh_projects_button = gr.Button("Refresh Projects List")
201
- projects_table = gr.Dataframe(
202
- value=admin_manager.list_projects(),
203
- headers=["project_slug", "name", "dataset_repo_id", "active"],
204
- interactive=False,
205
- )
206
-
207
- refresh_projects_button.click(
208
- fn=lambda: admin_manager.list_projects(),
209
- outputs=[projects_table],
210
- )
211
-
212
- # Users Tab
213
- with gr.Tab("Users"):
214
- gr.Markdown("## Manage User Access")
215
-
216
- with gr.Row():
217
- username_input = gr.Textbox(
218
- label="Username",
219
- placeholder="e.g., validator_001",
220
- lines=1,
221
- )
222
-
223
- project_select = gr.Dropdown(
224
- choices=[p["project_slug"] for p in admin_manager.list_projects()],
225
- label="Project",
226
- )
227
-
228
- role_select = gr.Dropdown(
229
- choices=["admin", "validator"],
230
- value="validator",
231
- label="Role",
232
- )
233
-
234
- user_message = gr.Markdown()
235
-
236
- assign_button = gr.Button("Assign User", variant="primary")
237
- remove_button = gr.Button("Remove User", variant="stop")
238
-
239
- def assign_user(username: str, project_slug: str, role: str) -> str:
240
- success, msg = admin_manager.assign_user_to_project(username, project_slug, role)
241
- return msg
242
-
243
- def remove_user(username: str, project_slug: str) -> str:
244
- success, msg = admin_manager.remove_user_from_project(username, project_slug)
245
- return msg
246
-
247
- assign_button.click(
248
- fn=assign_user,
249
- inputs=[username_input, project_select, role_select],
250
- outputs=[user_message],
251
- )
252
-
253
- remove_button.click(
254
- fn=remove_user,
255
- inputs=[username_input, project_select],
256
- outputs=[user_message],
257
- )
258
-
259
- # Users per project view
260
- with gr.Row():
261
- project_filter = gr.Dropdown(
262
- choices=[p["project_slug"] for p in admin_manager.list_projects()],
263
- label="View Users for Project",
264
- )
265
-
266
- users_table = gr.Dataframe(
267
- value=[],
268
- headers=["username", "role"],
269
- interactive=False,
270
- )
271
-
272
- def update_users_table(project_slug: str):
273
- return admin_manager.list_users_for_project(project_slug)
274
-
275
- project_filter.change(
276
- fn=update_users_table,
277
- inputs=[project_filter],
278
- outputs=[users_table],
279
- )
280
-
281
- return admin_block
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/ui/app_factory.py DELETED
@@ -1,1542 +0,0 @@
1
- import gradio as gr
2
- import tempfile
3
- from datetime import date, datetime
4
- from pathlib import Path
5
- from typing import Protocol
6
-
7
- from src.cache.ephemeral_cache_manager import EphemeralCacheManager
8
- from src.domain.models import Detection, Project, Role
9
- from src.repositories.append_only_validation_repository import AppendOnlyValidationRepository, OptimisticLockError
10
- from src.repositories.in_memory_detection_repository import InMemoryDetectionRepository
11
- from src.services.audio_fetch_service import AudioFetchService
12
- from src.services.detection_queue_service import DetectionQueueService
13
- from src.services.validation_service import ValidationService
14
- from src.auth.auth_service import AuthService
15
- from src.ui.login_page import create_login_page
16
- from src.ui.project_selector import create_project_selector
17
- from src.ui.admin_panel import AdminPanelManager, create_admin_panel
18
-
19
-
20
- class _AudioFetchResultProtocol(Protocol):
21
- cache_key: str
22
- local_path: str
23
- source: str
24
-
25
-
26
- class _AudioServiceProtocol(Protocol):
27
- def fetch(self, dataset_repo: str, audio_id: str) -> _AudioFetchResultProtocol: ...
28
-
29
- def cleanup_after_validation(self, cache_key: str) -> None: ...
30
-
31
-
32
- class _ValidationServiceProtocol(Protocol):
33
- def validate_detection(
34
- self,
35
- project_slug: str,
36
- detection_key: str,
37
- status: str,
38
- validator: str,
39
- notes: str = "",
40
- corrected_species: str | None = None,
41
- expected_version: int | None = None,
42
- ) -> object: ...
43
-
44
-
45
- class _ValidationReadRepositoryProtocol(Protocol):
46
- def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]: ...
47
-
48
- def list_events(self, project_slug: str) -> list[dict[str, object]]: ...
49
-
50
-
51
- class _QueueServiceProtocol(Protocol):
52
- def get_page(
53
- self,
54
- project_slug: str,
55
- page: int,
56
- page_size: int,
57
- scientific_name: str | None = None,
58
- min_confidence: float | None = None,
59
- max_confidence: float | None = None,
60
- ) -> object: ...
61
-
62
-
63
- def _seed_service() -> DetectionQueueService:
64
- repo = InMemoryDetectionRepository()
65
- demo_items = [
66
- Detection(
67
- detection_key="0000000000001001",
68
- audio_id="audio_1001",
69
- scientific_name="Cyanocorax cyanopogon",
70
- confidence=0.93,
71
- start_time=1.2,
72
- end_time=2.5,
73
- ),
74
- Detection(
75
- detection_key="0000000000001002",
76
- audio_id="audio_1002",
77
- scientific_name="Ramphastos toco",
78
- confidence=0.88,
79
- start_time=0.8,
80
- end_time=2.1,
81
- ),
82
- Detection(
83
- detection_key="0000000000001003",
84
- audio_id="audio_1003",
85
- scientific_name="Cyanocorax cyanopogon",
86
- confidence=0.72,
87
- start_time=3.1,
88
- end_time=4.0,
89
- ),
90
- Detection(
91
- detection_key="0000000000001004",
92
- audio_id="audio_1004",
93
- scientific_name="Psarocolius decumanus",
94
- confidence=0.67,
95
- start_time=5.0,
96
- end_time=6.3,
97
- ),
98
- ]
99
- repo.seed("demo-project", demo_items)
100
- return DetectionQueueService(repo)
101
-
102
-
103
- def _page_to_table(
104
- service: _QueueServiceProtocol,
105
- snapshot_reader: _ValidationReadRepositoryProtocol,
106
- project_slug: str,
107
- page: int,
108
- scientific_name: str,
109
- min_confidence: float,
110
- validator_filter: str = "",
111
- status_filter: str = "all",
112
- updated_after: object = None,
113
- conflict_detection_key: str = "",
114
- show_conflicts_only: bool = False,
115
- ):
116
- filter_name = scientific_name.strip() if scientific_name.strip() else None
117
- page_obj = service.get_page(
118
- project_slug="demo-project",
119
- page=page,
120
- page_size=2,
121
- scientific_name=filter_name,
122
- min_confidence=min_confidence,
123
- )
124
-
125
- snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
126
-
127
- normalized_status_filter = status_filter.strip().lower() if status_filter else "all"
128
- normalized_validator_filter = validator_filter.strip().lower()
129
- updated_after_date: date | None = None
130
- if updated_after is not None:
131
- if isinstance(updated_after, datetime):
132
- updated_after_date = updated_after.date()
133
- elif isinstance(updated_after, date):
134
- updated_after_date = updated_after
135
- elif isinstance(updated_after, (int, float)):
136
- updated_after_date = datetime.fromtimestamp(float(updated_after)).date()
137
- else:
138
- updated_after_text = str(updated_after).strip()
139
- if updated_after_text:
140
- try:
141
- updated_after_date = datetime.strptime(updated_after_text, "%Y-%m-%d").date()
142
- except ValueError:
143
- try:
144
- updated_after_date = datetime.fromisoformat(updated_after_text.replace("Z", "+00:00")).date()
145
- except ValueError:
146
- updated_after_date = None
147
-
148
- rows = [
149
- [
150
- item.detection_key,
151
- item.audio_id,
152
- item.scientific_name,
153
- round(item.confidence, 3),
154
- item.start_time,
155
- item.end_time,
156
- str(snapshot.get(item.detection_key, {}).get("status", "pending")),
157
- int(snapshot.get(item.detection_key, {}).get("version", 0)),
158
- "CONFLICT" if conflict_detection_key and item.detection_key == conflict_detection_key else "",
159
- "HIGH" if conflict_detection_key and item.detection_key == conflict_detection_key else "",
160
- ]
161
- for item in page_obj.items
162
- ]
163
-
164
- if normalized_validator_filter:
165
- rows = [
166
- row
167
- for row in rows
168
- if normalized_validator_filter in str(snapshot.get(str(row[0]), {}).get("validator", "")).strip().lower()
169
- ]
170
-
171
- if normalized_status_filter and normalized_status_filter != "all":
172
- rows = [row for row in rows if str(row[6]).strip().lower() == normalized_status_filter]
173
-
174
- if updated_after_date:
175
- filtered_rows: list[list[object]] = []
176
- for row in rows:
177
- snapshot_item = snapshot.get(str(row[0]), {})
178
- updated_at_value = str(snapshot_item.get("updated_at", "")).strip()
179
- if not updated_at_value:
180
- continue
181
- try:
182
- item_date = datetime.fromisoformat(updated_at_value.replace("Z", "+00:00")).date()
183
- if item_date >= updated_after_date:
184
- filtered_rows.append(row)
185
- except ValueError:
186
- continue
187
- rows = filtered_rows
188
-
189
- if show_conflicts_only:
190
- rows = [row for row in rows if str(row[8]) == "CONFLICT"]
191
-
192
- status = f"Pagina {page_obj.page}/{page_obj.total_pages} | Total base: {page_obj.total_items} | Exibidos: {len(rows)}"
193
- if show_conflicts_only:
194
- status = f"{status} | Apenas conflitos: {len(rows)} item(ns)"
195
- return rows, status, page_obj.page
196
-
197
-
198
- def _build_validation_report(snapshot_reader: _ValidationReadRepositoryProtocol, project_slug: str) -> str:
199
- snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
200
- events = snapshot_reader.list_events(project_slug=project_slug)
201
-
202
- counts: dict[str, int] = {}
203
- for payload in snapshot.values():
204
- status_value = str(payload.get("status", "unknown"))
205
- counts[status_value] = counts.get(status_value, 0) + 1
206
-
207
- parts = [
208
- f"Projeto: {project_slug}",
209
- f"Eventos append-only: {len(events)}",
210
- f"Deteccoes com estado atual: {len(snapshot)}",
211
- ]
212
- if counts:
213
- summary = ", ".join(f"{key}={value}" for key, value in sorted(counts.items()))
214
- parts.append(f"Status atual: {summary}")
215
- else:
216
- parts.append("Status atual: sem validacoes")
217
- return " | ".join(parts)
218
-
219
-
220
- def _extract_audio_id(rows: object, selected_index: int) -> str:
221
- normalized_rows: list[list[object]]
222
-
223
- if hasattr(rows, "values"):
224
- normalized_rows = [list(item) for item in rows.values.tolist()]
225
- else:
226
- normalized_rows = [list(item) for item in rows] if rows else []
227
-
228
- if not normalized_rows:
229
- raise ValueError("Nenhuma deteccao carregada na tabela")
230
- if selected_index < 0 or selected_index >= len(normalized_rows):
231
- raise ValueError("Selecione uma deteccao valida na tabela")
232
-
233
- value = normalized_rows[selected_index][1]
234
- audio_id = str(value).strip()
235
- if not audio_id:
236
- raise ValueError("audio_id invalido na deteccao selecionada")
237
- return audio_id
238
-
239
-
240
- def _extract_detection_key(rows: object, selected_index: int) -> str:
241
- normalized_rows: list[list[object]]
242
-
243
- if hasattr(rows, "values"):
244
- normalized_rows = [list(item) for item in rows.values.tolist()]
245
- else:
246
- normalized_rows = [list(item) for item in rows] if rows else []
247
-
248
- if not normalized_rows:
249
- raise ValueError("Nenhuma deteccao carregada na tabela")
250
- if selected_index < 0 or selected_index >= len(normalized_rows):
251
- raise ValueError("Selecione uma deteccao valida na tabela")
252
-
253
- value = normalized_rows[selected_index][0]
254
- detection_key = str(value).strip()
255
- if not detection_key:
256
- raise ValueError("detection_key invalido na deteccao selecionada")
257
- return detection_key
258
-
259
-
260
- def _find_detection_row_index(rows: object, detection_key: str) -> int:
261
- normalized_rows: list[list[object]]
262
-
263
- if hasattr(rows, "values"):
264
- normalized_rows = [list(item) for item in rows.values.tolist()]
265
- else:
266
- normalized_rows = [list(item) for item in rows] if rows else []
267
-
268
- for index, row in enumerate(normalized_rows):
269
- if str(row[0]).strip() == detection_key:
270
- return index
271
- return 0
272
-
273
-
274
- def _extract_expected_version(rows: object, selected_index: int) -> int:
275
- normalized_rows: list[list[object]]
276
-
277
- if hasattr(rows, "values"):
278
- normalized_rows = [list(item) for item in rows.values.tolist()]
279
- else:
280
- normalized_rows = [list(item) for item in rows] if rows else []
281
-
282
- if not normalized_rows:
283
- raise ValueError("Nenhuma deteccao carregada na tabela")
284
- if selected_index < 0 or selected_index >= len(normalized_rows):
285
- raise ValueError("Selecione uma deteccao valida na tabela")
286
-
287
- value = normalized_rows[selected_index][7]
288
- return int(value)
289
-
290
-
291
- def _fetch_selected_audio(
292
- audio_service: _AudioServiceProtocol,
293
- dataset_repo: str,
294
- rows: object,
295
- selected_index: int,
296
- previous_cache_key: str,
297
- ) -> tuple[str | None, str, str]:
298
- repo = dataset_repo.strip()
299
- if not repo:
300
- return None, "", "Informe dataset repo no formato owner/repo"
301
-
302
- try:
303
- audio_id = _extract_audio_id(rows=rows, selected_index=selected_index)
304
- result = audio_service.fetch(dataset_repo=repo, audio_id=audio_id)
305
- status = f"Audio carregado ({result.source}) para audio_id={audio_id}"
306
- return result.local_path, result.cache_key, status
307
- except Exception as exc:
308
- if previous_cache_key:
309
- return None, previous_cache_key, f"Falha ao carregar audio: {exc}"
310
- return None, "", f"Falha ao carregar audio: {exc}"
311
-
312
-
313
- def _cleanup_selected_audio(audio_service: _AudioServiceProtocol, cache_key: str) -> tuple[str, str | None]:
314
- if not cache_key:
315
- return "Nenhum audio em cache para limpar", None
316
-
317
- audio_service.cleanup_after_validation(cache_key=cache_key)
318
- return "Cache de audio limpo apos validacao", None
319
-
320
-
321
- def _save_selected_validation(
322
- validation_service: _ValidationServiceProtocol,
323
- audio_service: _AudioServiceProtocol,
324
- project_slug: str,
325
- rows: object,
326
- selected_index: int,
327
- status_value: str,
328
- validator: str,
329
- notes: str,
330
- cache_key: str,
331
- ) -> tuple[str, str, str | None]:
332
- validator_name = validator.strip()
333
- if not validator_name:
334
- return "Informe o nome do validador", cache_key, None
335
-
336
- try:
337
- detection_key = _extract_detection_key(rows=rows, selected_index=selected_index)
338
- expected_version = _extract_expected_version(rows=rows, selected_index=selected_index)
339
- _ = validation_service.validate_detection(
340
- project_slug=project_slug,
341
- detection_key=detection_key,
342
- status=status_value,
343
- validator=validator_name,
344
- notes=notes.strip(),
345
- expected_version=expected_version,
346
- )
347
- if cache_key:
348
- audio_service.cleanup_after_validation(cache_key=cache_key)
349
- return f"Validacao salva: {detection_key} -> {status_value}", "", None
350
- except OptimisticLockError as exc:
351
- return (
352
- "Conflito de concorrencia: esta deteccao foi atualizada por outro validador "
353
- f"(detection_key={exc.detection_key}, versao atual={exc.current_version}, esperada={exc.expected_version}). "
354
- "Recarregue a tabela.",
355
- cache_key,
356
- None,
357
- )
358
- except Exception as exc:
359
- return f"Falha ao salvar validacao: {exc}", cache_key, None
360
-
361
-
362
- def _save_selected_validation_with_refresh(
363
- validation_service: _ValidationServiceProtocol,
364
- audio_service: _AudioServiceProtocol,
365
- queue_service: _QueueServiceProtocol,
366
- snapshot_reader: _ValidationReadRepositoryProtocol,
367
- project_slug: str,
368
- rows: object,
369
- selected_index: int,
370
- status_value: str,
371
- validator: str,
372
- notes: str,
373
- cache_key: str,
374
- page: int,
375
- scientific_name: str,
376
- min_confidence: float,
377
- validator_filter: str,
378
- status_filter: str,
379
- updated_after: object,
380
- show_conflicts_only: bool,
381
- ) -> tuple[str, str, str | None, list[list[object]], int, int, str, str]:
382
- selected_key = ""
383
- try:
384
- selected_key = _extract_detection_key(rows=rows, selected_index=selected_index)
385
- except Exception:
386
- selected_key = ""
387
-
388
- save_status, updated_cache_key, audio_path = _save_selected_validation(
389
- validation_service=validation_service,
390
- audio_service=audio_service,
391
- project_slug=project_slug,
392
- rows=rows,
393
- selected_index=selected_index,
394
- status_value=status_value,
395
- validator=validator,
396
- notes=notes,
397
- cache_key=cache_key,
398
- )
399
-
400
- refreshed_rows, page_status, refreshed_page = _page_to_table(
401
- service=queue_service,
402
- snapshot_reader=snapshot_reader,
403
- project_slug=project_slug,
404
- page=page,
405
- scientific_name=scientific_name,
406
- min_confidence=min_confidence,
407
- validator_filter=validator_filter,
408
- status_filter=status_filter,
409
- updated_after=updated_after,
410
- show_conflicts_only=show_conflicts_only,
411
- )
412
-
413
- if selected_key:
414
- refreshed_index = _find_detection_row_index(refreshed_rows, selected_key)
415
- else:
416
- refreshed_index = 0
417
-
418
- if "Conflito de concorrencia" in save_status:
419
- conflict_key = selected_key
420
- refreshed_rows, page_status, refreshed_page = _page_to_table(
421
- service=queue_service,
422
- snapshot_reader=snapshot_reader,
423
- project_slug=project_slug,
424
- page=refreshed_page,
425
- scientific_name=scientific_name,
426
- min_confidence=min_confidence,
427
- validator_filter=validator_filter,
428
- status_filter=status_filter,
429
- updated_after=updated_after,
430
- conflict_detection_key=conflict_key,
431
- show_conflicts_only=show_conflicts_only,
432
- )
433
- refreshed_index = _find_detection_row_index(refreshed_rows, selected_key) if selected_key else 0
434
- pending_status_value = status_value
435
- status = f"{save_status} Tabela recarregada para resolver conflito."
436
- else:
437
- conflict_key = ""
438
- pending_status_value = ""
439
- status = f"{save_status} | {page_status}"
440
-
441
- return (
442
- status,
443
- updated_cache_key,
444
- audio_path,
445
- refreshed_rows,
446
- refreshed_page,
447
- refreshed_index,
448
- pending_status_value,
449
- conflict_key,
450
- )
451
-
452
-
453
- def _reapply_last_conflict_validation_with_refresh(
454
- validation_service: _ValidationServiceProtocol,
455
- audio_service: _AudioServiceProtocol,
456
- queue_service: _QueueServiceProtocol,
457
- snapshot_reader: _ValidationReadRepositoryProtocol,
458
- project_slug: str,
459
- rows: object,
460
- selected_index: int,
461
- pending_status_value: str,
462
- conflict_detection_key: str,
463
- validator: str,
464
- notes: str,
465
- cache_key: str,
466
- page: int,
467
- scientific_name: str,
468
- min_confidence: float,
469
- validator_filter: str,
470
- status_filter: str,
471
- updated_after: object,
472
- show_conflicts_only: bool,
473
- ) -> tuple[str, str, str | None, list[list[object]], int, int, str, str]:
474
- if not pending_status_value:
475
- refreshed_rows, page_status, refreshed_page = _page_to_table(
476
- service=queue_service,
477
- snapshot_reader=snapshot_reader,
478
- project_slug=project_slug,
479
- page=page,
480
- scientific_name=scientific_name,
481
- min_confidence=min_confidence,
482
- validator_filter=validator_filter,
483
- status_filter=status_filter,
484
- updated_after=updated_after,
485
- show_conflicts_only=show_conflicts_only,
486
- )
487
- return (
488
- f"Nenhuma validacao pendente para reaplicar | {page_status}",
489
- cache_key,
490
- None,
491
- refreshed_rows,
492
- refreshed_page,
493
- selected_index,
494
- "",
495
- "",
496
- )
497
-
498
- target_index = _find_detection_row_index(rows, conflict_detection_key) if conflict_detection_key else selected_index
499
- return _save_selected_validation_with_refresh(
500
- validation_service=validation_service,
501
- audio_service=audio_service,
502
- queue_service=queue_service,
503
- snapshot_reader=snapshot_reader,
504
- project_slug=project_slug,
505
- rows=rows,
506
- selected_index=target_index,
507
- status_value=pending_status_value,
508
- validator=validator,
509
- notes=notes,
510
- cache_key=cache_key,
511
- page=page,
512
- scientific_name=scientific_name,
513
- min_confidence=min_confidence,
514
- validator_filter=validator_filter,
515
- status_filter=status_filter,
516
- updated_after=updated_after,
517
- show_conflicts_only=show_conflicts_only,
518
- )
519
-
520
-
521
- def _batch_validate_conflicts(
522
- validation_service: _ValidationServiceProtocol,
523
- audio_service: _AudioServiceProtocol,
524
- queue_service: _QueueServiceProtocol,
525
- snapshot_reader: _ValidationReadRepositoryProtocol,
526
- project_slug: str,
527
- rows: object,
528
- status_value: str,
529
- validator: str,
530
- notes: str,
531
- cache_key: str,
532
- page: int,
533
- scientific_name: str,
534
- min_confidence: float,
535
- validator_filter: str,
536
- status_filter: str,
537
- updated_after: object,
538
- ) -> tuple[str, str, str | None, list[list[object]], int]:
539
- """Apply the same validation status to all visible conflicts in the table."""
540
- validator_name = validator.strip()
541
- if not validator_name:
542
- return "Informe o nome do validador", "", None, [], page
543
-
544
- normalized_rows: list[list[object]]
545
- if hasattr(rows, "values"):
546
- normalized_rows = [list(item) for item in rows.values.tolist()]
547
- else:
548
- normalized_rows = [list(item) for item in rows] if rows else []
549
-
550
- if not normalized_rows:
551
- return "Nenhuma deteccao com conflito para validar", "", None, [], page
552
-
553
- conflict_rows = [row for row in normalized_rows if str(row[8]) == "CONFLICT"]
554
- if not conflict_rows:
555
- return "Nenhuma deteccao com conflito identificada na tabela", "", None, normalized_rows, page
556
-
557
- success_count = 0
558
- failure_count = 0
559
- conflict_count = 0
560
-
561
- for row in conflict_rows:
562
- try:
563
- detection_key = str(row[0]).strip()
564
- expected_version = int(row[7])
565
-
566
- _ = validation_service.validate_detection(
567
- project_slug=project_slug,
568
- detection_key=detection_key,
569
- status=status_value,
570
- validator=validator_name,
571
- notes=notes.strip(),
572
- expected_version=expected_version,
573
- )
574
- success_count += 1
575
- if cache_key:
576
- audio_service.cleanup_after_validation(cache_key=cache_key)
577
- except OptimisticLockError:
578
- conflict_count += 1
579
- except Exception:
580
- failure_count += 1
581
-
582
- refreshed_rows, page_status, refreshed_page = _page_to_table(
583
- service=queue_service,
584
- snapshot_reader=snapshot_reader,
585
- project_slug=project_slug,
586
- page=page,
587
- scientific_name=scientific_name,
588
- min_confidence=min_confidence,
589
- validator_filter=validator_filter,
590
- status_filter=status_filter,
591
- updated_after=updated_after,
592
- show_conflicts_only=False,
593
- )
594
-
595
- summary = f"Processados {len(conflict_rows)} conflitos: {success_count} sucesso, {conflict_count} novos conflitos, {failure_count} falhas"
596
- status = f"{summary} | {page_status}"
597
-
598
- return status, "", None, refreshed_rows, refreshed_page
599
-
600
-
601
- def _batch_reapply_all_pending(
602
- validation_service: _ValidationServiceProtocol,
603
- audio_service: _AudioServiceProtocol,
604
- queue_service: _QueueServiceProtocol,
605
- snapshot_reader: _ValidationReadRepositoryProtocol,
606
- project_slug: str,
607
- rows: object,
608
- pending_statuses: dict[str, str],
609
- validator: str,
610
- notes: str,
611
- cache_key: str,
612
- page: int,
613
- scientific_name: str,
614
- min_confidence: float,
615
- validator_filter: str,
616
- status_filter: str,
617
- updated_after: object,
618
- ) -> tuple[str, str, str | None, list[list[object]], int]:
619
- """Reapply all pending validations (stored conflicts) with current version."""
620
- if not pending_statuses:
621
- return "Nenhuma validacao pendente para reaplicar", "", None, [], page
622
-
623
- validator_name = validator.strip()
624
- if not validator_name:
625
- return "Informe o nome do validador", "", None, [], page
626
-
627
- success_count = 0
628
- conflict_count = 0
629
- failure_count = 0
630
-
631
- snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
632
-
633
- for detection_key, status_value in pending_statuses.items():
634
- try:
635
- current_version = int(snapshot.get(detection_key, {}).get("version", 0))
636
-
637
- _ = validation_service.validate_detection(
638
- project_slug=project_slug,
639
- detection_key=detection_key,
640
- status=status_value,
641
- validator=validator_name,
642
- notes=notes.strip(),
643
- expected_version=current_version,
644
- )
645
- success_count += 1
646
- if cache_key:
647
- audio_service.cleanup_after_validation(cache_key=cache_key)
648
- except OptimisticLockError:
649
- conflict_count += 1
650
- except Exception:
651
- failure_count += 1
652
-
653
- refreshed_rows, page_status, refreshed_page = _page_to_table(
654
- service=queue_service,
655
- snapshot_reader=snapshot_reader,
656
- project_slug=project_slug,
657
- page=page,
658
- scientific_name=scientific_name,
659
- min_confidence=min_confidence,
660
- validator_filter=validator_filter,
661
- status_filter=status_filter,
662
- updated_after=updated_after,
663
- show_conflicts_only=False,
664
- )
665
-
666
- summary = f"Reaplicadas {len(pending_statuses)} validacoes: {success_count} sucesso, {conflict_count} novos conflitos, {failure_count} falhas"
667
- status = f"{summary} | {page_status}"
668
-
669
- return status, "", None, refreshed_rows, refreshed_page
670
-
671
-
672
- def build_demo_app(project_slug: str = "demo-project") -> gr.Blocks:
673
- """Build the demo validation app for a given project.
674
-
675
- Args:
676
- project_slug: Project identifier (default: demo-project)
677
-
678
- Returns:
679
- Gradio Blocks with validation interface
680
- """
681
- service = _seed_service()
682
- audio_service = AudioFetchService(EphemeralCacheManager(ttl_seconds=300, max_files=128))
683
- validation_base_dir = str(Path(tempfile.gettempdir()) / "birdnet-validator-validations")
684
- validation_repository = AppendOnlyValidationRepository(base_dir=validation_base_dir)
685
- validation_service = ValidationService(validation_repository)
686
-
687
- with gr.Blocks(title="BirdNET Validator HF") as demo:
688
- gr.Markdown("# BirdNET Validator HF")
689
- gr.Markdown("Sprint 2: fila paginada + audio sob demanda com cache efemero.")
690
-
691
- dataset_repo = gr.Textbox(label="Dataset repo", value="SEU_USUARIO/birdnet-projeto-dataset")
692
-
693
- with gr.Row():
694
- species_filter = gr.Textbox(label="Filtro especie", placeholder="Ex: Cyanocorax cyanopogon")
695
- min_confidence = gr.Slider(label="Confianca minima", minimum=0.0, maximum=1.0, step=0.01, value=0.0)
696
- show_conflicts_only = gr.Checkbox(label="Mostrar apenas conflitos", value=False)
697
-
698
- with gr.Row():
699
- validator_filter = gr.Textbox(label="Filtro validador", placeholder="Ex: validator-demo")
700
- validation_status_filter = gr.Dropdown(
701
- label="Filtro status",
702
- choices=["all", "pending", "positive", "negative", "uncertain", "skip"],
703
- value="all",
704
- )
705
- updated_after_filter = gr.DateTime(label="Atualizado a partir de", include_time=False, type="string")
706
-
707
- with gr.Row():
708
- prev_btn = gr.Button("Pagina anterior")
709
- next_btn = gr.Button("Proxima pagina")
710
- refresh_btn = gr.Button("Aplicar filtros")
711
-
712
- page_state = gr.State(value=1)
713
- table = gr.Dataframe(
714
- headers=[
715
- "detection_key",
716
- "audio_id",
717
- "scientific_name",
718
- "confidence",
719
- "start_time",
720
- "end_time",
721
- "validation_status",
722
- "version",
723
- "conflict_flag",
724
- "conflict_severity",
725
- ],
726
- label="Deteccoes",
727
- interactive=False,
728
- )
729
- selected_index = gr.Number(label="Linha selecionada", value=0, precision=0)
730
-
731
- with gr.Row():
732
- load_audio_btn = gr.Button("Carregar audio selecionado")
733
- clear_audio_btn = gr.Button("Limpar cache apos validacao")
734
-
735
- with gr.Row():
736
- validator_name = gr.Textbox(label="Validador", value="validator-demo")
737
- validation_notes = gr.Textbox(label="Notas", placeholder="Opcional")
738
-
739
- with gr.Row():
740
- approve_btn = gr.Button("Validar positivo")
741
- reject_btn = gr.Button("Validar negativo")
742
- uncertain_btn = gr.Button("Indeterminado")
743
- skip_btn = gr.Button("Pular")
744
- reapply_btn = gr.Button("Reaplicar validacao apos conflito")
745
-
746
- with gr.Row():
747
- batch_approve_conflicts_btn = gr.Button("Aprovar todos os conflitos")
748
- batch_reject_conflicts_btn = gr.Button("Rejeitar todos os conflitos")
749
-
750
- report_btn = gr.Button("Gerar relatorio de validacoes")
751
-
752
- audio_player = gr.Audio(label="Audio sob demanda", type="filepath")
753
- cache_key_state = gr.State(value="")
754
- pending_status_state = gr.State(value="")
755
- conflict_detection_key_state = gr.State(value="")
756
- status = gr.Textbox(label="Status", interactive=False)
757
- report_box = gr.Textbox(label="Relatorio", interactive=False)
758
-
759
- # Keyboard shortcuts: 1=positive, 2=negative, 3=uncertain, 4=skip, R=reapply
760
- keyboard_shortcuts_info = gr.HTML(
761
- value="<div style='font-size: 12px; color: #666; padding: 8px; background-color: #f5f5f5; border-radius: 4px; margin-bottom: 10px;'>"
762
- "<strong>Atalhos de teclado:</strong> 1=Positivo | 2=Negativo | 3=Indeterminado | 4=Pular | R=Reaplicar"
763
- "</div>"
764
- "<script>"
765
- "document.addEventListener('keydown', function(event) {"
766
- " if (event.target.tagName === 'INPUT' || event.target.tagName === 'TEXTAREA') return;"
767
- " const key = event.key.toLowerCase();"
768
- " let buttonText = null;"
769
- " if (key === '1') buttonText = 'Validar positivo';"
770
- " else if (key === '2') buttonText = 'Validar negativo';"
771
- " else if (key === '3') buttonText = 'Indeterminado';"
772
- " else if (key === '4') buttonText = 'Pular';"
773
- " else if (key === 'r') buttonText = 'Reaplicar validacao apos conflito';"
774
- " if (buttonText) {"
775
- " event.preventDefault();"
776
- " const buttons = document.querySelectorAll('button');"
777
- " for (const btn of buttons) {"
778
- " if (btn.textContent.includes(buttonText)) {"
779
- " btn.click();"
780
- " break;"
781
- " }"
782
- " }"
783
- " }"
784
- "});"
785
- "</script>"
786
- )
787
-
788
- def refresh(
789
- page: int,
790
- species: str,
791
- confidence: float,
792
- validator_filter_value: str,
793
- status_filter_value: str,
794
- updated_after_value: object,
795
- only_conflicts: bool,
796
- ):
797
- return _page_to_table(
798
- service=service,
799
- snapshot_reader=validation_repository,
800
- project_slug=project_slug,
801
- page=page,
802
- scientific_name=species,
803
- min_confidence=confidence,
804
- validator_filter=validator_filter_value,
805
- status_filter=status_filter_value,
806
- updated_after=updated_after_value,
807
- show_conflicts_only=only_conflicts,
808
- )
809
-
810
- def go_next(
811
- page: int,
812
- species: str,
813
- confidence: float,
814
- validator_filter_value: str,
815
- status_filter_value: str,
816
- updated_after_value: object,
817
- only_conflicts: bool,
818
- ):
819
- return refresh(
820
- page + 1,
821
- species,
822
- confidence,
823
- validator_filter_value,
824
- status_filter_value,
825
- updated_after_value,
826
- only_conflicts,
827
- )
828
-
829
- def go_prev(
830
- page: int,
831
- species: str,
832
- confidence: float,
833
- validator_filter_value: str,
834
- status_filter_value: str,
835
- updated_after_value: object,
836
- only_conflicts: bool,
837
- ):
838
- return refresh(
839
- max(1, page - 1),
840
- species,
841
- confidence,
842
- validator_filter_value,
843
- status_filter_value,
844
- updated_after_value,
845
- only_conflicts,
846
- )
847
-
848
- def on_select(evt: gr.SelectData):
849
- if isinstance(evt.index, tuple):
850
- return int(evt.index[0])
851
- if isinstance(evt.index, int):
852
- return int(evt.index)
853
- return 0
854
-
855
- demo.load(
856
- fn=refresh,
857
- inputs=[
858
- page_state,
859
- species_filter,
860
- min_confidence,
861
- validator_filter,
862
- validation_status_filter,
863
- updated_after_filter,
864
- show_conflicts_only,
865
- ],
866
- outputs=[table, status, page_state],
867
- )
868
- refresh_btn.click(
869
- fn=lambda species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: refresh(
870
- 1,
871
- species,
872
- confidence,
873
- validator_filter_value,
874
- status_filter_value,
875
- updated_after_value,
876
- only_conflicts,
877
- ),
878
- inputs=[
879
- species_filter,
880
- min_confidence,
881
- validator_filter,
882
- validation_status_filter,
883
- updated_after_filter,
884
- show_conflicts_only,
885
- ],
886
- outputs=[table, status, page_state],
887
- )
888
- next_btn.click(
889
- fn=go_next,
890
- inputs=[
891
- page_state,
892
- species_filter,
893
- min_confidence,
894
- validator_filter,
895
- validation_status_filter,
896
- updated_after_filter,
897
- show_conflicts_only,
898
- ],
899
- outputs=[table, status, page_state],
900
- )
901
- prev_btn.click(
902
- fn=go_prev,
903
- inputs=[
904
- page_state,
905
- species_filter,
906
- min_confidence,
907
- validator_filter,
908
- validation_status_filter,
909
- updated_after_filter,
910
- show_conflicts_only,
911
- ],
912
- outputs=[table, status, page_state],
913
- )
914
- table.select(fn=on_select, inputs=None, outputs=[selected_index])
915
- load_audio_btn.click(
916
- fn=lambda repo, rows, idx, cache_key: _fetch_selected_audio(
917
- audio_service=audio_service,
918
- dataset_repo=repo,
919
- rows=rows,
920
- selected_index=int(idx),
921
- previous_cache_key=cache_key,
922
- ),
923
- inputs=[dataset_repo, table, selected_index, cache_key_state],
924
- outputs=[audio_player, cache_key_state, status],
925
- )
926
- clear_audio_btn.click(
927
- fn=lambda cache_key: _cleanup_selected_audio(audio_service=audio_service, cache_key=cache_key),
928
- inputs=[cache_key_state],
929
- outputs=[status, audio_player],
930
- )
931
- approve_btn.click(
932
- fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
933
- validation_service=validation_service,
934
- audio_service=audio_service,
935
- queue_service=service,
936
- snapshot_reader=validation_repository,
937
- project_slug=project_slug,
938
- rows=rows,
939
- selected_index=int(idx),
940
- status_value="positive",
941
- validator=name,
942
- notes=notes,
943
- cache_key=cache_key,
944
- page=int(page),
945
- scientific_name=species,
946
- min_confidence=float(confidence),
947
- validator_filter=validator_filter_value,
948
- status_filter=status_filter_value,
949
- updated_after=updated_after_value,
950
- show_conflicts_only=bool(only_conflicts),
951
- ),
952
- inputs=[
953
- table,
954
- selected_index,
955
- validator_name,
956
- validation_notes,
957
- cache_key_state,
958
- page_state,
959
- species_filter,
960
- min_confidence,
961
- validator_filter,
962
- validation_status_filter,
963
- updated_after_filter,
964
- show_conflicts_only,
965
- ],
966
- outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
967
- )
968
- reject_btn.click(
969
- fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
970
- validation_service=validation_service,
971
- audio_service=audio_service,
972
- queue_service=service,
973
- snapshot_reader=validation_repository,
974
- project_slug=project_slug,
975
- rows=rows,
976
- selected_index=int(idx),
977
- status_value="negative",
978
- validator=name,
979
- notes=notes,
980
- cache_key=cache_key,
981
- page=int(page),
982
- scientific_name=species,
983
- min_confidence=float(confidence),
984
- validator_filter=validator_filter_value,
985
- status_filter=status_filter_value,
986
- updated_after=updated_after_value,
987
- show_conflicts_only=bool(only_conflicts),
988
- ),
989
- inputs=[
990
- table,
991
- selected_index,
992
- validator_name,
993
- validation_notes,
994
- cache_key_state,
995
- page_state,
996
- species_filter,
997
- min_confidence,
998
- validator_filter,
999
- validation_status_filter,
1000
- updated_after_filter,
1001
- show_conflicts_only,
1002
- ],
1003
- outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
1004
- )
1005
- uncertain_btn.click(
1006
- fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
1007
- validation_service=validation_service,
1008
- audio_service=audio_service,
1009
- queue_service=service,
1010
- snapshot_reader=validation_repository,
1011
- project_slug=project_slug,
1012
- rows=rows,
1013
- selected_index=int(idx),
1014
- status_value="uncertain",
1015
- validator=name,
1016
- notes=notes,
1017
- cache_key=cache_key,
1018
- page=int(page),
1019
- scientific_name=species,
1020
- min_confidence=float(confidence),
1021
- validator_filter=validator_filter_value,
1022
- status_filter=status_filter_value,
1023
- updated_after=updated_after_value,
1024
- show_conflicts_only=bool(only_conflicts),
1025
- ),
1026
- inputs=[
1027
- table,
1028
- selected_index,
1029
- validator_name,
1030
- validation_notes,
1031
- cache_key_state,
1032
- page_state,
1033
- species_filter,
1034
- min_confidence,
1035
- validator_filter,
1036
- validation_status_filter,
1037
- updated_after_filter,
1038
- show_conflicts_only,
1039
- ],
1040
- outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
1041
- )
1042
- skip_btn.click(
1043
- fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
1044
- validation_service=validation_service,
1045
- audio_service=audio_service,
1046
- queue_service=service,
1047
- snapshot_reader=validation_repository,
1048
- project_slug=project_slug,
1049
- rows=rows,
1050
- selected_index=int(idx),
1051
- status_value="skip",
1052
- validator=name,
1053
- notes=notes,
1054
- cache_key=cache_key,
1055
- page=int(page),
1056
- scientific_name=species,
1057
- min_confidence=float(confidence),
1058
- validator_filter=validator_filter_value,
1059
- status_filter=status_filter_value,
1060
- updated_after=updated_after_value,
1061
- show_conflicts_only=bool(only_conflicts),
1062
- ),
1063
- inputs=[
1064
- table,
1065
- selected_index,
1066
- validator_name,
1067
- validation_notes,
1068
- cache_key_state,
1069
- page_state,
1070
- species_filter,
1071
- min_confidence,
1072
- validator_filter,
1073
- validation_status_filter,
1074
- updated_after_filter,
1075
- show_conflicts_only,
1076
- ],
1077
- outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
1078
- )
1079
- reapply_btn.click(
1080
- fn=lambda rows, idx, pending_status, conflict_key, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _reapply_last_conflict_validation_with_refresh(
1081
- validation_service=validation_service,
1082
- audio_service=audio_service,
1083
- queue_service=service,
1084
- snapshot_reader=validation_repository,
1085
- project_slug=project_slug,
1086
- rows=rows,
1087
- selected_index=int(idx),
1088
- pending_status_value=pending_status,
1089
- conflict_detection_key=conflict_key,
1090
- validator=name,
1091
- notes=notes,
1092
- cache_key=cache_key,
1093
- page=int(page),
1094
- scientific_name=species,
1095
- min_confidence=float(confidence),
1096
- validator_filter=validator_filter_value,
1097
- status_filter=status_filter_value,
1098
- updated_after=updated_after_value,
1099
- show_conflicts_only=bool(only_conflicts),
1100
- ),
1101
- inputs=[
1102
- table,
1103
- selected_index,
1104
- pending_status_state,
1105
- conflict_detection_key_state,
1106
- validator_name,
1107
- validation_notes,
1108
- cache_key_state,
1109
- page_state,
1110
- species_filter,
1111
- min_confidence,
1112
- validator_filter,
1113
- validation_status_filter,
1114
- updated_after_filter,
1115
- show_conflicts_only,
1116
- ],
1117
- outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
1118
- )
1119
- batch_approve_conflicts_btn.click(
1120
- fn=lambda rows, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value: _batch_validate_conflicts(
1121
- validation_service=validation_service,
1122
- audio_service=audio_service,
1123
- queue_service=service,
1124
- snapshot_reader=validation_repository,
1125
- project_slug=project_slug,
1126
- rows=rows,
1127
- status_value="positive",
1128
- validator=name,
1129
- notes=notes,
1130
- cache_key=cache_key,
1131
- page=int(page),
1132
- scientific_name=species,
1133
- min_confidence=float(confidence),
1134
- validator_filter=validator_filter_value,
1135
- status_filter=status_filter_value,
1136
- updated_after=updated_after_value,
1137
- ),
1138
- inputs=[
1139
- table,
1140
- validator_name,
1141
- validation_notes,
1142
- cache_key_state,
1143
- page_state,
1144
- species_filter,
1145
- min_confidence,
1146
- validator_filter,
1147
- validation_status_filter,
1148
- updated_after_filter,
1149
- ],
1150
- outputs=[status, cache_key_state, audio_player, table, page_state],
1151
- )
1152
- batch_reject_conflicts_btn.click(
1153
- fn=lambda rows, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value: _batch_validate_conflicts(
1154
- validation_service=validation_service,
1155
- audio_service=audio_service,
1156
- queue_service=service,
1157
- snapshot_reader=validation_repository,
1158
- project_slug=project_slug,
1159
- rows=rows,
1160
- status_value="negative",
1161
- validator=name,
1162
- notes=notes,
1163
- cache_key=cache_key,
1164
- page=int(page),
1165
- scientific_name=species,
1166
- min_confidence=float(confidence),
1167
- validator_filter=validator_filter_value,
1168
- status_filter=status_filter_value,
1169
- updated_after=updated_after_value,
1170
- ),
1171
- inputs=[
1172
- table,
1173
- validator_name,
1174
- validation_notes,
1175
- cache_key_state,
1176
- page_state,
1177
- species_filter,
1178
- min_confidence,
1179
- validator_filter,
1180
- validation_status_filter,
1181
- updated_after_filter,
1182
- ],
1183
- outputs=[status, cache_key_state, audio_player, table, page_state],
1184
- )
1185
- report_btn.click(
1186
- fn=lambda: _build_validation_report(validation_repository, project_slug),
1187
- inputs=None,
1188
- outputs=[report_box],
1189
- )
1190
-
1191
- return demo
1192
-
1193
-
1194
- def create_app() -> gr.Blocks:
1195
- """Build the BirdNET Validator app with multi-project auth integration.
1196
-
1197
- Returns multi-tab interface with:
1198
- - Login tab for user authentication
1199
- - Project selection for authorized projects
1200
- - Admin panel for project/user management (admin only)
1201
- - Validation interface for selected project
1202
-
1203
- Returns:
1204
- Gradio Blocks with full auth-integrated app
1205
- """
1206
- # Initialize auth service
1207
- auth_service = AuthService(session_ttl_minutes=120)
1208
-
1209
- # Setup demo users (in production: database/LDAP/OAuth)
1210
- auth_service.register_user_project_access(
1211
- "admin_user",
1212
- {"kenya-2024": Role.admin, "nairobi-2023": Role.admin},
1213
- )
1214
- auth_service.register_user_project_access(
1215
- "validator_demo",
1216
- {"demo-project": Role.validator, "kenya-2024": Role.validator},
1217
- )
1218
- auth_service.register_user_project_access(
1219
- "validator_other",
1220
- {"nairobi-2023": Role.validator},
1221
- )
1222
-
1223
- # Initialize admin panel manager
1224
- admin_manager = AdminPanelManager(auth_service)
1225
-
1226
- # Register projects
1227
- admin_manager.register_project(
1228
- Project(
1229
- project_slug="kenya-2024",
1230
- name="Kenya Survey 2024",
1231
- dataset_repo_id="birdnet/kenya-2024-dataset",
1232
- active=True,
1233
- )
1234
- )
1235
- admin_manager.register_project(
1236
- Project(
1237
- project_slug="nairobi-2023",
1238
- name="Nairobi Survey 2023",
1239
- dataset_repo_id="birdnet/nairobi-2023-dataset",
1240
- active=True,
1241
- )
1242
- )
1243
- admin_manager.register_project(
1244
- Project(
1245
- project_slug="demo-project",
1246
- name="Demo Project",
1247
- dataset_repo_id="birdnet/demo-dataset",
1248
- active=True,
1249
- )
1250
- )
1251
-
1252
- with gr.Blocks(title="BirdNET Validator HF - Multi-Project") as wrapper:
1253
- gr.Markdown("# BirdNET Validator HF - Sprint 4: Segurança Multi-Projeto")
1254
- gr.Markdown("**Versão com autenticação, autorização por projeto, e painel admin**")
1255
-
1256
- # Session state
1257
- session_state = gr.State(value=None)
1258
- selected_project_state = gr.State(value=None)
1259
-
1260
- def _project_rows() -> list[list[object]]:
1261
- projects = admin_manager.list_projects()
1262
- return [
1263
- [
1264
- p["project_slug"],
1265
- p["name"],
1266
- p["dataset_repo_id"],
1267
- "yes" if bool(p["active"]) else "no",
1268
- ]
1269
- for p in projects
1270
- ]
1271
-
1272
- def _project_slugs() -> list[str]:
1273
- return [p["project_slug"] for p in admin_manager.list_projects()]
1274
-
1275
- with gr.Tabs():
1276
- # ===== TAB 1: Login =====
1277
- with gr.Tab("🔐 Login", id="login_tab"):
1278
- login_block, username_input, session_output, login_button, error_message = (
1279
- create_login_page(auth_service)
1280
- )
1281
-
1282
- # Store session ID when login succeeds
1283
- def handle_login_success(session_id: str):
1284
- """Process successful login and store session."""
1285
- if session_id:
1286
- return auth_service.get_session(session_id)
1287
- return None
1288
-
1289
- session_output.change(
1290
- fn=handle_login_success,
1291
- inputs=[session_output],
1292
- outputs=[session_state],
1293
- )
1294
-
1295
- # ===== TAB 2: Admin Panel =====
1296
- with gr.Tab("⚙️ Admin", id="admin_tab"):
1297
- admin_info = gr.Markdown(value="⚠️ Faça login primeiro")
1298
-
1299
- def create_admin_display(session):
1300
- """Show admin panel or access denied message."""
1301
- if session is None:
1302
- return (
1303
- "❌ **Not Logged In** — Please login first in the Login tab.",
1304
- gr.update(visible=False),
1305
- )
1306
- if session.role.value != "admin":
1307
- return (
1308
- "❌ **Access Denied** — Only administrators can access this panel.",
1309
- gr.update(visible=False),
1310
- )
1311
- return (
1312
- f"✅ **Admin Panel** — Welcome {session.username}! Click below to manage projects and users.",
1313
- gr.update(visible=True),
1314
- )
1315
-
1316
- with gr.Group(visible=False) as admin_controls:
1317
- gr.Markdown("#### Projetos Cadastrados")
1318
-
1319
- with gr.Row():
1320
- create_project_slug = gr.Textbox(
1321
- label="Novo Project Slug",
1322
- placeholder="ex: amazonas-2026",
1323
- )
1324
- create_project_name = gr.Textbox(
1325
- label="Nome do Projeto",
1326
- placeholder="ex: Amazonas Survey 2026",
1327
- )
1328
- create_project_repo = gr.Textbox(
1329
- label="HF Dataset Repo ID",
1330
- placeholder="ex: birdnet/amazonas-2026-dataset",
1331
- )
1332
-
1333
- create_project_message = gr.Markdown()
1334
-
1335
- def create_project(session, slug: str, name: str, repo_id: str):
1336
- if session is None or session.role.value != "admin":
1337
- return "❌ Access denied. Apenas admin pode criar projetos.", gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
1338
-
1339
- slug = (slug or "").strip()
1340
- name = (name or "").strip()
1341
- repo_id = (repo_id or "").strip()
1342
- if not slug or not name or not repo_id:
1343
- return "⚠️ Preencha slug, nome e repo id.", gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
1344
-
1345
- created = admin_manager.register_project(
1346
- Project(
1347
- project_slug=slug,
1348
- name=name,
1349
- dataset_repo_id=repo_id,
1350
- active=True,
1351
- )
1352
- )
1353
- if not created:
1354
- return (
1355
- f"⚠️ Projeto '{slug}' já existe.",
1356
- _project_rows(),
1357
- gr.update(choices=_project_slugs()),
1358
- gr.update(),
1359
- gr.update(),
1360
- gr.update(),
1361
- )
1362
-
1363
- return (
1364
- f"✅ Projeto '{slug}' criado com sucesso.",
1365
- _project_rows(),
1366
- gr.update(choices=_project_slugs(), value=slug),
1367
- gr.update(value=""),
1368
- gr.update(value=""),
1369
- gr.update(value=""),
1370
- )
1371
-
1372
- create_project_btn = gr.Button("➕ Criar Projeto", variant="primary")
1373
- projects_table = gr.Dataframe(
1374
- value=_project_rows(),
1375
- headers=["project_slug", "name", "dataset_repo_id", "active"],
1376
- interactive=False,
1377
- )
1378
- refresh_projects_btn = gr.Button("🔄 Atualizar Lista")
1379
-
1380
- def refresh_projects(session):
1381
- if session is None or session.role.value != "admin":
1382
- return []
1383
- return _project_rows()
1384
-
1385
- refresh_projects_btn.click(
1386
- fn=refresh_projects,
1387
- inputs=[session_state],
1388
- outputs=[projects_table],
1389
- )
1390
-
1391
- with gr.Group(visible=False) as admin_users_controls:
1392
- gr.Markdown("#### Assignar Usuário a Projeto")
1393
- with gr.Row():
1394
- admin_username = gr.Textbox(
1395
- label="Usuário", placeholder="validator_001"
1396
- )
1397
- admin_project = gr.Dropdown(
1398
- choices=_project_slugs(),
1399
- label="Projeto",
1400
- )
1401
- admin_role = gr.Dropdown(
1402
- choices=["admin", "validator"],
1403
- value="validator",
1404
- label="Role",
1405
- )
1406
-
1407
- admin_message = gr.Markdown()
1408
-
1409
- def assign_user(session, username: str, project: str, role: str):
1410
- if session is None or session.role.value != "admin":
1411
- return "❌ Access denied. Apenas admin pode atribuir usuários.", gr.update(), gr.update(), gr.update()
1412
- success, msg = admin_manager.assign_user_to_project(
1413
- username, project, role
1414
- )
1415
- if success:
1416
- return msg, gr.update(value=""), gr.update(value=None), gr.update(value="validator")
1417
- return msg, gr.update(), gr.update(), gr.update()
1418
-
1419
- assign_btn = gr.Button("✅ Assignar", variant="primary")
1420
- assign_btn.click(
1421
- fn=assign_user,
1422
- inputs=[session_state, admin_username, admin_project, admin_role],
1423
- outputs=[admin_message, admin_username, admin_project, admin_role],
1424
- )
1425
-
1426
- create_project_btn.click(
1427
- fn=create_project,
1428
- inputs=[session_state, create_project_slug, create_project_name, create_project_repo],
1429
- outputs=[
1430
- create_project_message,
1431
- projects_table,
1432
- admin_project,
1433
- create_project_slug,
1434
- create_project_name,
1435
- create_project_repo,
1436
- ],
1437
- )
1438
-
1439
- session_state.change(
1440
- fn=create_admin_display,
1441
- inputs=[session_state],
1442
- outputs=[admin_info, admin_controls],
1443
- )
1444
-
1445
- session_state.change(
1446
- fn=lambda s: gr.update(visible=bool(s is not None and s.role.value == "admin")),
1447
- inputs=[session_state],
1448
- outputs=[admin_users_controls],
1449
- )
1450
-
1451
- # ===== TAB 3: Project Selection =====
1452
- with gr.Tab("📁 Selecionar Projeto", id="project_tab"):
1453
- project_info_display = gr.Markdown(
1454
- value="⚠️ Faça login primeiro na aba **Login**"
1455
- )
1456
- project_selector = gr.Dropdown(
1457
- choices=[],
1458
- label="Projeto Autorizado",
1459
- interactive=False,
1460
- )
1461
-
1462
- def update_project_selector(session):
1463
- """Update project dropdown when user logs in."""
1464
- if session is None:
1465
- return (
1466
- gr.Dropdown(choices=[], value=None, interactive=False),
1467
- "❌ Not logged in. Please login first.",
1468
- None,
1469
- )
1470
-
1471
- projects = session.authorized_projects
1472
- if not projects:
1473
- return (
1474
- gr.Dropdown(choices=[], value=None, interactive=False),
1475
- "⚠️ **No Projects Assigned**\n\nYou don't have access to any projects yet. Contact an administrator.",
1476
- None,
1477
- )
1478
-
1479
- selected = projects[0]
1480
- role = auth_service.get_user_role_for_project(session.username, selected)
1481
- role_label = role.value.upper() if role else "UNKNOWN"
1482
- return (
1483
- gr.Dropdown(choices=projects, value=selected, interactive=True),
1484
- f"📁 **Project:** {selected} | **Your Role:** {role_label}",
1485
- selected,
1486
- )
1487
-
1488
- session_state.change(
1489
- fn=update_project_selector,
1490
- inputs=[session_state],
1491
- outputs=[project_selector, project_info_display, selected_project_state],
1492
- )
1493
-
1494
- def update_selected_project(selected: str, session):
1495
- """Update state when project is selected."""
1496
- if session and selected:
1497
- return selected
1498
- return None
1499
-
1500
- project_selector.change(
1501
- fn=update_selected_project,
1502
- inputs=[project_selector, session_state],
1503
- outputs=[selected_project_state],
1504
- )
1505
-
1506
- # ===== TAB 4: Validation =====
1507
- with gr.Tab("✓ Validação", id="validation_tab"):
1508
- validation_status = gr.Markdown(
1509
- value="ℹ️ Faça login e selecione um projeto para começar"
1510
- )
1511
-
1512
- def get_validation_status(session, selected_project):
1513
- """Show status message based on login/project state."""
1514
- if session is None:
1515
- return "❌ **Não autenticado** — Faça login na aba **Login** primeiro"
1516
- if selected_project is None:
1517
- return f"⚠️ **Projeto não selecionado** — Selecione um projeto na aba **Selecionar Projeto**"
1518
- return f"✅ **Pronto para validar** — Projeto: **{selected_project}** | Usuário: **{session.username}**\n\n(Validação será carregada abaixo)"
1519
-
1520
- session_state.change(
1521
- fn=lambda s, p: get_validation_status(s, p),
1522
- inputs=[session_state, selected_project_state],
1523
- outputs=[validation_status],
1524
- )
1525
-
1526
- selected_project_state.change(
1527
- fn=lambda s, p: get_validation_status(s, p),
1528
- inputs=[session_state, selected_project_state],
1529
- outputs=[validation_status],
1530
- )
1531
-
1532
- # Note: In a real implementation, we would load build_demo_app(selected_project) here
1533
- # For now, we show a placeholder since Gradio tabs don't support dynamic nested Blocks
1534
- gr.Markdown("---")
1535
- validation_placeholder = gr.Textbox(
1536
- label="Validação",
1537
- value="📋 Interface de validação será carregada aqui para o projeto selecionado",
1538
- interactive=False,
1539
- lines=10,
1540
- )
1541
-
1542
- return wrapper
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/ui/login_page.py DELETED
@@ -1,66 +0,0 @@
1
- """Gradio login page component for multi-project authorization."""
2
-
3
- from typing import Optional, Tuple
4
-
5
- import gradio as gr
6
-
7
- from src.auth.auth_service import AuthService, Session
8
-
9
-
10
- def create_login_page(auth_service: AuthService) -> Tuple[gr.Blocks, gr.Textbox, gr.Textbox, gr.Button, gr.Markdown]:
11
- """Create a Gradio login page with username input and session tracking.
12
-
13
- Args:
14
- auth_service: AuthService instance for login validation
15
-
16
- Returns:
17
- Tuple of (login_block, username_input, session_output, login_button, error_message)
18
- """
19
-
20
- with gr.Blocks(title="BirdNET Validator - Login") as login_block:
21
- gr.Markdown("# BirdNET Validation Platform")
22
- gr.Markdown("Login to access multi-project validation workflows")
23
-
24
- username_input = gr.Textbox(
25
- label="Username",
26
- placeholder="Enter your username",
27
- lines=1,
28
- )
29
-
30
- error_message = gr.Markdown()
31
-
32
- login_button = gr.Button("Login", variant="primary", scale=1)
33
-
34
- session_output = gr.Textbox(
35
- label="Session ID",
36
- interactive=False,
37
- visible=False,
38
- )
39
-
40
- def perform_login(username: str) -> Tuple[str, str]:
41
- """Attempt login and return session ID or error message.
42
-
43
- Args:
44
- username: Username to authenticate
45
-
46
- Returns:
47
- Tuple of (session_id, error_message)
48
- """
49
- if not username or not username.strip():
50
- return "", "❌ Please enter a username"
51
-
52
- username = username.strip()
53
- session = auth_service.login(username)
54
-
55
- if session is None:
56
- return "", f"❌ User '{username}' not found or inactive"
57
-
58
- return session.session_id, f"✅ Welcome, {username}! (Admin)" if session.role.value == "admin" else f"✅ Welcome, {username}! (Validator)"
59
-
60
- login_button.click(
61
- fn=perform_login,
62
- inputs=[username_input],
63
- outputs=[session_output, error_message],
64
- )
65
-
66
- return login_block, username_input, session_output, login_button, error_message
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/ui/project_selector.py DELETED
@@ -1,77 +0,0 @@
1
- """Project selector component for switching between authorized projects."""
2
-
3
- from typing import List, Tuple
4
-
5
- import gradio as gr
6
-
7
- from src.auth.auth_service import AuthService, Session
8
-
9
-
10
- def create_project_selector(
11
- auth_service: AuthService, current_session: Session
12
- ) -> Tuple[gr.Dropdown, gr.Markdown]:
13
- """Create a project selector dropdown for the current user.
14
-
15
- Args:
16
- auth_service: AuthService instance
17
- current_session: Current user's session
18
-
19
- Returns:
20
- Tuple of (project_dropdown, project_info_display)
21
- """
22
- authorized_projects = current_session.authorized_projects
23
-
24
- if not authorized_projects:
25
- with gr.Group():
26
- gr.Markdown("⚠️ **No Projects Assigned**\n\nYou don't have access to any projects yet. Contact an administrator.")
27
-
28
- return (
29
- gr.Dropdown(choices=[], label="Project", interactive=False),
30
- gr.Markdown(""),
31
- )
32
-
33
- with gr.Group():
34
- project_dropdown = gr.Dropdown(
35
- choices=authorized_projects,
36
- value=authorized_projects[0],
37
- label="Select Project",
38
- interactive=True,
39
- )
40
-
41
- project_info = gr.Markdown()
42
-
43
- def update_project_info(selected_project: str) -> str:
44
- """Display info about the selected project.
45
-
46
- Args:
47
- selected_project: Project slug
48
-
49
- Returns:
50
- Markdown formatted project info
51
- """
52
- role = auth_service.get_user_role_for_project(
53
- current_session.username, selected_project
54
- )
55
- if role:
56
- return f"📁 **Project:** {selected_project} | **Your Role:** {role.value.upper()}"
57
- return f"📁 **Project:** {selected_project}"
58
-
59
- # Initial display
60
- initial_project = authorized_projects[0]
61
- role = auth_service.get_user_role_for_project(
62
- current_session.username, initial_project
63
- )
64
- initial_info = (
65
- f"📁 **Project:** {initial_project} | **Your Role:** {role.value.upper()}"
66
- if role
67
- else f"📁 **Project:** {initial_project}"
68
- )
69
- project_info.value = initial_info
70
-
71
- project_dropdown.change(
72
- fn=update_project_info,
73
- inputs=[project_dropdown],
74
- outputs=[project_info],
75
- )
76
-
77
- return project_dropdown, project_info
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BirdNET Validator App/src/ui/upload_app_factory.py DELETED
@@ -1,177 +0,0 @@
1
- import json
2
- from pathlib import Path
3
- from typing import Any
4
-
5
- import gradio as gr
6
- from huggingface_hub import HfApi
7
-
8
- from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
9
-
10
-
11
- def _as_pretty_json(payload: dict[str, Any]) -> str:
12
- return json.dumps(payload, indent=2, ensure_ascii=False)
13
-
14
-
15
- def _build_api(token: str) -> HfApi:
16
- clean_token = (token or "").strip()
17
- if clean_token:
18
- return HfApi(token=clean_token)
19
- return HfApi()
20
-
21
-
22
- def build_upload_app() -> gr.Blocks:
23
- with gr.Blocks(title="BirdNET Segments Uploader") as demo:
24
- gr.Markdown("# BirdNET Segments Uploader")
25
- gr.Markdown(
26
- "Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
27
- "Recomendado: executar dry-run antes do upload real."
28
- )
29
-
30
- with gr.Row():
31
- project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
32
- dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
33
-
34
- detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
35
- segments_root = gr.Textbox(
36
- label="Pasta raiz de segmentos",
37
- placeholder=r"ex: C:\dados\BirdNET Segments",
38
- )
39
-
40
- with gr.Accordion("Configuracao avancada", open=False):
41
- hf_token = gr.Textbox(
42
- label="HF Token (opcional)",
43
- type="password",
44
- placeholder="Se vazio, usa sessao ja autenticada no ambiente",
45
- )
46
- with gr.Row():
47
- batch_size = gr.Number(label="Batch size", value=200, precision=0)
48
- shard_size = gr.Number(label="Shard size", value=10000, precision=0)
49
- with gr.Row():
50
- max_retries = gr.Number(label="Max retries", value=3, precision=0)
51
- retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
52
- resume_state_file = gr.Textbox(
53
- label="Resume state file",
54
- value=".ingest-segments-state.json",
55
- )
56
- report_file = gr.Textbox(
57
- label="Report file local (opcional)",
58
- placeholder="ex: .ingest-run-report.json",
59
- )
60
-
61
- with gr.Row():
62
- dry_run_button = gr.Button("Dry-run", variant="secondary")
63
- upload_button = gr.Button("Upload real", variant="primary")
64
-
65
- status = gr.Markdown(value="Pronto")
66
- result_json = gr.Code(label="Resultado JSON", language="json")
67
-
68
- def run_dry_run(
69
- project: str,
70
- repo: str,
71
- csv_path: str | None,
72
- segments_path: str,
73
- report_path: str,
74
- ) -> tuple[str, str]:
75
- project = (project or "").strip()
76
- if not project:
77
- return "❌ Informe o project slug", "{}"
78
-
79
- if not csv_path:
80
- return "❌ Selecione o CSV de deteccoes", "{}"
81
-
82
- if not segments_path or not Path(segments_path).exists():
83
- return "❌ Pasta de segmentos invalida", "{}"
84
-
85
- try:
86
- result = run_ingest_segments_dry_run(
87
- project_slug=project,
88
- detections_csv=csv_path,
89
- segments_root=segments_path,
90
- )
91
- if report_path.strip():
92
- Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
93
- summary = (
94
- "✅ Dry-run finalizado | "
95
- f"Matched: {result['matched_rows']} | "
96
- f"Unmatched: {result['unmatched_rows']} | "
97
- f"Ambiguous: {result['ambiguous_rows']}"
98
- )
99
- return summary, _as_pretty_json(result)
100
- except Exception as exc:
101
- return f"❌ Dry-run falhou: {exc}", "{}"
102
-
103
- def run_upload(
104
- project: str,
105
- repo: str,
106
- csv_path: str | None,
107
- segments_path: str,
108
- token: str,
109
- batch: float,
110
- shard: float,
111
- retries: float,
112
- backoff: float,
113
- resume_file: str,
114
- report_path: str,
115
- ) -> tuple[str, str]:
116
- project = (project or "").strip()
117
- repo = (repo or "").strip()
118
- if not project:
119
- return "❌ Informe o project slug", "{}"
120
- if not repo or "/" not in repo:
121
- return "❌ Informe dataset repo no formato owner/repo", "{}"
122
- if not csv_path:
123
- return "❌ Selecione o CSV de deteccoes", "{}"
124
- if not segments_path or not Path(segments_path).exists():
125
- return "❌ Pasta de segmentos invalida", "{}"
126
-
127
- try:
128
- api = _build_api(token)
129
- result = ingest_segments_to_hf(
130
- api=api,
131
- project_slug=project,
132
- dataset_repo=repo,
133
- detections_csv=csv_path,
134
- segments_root=segments_path,
135
- batch_size=int(batch),
136
- shard_size=int(shard),
137
- max_retries=int(retries),
138
- retry_backoff_seconds=float(backoff),
139
- resume_state_file=resume_file,
140
- )
141
- if report_path.strip():
142
- Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
143
- summary = (
144
- "✅ Upload finalizado | "
145
- f"Uploaded now: {result['uploaded_audio_now']} | "
146
- f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
147
- f"Failed uploads: {result['failed_uploads']}"
148
- )
149
- return summary, _as_pretty_json(result)
150
- except Exception as exc:
151
- return f"❌ Upload falhou: {exc}", "{}"
152
-
153
- dry_run_button.click(
154
- fn=run_dry_run,
155
- inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
156
- outputs=[status, result_json],
157
- )
158
-
159
- upload_button.click(
160
- fn=run_upload,
161
- inputs=[
162
- project_slug,
163
- dataset_repo,
164
- detections_csv,
165
- segments_root,
166
- hf_token,
167
- batch_size,
168
- shard_size,
169
- max_retries,
170
- retry_backoff_seconds,
171
- resume_state_file,
172
- report_file,
173
- ],
174
- outputs=[status, result_json],
175
- )
176
-
177
- return demo
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
README.md CHANGED
@@ -1,82 +1,64 @@
1
- # BirdNET Validator HF App
2
-
3
- Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
4
- - Multi-projeto
5
- - Audio sob demanda
6
- - Persistencia de validacoes
7
- - Escalabilidade para datasets grandes
8
-
9
- ## Desenvolvimento local
10
- 1. Criar ambiente virtual Python 3.11+
11
- 2. Instalar dependencias:
12
- pip install -r requirements.txt
13
- 3. Rodar app:
14
- python app.py
15
- 4. Rodar uploader (app separado para ingestao):
16
- python app_uploader.py
17
-
18
- ## Fluxo recomendado (separado)
19
- 1. Uploader app (porta 7862 por padrao):
20
- - Rodar dry-run com CSV + pasta de segmentos
21
- - Rodar upload real para owner/repo do projeto
22
- 2. Validator app (porta 7860):
23
- - Validar deteccoes do projeto com dados ja publicados
24
-
25
- Essa separacao evita impacto de performance na validacao durante uploads grandes.
26
-
27
- ## CLI de dataset (Sprint 1)
28
- Comandos principais:
29
-
30
- 1. Criar estrutura do projeto no dataset Hugging Face:
31
- python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
32
-
33
- 2. Gerar manifesto e shards iniciais de indice:
34
- python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
35
-
36
- 3. Fazer upload de audio em lotes com retry e resume:
37
- python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
38
-
39
- 4. Verificar consistencia basica do projeto:
40
- python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
41
-
42
- 5. Ingestao BirdNET por projeto (dry-run):
43
- python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
44
-
45
- 6. Ingestao BirdNET por projeto (upload real):
46
- python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
47
-
48
- Estrutura base criada por projeto/dataset:
49
- - audio/
50
- - index/
51
- - index/shards/
52
- - validations/
53
- - audit/
54
- - manifest.json
55
-
56
- ## Estrutura
57
- - src/domain: modelos de dominio
58
- - src/repositories: contratos de persistencia
59
- - src/services: servicos de aplicacao
60
- - src/auth: autenticacao e autorizacao
61
- - src/ui: montagem da interface
62
- - src/cache: cache efemero local
63
- - cli: comandos de ingestao/publicacao
64
- - tests: testes unitarios e integracao
65
-
66
- ## Estado atual de implementacao
67
- - Sprint 1: CLI create-project, sync-audio, build-index, verify-project
68
- - Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
69
- - Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
70
- - Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
71
- - Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
72
- - Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
73
- - Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
74
- - Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
75
- - Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
76
- - Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
77
- - Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
78
- - Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
79
- - Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
80
- - Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
81
- - Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
82
- - Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
 
1
+ # BirdNET-Validator-App
2
+
3
+ BirdNET-Validator-App is a Gradio application for human validation of BirdNET detections in multi-project workflows.
4
+
5
+ It provides a practical review interface with authentication, project-level authorization, paginated queues, on-demand audio, and append-only validation history.
6
+
7
+ ## Objective
8
+
9
+ This project focuses on high-confidence validation operations by combining:
10
+
11
+ - controlled access by user and project
12
+ - rapid decision flow for detections
13
+ - auditability through append-only events
14
+ - conflict-aware updates for concurrent validators
15
+
16
+ ## Key Usability Features
17
+
18
+ 1. Multi-project login with role-based access
19
+ 2. Project selection for authorized datasets
20
+ 3. Detection queue with filters and pagination
21
+ 4. Audio loading on demand with ephemeral cache
22
+ 5. Fast validation actions (positive, negative, uncertain, skip)
23
+ 6. Concurrency-safe writes with optimistic lock feedback
24
+ 7. Conflict resolution support and validation reporting
25
+
26
+ ## Quick Start (Local)
27
+
28
+ 1. Create and activate a Python 3.11+ virtual environment.
29
+ 2. Install dependencies:
30
+
31
+ ```bash
32
+ pip install -r requirements.txt
33
+ ```
34
+
35
+ 3. Run the app:
36
+
37
+ ```bash
38
+ python app.py
39
+ ```
40
+
41
+ Default port is 7860.
42
+
43
+ ## Typical Validation Flow
44
+
45
+ 1. Login with a valid user.
46
+ 2. Select an authorized project.
47
+ 3. Load queue items and apply filters when needed.
48
+ 4. Listen to selected audio and submit validation status.
49
+ 5. Resolve conflicts when concurrent updates occur.
50
+ 6. Export or inspect project validation report.
51
+
52
+ ## Repository Structure
53
+
54
+ - `app.py`: validator app entrypoint
55
+ - `src/auth`: authentication/session and ACL logic
56
+ - `src/domain`: domain models
57
+ - `src/repositories`: in-memory and append-only persistence
58
+ - `src/services`: queue, validation, and audio fetch services
59
+ - `src/ui`: Gradio interface composition and callbacks
60
+ - `tests`: unit and integration test suites
61
+
62
+ ## Project Status
63
+
64
+ Validator workflow is active and under continuous development, with emphasis on reliability, auditability, and operator productivity.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app_uploader.py DELETED
@@ -1,9 +0,0 @@
1
- import os
2
-
3
- from src.ui.upload_app_factory import build_upload_app
4
-
5
-
6
- if __name__ == "__main__":
7
- app = build_upload_app()
8
- port = int(os.getenv("BIRDNET_UPLOADER_PORT", "7862"))
9
- app.launch(server_name="0.0.0.0", server_port=port)
 
 
 
 
 
 
 
 
 
 
cli/hf_dataset_cli.py DELETED
@@ -1,969 +0,0 @@
1
- import argparse
2
- import hashlib
3
- import json
4
- import math
5
- import re
6
- import tempfile
7
- import time
8
- from dataclasses import asdict, dataclass
9
- from datetime import UTC, datetime
10
- from io import BytesIO
11
- from pathlib import Path
12
- from typing import Any
13
-
14
- import pandas as pd
15
- from huggingface_hub import HfApi, hf_hub_download
16
-
17
-
18
- SCHEMA_VERSION = "1.0.0"
19
- REQUIRED_PROJECT_PREFIXES = (
20
- "audio/",
21
- "index/",
22
- "index/shards/",
23
- "validations/",
24
- "audit/",
25
- )
26
- REQUIRED_PLACEHOLDER_FILES = (
27
- "audio/.gitkeep",
28
- "index/.gitkeep",
29
- "index/shards/.gitkeep",
30
- "validations/.gitkeep",
31
- "audit/.gitkeep",
32
- )
33
- REQUIRED_DETECTIONS_COLUMNS = (
34
- "detection_key",
35
- "audio_id",
36
- "scientific_name",
37
- "confidence",
38
- "start_time",
39
- "end_time",
40
- )
41
- SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
42
- REQUIRED_BIRDNET_COLUMNS = (
43
- "source_file",
44
- "scientific_name",
45
- "confidence",
46
- "start_time",
47
- "end_time",
48
- )
49
- SEGMENT_FILENAME_PATTERN = re.compile(
50
- r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
51
- )
52
-
53
-
54
- @dataclass(slots=True)
55
- class ShardMetadata:
56
- path: str
57
- rows: int
58
- sha256: str
59
- size_bytes: int
60
-
61
-
62
- @dataclass(slots=True)
63
- class SegmentFileRecord:
64
- scientific_name: str
65
- source_stem: str
66
- start_time: float
67
- end_time: float
68
- confidence_pct: int
69
- absolute_path: Path
70
- relative_path: str
71
-
72
-
73
- def _utcnow_iso() -> str:
74
- return datetime.now(UTC).replace(microsecond=0).isoformat()
75
-
76
-
77
- def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
78
- api.upload_file(
79
- path_or_fileobj=BytesIO(content.encode("utf-8")),
80
- path_in_repo=path_in_repo,
81
- repo_id=dataset_repo,
82
- repo_type="dataset",
83
- )
84
-
85
-
86
- def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
87
- api.upload_file(
88
- path_or_fileobj=BytesIO(b""),
89
- path_in_repo=path_in_repo,
90
- repo_id=dataset_repo,
91
- repo_type="dataset",
92
- )
93
-
94
-
95
- def ensure_project_dataset_structure(
96
- api: HfApi,
97
- project_slug: str,
98
- dataset_repo: str,
99
- create_private_repo: bool,
100
- ) -> dict[str, Any]:
101
- api.create_repo(
102
- repo_id=dataset_repo,
103
- repo_type="dataset",
104
- private=create_private_repo,
105
- exist_ok=True,
106
- )
107
-
108
- files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
109
- created_paths: list[str] = []
110
-
111
- for placeholder in REQUIRED_PLACEHOLDER_FILES:
112
- if placeholder not in files:
113
- _upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
114
- created_paths.append(placeholder)
115
-
116
- manifest_path = "manifest.json"
117
- if manifest_path not in files:
118
- manifest = {
119
- "schema_version": SCHEMA_VERSION,
120
- "project_slug": project_slug,
121
- "dataset_repo_id": dataset_repo,
122
- "created_at": _utcnow_iso(),
123
- "updated_at": _utcnow_iso(),
124
- "index": {
125
- "total_detections": 0,
126
- "total_audio_files": 0,
127
- "shard_size": 0,
128
- "shards": [],
129
- },
130
- }
131
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
132
- created_paths.append(manifest_path)
133
-
134
- return {
135
- "dataset_repo": dataset_repo,
136
- "project_slug": project_slug,
137
- "created_paths": created_paths,
138
- }
139
-
140
-
141
- def load_detections_table(detections_file: str) -> pd.DataFrame:
142
- input_path = Path(detections_file)
143
- if not input_path.exists():
144
- raise FileNotFoundError(f"Detections file not found: {detections_file}")
145
-
146
- suffix = input_path.suffix.lower()
147
- if suffix == ".csv":
148
- frame = pd.read_csv(input_path)
149
- elif suffix in {".jsonl", ".ndjson"}:
150
- frame = pd.read_json(input_path, lines=True)
151
- elif suffix == ".parquet":
152
- frame = pd.read_parquet(input_path)
153
- else:
154
- raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
155
-
156
- missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
157
- if missing_columns:
158
- raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
159
-
160
- if frame.empty:
161
- return frame
162
-
163
- deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
164
- return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
165
-
166
-
167
- def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
168
- input_path = Path(detections_csv)
169
- if not input_path.exists():
170
- raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
171
-
172
- frame = pd.read_csv(input_path)
173
- missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
174
- if missing_columns:
175
- raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
176
-
177
- if frame.empty:
178
- return frame
179
-
180
- normalized = frame.copy()
181
- normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
182
- normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
183
- normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
184
- normalized["species_key"] = normalized["scientific_name"].str.casefold()
185
- normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
186
- normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
187
- normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
188
-
189
- normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
190
- return normalized
191
-
192
-
193
- def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
194
- payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
195
- return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
196
-
197
-
198
- def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
199
- stem = Path(file_name).stem
200
- match = SEGMENT_FILENAME_PATTERN.match(stem)
201
- if not match:
202
- return None
203
-
204
- source_stem = match.group("source_stem")
205
- start_time = float(match.group("start"))
206
- end_time = float(match.group("end"))
207
- confidence_pct = int(match.group("confidence_pct"))
208
- return source_stem, start_time, end_time, confidence_pct
209
-
210
-
211
- def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
212
- root = Path(segments_root)
213
- if not root.exists() or not root.is_dir():
214
- raise FileNotFoundError(f"Segments root not found: {segments_root}")
215
-
216
- records: list[SegmentFileRecord] = []
217
- for absolute_path in root.rglob("*"):
218
- if not absolute_path.is_file():
219
- continue
220
- if absolute_path.suffix.lower() != ".wav":
221
- continue
222
-
223
- rel = absolute_path.relative_to(root)
224
- scientific_name = rel.parts[0] if rel.parts else ""
225
- parsed = parse_segment_filename(absolute_path.name)
226
- if not parsed:
227
- continue
228
- source_stem, start_time, end_time, confidence_pct = parsed
229
- records.append(
230
- SegmentFileRecord(
231
- scientific_name=scientific_name,
232
- source_stem=source_stem,
233
- start_time=start_time,
234
- end_time=end_time,
235
- confidence_pct=confidence_pct,
236
- absolute_path=absolute_path,
237
- relative_path=rel.as_posix(),
238
- )
239
- )
240
-
241
- return records
242
-
243
-
244
- def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
245
- frame = load_birdnet_detections_csv(detections_csv=detections_csv)
246
- segment_records = discover_segment_records(segments_root=segments_root)
247
-
248
- by_species: dict[str, list[SegmentFileRecord]] = {}
249
- for record in segment_records:
250
- by_species.setdefault(record.scientific_name.casefold(), []).append(record)
251
-
252
- matched_rows: list[dict[str, Any]] = []
253
- unmatched_rows: list[dict[str, Any]] = []
254
- ambiguous_rows: list[dict[str, Any]] = []
255
-
256
- for row in frame.to_dict(orient="records"):
257
- species_key = str(row.get("species_key", "")).casefold()
258
- source_stem = str(row.get("source_stem", "")).strip()
259
- start_time = float(row["start_time"])
260
- end_time = float(row["end_time"])
261
- confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
262
-
263
- species_candidates = by_species.get(species_key, [])
264
- exact_matches = [
265
- record
266
- for record in species_candidates
267
- if record.source_stem == source_stem
268
- and abs(record.start_time - start_time) < 1e-6
269
- and abs(record.end_time - end_time) < 1e-6
270
- ]
271
-
272
- if not exact_matches:
273
- unmatched_rows.append(
274
- {
275
- "source_file": row.get("source_file", ""),
276
- "scientific_name": row.get("scientific_name", ""),
277
- "start_time": start_time,
278
- "end_time": end_time,
279
- }
280
- )
281
- continue
282
-
283
- selected = exact_matches[0]
284
- if len(exact_matches) > 1:
285
- expected_pct = int(round(confidence * 100))
286
- exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
287
- selected = exact_matches[0]
288
- ambiguous_rows.append(
289
- {
290
- "source_file": row.get("source_file", ""),
291
- "scientific_name": row.get("scientific_name", ""),
292
- "start_time": start_time,
293
- "end_time": end_time,
294
- "candidates": len(exact_matches),
295
- }
296
- )
297
-
298
- matched_rows.append(
299
- {
300
- "project_slug": project_slug,
301
- "detection_key": _build_detection_key(
302
- project_slug=project_slug,
303
- source_file=str(row.get("source_file", "")),
304
- scientific_name=str(row.get("scientific_name", "")),
305
- start_time=start_time,
306
- end_time=end_time,
307
- ),
308
- "audio_id": source_stem,
309
- "segment_filename": selected.absolute_path.name,
310
- "segment_relpath": selected.relative_path,
311
- "scientific_name": row.get("scientific_name", ""),
312
- "common_name": row.get("common_name", ""),
313
- "confidence": confidence,
314
- "start_time": start_time,
315
- "end_time": end_time,
316
- "exact_start": row.get("exact_start", start_time),
317
- "exact_end": row.get("exact_end", end_time),
318
- "locality": row.get("locality", ""),
319
- "point": row.get("point", ""),
320
- "date_folder": row.get("date_folder", ""),
321
- "min_freq": row.get("min_freq", None),
322
- "max_freq": row.get("max_freq", None),
323
- "box_source": row.get("box_source", ""),
324
- "label": row.get("label", ""),
325
- "source_file": row.get("source_file", ""),
326
- "schema_version": SCHEMA_VERSION,
327
- "ingested_at": _utcnow_iso(),
328
- }
329
- )
330
-
331
- return {
332
- "frame": frame,
333
- "segment_records_total": len(segment_records),
334
- "matched_rows": matched_rows,
335
- "unmatched_rows": unmatched_rows,
336
- "ambiguous_rows": ambiguous_rows,
337
- }
338
-
339
-
340
- def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
341
- match_result = _match_birdnet_rows(
342
- project_slug=project_slug,
343
- detections_csv=detections_csv,
344
- segments_root=segments_root,
345
- )
346
- frame = match_result["frame"]
347
- matched_rows = match_result["matched_rows"]
348
- unmatched_rows = match_result["unmatched_rows"]
349
- ambiguous_rows = match_result["ambiguous_rows"]
350
-
351
- unique_audio = {row["audio_id"] for row in matched_rows}
352
- result = {
353
- "mode": "dry-run",
354
- "project_slug": project_slug,
355
- "detections_csv": detections_csv,
356
- "segments_root": segments_root,
357
- "csv_rows_total": int(len(frame)),
358
- "segments_found_total": int(match_result["segment_records_total"]),
359
- "matched_rows": len(matched_rows),
360
- "unmatched_rows": len(unmatched_rows),
361
- "ambiguous_rows": len(ambiguous_rows),
362
- "unique_audio_ids": len(unique_audio),
363
- "sample_unmatched": unmatched_rows[:20],
364
- "sample_ambiguous": ambiguous_rows[:20],
365
- }
366
- return result
367
-
368
-
369
- def _file_sha256(file_path: Path) -> str:
370
- hasher = hashlib.sha256()
371
- with file_path.open("rb") as handle:
372
- for chunk in iter(lambda: handle.read(65536), b""):
373
- hasher.update(chunk)
374
- return hasher.hexdigest()
375
-
376
-
377
- def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
378
- if shard_size <= 0:
379
- raise ValueError("shard_size must be greater than zero")
380
-
381
- output_dir.mkdir(parents=True, exist_ok=True)
382
- shards: list[ShardMetadata] = []
383
-
384
- if frame.empty:
385
- return shards
386
-
387
- total_rows = len(frame)
388
- total_shards = math.ceil(total_rows / shard_size)
389
-
390
- for index in range(total_shards):
391
- start = index * shard_size
392
- end = min(start + shard_size, total_rows)
393
- shard_frame = frame.iloc[start:end]
394
- shard_filename = f"shard-{index:05d}.parquet"
395
- local_path = output_dir / shard_filename
396
- shard_frame.to_parquet(local_path, index=False)
397
-
398
- shards.append(
399
- ShardMetadata(
400
- path=f"index/shards/{shard_filename}",
401
- rows=len(shard_frame),
402
- sha256=_file_sha256(local_path),
403
- size_bytes=local_path.stat().st_size,
404
- )
405
- )
406
-
407
- return shards
408
-
409
-
410
- def build_manifest_payload(
411
- project_slug: str,
412
- dataset_repo: str,
413
- frame: pd.DataFrame,
414
- shard_size: int,
415
- shard_metadata: list[ShardMetadata],
416
- ) -> dict[str, Any]:
417
- now = _utcnow_iso()
418
- return {
419
- "schema_version": SCHEMA_VERSION,
420
- "project_slug": project_slug,
421
- "dataset_repo_id": dataset_repo,
422
- "updated_at": now,
423
- "index": {
424
- "total_detections": len(frame),
425
- "total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
426
- "shard_size": shard_size,
427
- "shards": [asdict(item) for item in shard_metadata],
428
- },
429
- }
430
-
431
-
432
- def discover_audio_files(local_audio_dir: str) -> list[Path]:
433
- audio_root = Path(local_audio_dir)
434
- if not audio_root.exists() or not audio_root.is_dir():
435
- raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
436
-
437
- files = [
438
- path
439
- for path in audio_root.rglob("*")
440
- if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
441
- ]
442
- return sorted(files)
443
-
444
-
445
- def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
446
- if chunk_size <= 0:
447
- raise ValueError("chunk_size must be greater than zero")
448
- return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
449
-
450
-
451
- def _load_resume_state(state_file: Path) -> dict[str, Any]:
452
- if not state_file.exists():
453
- return {"uploaded": [], "failed": []}
454
-
455
- payload = json.loads(state_file.read_text(encoding="utf-8"))
456
- uploaded = payload.get("uploaded", [])
457
- failed = payload.get("failed", [])
458
- return {
459
- "uploaded": uploaded if isinstance(uploaded, list) else [],
460
- "failed": failed if isinstance(failed, list) else [],
461
- }
462
-
463
-
464
- def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
465
- state_file.parent.mkdir(parents=True, exist_ok=True)
466
- payload = {
467
- "updated_at": _utcnow_iso(),
468
- "uploaded": sorted(uploaded),
469
- "failed": sorted(failed),
470
- }
471
- state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
472
-
473
-
474
- def _upload_audio_with_retry(
475
- api: HfApi,
476
- dataset_repo: str,
477
- local_path: Path,
478
- path_in_repo: str,
479
- max_retries: int,
480
- retry_backoff_seconds: float,
481
- ) -> None:
482
- attempts = 0
483
- while True:
484
- attempts += 1
485
- try:
486
- api.upload_file(
487
- path_or_fileobj=str(local_path),
488
- path_in_repo=path_in_repo,
489
- repo_id=dataset_repo,
490
- repo_type="dataset",
491
- )
492
- return
493
- except Exception:
494
- if attempts > max_retries:
495
- raise
496
- time.sleep(retry_backoff_seconds * attempts)
497
-
498
-
499
- def ingest_segments_to_hf(
500
- api: HfApi,
501
- project_slug: str,
502
- dataset_repo: str,
503
- detections_csv: str,
504
- segments_root: str,
505
- batch_size: int,
506
- shard_size: int,
507
- max_retries: int,
508
- retry_backoff_seconds: float,
509
- resume_state_file: str,
510
- ) -> dict[str, Any]:
511
- ensure_project_dataset_structure(
512
- api=api,
513
- project_slug=project_slug,
514
- dataset_repo=dataset_repo,
515
- create_private_repo=False,
516
- )
517
-
518
- match_result = _match_birdnet_rows(
519
- project_slug=project_slug,
520
- detections_csv=detections_csv,
521
- segments_root=segments_root,
522
- )
523
- frame = match_result["frame"]
524
- matched_rows = match_result["matched_rows"]
525
- unmatched_rows = match_result["unmatched_rows"]
526
- ambiguous_rows = match_result["ambiguous_rows"]
527
-
528
- # Deduplicate by remote path so retries/resume work on a stable key.
529
- path_to_local: dict[str, Path] = {}
530
- for row in matched_rows:
531
- repo_path = f"audio/segments/{row['segment_relpath']}"
532
- if repo_path in path_to_local:
533
- continue
534
- path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
535
-
536
- state_path = Path(resume_state_file)
537
- state_payload = _load_resume_state(state_file=state_path)
538
- uploaded_state = set(str(item) for item in state_payload["uploaded"])
539
- failed_state = set(str(item) for item in state_payload["failed"])
540
-
541
- remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
542
-
543
- pending: list[tuple[Path, str]] = []
544
- skipped_existing = 0
545
- for repo_path, local_path in path_to_local.items():
546
- if repo_path in remote_files:
547
- uploaded_state.add(repo_path)
548
- skipped_existing += 1
549
- continue
550
- if repo_path in uploaded_state:
551
- continue
552
- pending.append((local_path, repo_path))
553
-
554
- uploaded_now: set[str] = set()
555
- failed_now: set[str] = set()
556
-
557
- for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
558
- batch_uploaded = 0
559
- batch_failed = 0
560
- for local_path, repo_path in batch:
561
- try:
562
- _upload_audio_with_retry(
563
- api=api,
564
- dataset_repo=dataset_repo,
565
- local_path=local_path,
566
- path_in_repo=repo_path,
567
- max_retries=max_retries,
568
- retry_backoff_seconds=retry_backoff_seconds,
569
- )
570
- uploaded_state.add(repo_path)
571
- failed_state.discard(repo_path)
572
- uploaded_now.add(repo_path)
573
- batch_uploaded += 1
574
- except Exception:
575
- failed_state.add(repo_path)
576
- failed_now.add(repo_path)
577
- batch_failed += 1
578
-
579
- print(
580
- json.dumps(
581
- {
582
- "event": "ingest-segments-audio-batch",
583
- "batch_index": batch_index,
584
- "batch_size": len(batch),
585
- "uploaded": batch_uploaded,
586
- "failed": batch_failed,
587
- }
588
- )
589
- )
590
- _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
591
-
592
- index_frame = pd.DataFrame(matched_rows)
593
- if not index_frame.empty:
594
- index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
595
-
596
- with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
597
- temp_dir = Path(temp_dir_str)
598
- shard_dir = temp_dir / "index" / "shards"
599
- shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
600
-
601
- for item in shard_metadata:
602
- local_path = temp_dir / item.path
603
- api.upload_file(
604
- path_or_fileobj=str(local_path),
605
- path_in_repo=item.path,
606
- repo_id=dataset_repo,
607
- repo_type="dataset",
608
- )
609
-
610
- manifest = build_manifest_payload(
611
- project_slug=project_slug,
612
- dataset_repo=dataset_repo,
613
- frame=index_frame,
614
- shard_size=shard_size,
615
- shard_metadata=shard_metadata,
616
- )
617
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
618
-
619
- run_report = {
620
- "mode": "execute",
621
- "project_slug": project_slug,
622
- "dataset_repo": dataset_repo,
623
- "detections_csv": detections_csv,
624
- "segments_root": segments_root,
625
- "csv_rows_total": int(len(frame)),
626
- "segments_found_total": int(match_result["segment_records_total"]),
627
- "matched_rows": len(matched_rows),
628
- "unmatched_rows": len(unmatched_rows),
629
- "ambiguous_rows": len(ambiguous_rows),
630
- "unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
631
- "pending_audio_uploads": len(pending),
632
- "uploaded_audio_now": len(uploaded_now),
633
- "uploaded_audio_skipped_existing": skipped_existing,
634
- "failed_uploads": len(failed_now),
635
- "resume_state_file": str(state_path),
636
- "index_rows_written": len(index_frame),
637
- "shards_written": len(shard_metadata),
638
- "sample_unmatched": unmatched_rows[:20],
639
- "sample_ambiguous": ambiguous_rows[:20],
640
- }
641
-
642
- audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
643
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
644
- return run_report
645
-
646
-
647
- def sync_audio_batches(
648
- api: HfApi,
649
- project_slug: str,
650
- dataset_repo: str,
651
- local_audio_dir: str,
652
- batch_size: int,
653
- max_retries: int,
654
- retry_backoff_seconds: float,
655
- resume_state_file: str,
656
- ) -> dict[str, Any]:
657
- ensure_project_dataset_structure(
658
- api=api,
659
- project_slug=project_slug,
660
- dataset_repo=dataset_repo,
661
- create_private_repo=False,
662
- )
663
-
664
- all_files = discover_audio_files(local_audio_dir=local_audio_dir)
665
- state_path = Path(resume_state_file)
666
- state_payload = _load_resume_state(state_file=state_path)
667
- uploaded_state = set(str(item) for item in state_payload["uploaded"])
668
-
669
- remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
670
-
671
- pending: list[tuple[Path, str]] = []
672
- for path in all_files:
673
- rel = path.relative_to(Path(local_audio_dir)).as_posix()
674
- repo_path = f"audio/{rel}"
675
-
676
- if repo_path in remote_files:
677
- uploaded_state.add(repo_path)
678
- continue
679
-
680
- if repo_path in uploaded_state:
681
- continue
682
-
683
- pending.append((path, repo_path))
684
-
685
- pending_paths = [item[0] for item in pending]
686
- by_local_to_repo = {local: repo for local, repo in pending}
687
- batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
688
-
689
- uploaded_now: set[str] = set()
690
- failed: set[str] = set()
691
-
692
- for batch_index, batch in enumerate(batches, start=1):
693
- batch_uploaded = 0
694
- batch_failed = 0
695
-
696
- for local_path in batch:
697
- repo_path = by_local_to_repo[local_path]
698
- try:
699
- _upload_audio_with_retry(
700
- api=api,
701
- dataset_repo=dataset_repo,
702
- local_path=local_path,
703
- path_in_repo=repo_path,
704
- max_retries=max_retries,
705
- retry_backoff_seconds=retry_backoff_seconds,
706
- )
707
- uploaded_now.add(repo_path)
708
- uploaded_state.add(repo_path)
709
- batch_uploaded += 1
710
- except Exception:
711
- failed.add(repo_path)
712
- batch_failed += 1
713
-
714
- print(
715
- json.dumps(
716
- {
717
- "event": "sync-audio-batch",
718
- "batch_index": batch_index,
719
- "batch_size": len(batch),
720
- "uploaded": batch_uploaded,
721
- "failed": batch_failed,
722
- }
723
- )
724
- )
725
- _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
726
-
727
- return {
728
- "project_slug": project_slug,
729
- "dataset_repo": dataset_repo,
730
- "total_local_audio_files": len(all_files),
731
- "pending_uploads": len(pending),
732
- "uploaded_now": len(uploaded_now),
733
- "failed": len(failed),
734
- "resume_state_file": str(state_path),
735
- }
736
-
737
-
738
- def build_and_upload_index(
739
- api: HfApi,
740
- project_slug: str,
741
- dataset_repo: str,
742
- detections_file: str,
743
- shard_size: int,
744
- ) -> dict[str, Any]:
745
- ensure_project_dataset_structure(
746
- api=api,
747
- project_slug=project_slug,
748
- dataset_repo=dataset_repo,
749
- create_private_repo=False,
750
- )
751
-
752
- frame = load_detections_table(detections_file=detections_file)
753
-
754
- with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
755
- temp_dir = Path(temp_dir_str)
756
- shard_dir = temp_dir / "index" / "shards"
757
- metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
758
-
759
- for item in metadata:
760
- local_path = temp_dir / item.path
761
- api.upload_file(
762
- path_or_fileobj=str(local_path),
763
- path_in_repo=item.path,
764
- repo_id=dataset_repo,
765
- repo_type="dataset",
766
- )
767
-
768
- manifest = build_manifest_payload(
769
- project_slug=project_slug,
770
- dataset_repo=dataset_repo,
771
- frame=frame,
772
- shard_size=shard_size,
773
- shard_metadata=metadata,
774
- )
775
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
776
-
777
- return {
778
- "project_slug": project_slug,
779
- "dataset_repo": dataset_repo,
780
- "total_detections": manifest["index"]["total_detections"],
781
- "total_audio_files": manifest["index"]["total_audio_files"],
782
- "total_shards": len(metadata),
783
- }
784
-
785
-
786
- def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
787
- errors: list[str] = []
788
-
789
- for prefix in REQUIRED_PROJECT_PREFIXES:
790
- if not any(path.startswith(prefix) for path in repo_files):
791
- errors.append(f"Missing prefix in dataset repo: {prefix}")
792
-
793
- if manifest_payload.get("schema_version") != SCHEMA_VERSION:
794
- errors.append(
795
- f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
796
- )
797
-
798
- if manifest_payload.get("project_slug") != project_slug:
799
- errors.append(
800
- f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
801
- )
802
-
803
- index_section = manifest_payload.get("index", {})
804
- shards = index_section.get("shards", [])
805
- if not isinstance(shards, list):
806
- errors.append("Manifest index.shards must be a list")
807
- return errors
808
-
809
- for shard in shards:
810
- shard_path = shard.get("path")
811
- if not shard_path or not isinstance(shard_path, str):
812
- errors.append("Shard entry missing path")
813
- continue
814
- if shard_path not in repo_files:
815
- errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
816
-
817
- return errors
818
-
819
-
820
- def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
821
- repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
822
- if "manifest.json" not in repo_files:
823
- return {
824
- "ok": False,
825
- "errors": ["manifest.json not found in dataset repository"],
826
- }
827
-
828
- manifest_local = hf_hub_download(
829
- repo_id=dataset_repo,
830
- repo_type="dataset",
831
- filename="manifest.json",
832
- )
833
- manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
834
- errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
835
-
836
- return {
837
- "ok": len(errors) == 0,
838
- "errors": errors,
839
- "total_files": len(repo_files),
840
- "total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
841
- }
842
-
843
-
844
- def build_parser() -> argparse.ArgumentParser:
845
- parser = argparse.ArgumentParser(description="HF dataset project CLI")
846
- sub = parser.add_subparsers(dest="command", required=True)
847
-
848
- create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
849
- create_project.add_argument("--project-slug", required=True)
850
- create_project.add_argument("--dataset-repo", required=True)
851
- create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
852
-
853
- build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
854
- build_index.add_argument("--project-slug", required=True)
855
- build_index.add_argument("--dataset-repo", required=True)
856
- build_index.add_argument("--detections-file", required=True)
857
- build_index.add_argument("--shard-size", type=int, default=10000)
858
-
859
- sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
860
- sync_audio.add_argument("--project-slug", required=True)
861
- sync_audio.add_argument("--dataset-repo", required=True)
862
- sync_audio.add_argument("--local-audio-dir", required=True)
863
- sync_audio.add_argument("--batch-size", type=int, default=100)
864
- sync_audio.add_argument("--max-retries", type=int, default=3)
865
- sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
866
- sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
867
-
868
- verify_project = sub.add_parser("verify-project", help="Verify project integrity")
869
- verify_project.add_argument("--project-slug", required=True)
870
- verify_project.add_argument("--dataset-repo", required=True)
871
-
872
- ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
873
- ingest_segments.add_argument("--project-slug", required=True)
874
- ingest_segments.add_argument("--dataset-repo", required=True)
875
- ingest_segments.add_argument("--detections-csv", required=True)
876
- ingest_segments.add_argument("--segments-root", required=True)
877
- ingest_segments.add_argument("--batch-size", type=int, default=200)
878
- ingest_segments.add_argument("--shard-size", type=int, default=10000)
879
- ingest_segments.add_argument("--max-retries", type=int, default=3)
880
- ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
881
- ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
882
- ingest_segments.add_argument("--dry-run", action="store_true")
883
- ingest_segments.add_argument("--report-file", default="")
884
-
885
- return parser
886
-
887
-
888
- def main() -> int:
889
- parser = build_parser()
890
- args = parser.parse_args()
891
- api = HfApi()
892
-
893
- if args.command == "create-project":
894
- result = ensure_project_dataset_structure(
895
- api=api,
896
- project_slug=args.project_slug,
897
- dataset_repo=args.dataset_repo,
898
- create_private_repo=bool(args.private),
899
- )
900
- print(json.dumps(result, indent=2))
901
- return 0
902
-
903
- if args.command == "build-index":
904
- result = build_and_upload_index(
905
- api=api,
906
- project_slug=args.project_slug,
907
- dataset_repo=args.dataset_repo,
908
- detections_file=args.detections_file,
909
- shard_size=args.shard_size,
910
- )
911
- print(json.dumps(result, indent=2))
912
- return 0
913
-
914
- if args.command == "sync-audio":
915
- result = sync_audio_batches(
916
- api=api,
917
- project_slug=args.project_slug,
918
- dataset_repo=args.dataset_repo,
919
- local_audio_dir=args.local_audio_dir,
920
- batch_size=args.batch_size,
921
- max_retries=args.max_retries,
922
- retry_backoff_seconds=args.retry_backoff_seconds,
923
- resume_state_file=args.resume_state_file,
924
- )
925
- print(json.dumps(result, indent=2))
926
- return 0 if result["failed"] == 0 else 1
927
-
928
- if args.command == "verify-project":
929
- result = verify_project(
930
- api=api,
931
- project_slug=args.project_slug,
932
- dataset_repo=args.dataset_repo,
933
- )
934
- print(json.dumps(result, indent=2))
935
- return 0 if result["ok"] else 1
936
-
937
- if args.command == "ingest-segments":
938
- if args.dry_run:
939
- result = run_ingest_segments_dry_run(
940
- project_slug=args.project_slug,
941
- detections_csv=args.detections_csv,
942
- segments_root=args.segments_root,
943
- )
944
- else:
945
- result = ingest_segments_to_hf(
946
- api=api,
947
- project_slug=args.project_slug,
948
- dataset_repo=args.dataset_repo,
949
- detections_csv=args.detections_csv,
950
- segments_root=args.segments_root,
951
- batch_size=args.batch_size,
952
- shard_size=args.shard_size,
953
- max_retries=args.max_retries,
954
- retry_backoff_seconds=args.retry_backoff_seconds,
955
- resume_state_file=args.resume_state_file,
956
- )
957
-
958
- if args.report_file:
959
- Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
960
-
961
- print(json.dumps(result, indent=2))
962
- return 0 if result.get("failed_uploads", 0) == 0 else 1
963
-
964
- parser.error("Unknown command")
965
- return 2
966
-
967
-
968
- if __name__ == "__main__":
969
- raise SystemExit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docs/HF_SEGMENTS_INGESTION_PLAN.md DELETED
@@ -1,194 +0,0 @@
1
- # HF Segments Ingestion Plan
2
-
3
- ## Objective
4
- Build a dedicated ingestion tool (separate from validation UI) that accepts:
5
- 1. A BirdNET detections CSV
6
- 2. A root segments folder (including subfolders)
7
-
8
- Then uploads both audio segments and a normalized metadata index to a Hugging Face dataset repo.
9
-
10
- ## Why This Comes Next
11
- - Validation UI is already stable for auth/admin/ACL.
12
- - Real operations depend on real data availability.
13
- - Ingestion first avoids UI rework over mock assumptions.
14
-
15
- ## Real Dataset Observations (from local reference)
16
- - Segments root folders (species folders): 293
17
- - Segment files: 146562
18
- - Extension profile: `.wav` only (plus one `_progress_log.txt`)
19
- - Filename pattern sample:
20
- - `Accipiter striatus/Catim_LS-1500_U10_20250221_060600_0.0-3.0s_85%.wav`
21
- - `Cyanocorax cyanopogon/Aiuab_LN-0500_U16_20260121_153400_42.0-45.0s_98%.wav`
22
-
23
- ## Input CSV (reference provided)
24
- Observed columns include:
25
- - `locality`
26
- - `point`
27
- - `date_folder`
28
- - `source_file`
29
- - `scientific_name`
30
- - `common_name`
31
- - `confidence`
32
- - `start_time`
33
- - `end_time`
34
- - `exact_start`
35
- - `exact_end`
36
- - `min_freq`
37
- - `max_freq`
38
- - `box_source`
39
- - `label`
40
-
41
- ## Proposed HF Dataset Layout
42
- - `audio/segments/<scientific_name>/<segment_filename>.wav`
43
- - `index/shards/*.parquet` (normalized row index for fast reads)
44
- - `manifest.json` (schema/version/counters)
45
- - `audit/ingestion-runs/<run_id>.json` (run report)
46
- - `audit/unmatched/<run_id>.parquet` (rows not linked to files)
47
-
48
- ## Canonical Metadata Row (Normalized)
49
- Required fields for each detection segment:
50
- - `project_slug`
51
- - `detection_key` (stable deterministic key)
52
- - `audio_id` (source audio id, from `source_file` stem)
53
- - `segment_filename`
54
- - `segment_relpath`
55
- - `scientific_name`
56
- - `common_name`
57
- - `confidence`
58
- - `start_time`
59
- - `end_time`
60
- - `exact_start`
61
- - `exact_end`
62
- - `locality`
63
- - `point`
64
- - `date_folder`
65
- - `min_freq`
66
- - `max_freq`
67
- - `box_source`
68
- - `label`
69
- - `source_file`
70
- - `ingested_at`
71
- - `schema_version`
72
-
73
- Optional convenience fields:
74
- - `segment_duration`
75
- - `confidence_pct_rounded`
76
- - `match_strategy`
77
- - `match_score`
78
-
79
- ## Matching Strategy (CSV <-> Segment Files)
80
- Primary join (deterministic):
81
- 1. species folder = `scientific_name`
82
- 2. segment filename starts with `source_file` stem
83
- 3. filename contains `<start_time>-<end_time>s`
84
-
85
- Tolerance rules:
86
- - Use decimal normalization (`3` vs `3.0`) for time token search.
87
- - Confidence in filename (`85%`) is optional for join, used as tie-breaker only.
88
-
89
- Tie-break policy:
90
- - If more than one segment matches, pick exact time token first.
91
- - If still ambiguous, pick closest confidence percentage.
92
- - Persist ambiguity in audit file.
93
-
94
- ## Deterministic detection_key
95
- Generate with stable hash to satisfy model constraints and dedupe safely:
96
- - input: `project_slug|source_file|scientific_name|start_time|end_time`
97
- - `detection_key = sha256(input).hexdigest()[:24]`
98
-
99
- Benefits:
100
- - deterministic across reruns
101
- - longer than minimum length constraints
102
- - independent from local filesystem ordering
103
-
104
- ## CLI Proposal (new command)
105
- Add command to `cli/hf_dataset_cli.py`:
106
- - `ingest-segments`
107
-
108
- Arguments:
109
- - `--project-slug`
110
- - `--dataset-repo`
111
- - `--detections-csv`
112
- - `--segments-root`
113
- - `--batch-size` (default 200)
114
- - `--shard-size` (default 10000)
115
- - `--dry-run`
116
- - `--resume-state-file` (default `.ingest-segments-state.json`)
117
- - `--max-retries` (default 3)
118
- - `--retry-backoff-seconds` (default 1.0)
119
-
120
- ## Command Behavior
121
- Dry-run mode:
122
- - validates input schema
123
- - computes matching stats
124
- - reports would-upload counts
125
- - writes no remote files
126
-
127
- Execute mode:
128
- 1. ensure dataset scaffold
129
- 2. parse CSV and normalize rows
130
- 3. discover segments recursively
131
- 4. match rows to segment files
132
- 5. upload segment audio in batches with resume/retry
133
- 6. build/upload index shards from matched rows
134
- 7. update `manifest.json`
135
- 8. write audit artifacts
136
-
137
- ## Operational Report (stdout JSON)
138
- Return key counters:
139
- - `csv_rows_total`
140
- - `segments_found_total`
141
- - `matched_rows`
142
- - `unmatched_rows`
143
- - `ambiguous_rows`
144
- - `uploaded_audio_now`
145
- - `uploaded_audio_skipped_existing`
146
- - `index_rows_written`
147
- - `shards_written`
148
- - `failed_uploads`
149
- - `duration_seconds`
150
-
151
- ## Failure Policy
152
- Hard-fail:
153
- - missing required CSV columns
154
- - unreadable segments root
155
- - HF auth/repo unreachable
156
-
157
- Soft-fail with audit:
158
- - unmatched rows
159
- - ambiguous matches
160
- - partial audio upload failures
161
-
162
- Exit code:
163
- - `0` when no upload failures (unmatched allowed with warning)
164
- - `1` when upload/index write failures occur
165
-
166
- ## Performance Notes
167
- Given ~146k WAV files:
168
- - keep file discovery cached in-memory map by species + source_file stem
169
- - avoid O(N*M) matching scans
170
- - process CSV in chunks where possible
171
- - upload in batches and persist resume state after each batch
172
-
173
- ## Security and Separation
174
- - Ingestion tool remains separate from validation UI.
175
- - Validation app reads index/audio already published to HF.
176
- - ACL remains project-scoped at validation stage.
177
-
178
- ## Implementation Phases
179
- Phase 1 (MVP ingestion):
180
- - `ingest-segments --dry-run`
181
- - deterministic key generation
182
- - CSV normalization + matching report
183
-
184
- Phase 2 (publish):
185
- - batched audio upload + resume/retry
186
- - index shard generation and manifest update
187
-
188
- Phase 3 (hardening):
189
- - audit files for unmatched/ambiguous
190
- - duplicate handling policy and idempotency checks
191
- - integration tests with fixture CSV + synthetic segment tree
192
-
193
- ## Suggested Next Action
194
- Implement Phase 1 first (`dry-run`) so we can validate matching quality on your real CSV and folder before any remote upload.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
pyproject.toml CHANGED
@@ -1,5 +1,5 @@
1
  [project]
2
- name = "BirdNET-validator-App"
3
  version = "0.1.0"
4
  description = "BirdNET detection validator for Hugging Face Spaces"
5
  readme = "README.md"
 
1
  [project]
2
+ name = "BirdNET-Validator-App"
3
  version = "0.1.0"
4
  description = "BirdNET detection validator for Hugging Face Spaces"
5
  readme = "README.md"
requirements.txt CHANGED
@@ -1,7 +1,4 @@
1
  gradio>=5.0.0
2
  huggingface_hub>=0.30.0
3
  pydantic>=2.8.0
4
- pandas>=2.2.0
5
- pyarrow>=17.0.0
6
- soundfile>=0.12.1
7
  pytest>=8.3.0
 
1
  gradio>=5.0.0
2
  huggingface_hub>=0.30.0
3
  pydantic>=2.8.0
 
 
 
4
  pytest>=8.3.0
src/ui/upload_app_factory.py DELETED
@@ -1,177 +0,0 @@
1
- import json
2
- from pathlib import Path
3
- from typing import Any
4
-
5
- import gradio as gr
6
- from huggingface_hub import HfApi
7
-
8
- from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
9
-
10
-
11
- def _as_pretty_json(payload: dict[str, Any]) -> str:
12
- return json.dumps(payload, indent=2, ensure_ascii=False)
13
-
14
-
15
- def _build_api(token: str) -> HfApi:
16
- clean_token = (token or "").strip()
17
- if clean_token:
18
- return HfApi(token=clean_token)
19
- return HfApi()
20
-
21
-
22
- def build_upload_app() -> gr.Blocks:
23
- with gr.Blocks(title="BirdNET Segments Uploader") as demo:
24
- gr.Markdown("# BirdNET Segments Uploader")
25
- gr.Markdown(
26
- "Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
27
- "Recomendado: executar dry-run antes do upload real."
28
- )
29
-
30
- with gr.Row():
31
- project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
32
- dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
33
-
34
- detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
35
- segments_root = gr.Textbox(
36
- label="Pasta raiz de segmentos",
37
- placeholder=r"ex: C:\dados\BirdNET Segments",
38
- )
39
-
40
- with gr.Accordion("Configuracao avancada", open=False):
41
- hf_token = gr.Textbox(
42
- label="HF Token (opcional)",
43
- type="password",
44
- placeholder="Se vazio, usa sessao ja autenticada no ambiente",
45
- )
46
- with gr.Row():
47
- batch_size = gr.Number(label="Batch size", value=200, precision=0)
48
- shard_size = gr.Number(label="Shard size", value=10000, precision=0)
49
- with gr.Row():
50
- max_retries = gr.Number(label="Max retries", value=3, precision=0)
51
- retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
52
- resume_state_file = gr.Textbox(
53
- label="Resume state file",
54
- value=".ingest-segments-state.json",
55
- )
56
- report_file = gr.Textbox(
57
- label="Report file local (opcional)",
58
- placeholder="ex: .ingest-run-report.json",
59
- )
60
-
61
- with gr.Row():
62
- dry_run_button = gr.Button("Dry-run", variant="secondary")
63
- upload_button = gr.Button("Upload real", variant="primary")
64
-
65
- status = gr.Markdown(value="Pronto")
66
- result_json = gr.Code(label="Resultado JSON", language="json")
67
-
68
- def run_dry_run(
69
- project: str,
70
- repo: str,
71
- csv_path: str | None,
72
- segments_path: str,
73
- report_path: str,
74
- ) -> tuple[str, str]:
75
- project = (project or "").strip()
76
- if not project:
77
- return "❌ Informe o project slug", "{}"
78
-
79
- if not csv_path:
80
- return "❌ Selecione o CSV de deteccoes", "{}"
81
-
82
- if not segments_path or not Path(segments_path).exists():
83
- return "❌ Pasta de segmentos invalida", "{}"
84
-
85
- try:
86
- result = run_ingest_segments_dry_run(
87
- project_slug=project,
88
- detections_csv=csv_path,
89
- segments_root=segments_path,
90
- )
91
- if report_path.strip():
92
- Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
93
- summary = (
94
- "✅ Dry-run finalizado | "
95
- f"Matched: {result['matched_rows']} | "
96
- f"Unmatched: {result['unmatched_rows']} | "
97
- f"Ambiguous: {result['ambiguous_rows']}"
98
- )
99
- return summary, _as_pretty_json(result)
100
- except Exception as exc:
101
- return f"❌ Dry-run falhou: {exc}", "{}"
102
-
103
- def run_upload(
104
- project: str,
105
- repo: str,
106
- csv_path: str | None,
107
- segments_path: str,
108
- token: str,
109
- batch: float,
110
- shard: float,
111
- retries: float,
112
- backoff: float,
113
- resume_file: str,
114
- report_path: str,
115
- ) -> tuple[str, str]:
116
- project = (project or "").strip()
117
- repo = (repo or "").strip()
118
- if not project:
119
- return "❌ Informe o project slug", "{}"
120
- if not repo or "/" not in repo:
121
- return "❌ Informe dataset repo no formato owner/repo", "{}"
122
- if not csv_path:
123
- return "❌ Selecione o CSV de deteccoes", "{}"
124
- if not segments_path or not Path(segments_path).exists():
125
- return "❌ Pasta de segmentos invalida", "{}"
126
-
127
- try:
128
- api = _build_api(token)
129
- result = ingest_segments_to_hf(
130
- api=api,
131
- project_slug=project,
132
- dataset_repo=repo,
133
- detections_csv=csv_path,
134
- segments_root=segments_path,
135
- batch_size=int(batch),
136
- shard_size=int(shard),
137
- max_retries=int(retries),
138
- retry_backoff_seconds=float(backoff),
139
- resume_state_file=resume_file,
140
- )
141
- if report_path.strip():
142
- Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
143
- summary = (
144
- "✅ Upload finalizado | "
145
- f"Uploaded now: {result['uploaded_audio_now']} | "
146
- f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
147
- f"Failed uploads: {result['failed_uploads']}"
148
- )
149
- return summary, _as_pretty_json(result)
150
- except Exception as exc:
151
- return f"❌ Upload falhou: {exc}", "{}"
152
-
153
- dry_run_button.click(
154
- fn=run_dry_run,
155
- inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
156
- outputs=[status, result_json],
157
- )
158
-
159
- upload_button.click(
160
- fn=run_upload,
161
- inputs=[
162
- project_slug,
163
- dataset_repo,
164
- detections_csv,
165
- segments_root,
166
- hf_token,
167
- batch_size,
168
- shard_size,
169
- max_retries,
170
- retry_backoff_seconds,
171
- resume_state_file,
172
- report_file,
173
- ],
174
- outputs=[status, result_json],
175
- )
176
-
177
- return demo
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests/unit/test_hf_dataset_cli.py DELETED
@@ -1,231 +0,0 @@
1
- import json
2
- from pathlib import Path
3
-
4
- import pandas as pd
5
- import pytest
6
-
7
- from cli.hf_dataset_cli import (
8
- SCHEMA_VERSION,
9
- _chunk_items,
10
- _load_resume_state,
11
- _save_resume_state,
12
- build_manifest_payload,
13
- build_shards_in_directory,
14
- collect_verify_errors,
15
- discover_audio_files,
16
- load_detections_table,
17
- sync_audio_batches,
18
- )
19
-
20
-
21
- def _sample_rows() -> list[dict[str, object]]:
22
- return [
23
- {
24
- "detection_key": "aaaaaaaaaaaaaaaa",
25
- "audio_id": "audio_1",
26
- "scientific_name": "Cyanocorax cyanopogon",
27
- "confidence": 0.91,
28
- "start_time": 0.0,
29
- "end_time": 1.2,
30
- },
31
- {
32
- "detection_key": "bbbbbbbbbbbbbbbb",
33
- "audio_id": "audio_2",
34
- "scientific_name": "Ramphastos toco",
35
- "confidence": 0.88,
36
- "start_time": 0.5,
37
- "end_time": 2.2,
38
- },
39
- {
40
- "detection_key": "cccccccccccccccc",
41
- "audio_id": "audio_2",
42
- "scientific_name": "Psarocolius decumanus",
43
- "confidence": 0.77,
44
- "start_time": 1.1,
45
- "end_time": 2.8,
46
- },
47
- ]
48
-
49
-
50
- def test_load_detections_table_csv(tmp_path: Path) -> None:
51
- csv_path = tmp_path / "detections.csv"
52
- pd.DataFrame(_sample_rows()).to_csv(csv_path, index=False)
53
-
54
- loaded = load_detections_table(str(csv_path))
55
-
56
- assert len(loaded) == 3
57
- assert list(loaded.columns).count("detection_key") == 1
58
-
59
-
60
- def test_load_detections_missing_columns(tmp_path: Path) -> None:
61
- csv_path = tmp_path / "bad.csv"
62
- pd.DataFrame([{"detection_key": "aaaaaaaaaaaaaaaa", "audio_id": "audio_1"}]).to_csv(csv_path, index=False)
63
-
64
- with pytest.raises(ValueError):
65
- _ = load_detections_table(str(csv_path))
66
-
67
-
68
- def test_build_shards_in_directory(tmp_path: Path) -> None:
69
- frame = pd.DataFrame(_sample_rows())
70
- output_dir = tmp_path / "index" / "shards"
71
-
72
- shards = build_shards_in_directory(frame=frame, output_dir=output_dir, shard_size=2)
73
-
74
- assert len(shards) == 2
75
- assert (output_dir / "shard-00000.parquet").exists()
76
- assert (output_dir / "shard-00001.parquet").exists()
77
- assert shards[0].rows == 2
78
- assert shards[1].rows == 1
79
-
80
-
81
- def test_manifest_payload_has_expected_stats() -> None:
82
- frame = pd.DataFrame(_sample_rows())
83
- payload = build_manifest_payload(
84
- project_slug="ppbio-rabeca",
85
- dataset_repo="org/birdnet-ppbio-rabeca-dataset",
86
- frame=frame,
87
- shard_size=1000,
88
- shard_metadata=[],
89
- )
90
-
91
- assert payload["schema_version"] == SCHEMA_VERSION
92
- assert payload["index"]["total_detections"] == 3
93
- assert payload["index"]["total_audio_files"] == 2
94
-
95
-
96
- def test_collect_verify_errors_for_missing_prefix_and_shard() -> None:
97
- manifest_payload = {
98
- "schema_version": SCHEMA_VERSION,
99
- "project_slug": "ppbio-rabeca",
100
- "index": {
101
- "shards": [
102
- {
103
- "path": "index/shards/shard-00000.parquet",
104
- "rows": 2,
105
- "sha256": "abc",
106
- "size_bytes": 10,
107
- }
108
- ]
109
- },
110
- }
111
-
112
- repo_files = {
113
- "manifest.json",
114
- "audio/.gitkeep",
115
- "index/.gitkeep",
116
- "validations/.gitkeep",
117
- "audit/.gitkeep",
118
- }
119
-
120
- errors = collect_verify_errors(
121
- repo_files=repo_files,
122
- manifest_payload=manifest_payload,
123
- project_slug="ppbio-rabeca",
124
- )
125
-
126
- assert any("index/shards/" in message for message in errors)
127
- assert any("Shard referenced in manifest" in message for message in errors)
128
-
129
-
130
- def test_manifest_json_roundtrip(tmp_path: Path) -> None:
131
- payload = {
132
- "schema_version": SCHEMA_VERSION,
133
- "project_slug": "ppbio-rabeca",
134
- "index": {"shards": []},
135
- }
136
- manifest_path = tmp_path / "manifest.json"
137
- manifest_path.write_text(json.dumps(payload), encoding="utf-8")
138
-
139
- loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
140
- assert loaded["project_slug"] == "ppbio-rabeca"
141
-
142
-
143
- def test_discover_audio_files_filters_supported_extensions(tmp_path: Path) -> None:
144
- audio_dir = tmp_path / "audio"
145
- audio_dir.mkdir(parents=True)
146
- (audio_dir / "a.wav").write_bytes(b"wav")
147
- (audio_dir / "b.mp3").write_bytes(b"mp3")
148
- (audio_dir / "c.txt").write_text("ignore", encoding="utf-8")
149
-
150
- files = discover_audio_files(str(audio_dir))
151
- names = [item.name for item in files]
152
-
153
- assert names == ["a.wav", "b.mp3"]
154
-
155
-
156
- def test_chunk_items() -> None:
157
- files = [Path(f"f{i}.wav") for i in range(5)]
158
- chunks = _chunk_items(files, 2)
159
-
160
- assert len(chunks) == 3
161
- assert len(chunks[0]) == 2
162
- assert len(chunks[1]) == 2
163
- assert len(chunks[2]) == 1
164
-
165
-
166
- def test_resume_state_roundtrip(tmp_path: Path) -> None:
167
- state_file = tmp_path / "state.json"
168
- _save_resume_state(state_file=state_file, uploaded={"audio/a.wav"}, failed={"audio/b.wav"})
169
-
170
- loaded = _load_resume_state(state_file=state_file)
171
-
172
- assert loaded["uploaded"] == ["audio/a.wav"]
173
- assert loaded["failed"] == ["audio/b.wav"]
174
-
175
-
176
- def test_sync_audio_batches_skips_remote_and_checkpointed(tmp_path: Path) -> None:
177
- class FakeApi:
178
- def __init__(self) -> None:
179
- self.files = {
180
- "audio/existing.wav",
181
- "manifest.json",
182
- "audio/.gitkeep",
183
- "index/.gitkeep",
184
- "index/shards/.gitkeep",
185
- "validations/.gitkeep",
186
- "audit/.gitkeep",
187
- }
188
- self.uploaded: list[str] = []
189
-
190
- def create_repo(self, **kwargs: object) -> None:
191
- _ = kwargs
192
-
193
- def list_repo_files(self, **kwargs: object) -> list[str]:
194
- _ = kwargs
195
- return sorted(self.files)
196
-
197
- def upload_file(self, *, path_or_fileobj: object, path_in_repo: str, **kwargs: object) -> None:
198
- _ = path_or_fileobj
199
- _ = kwargs
200
- self.uploaded.append(path_in_repo)
201
- self.files.add(path_in_repo)
202
-
203
- audio_dir = tmp_path / "audio"
204
- audio_dir.mkdir(parents=True)
205
- (audio_dir / "existing.wav").write_bytes(b"x")
206
- (audio_dir / "new.wav").write_bytes(b"x")
207
- (audio_dir / "checkpointed.wav").write_bytes(b"x")
208
-
209
- state_file = tmp_path / "resume.json"
210
- state_file.write_text(
211
- json.dumps({"uploaded": ["audio/checkpointed.wav"], "failed": []}),
212
- encoding="utf-8",
213
- )
214
-
215
- api = FakeApi()
216
- result = sync_audio_batches(
217
- api=api,
218
- project_slug="ppbio-rabeca",
219
- dataset_repo="org/birdnet-ppbio-rabeca-dataset",
220
- local_audio_dir=str(audio_dir),
221
- batch_size=2,
222
- max_retries=1,
223
- retry_backoff_seconds=0,
224
- resume_state_file=str(state_file),
225
- )
226
-
227
- assert result["total_local_audio_files"] == 3
228
- assert result["pending_uploads"] == 1
229
- assert result["uploaded_now"] == 1
230
- assert result["failed"] == 0
231
- assert api.uploaded == ["audio/new.wav"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests/unit/test_hf_ingest_segments_cli.py DELETED
@@ -1,127 +0,0 @@
1
- from pathlib import Path
2
-
3
- import pandas as pd
4
-
5
- from cli.hf_dataset_cli import (
6
- _build_detection_key,
7
- build_parser,
8
- parse_segment_filename,
9
- run_ingest_segments_dry_run,
10
- )
11
-
12
-
13
- def test_parse_segment_filename_extracts_components():
14
- parsed = parse_segment_filename("Aiuab_LN-0500_U16_20260121_153400_42.0-45.0s_98%.wav")
15
- assert parsed is not None
16
- source_stem, start_time, end_time, confidence_pct = parsed
17
- assert source_stem == "Aiuab_LN-0500_U16_20260121_153400"
18
- assert start_time == 42.0
19
- assert end_time == 45.0
20
- assert confidence_pct == 98
21
-
22
-
23
- def test_detection_key_is_stable_and_long_enough():
24
- key1 = _build_detection_key(
25
- project_slug="ppbio-aiuaba",
26
- source_file="Aiuab_LN-0500_U16_20260121_153400.wav",
27
- scientific_name="Cyanocorax cyanopogon",
28
- start_time=42.0,
29
- end_time=45.0,
30
- )
31
- key2 = _build_detection_key(
32
- project_slug="ppbio-aiuaba",
33
- source_file="Aiuab_LN-0500_U16_20260121_153400.wav",
34
- scientific_name="Cyanocorax cyanopogon",
35
- start_time=42.0,
36
- end_time=45.0,
37
- )
38
- assert key1 == key2
39
- assert len(key1) >= 16
40
-
41
-
42
- def test_run_ingest_segments_dry_run_matches_rows(tmp_path: Path):
43
- segments_root = tmp_path / "segments"
44
- species_dir = segments_root / "Cyanocorax cyanopogon"
45
- species_dir.mkdir(parents=True, exist_ok=True)
46
-
47
- (species_dir / "Aiuab_LN-0500_U16_20260121_153400_42.0-45.0s_98%.wav").write_bytes(b"dummy")
48
- (species_dir / "Aiuab_LN-0500_U16_20260121_153400_45.0-48.0s_88%.wav").write_bytes(b"dummy")
49
-
50
- frame = pd.DataFrame(
51
- [
52
- {
53
- "locality": "PPBIO Aiuaba",
54
- "point": "Aiuab_LN-0500_U16",
55
- "date_folder": "20260121",
56
- "source_file": "Aiuab_LN-0500_U16_20260121_153400.wav",
57
- "scientific_name": "Cyanocorax cyanopogon",
58
- "common_name": "White-naped Jay",
59
- "confidence": 0.98,
60
- "start_time": 42.0,
61
- "end_time": 45.0,
62
- "exact_start": 42.0,
63
- "exact_end": 45.0,
64
- "min_freq": 0,
65
- "max_freq": 15000,
66
- "box_source": "BirdNET_Original",
67
- "label": "Cyanocorax cyanopogon_White-naped Jay",
68
- },
69
- {
70
- "locality": "PPBIO Aiuaba",
71
- "point": "Aiuab_LN-0500_U16",
72
- "date_folder": "20260121",
73
- "source_file": "Aiuab_LN-0500_U16_20260121_153400.wav",
74
- "scientific_name": "Cyanocorax cyanopogon",
75
- "common_name": "White-naped Jay",
76
- "confidence": 0.50,
77
- "start_time": 0.0,
78
- "end_time": 3.0,
79
- "exact_start": 0.0,
80
- "exact_end": 3.0,
81
- "min_freq": 0,
82
- "max_freq": 15000,
83
- "box_source": "BirdNET_Original",
84
- "label": "Cyanocorax cyanopogon_White-naped Jay",
85
- },
86
- ]
87
- )
88
-
89
- csv_path = tmp_path / "detections.csv"
90
- frame.to_csv(csv_path, index=False)
91
-
92
- result = run_ingest_segments_dry_run(
93
- project_slug="ppbio-aiuaba",
94
- detections_csv=str(csv_path),
95
- segments_root=str(segments_root),
96
- )
97
-
98
- assert result["mode"] == "dry-run"
99
- assert result["csv_rows_total"] == 2
100
- assert result["segments_found_total"] == 2
101
- assert result["matched_rows"] == 1
102
- assert result["unmatched_rows"] == 1
103
-
104
-
105
- def test_build_parser_ingest_segments_defaults():
106
- parser = build_parser()
107
- args = parser.parse_args(
108
- [
109
- "ingest-segments",
110
- "--project-slug",
111
- "ppbio-aiuaba",
112
- "--dataset-repo",
113
- "owner/repo",
114
- "--detections-csv",
115
- "detections.csv",
116
- "--segments-root",
117
- "segments",
118
- ]
119
- )
120
-
121
- assert args.command == "ingest-segments"
122
- assert args.batch_size == 200
123
- assert args.shard_size == 10000
124
- assert args.max_retries == 3
125
- assert args.retry_backoff_seconds == 1.0
126
- assert args.resume_state_file == ".ingest-segments-state.json"
127
- assert args.dry_run is False