jrrribeiro commited on
Commit
f5bcf37
Β·
1 Parent(s): 837d92e

chore: bootstrap BirdNET uploader standalone repo

Browse files
Files changed (35) hide show
  1. BirdNET Uploader App/README.md +84 -0
  2. BirdNET Uploader App/app.py +9 -0
  3. BirdNET Uploader App/cli/hf_dataset_cli.py +969 -0
  4. BirdNET Uploader App/pyproject.toml +9 -0
  5. BirdNET Uploader App/requirements.txt +7 -0
  6. BirdNET Uploader App/src/__init__.py +0 -0
  7. BirdNET Uploader App/src/ui/__init__.py +0 -0
  8. BirdNET Uploader App/src/ui/upload_app_factory.py +177 -0
  9. BirdNET Validator App/README.md +83 -0
  10. BirdNET Validator App/app.py +6 -0
  11. BirdNET Validator App/pyproject.toml +9 -0
  12. BirdNET Validator App/requirements.txt +7 -0
  13. BirdNET Validator App/src/__init__.py +0 -0
  14. BirdNET Validator App/src/auth/__init__.py +0 -0
  15. BirdNET Validator App/src/auth/auth_service.py +200 -0
  16. BirdNET Validator App/src/cache/__init__.py +0 -0
  17. BirdNET Validator App/src/cache/ephemeral_cache_manager.py +82 -0
  18. BirdNET Validator App/src/domain/__init__.py +0 -0
  19. BirdNET Validator App/src/domain/models.py +46 -0
  20. BirdNET Validator App/src/repositories/__init__.py +0 -0
  21. BirdNET Validator App/src/repositories/append_only_validation_repository.py +89 -0
  22. BirdNET Validator App/src/repositories/contracts.py +35 -0
  23. BirdNET Validator App/src/repositories/in_memory_detection_repository.py +73 -0
  24. BirdNET Validator App/src/repositories/in_memory_validation_repository.py +25 -0
  25. BirdNET Validator App/src/services/__init__.py +0 -0
  26. BirdNET Validator App/src/services/audio_fetch_service.py +54 -0
  27. BirdNET Validator App/src/services/detection_queue_service.py +64 -0
  28. BirdNET Validator App/src/services/validation_service.py +39 -0
  29. BirdNET Validator App/src/telemetry/__init__.py +0 -0
  30. BirdNET Validator App/src/ui/__init__.py +0 -0
  31. BirdNET Validator App/src/ui/admin_panel.py +281 -0
  32. BirdNET Validator App/src/ui/app_factory.py +1542 -0
  33. BirdNET Validator App/src/ui/login_page.py +66 -0
  34. BirdNET Validator App/src/ui/project_selector.py +77 -0
  35. BirdNET Validator App/src/ui/upload_app_factory.py +177 -0
BirdNET Uploader App/README.md ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # BirdNET Uploader App
2
+
3
+ Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
4
+ - Multi-projeto
5
+ - Audio sob demanda
6
+ - Persistencia de validacoes
7
+ - Escalabilidade para datasets grandes
8
+
9
+ ## Desenvolvimento local
10
+ 1. Criar ambiente virtual Python 3.11+
11
+ 2. Instalar dependencias:
12
+ pip install -r requirements.txt
13
+ 3. Rodar app:
14
+ python app.py
15
+
16
+ ## Objetivo desta pasta
17
+ Contem apenas os arquivos necessarios para o app de upload/ingestao.
18
+ Use esta pasta para criar/publicar o repositorio do uploader de forma independente.
19
+
20
+ ## Fluxo recomendado (separado)
21
+ 1. Uploader app (porta 7862 por padrao):
22
+ - Rodar dry-run com CSV + pasta de segmentos
23
+ - Rodar upload real para owner/repo do projeto
24
+ 2. Validator app (porta 7860):
25
+ - Validar deteccoes do projeto com dados ja publicados
26
+
27
+ Essa separacao evita impacto de performance na validacao durante uploads grandes.
28
+
29
+ ## CLI de dataset (Sprint 1)
30
+ Comandos principais:
31
+
32
+ 1. Criar estrutura do projeto no dataset Hugging Face:
33
+ python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
34
+
35
+ 2. Gerar manifesto e shards iniciais de indice:
36
+ python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
37
+
38
+ 3. Fazer upload de audio em lotes com retry e resume:
39
+ python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
40
+
41
+ 4. Verificar consistencia basica do projeto:
42
+ python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
43
+
44
+ 5. Ingestao BirdNET por projeto (dry-run):
45
+ python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
46
+
47
+ 6. Ingestao BirdNET por projeto (upload real):
48
+ python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
49
+
50
+ Estrutura base criada por projeto/dataset:
51
+ - audio/
52
+ - index/
53
+ - index/shards/
54
+ - validations/
55
+ - audit/
56
+ - manifest.json
57
+
58
+ ## Estrutura
59
+ - src/domain: modelos de dominio
60
+ - src/repositories: contratos de persistencia
61
+ - src/services: servicos de aplicacao
62
+ - src/auth: autenticacao e autorizacao
63
+ - src/ui: montagem da interface
64
+ - src/cache: cache efemero local
65
+ - cli: comandos de ingestao/publicacao
66
+ - tests: testes unitarios e integracao
67
+
68
+ ## Estado atual de implementacao
69
+ - Sprint 1: CLI create-project, sync-audio, build-index, verify-project
70
+ - Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
71
+ - Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
72
+ - Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
73
+ - Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
74
+ - Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
75
+ - Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
76
+ - Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
77
+ - Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
78
+ - Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
79
+ - Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
80
+ - Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
81
+ - Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
82
+ - Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
83
+ - Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
84
+ - Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
BirdNET Uploader App/app.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ from src.ui.upload_app_factory import build_upload_app
4
+
5
+
6
+ if __name__ == "__main__":
7
+ app = build_upload_app()
8
+ port = int(os.getenv("BIRDNET_UPLOADER_PORT", "7862"))
9
+ app.launch(server_name="0.0.0.0", server_port=port)
BirdNET Uploader App/cli/hf_dataset_cli.py ADDED
@@ -0,0 +1,969 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import hashlib
3
+ import json
4
+ import math
5
+ import re
6
+ import tempfile
7
+ import time
8
+ from dataclasses import asdict, dataclass
9
+ from datetime import UTC, datetime
10
+ from io import BytesIO
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ import pandas as pd
15
+ from huggingface_hub import HfApi, hf_hub_download
16
+
17
+
18
+ SCHEMA_VERSION = "1.0.0"
19
+ REQUIRED_PROJECT_PREFIXES = (
20
+ "audio/",
21
+ "index/",
22
+ "index/shards/",
23
+ "validations/",
24
+ "audit/",
25
+ )
26
+ REQUIRED_PLACEHOLDER_FILES = (
27
+ "audio/.gitkeep",
28
+ "index/.gitkeep",
29
+ "index/shards/.gitkeep",
30
+ "validations/.gitkeep",
31
+ "audit/.gitkeep",
32
+ )
33
+ REQUIRED_DETECTIONS_COLUMNS = (
34
+ "detection_key",
35
+ "audio_id",
36
+ "scientific_name",
37
+ "confidence",
38
+ "start_time",
39
+ "end_time",
40
+ )
41
+ SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
42
+ REQUIRED_BIRDNET_COLUMNS = (
43
+ "source_file",
44
+ "scientific_name",
45
+ "confidence",
46
+ "start_time",
47
+ "end_time",
48
+ )
49
+ SEGMENT_FILENAME_PATTERN = re.compile(
50
+ r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
51
+ )
52
+
53
+
54
+ @dataclass(slots=True)
55
+ class ShardMetadata:
56
+ path: str
57
+ rows: int
58
+ sha256: str
59
+ size_bytes: int
60
+
61
+
62
+ @dataclass(slots=True)
63
+ class SegmentFileRecord:
64
+ scientific_name: str
65
+ source_stem: str
66
+ start_time: float
67
+ end_time: float
68
+ confidence_pct: int
69
+ absolute_path: Path
70
+ relative_path: str
71
+
72
+
73
+ def _utcnow_iso() -> str:
74
+ return datetime.now(UTC).replace(microsecond=0).isoformat()
75
+
76
+
77
+ def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
78
+ api.upload_file(
79
+ path_or_fileobj=BytesIO(content.encode("utf-8")),
80
+ path_in_repo=path_in_repo,
81
+ repo_id=dataset_repo,
82
+ repo_type="dataset",
83
+ )
84
+
85
+
86
+ def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
87
+ api.upload_file(
88
+ path_or_fileobj=BytesIO(b""),
89
+ path_in_repo=path_in_repo,
90
+ repo_id=dataset_repo,
91
+ repo_type="dataset",
92
+ )
93
+
94
+
95
+ def ensure_project_dataset_structure(
96
+ api: HfApi,
97
+ project_slug: str,
98
+ dataset_repo: str,
99
+ create_private_repo: bool,
100
+ ) -> dict[str, Any]:
101
+ api.create_repo(
102
+ repo_id=dataset_repo,
103
+ repo_type="dataset",
104
+ private=create_private_repo,
105
+ exist_ok=True,
106
+ )
107
+
108
+ files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
109
+ created_paths: list[str] = []
110
+
111
+ for placeholder in REQUIRED_PLACEHOLDER_FILES:
112
+ if placeholder not in files:
113
+ _upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
114
+ created_paths.append(placeholder)
115
+
116
+ manifest_path = "manifest.json"
117
+ if manifest_path not in files:
118
+ manifest = {
119
+ "schema_version": SCHEMA_VERSION,
120
+ "project_slug": project_slug,
121
+ "dataset_repo_id": dataset_repo,
122
+ "created_at": _utcnow_iso(),
123
+ "updated_at": _utcnow_iso(),
124
+ "index": {
125
+ "total_detections": 0,
126
+ "total_audio_files": 0,
127
+ "shard_size": 0,
128
+ "shards": [],
129
+ },
130
+ }
131
+ _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
132
+ created_paths.append(manifest_path)
133
+
134
+ return {
135
+ "dataset_repo": dataset_repo,
136
+ "project_slug": project_slug,
137
+ "created_paths": created_paths,
138
+ }
139
+
140
+
141
+ def load_detections_table(detections_file: str) -> pd.DataFrame:
142
+ input_path = Path(detections_file)
143
+ if not input_path.exists():
144
+ raise FileNotFoundError(f"Detections file not found: {detections_file}")
145
+
146
+ suffix = input_path.suffix.lower()
147
+ if suffix == ".csv":
148
+ frame = pd.read_csv(input_path)
149
+ elif suffix in {".jsonl", ".ndjson"}:
150
+ frame = pd.read_json(input_path, lines=True)
151
+ elif suffix == ".parquet":
152
+ frame = pd.read_parquet(input_path)
153
+ else:
154
+ raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
155
+
156
+ missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
157
+ if missing_columns:
158
+ raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
159
+
160
+ if frame.empty:
161
+ return frame
162
+
163
+ deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
164
+ return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
165
+
166
+
167
+ def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
168
+ input_path = Path(detections_csv)
169
+ if not input_path.exists():
170
+ raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
171
+
172
+ frame = pd.read_csv(input_path)
173
+ missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
174
+ if missing_columns:
175
+ raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
176
+
177
+ if frame.empty:
178
+ return frame
179
+
180
+ normalized = frame.copy()
181
+ normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
182
+ normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
183
+ normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
184
+ normalized["species_key"] = normalized["scientific_name"].str.casefold()
185
+ normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
186
+ normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
187
+ normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
188
+
189
+ normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
190
+ return normalized
191
+
192
+
193
+ def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
194
+ payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
195
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
196
+
197
+
198
+ def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
199
+ stem = Path(file_name).stem
200
+ match = SEGMENT_FILENAME_PATTERN.match(stem)
201
+ if not match:
202
+ return None
203
+
204
+ source_stem = match.group("source_stem")
205
+ start_time = float(match.group("start"))
206
+ end_time = float(match.group("end"))
207
+ confidence_pct = int(match.group("confidence_pct"))
208
+ return source_stem, start_time, end_time, confidence_pct
209
+
210
+
211
+ def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
212
+ root = Path(segments_root)
213
+ if not root.exists() or not root.is_dir():
214
+ raise FileNotFoundError(f"Segments root not found: {segments_root}")
215
+
216
+ records: list[SegmentFileRecord] = []
217
+ for absolute_path in root.rglob("*"):
218
+ if not absolute_path.is_file():
219
+ continue
220
+ if absolute_path.suffix.lower() != ".wav":
221
+ continue
222
+
223
+ rel = absolute_path.relative_to(root)
224
+ scientific_name = rel.parts[0] if rel.parts else ""
225
+ parsed = parse_segment_filename(absolute_path.name)
226
+ if not parsed:
227
+ continue
228
+ source_stem, start_time, end_time, confidence_pct = parsed
229
+ records.append(
230
+ SegmentFileRecord(
231
+ scientific_name=scientific_name,
232
+ source_stem=source_stem,
233
+ start_time=start_time,
234
+ end_time=end_time,
235
+ confidence_pct=confidence_pct,
236
+ absolute_path=absolute_path,
237
+ relative_path=rel.as_posix(),
238
+ )
239
+ )
240
+
241
+ return records
242
+
243
+
244
+ def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
245
+ frame = load_birdnet_detections_csv(detections_csv=detections_csv)
246
+ segment_records = discover_segment_records(segments_root=segments_root)
247
+
248
+ by_species: dict[str, list[SegmentFileRecord]] = {}
249
+ for record in segment_records:
250
+ by_species.setdefault(record.scientific_name.casefold(), []).append(record)
251
+
252
+ matched_rows: list[dict[str, Any]] = []
253
+ unmatched_rows: list[dict[str, Any]] = []
254
+ ambiguous_rows: list[dict[str, Any]] = []
255
+
256
+ for row in frame.to_dict(orient="records"):
257
+ species_key = str(row.get("species_key", "")).casefold()
258
+ source_stem = str(row.get("source_stem", "")).strip()
259
+ start_time = float(row["start_time"])
260
+ end_time = float(row["end_time"])
261
+ confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
262
+
263
+ species_candidates = by_species.get(species_key, [])
264
+ exact_matches = [
265
+ record
266
+ for record in species_candidates
267
+ if record.source_stem == source_stem
268
+ and abs(record.start_time - start_time) < 1e-6
269
+ and abs(record.end_time - end_time) < 1e-6
270
+ ]
271
+
272
+ if not exact_matches:
273
+ unmatched_rows.append(
274
+ {
275
+ "source_file": row.get("source_file", ""),
276
+ "scientific_name": row.get("scientific_name", ""),
277
+ "start_time": start_time,
278
+ "end_time": end_time,
279
+ }
280
+ )
281
+ continue
282
+
283
+ selected = exact_matches[0]
284
+ if len(exact_matches) > 1:
285
+ expected_pct = int(round(confidence * 100))
286
+ exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
287
+ selected = exact_matches[0]
288
+ ambiguous_rows.append(
289
+ {
290
+ "source_file": row.get("source_file", ""),
291
+ "scientific_name": row.get("scientific_name", ""),
292
+ "start_time": start_time,
293
+ "end_time": end_time,
294
+ "candidates": len(exact_matches),
295
+ }
296
+ )
297
+
298
+ matched_rows.append(
299
+ {
300
+ "project_slug": project_slug,
301
+ "detection_key": _build_detection_key(
302
+ project_slug=project_slug,
303
+ source_file=str(row.get("source_file", "")),
304
+ scientific_name=str(row.get("scientific_name", "")),
305
+ start_time=start_time,
306
+ end_time=end_time,
307
+ ),
308
+ "audio_id": source_stem,
309
+ "segment_filename": selected.absolute_path.name,
310
+ "segment_relpath": selected.relative_path,
311
+ "scientific_name": row.get("scientific_name", ""),
312
+ "common_name": row.get("common_name", ""),
313
+ "confidence": confidence,
314
+ "start_time": start_time,
315
+ "end_time": end_time,
316
+ "exact_start": row.get("exact_start", start_time),
317
+ "exact_end": row.get("exact_end", end_time),
318
+ "locality": row.get("locality", ""),
319
+ "point": row.get("point", ""),
320
+ "date_folder": row.get("date_folder", ""),
321
+ "min_freq": row.get("min_freq", None),
322
+ "max_freq": row.get("max_freq", None),
323
+ "box_source": row.get("box_source", ""),
324
+ "label": row.get("label", ""),
325
+ "source_file": row.get("source_file", ""),
326
+ "schema_version": SCHEMA_VERSION,
327
+ "ingested_at": _utcnow_iso(),
328
+ }
329
+ )
330
+
331
+ return {
332
+ "frame": frame,
333
+ "segment_records_total": len(segment_records),
334
+ "matched_rows": matched_rows,
335
+ "unmatched_rows": unmatched_rows,
336
+ "ambiguous_rows": ambiguous_rows,
337
+ }
338
+
339
+
340
+ def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
341
+ match_result = _match_birdnet_rows(
342
+ project_slug=project_slug,
343
+ detections_csv=detections_csv,
344
+ segments_root=segments_root,
345
+ )
346
+ frame = match_result["frame"]
347
+ matched_rows = match_result["matched_rows"]
348
+ unmatched_rows = match_result["unmatched_rows"]
349
+ ambiguous_rows = match_result["ambiguous_rows"]
350
+
351
+ unique_audio = {row["audio_id"] for row in matched_rows}
352
+ result = {
353
+ "mode": "dry-run",
354
+ "project_slug": project_slug,
355
+ "detections_csv": detections_csv,
356
+ "segments_root": segments_root,
357
+ "csv_rows_total": int(len(frame)),
358
+ "segments_found_total": int(match_result["segment_records_total"]),
359
+ "matched_rows": len(matched_rows),
360
+ "unmatched_rows": len(unmatched_rows),
361
+ "ambiguous_rows": len(ambiguous_rows),
362
+ "unique_audio_ids": len(unique_audio),
363
+ "sample_unmatched": unmatched_rows[:20],
364
+ "sample_ambiguous": ambiguous_rows[:20],
365
+ }
366
+ return result
367
+
368
+
369
+ def _file_sha256(file_path: Path) -> str:
370
+ hasher = hashlib.sha256()
371
+ with file_path.open("rb") as handle:
372
+ for chunk in iter(lambda: handle.read(65536), b""):
373
+ hasher.update(chunk)
374
+ return hasher.hexdigest()
375
+
376
+
377
+ def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
378
+ if shard_size <= 0:
379
+ raise ValueError("shard_size must be greater than zero")
380
+
381
+ output_dir.mkdir(parents=True, exist_ok=True)
382
+ shards: list[ShardMetadata] = []
383
+
384
+ if frame.empty:
385
+ return shards
386
+
387
+ total_rows = len(frame)
388
+ total_shards = math.ceil(total_rows / shard_size)
389
+
390
+ for index in range(total_shards):
391
+ start = index * shard_size
392
+ end = min(start + shard_size, total_rows)
393
+ shard_frame = frame.iloc[start:end]
394
+ shard_filename = f"shard-{index:05d}.parquet"
395
+ local_path = output_dir / shard_filename
396
+ shard_frame.to_parquet(local_path, index=False)
397
+
398
+ shards.append(
399
+ ShardMetadata(
400
+ path=f"index/shards/{shard_filename}",
401
+ rows=len(shard_frame),
402
+ sha256=_file_sha256(local_path),
403
+ size_bytes=local_path.stat().st_size,
404
+ )
405
+ )
406
+
407
+ return shards
408
+
409
+
410
+ def build_manifest_payload(
411
+ project_slug: str,
412
+ dataset_repo: str,
413
+ frame: pd.DataFrame,
414
+ shard_size: int,
415
+ shard_metadata: list[ShardMetadata],
416
+ ) -> dict[str, Any]:
417
+ now = _utcnow_iso()
418
+ return {
419
+ "schema_version": SCHEMA_VERSION,
420
+ "project_slug": project_slug,
421
+ "dataset_repo_id": dataset_repo,
422
+ "updated_at": now,
423
+ "index": {
424
+ "total_detections": len(frame),
425
+ "total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
426
+ "shard_size": shard_size,
427
+ "shards": [asdict(item) for item in shard_metadata],
428
+ },
429
+ }
430
+
431
+
432
+ def discover_audio_files(local_audio_dir: str) -> list[Path]:
433
+ audio_root = Path(local_audio_dir)
434
+ if not audio_root.exists() or not audio_root.is_dir():
435
+ raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
436
+
437
+ files = [
438
+ path
439
+ for path in audio_root.rglob("*")
440
+ if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
441
+ ]
442
+ return sorted(files)
443
+
444
+
445
+ def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
446
+ if chunk_size <= 0:
447
+ raise ValueError("chunk_size must be greater than zero")
448
+ return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
449
+
450
+
451
+ def _load_resume_state(state_file: Path) -> dict[str, Any]:
452
+ if not state_file.exists():
453
+ return {"uploaded": [], "failed": []}
454
+
455
+ payload = json.loads(state_file.read_text(encoding="utf-8"))
456
+ uploaded = payload.get("uploaded", [])
457
+ failed = payload.get("failed", [])
458
+ return {
459
+ "uploaded": uploaded if isinstance(uploaded, list) else [],
460
+ "failed": failed if isinstance(failed, list) else [],
461
+ }
462
+
463
+
464
+ def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
465
+ state_file.parent.mkdir(parents=True, exist_ok=True)
466
+ payload = {
467
+ "updated_at": _utcnow_iso(),
468
+ "uploaded": sorted(uploaded),
469
+ "failed": sorted(failed),
470
+ }
471
+ state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
472
+
473
+
474
+ def _upload_audio_with_retry(
475
+ api: HfApi,
476
+ dataset_repo: str,
477
+ local_path: Path,
478
+ path_in_repo: str,
479
+ max_retries: int,
480
+ retry_backoff_seconds: float,
481
+ ) -> None:
482
+ attempts = 0
483
+ while True:
484
+ attempts += 1
485
+ try:
486
+ api.upload_file(
487
+ path_or_fileobj=str(local_path),
488
+ path_in_repo=path_in_repo,
489
+ repo_id=dataset_repo,
490
+ repo_type="dataset",
491
+ )
492
+ return
493
+ except Exception:
494
+ if attempts > max_retries:
495
+ raise
496
+ time.sleep(retry_backoff_seconds * attempts)
497
+
498
+
499
+ def ingest_segments_to_hf(
500
+ api: HfApi,
501
+ project_slug: str,
502
+ dataset_repo: str,
503
+ detections_csv: str,
504
+ segments_root: str,
505
+ batch_size: int,
506
+ shard_size: int,
507
+ max_retries: int,
508
+ retry_backoff_seconds: float,
509
+ resume_state_file: str,
510
+ ) -> dict[str, Any]:
511
+ ensure_project_dataset_structure(
512
+ api=api,
513
+ project_slug=project_slug,
514
+ dataset_repo=dataset_repo,
515
+ create_private_repo=False,
516
+ )
517
+
518
+ match_result = _match_birdnet_rows(
519
+ project_slug=project_slug,
520
+ detections_csv=detections_csv,
521
+ segments_root=segments_root,
522
+ )
523
+ frame = match_result["frame"]
524
+ matched_rows = match_result["matched_rows"]
525
+ unmatched_rows = match_result["unmatched_rows"]
526
+ ambiguous_rows = match_result["ambiguous_rows"]
527
+
528
+ # Deduplicate by remote path so retries/resume work on a stable key.
529
+ path_to_local: dict[str, Path] = {}
530
+ for row in matched_rows:
531
+ repo_path = f"audio/segments/{row['segment_relpath']}"
532
+ if repo_path in path_to_local:
533
+ continue
534
+ path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
535
+
536
+ state_path = Path(resume_state_file)
537
+ state_payload = _load_resume_state(state_file=state_path)
538
+ uploaded_state = set(str(item) for item in state_payload["uploaded"])
539
+ failed_state = set(str(item) for item in state_payload["failed"])
540
+
541
+ remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
542
+
543
+ pending: list[tuple[Path, str]] = []
544
+ skipped_existing = 0
545
+ for repo_path, local_path in path_to_local.items():
546
+ if repo_path in remote_files:
547
+ uploaded_state.add(repo_path)
548
+ skipped_existing += 1
549
+ continue
550
+ if repo_path in uploaded_state:
551
+ continue
552
+ pending.append((local_path, repo_path))
553
+
554
+ uploaded_now: set[str] = set()
555
+ failed_now: set[str] = set()
556
+
557
+ for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
558
+ batch_uploaded = 0
559
+ batch_failed = 0
560
+ for local_path, repo_path in batch:
561
+ try:
562
+ _upload_audio_with_retry(
563
+ api=api,
564
+ dataset_repo=dataset_repo,
565
+ local_path=local_path,
566
+ path_in_repo=repo_path,
567
+ max_retries=max_retries,
568
+ retry_backoff_seconds=retry_backoff_seconds,
569
+ )
570
+ uploaded_state.add(repo_path)
571
+ failed_state.discard(repo_path)
572
+ uploaded_now.add(repo_path)
573
+ batch_uploaded += 1
574
+ except Exception:
575
+ failed_state.add(repo_path)
576
+ failed_now.add(repo_path)
577
+ batch_failed += 1
578
+
579
+ print(
580
+ json.dumps(
581
+ {
582
+ "event": "ingest-segments-audio-batch",
583
+ "batch_index": batch_index,
584
+ "batch_size": len(batch),
585
+ "uploaded": batch_uploaded,
586
+ "failed": batch_failed,
587
+ }
588
+ )
589
+ )
590
+ _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
591
+
592
+ index_frame = pd.DataFrame(matched_rows)
593
+ if not index_frame.empty:
594
+ index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
595
+
596
+ with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
597
+ temp_dir = Path(temp_dir_str)
598
+ shard_dir = temp_dir / "index" / "shards"
599
+ shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
600
+
601
+ for item in shard_metadata:
602
+ local_path = temp_dir / item.path
603
+ api.upload_file(
604
+ path_or_fileobj=str(local_path),
605
+ path_in_repo=item.path,
606
+ repo_id=dataset_repo,
607
+ repo_type="dataset",
608
+ )
609
+
610
+ manifest = build_manifest_payload(
611
+ project_slug=project_slug,
612
+ dataset_repo=dataset_repo,
613
+ frame=index_frame,
614
+ shard_size=shard_size,
615
+ shard_metadata=shard_metadata,
616
+ )
617
+ _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
618
+
619
+ run_report = {
620
+ "mode": "execute",
621
+ "project_slug": project_slug,
622
+ "dataset_repo": dataset_repo,
623
+ "detections_csv": detections_csv,
624
+ "segments_root": segments_root,
625
+ "csv_rows_total": int(len(frame)),
626
+ "segments_found_total": int(match_result["segment_records_total"]),
627
+ "matched_rows": len(matched_rows),
628
+ "unmatched_rows": len(unmatched_rows),
629
+ "ambiguous_rows": len(ambiguous_rows),
630
+ "unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
631
+ "pending_audio_uploads": len(pending),
632
+ "uploaded_audio_now": len(uploaded_now),
633
+ "uploaded_audio_skipped_existing": skipped_existing,
634
+ "failed_uploads": len(failed_now),
635
+ "resume_state_file": str(state_path),
636
+ "index_rows_written": len(index_frame),
637
+ "shards_written": len(shard_metadata),
638
+ "sample_unmatched": unmatched_rows[:20],
639
+ "sample_ambiguous": ambiguous_rows[:20],
640
+ }
641
+
642
+ audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
643
+ _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
644
+ return run_report
645
+
646
+
647
+ def sync_audio_batches(
648
+ api: HfApi,
649
+ project_slug: str,
650
+ dataset_repo: str,
651
+ local_audio_dir: str,
652
+ batch_size: int,
653
+ max_retries: int,
654
+ retry_backoff_seconds: float,
655
+ resume_state_file: str,
656
+ ) -> dict[str, Any]:
657
+ ensure_project_dataset_structure(
658
+ api=api,
659
+ project_slug=project_slug,
660
+ dataset_repo=dataset_repo,
661
+ create_private_repo=False,
662
+ )
663
+
664
+ all_files = discover_audio_files(local_audio_dir=local_audio_dir)
665
+ state_path = Path(resume_state_file)
666
+ state_payload = _load_resume_state(state_file=state_path)
667
+ uploaded_state = set(str(item) for item in state_payload["uploaded"])
668
+
669
+ remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
670
+
671
+ pending: list[tuple[Path, str]] = []
672
+ for path in all_files:
673
+ rel = path.relative_to(Path(local_audio_dir)).as_posix()
674
+ repo_path = f"audio/{rel}"
675
+
676
+ if repo_path in remote_files:
677
+ uploaded_state.add(repo_path)
678
+ continue
679
+
680
+ if repo_path in uploaded_state:
681
+ continue
682
+
683
+ pending.append((path, repo_path))
684
+
685
+ pending_paths = [item[0] for item in pending]
686
+ by_local_to_repo = {local: repo for local, repo in pending}
687
+ batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
688
+
689
+ uploaded_now: set[str] = set()
690
+ failed: set[str] = set()
691
+
692
+ for batch_index, batch in enumerate(batches, start=1):
693
+ batch_uploaded = 0
694
+ batch_failed = 0
695
+
696
+ for local_path in batch:
697
+ repo_path = by_local_to_repo[local_path]
698
+ try:
699
+ _upload_audio_with_retry(
700
+ api=api,
701
+ dataset_repo=dataset_repo,
702
+ local_path=local_path,
703
+ path_in_repo=repo_path,
704
+ max_retries=max_retries,
705
+ retry_backoff_seconds=retry_backoff_seconds,
706
+ )
707
+ uploaded_now.add(repo_path)
708
+ uploaded_state.add(repo_path)
709
+ batch_uploaded += 1
710
+ except Exception:
711
+ failed.add(repo_path)
712
+ batch_failed += 1
713
+
714
+ print(
715
+ json.dumps(
716
+ {
717
+ "event": "sync-audio-batch",
718
+ "batch_index": batch_index,
719
+ "batch_size": len(batch),
720
+ "uploaded": batch_uploaded,
721
+ "failed": batch_failed,
722
+ }
723
+ )
724
+ )
725
+ _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
726
+
727
+ return {
728
+ "project_slug": project_slug,
729
+ "dataset_repo": dataset_repo,
730
+ "total_local_audio_files": len(all_files),
731
+ "pending_uploads": len(pending),
732
+ "uploaded_now": len(uploaded_now),
733
+ "failed": len(failed),
734
+ "resume_state_file": str(state_path),
735
+ }
736
+
737
+
738
+ def build_and_upload_index(
739
+ api: HfApi,
740
+ project_slug: str,
741
+ dataset_repo: str,
742
+ detections_file: str,
743
+ shard_size: int,
744
+ ) -> dict[str, Any]:
745
+ ensure_project_dataset_structure(
746
+ api=api,
747
+ project_slug=project_slug,
748
+ dataset_repo=dataset_repo,
749
+ create_private_repo=False,
750
+ )
751
+
752
+ frame = load_detections_table(detections_file=detections_file)
753
+
754
+ with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
755
+ temp_dir = Path(temp_dir_str)
756
+ shard_dir = temp_dir / "index" / "shards"
757
+ metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
758
+
759
+ for item in metadata:
760
+ local_path = temp_dir / item.path
761
+ api.upload_file(
762
+ path_or_fileobj=str(local_path),
763
+ path_in_repo=item.path,
764
+ repo_id=dataset_repo,
765
+ repo_type="dataset",
766
+ )
767
+
768
+ manifest = build_manifest_payload(
769
+ project_slug=project_slug,
770
+ dataset_repo=dataset_repo,
771
+ frame=frame,
772
+ shard_size=shard_size,
773
+ shard_metadata=metadata,
774
+ )
775
+ _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
776
+
777
+ return {
778
+ "project_slug": project_slug,
779
+ "dataset_repo": dataset_repo,
780
+ "total_detections": manifest["index"]["total_detections"],
781
+ "total_audio_files": manifest["index"]["total_audio_files"],
782
+ "total_shards": len(metadata),
783
+ }
784
+
785
+
786
+ def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
787
+ errors: list[str] = []
788
+
789
+ for prefix in REQUIRED_PROJECT_PREFIXES:
790
+ if not any(path.startswith(prefix) for path in repo_files):
791
+ errors.append(f"Missing prefix in dataset repo: {prefix}")
792
+
793
+ if manifest_payload.get("schema_version") != SCHEMA_VERSION:
794
+ errors.append(
795
+ f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
796
+ )
797
+
798
+ if manifest_payload.get("project_slug") != project_slug:
799
+ errors.append(
800
+ f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
801
+ )
802
+
803
+ index_section = manifest_payload.get("index", {})
804
+ shards = index_section.get("shards", [])
805
+ if not isinstance(shards, list):
806
+ errors.append("Manifest index.shards must be a list")
807
+ return errors
808
+
809
+ for shard in shards:
810
+ shard_path = shard.get("path")
811
+ if not shard_path or not isinstance(shard_path, str):
812
+ errors.append("Shard entry missing path")
813
+ continue
814
+ if shard_path not in repo_files:
815
+ errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
816
+
817
+ return errors
818
+
819
+
820
+ def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
821
+ repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
822
+ if "manifest.json" not in repo_files:
823
+ return {
824
+ "ok": False,
825
+ "errors": ["manifest.json not found in dataset repository"],
826
+ }
827
+
828
+ manifest_local = hf_hub_download(
829
+ repo_id=dataset_repo,
830
+ repo_type="dataset",
831
+ filename="manifest.json",
832
+ )
833
+ manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
834
+ errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
835
+
836
+ return {
837
+ "ok": len(errors) == 0,
838
+ "errors": errors,
839
+ "total_files": len(repo_files),
840
+ "total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
841
+ }
842
+
843
+
844
+ def build_parser() -> argparse.ArgumentParser:
845
+ parser = argparse.ArgumentParser(description="HF dataset project CLI")
846
+ sub = parser.add_subparsers(dest="command", required=True)
847
+
848
+ create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
849
+ create_project.add_argument("--project-slug", required=True)
850
+ create_project.add_argument("--dataset-repo", required=True)
851
+ create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
852
+
853
+ build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
854
+ build_index.add_argument("--project-slug", required=True)
855
+ build_index.add_argument("--dataset-repo", required=True)
856
+ build_index.add_argument("--detections-file", required=True)
857
+ build_index.add_argument("--shard-size", type=int, default=10000)
858
+
859
+ sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
860
+ sync_audio.add_argument("--project-slug", required=True)
861
+ sync_audio.add_argument("--dataset-repo", required=True)
862
+ sync_audio.add_argument("--local-audio-dir", required=True)
863
+ sync_audio.add_argument("--batch-size", type=int, default=100)
864
+ sync_audio.add_argument("--max-retries", type=int, default=3)
865
+ sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
866
+ sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
867
+
868
+ verify_project = sub.add_parser("verify-project", help="Verify project integrity")
869
+ verify_project.add_argument("--project-slug", required=True)
870
+ verify_project.add_argument("--dataset-repo", required=True)
871
+
872
+ ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
873
+ ingest_segments.add_argument("--project-slug", required=True)
874
+ ingest_segments.add_argument("--dataset-repo", required=True)
875
+ ingest_segments.add_argument("--detections-csv", required=True)
876
+ ingest_segments.add_argument("--segments-root", required=True)
877
+ ingest_segments.add_argument("--batch-size", type=int, default=200)
878
+ ingest_segments.add_argument("--shard-size", type=int, default=10000)
879
+ ingest_segments.add_argument("--max-retries", type=int, default=3)
880
+ ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
881
+ ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
882
+ ingest_segments.add_argument("--dry-run", action="store_true")
883
+ ingest_segments.add_argument("--report-file", default="")
884
+
885
+ return parser
886
+
887
+
888
+ def main() -> int:
889
+ parser = build_parser()
890
+ args = parser.parse_args()
891
+ api = HfApi()
892
+
893
+ if args.command == "create-project":
894
+ result = ensure_project_dataset_structure(
895
+ api=api,
896
+ project_slug=args.project_slug,
897
+ dataset_repo=args.dataset_repo,
898
+ create_private_repo=bool(args.private),
899
+ )
900
+ print(json.dumps(result, indent=2))
901
+ return 0
902
+
903
+ if args.command == "build-index":
904
+ result = build_and_upload_index(
905
+ api=api,
906
+ project_slug=args.project_slug,
907
+ dataset_repo=args.dataset_repo,
908
+ detections_file=args.detections_file,
909
+ shard_size=args.shard_size,
910
+ )
911
+ print(json.dumps(result, indent=2))
912
+ return 0
913
+
914
+ if args.command == "sync-audio":
915
+ result = sync_audio_batches(
916
+ api=api,
917
+ project_slug=args.project_slug,
918
+ dataset_repo=args.dataset_repo,
919
+ local_audio_dir=args.local_audio_dir,
920
+ batch_size=args.batch_size,
921
+ max_retries=args.max_retries,
922
+ retry_backoff_seconds=args.retry_backoff_seconds,
923
+ resume_state_file=args.resume_state_file,
924
+ )
925
+ print(json.dumps(result, indent=2))
926
+ return 0 if result["failed"] == 0 else 1
927
+
928
+ if args.command == "verify-project":
929
+ result = verify_project(
930
+ api=api,
931
+ project_slug=args.project_slug,
932
+ dataset_repo=args.dataset_repo,
933
+ )
934
+ print(json.dumps(result, indent=2))
935
+ return 0 if result["ok"] else 1
936
+
937
+ if args.command == "ingest-segments":
938
+ if args.dry_run:
939
+ result = run_ingest_segments_dry_run(
940
+ project_slug=args.project_slug,
941
+ detections_csv=args.detections_csv,
942
+ segments_root=args.segments_root,
943
+ )
944
+ else:
945
+ result = ingest_segments_to_hf(
946
+ api=api,
947
+ project_slug=args.project_slug,
948
+ dataset_repo=args.dataset_repo,
949
+ detections_csv=args.detections_csv,
950
+ segments_root=args.segments_root,
951
+ batch_size=args.batch_size,
952
+ shard_size=args.shard_size,
953
+ max_retries=args.max_retries,
954
+ retry_backoff_seconds=args.retry_backoff_seconds,
955
+ resume_state_file=args.resume_state_file,
956
+ )
957
+
958
+ if args.report_file:
959
+ Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
960
+
961
+ print(json.dumps(result, indent=2))
962
+ return 0 if result.get("failed_uploads", 0) == 0 else 1
963
+
964
+ parser.error("Unknown command")
965
+ return 2
966
+
967
+
968
+ if __name__ == "__main__":
969
+ raise SystemExit(main())
BirdNET Uploader App/pyproject.toml ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ [project]
2
+ name = "BirdNET-validator-App"
3
+ version = "0.1.0"
4
+ description = "BirdNET detection validator for Hugging Face Spaces"
5
+ readme = "README.md"
6
+ requires-python = ">=3.11"
7
+
8
+ [tool.pytest.ini_options]
9
+ testpaths = ["tests"]
BirdNET Uploader App/requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ gradio>=5.0.0
2
+ huggingface_hub>=0.30.0
3
+ pydantic>=2.8.0
4
+ pandas>=2.2.0
5
+ pyarrow>=17.0.0
6
+ soundfile>=0.12.1
7
+ pytest>=8.3.0
BirdNET Uploader App/src/__init__.py ADDED
File without changes
BirdNET Uploader App/src/ui/__init__.py ADDED
File without changes
BirdNET Uploader App/src/ui/upload_app_factory.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from pathlib import Path
3
+ from typing import Any
4
+
5
+ import gradio as gr
6
+ from huggingface_hub import HfApi
7
+
8
+ from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
9
+
10
+
11
+ def _as_pretty_json(payload: dict[str, Any]) -> str:
12
+ return json.dumps(payload, indent=2, ensure_ascii=False)
13
+
14
+
15
+ def _build_api(token: str) -> HfApi:
16
+ clean_token = (token or "").strip()
17
+ if clean_token:
18
+ return HfApi(token=clean_token)
19
+ return HfApi()
20
+
21
+
22
+ def build_upload_app() -> gr.Blocks:
23
+ with gr.Blocks(title="BirdNET Segments Uploader") as demo:
24
+ gr.Markdown("# BirdNET Segments Uploader")
25
+ gr.Markdown(
26
+ "Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
27
+ "Recomendado: executar dry-run antes do upload real."
28
+ )
29
+
30
+ with gr.Row():
31
+ project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
32
+ dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
33
+
34
+ detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
35
+ segments_root = gr.Textbox(
36
+ label="Pasta raiz de segmentos",
37
+ placeholder=r"ex: C:\dados\BirdNET Segments",
38
+ )
39
+
40
+ with gr.Accordion("Configuracao avancada", open=False):
41
+ hf_token = gr.Textbox(
42
+ label="HF Token (opcional)",
43
+ type="password",
44
+ placeholder="Se vazio, usa sessao ja autenticada no ambiente",
45
+ )
46
+ with gr.Row():
47
+ batch_size = gr.Number(label="Batch size", value=200, precision=0)
48
+ shard_size = gr.Number(label="Shard size", value=10000, precision=0)
49
+ with gr.Row():
50
+ max_retries = gr.Number(label="Max retries", value=3, precision=0)
51
+ retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
52
+ resume_state_file = gr.Textbox(
53
+ label="Resume state file",
54
+ value=".ingest-segments-state.json",
55
+ )
56
+ report_file = gr.Textbox(
57
+ label="Report file local (opcional)",
58
+ placeholder="ex: .ingest-run-report.json",
59
+ )
60
+
61
+ with gr.Row():
62
+ dry_run_button = gr.Button("Dry-run", variant="secondary")
63
+ upload_button = gr.Button("Upload real", variant="primary")
64
+
65
+ status = gr.Markdown(value="Pronto")
66
+ result_json = gr.Code(label="Resultado JSON", language="json")
67
+
68
+ def run_dry_run(
69
+ project: str,
70
+ repo: str,
71
+ csv_path: str | None,
72
+ segments_path: str,
73
+ report_path: str,
74
+ ) -> tuple[str, str]:
75
+ project = (project or "").strip()
76
+ if not project:
77
+ return "❌ Informe o project slug", "{}"
78
+
79
+ if not csv_path:
80
+ return "❌ Selecione o CSV de deteccoes", "{}"
81
+
82
+ if not segments_path or not Path(segments_path).exists():
83
+ return "❌ Pasta de segmentos invalida", "{}"
84
+
85
+ try:
86
+ result = run_ingest_segments_dry_run(
87
+ project_slug=project,
88
+ detections_csv=csv_path,
89
+ segments_root=segments_path,
90
+ )
91
+ if report_path.strip():
92
+ Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
93
+ summary = (
94
+ "βœ… Dry-run finalizado | "
95
+ f"Matched: {result['matched_rows']} | "
96
+ f"Unmatched: {result['unmatched_rows']} | "
97
+ f"Ambiguous: {result['ambiguous_rows']}"
98
+ )
99
+ return summary, _as_pretty_json(result)
100
+ except Exception as exc:
101
+ return f"❌ Dry-run falhou: {exc}", "{}"
102
+
103
+ def run_upload(
104
+ project: str,
105
+ repo: str,
106
+ csv_path: str | None,
107
+ segments_path: str,
108
+ token: str,
109
+ batch: float,
110
+ shard: float,
111
+ retries: float,
112
+ backoff: float,
113
+ resume_file: str,
114
+ report_path: str,
115
+ ) -> tuple[str, str]:
116
+ project = (project or "").strip()
117
+ repo = (repo or "").strip()
118
+ if not project:
119
+ return "❌ Informe o project slug", "{}"
120
+ if not repo or "/" not in repo:
121
+ return "❌ Informe dataset repo no formato owner/repo", "{}"
122
+ if not csv_path:
123
+ return "❌ Selecione o CSV de deteccoes", "{}"
124
+ if not segments_path or not Path(segments_path).exists():
125
+ return "❌ Pasta de segmentos invalida", "{}"
126
+
127
+ try:
128
+ api = _build_api(token)
129
+ result = ingest_segments_to_hf(
130
+ api=api,
131
+ project_slug=project,
132
+ dataset_repo=repo,
133
+ detections_csv=csv_path,
134
+ segments_root=segments_path,
135
+ batch_size=int(batch),
136
+ shard_size=int(shard),
137
+ max_retries=int(retries),
138
+ retry_backoff_seconds=float(backoff),
139
+ resume_state_file=resume_file,
140
+ )
141
+ if report_path.strip():
142
+ Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
143
+ summary = (
144
+ "βœ… Upload finalizado | "
145
+ f"Uploaded now: {result['uploaded_audio_now']} | "
146
+ f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
147
+ f"Failed uploads: {result['failed_uploads']}"
148
+ )
149
+ return summary, _as_pretty_json(result)
150
+ except Exception as exc:
151
+ return f"❌ Upload falhou: {exc}", "{}"
152
+
153
+ dry_run_button.click(
154
+ fn=run_dry_run,
155
+ inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
156
+ outputs=[status, result_json],
157
+ )
158
+
159
+ upload_button.click(
160
+ fn=run_upload,
161
+ inputs=[
162
+ project_slug,
163
+ dataset_repo,
164
+ detections_csv,
165
+ segments_root,
166
+ hf_token,
167
+ batch_size,
168
+ shard_size,
169
+ max_retries,
170
+ retry_backoff_seconds,
171
+ resume_state_file,
172
+ report_file,
173
+ ],
174
+ outputs=[status, result_json],
175
+ )
176
+
177
+ return demo
BirdNET Validator App/README.md ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # BirdNET Validator App
2
+
3
+ Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
4
+ - Multi-projeto
5
+ - Audio sob demanda
6
+ - Persistencia de validacoes
7
+ - Escalabilidade para datasets grandes
8
+
9
+ ## Desenvolvimento local
10
+ 1. Criar ambiente virtual Python 3.11+
11
+ 2. Instalar dependencias:
12
+ pip install -r requirements.txt
13
+ 3. Rodar app:
14
+ python app.py
15
+
16
+ ## Objetivo desta pasta
17
+ Contem os arquivos do app de validacao para evolucao/publicacao independente.
18
+
19
+ ## Fluxo recomendado (separado)
20
+ 1. Uploader app (porta 7862 por padrao):
21
+ - Rodar dry-run com CSV + pasta de segmentos
22
+ - Rodar upload real para owner/repo do projeto
23
+ 2. Validator app (porta 7860):
24
+ - Validar deteccoes do projeto com dados ja publicados
25
+
26
+ Essa separacao evita impacto de performance na validacao durante uploads grandes.
27
+
28
+ ## CLI de dataset (Sprint 1)
29
+ Comandos principais:
30
+
31
+ 1. Criar estrutura do projeto no dataset Hugging Face:
32
+ python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
33
+
34
+ 2. Gerar manifesto e shards iniciais de indice:
35
+ python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
36
+
37
+ 3. Fazer upload de audio em lotes com retry e resume:
38
+ python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
39
+
40
+ 4. Verificar consistencia basica do projeto:
41
+ python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
42
+
43
+ 5. Ingestao BirdNET por projeto (dry-run):
44
+ python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
45
+
46
+ 6. Ingestao BirdNET por projeto (upload real):
47
+ python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
48
+
49
+ Estrutura base criada por projeto/dataset:
50
+ - audio/
51
+ - index/
52
+ - index/shards/
53
+ - validations/
54
+ - audit/
55
+ - manifest.json
56
+
57
+ ## Estrutura
58
+ - src/domain: modelos de dominio
59
+ - src/repositories: contratos de persistencia
60
+ - src/services: servicos de aplicacao
61
+ - src/auth: autenticacao e autorizacao
62
+ - src/ui: montagem da interface
63
+ - src/cache: cache efemero local
64
+ - cli: comandos de ingestao/publicacao
65
+ - tests: testes unitarios e integracao
66
+
67
+ ## Estado atual de implementacao
68
+ - Sprint 1: CLI create-project, sync-audio, build-index, verify-project
69
+ - Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
70
+ - Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
71
+ - Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
72
+ - Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
73
+ - Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
74
+ - Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
75
+ - Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
76
+ - Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
77
+ - Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
78
+ - Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
79
+ - Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
80
+ - Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
81
+ - Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
82
+ - Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
83
+ - Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
BirdNET Validator App/app.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ from src.ui.app_factory import create_app
2
+
3
+
4
+ if __name__ == "__main__":
5
+ app = create_app()
6
+ app.launch(server_name="0.0.0.0", server_port=7860)
BirdNET Validator App/pyproject.toml ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ [project]
2
+ name = "BirdNET-validator-App"
3
+ version = "0.1.0"
4
+ description = "BirdNET detection validator for Hugging Face Spaces"
5
+ readme = "README.md"
6
+ requires-python = ">=3.11"
7
+
8
+ [tool.pytest.ini_options]
9
+ testpaths = ["tests"]
BirdNET Validator App/requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ gradio>=5.0.0
2
+ huggingface_hub>=0.30.0
3
+ pydantic>=2.8.0
4
+ pandas>=2.2.0
5
+ pyarrow>=17.0.0
6
+ soundfile>=0.12.1
7
+ pytest>=8.3.0
BirdNET Validator App/src/__init__.py ADDED
File without changes
BirdNET Validator App/src/auth/__init__.py ADDED
File without changes
BirdNET Validator App/src/auth/auth_service.py ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AuthService: User authentication, session management, and project ACL."""
2
+
3
+ from dataclasses import dataclass, field
4
+ from datetime import datetime, timedelta
5
+ from typing import Dict, List, Optional
6
+ import uuid
7
+
8
+ from src.domain.models import Role, User
9
+
10
+
11
+ @dataclass
12
+ class UserProjectAccess:
13
+ """Maps a user to projects with specific roles."""
14
+
15
+ username: str
16
+ project_slugs: Dict[str, Role] # Maps project_slug -> role (admin or validator)
17
+ is_active: bool = True
18
+
19
+
20
+ @dataclass
21
+ class Session:
22
+ """Represents an authenticated user session."""
23
+
24
+ session_id: str
25
+ username: str
26
+ role: Role
27
+ authorized_projects: List[str] # Projects this user can access
28
+ created_at: datetime
29
+ last_activity: datetime
30
+ expires_at: datetime
31
+
32
+ def is_expired(self) -> bool:
33
+ """Check if session has expired."""
34
+ return datetime.utcnow() > self.expires_at
35
+
36
+ def update_activity(self, ttl_minutes: int = 120) -> None:
37
+ """Update last activity timestamp and extend expiration."""
38
+ self.last_activity = datetime.utcnow()
39
+ self.expires_at = datetime.utcnow() + timedelta(minutes=ttl_minutes)
40
+
41
+
42
+ class AuthService:
43
+ """Manages user authentication, sessions, and project access control."""
44
+
45
+ def __init__(self, session_ttl_minutes: int = 120):
46
+ """Initialize AuthService with session TTL.
47
+
48
+ Args:
49
+ session_ttl_minutes: Session time-to-live in minutes (default: 2 hours)
50
+ """
51
+ self.session_ttl_minutes = session_ttl_minutes
52
+ self._sessions: Dict[str, Session] = {} # session_id -> Session
53
+ self._user_access: Dict[str, UserProjectAccess] = {} # username -> UserProjectAccess
54
+
55
+ def register_user_project_access(
56
+ self, username: str, project_access: Dict[str, Role]
57
+ ) -> None:
58
+ """Register or update a user's access to projects.
59
+
60
+ Args:
61
+ username: User name
62
+ project_access: Dict mapping project_slug to Role (admin or validator)
63
+ """
64
+ self._user_access[username] = UserProjectAccess(
65
+ username=username,
66
+ project_slugs=project_access,
67
+ is_active=True,
68
+ )
69
+
70
+ def login(self, username: str) -> Optional[Session]:
71
+ """Authenticate a user and create a new session.
72
+
73
+ Args:
74
+ username: User name to authenticate
75
+
76
+ Returns:
77
+ Session if user exists and is active, None otherwise
78
+ """
79
+ if username not in self._user_access:
80
+ return None
81
+
82
+ access = self._user_access[username]
83
+ if not access.is_active:
84
+ return None
85
+
86
+ # Determine highest role across all projects (admin > validator)
87
+ role = Role.validator
88
+ for proj_role in access.project_slugs.values():
89
+ if proj_role == Role.admin:
90
+ role = Role.admin
91
+ break
92
+
93
+ session_id = str(uuid.uuid4())
94
+ now = datetime.utcnow()
95
+ session = Session(
96
+ session_id=session_id,
97
+ username=username,
98
+ role=role,
99
+ authorized_projects=list(access.project_slugs.keys()),
100
+ created_at=now,
101
+ last_activity=now,
102
+ expires_at=now + timedelta(minutes=self.session_ttl_minutes),
103
+ )
104
+
105
+ self._sessions[session_id] = session
106
+ return session
107
+
108
+ def get_session(self, session_id: str) -> Optional[Session]:
109
+ """Retrieve an active session by ID.
110
+
111
+ Args:
112
+ session_id: Session ID
113
+
114
+ Returns:
115
+ Session if valid and not expired, None otherwise
116
+ """
117
+ session = self._sessions.get(session_id)
118
+ if session is None:
119
+ return None
120
+
121
+ if session.is_expired():
122
+ self._sessions.pop(session_id, None)
123
+ return None
124
+
125
+ session.update_activity(self.session_ttl_minutes)
126
+ return session
127
+
128
+ def logout(self, session_id: str) -> None:
129
+ """Delete a session.
130
+
131
+ Args:
132
+ session_id: Session ID to invalidate
133
+ """
134
+ self._sessions.pop(session_id, None)
135
+
136
+ def is_user_authorized_for_project(self, username: str, project_slug: str) -> bool:
137
+ """Check if a user has access to a specific project.
138
+
139
+ Args:
140
+ username: User name
141
+ project_slug: Project slug
142
+
143
+ Returns:
144
+ True if user has access (admin or validator), False otherwise
145
+ """
146
+ access = self._user_access.get(username)
147
+ if access is None or not access.is_active:
148
+ return False
149
+
150
+ return project_slug in access.project_slugs
151
+
152
+ def get_user_role_for_project(self, username: str, project_slug: str) -> Optional[Role]:
153
+ """Get the specific role a user has for a project.
154
+
155
+ Args:
156
+ username: User name
157
+ project_slug: Project slug
158
+
159
+ Returns:
160
+ Role if user has access, None otherwise
161
+ """
162
+ access = self._user_access.get(username)
163
+ if access is None or not access.is_active:
164
+ return None
165
+
166
+ return access.project_slugs.get(project_slug)
167
+
168
+ def list_user_projects(self, username: str) -> List[str]:
169
+ """List all projects a user has access to.
170
+
171
+ Args:
172
+ username: User name
173
+
174
+ Returns:
175
+ List of project slugs
176
+ """
177
+ access = self._user_access.get(username)
178
+ if access is None or not access.is_active:
179
+ return []
180
+
181
+ return list(access.project_slugs.keys())
182
+
183
+ def cleanup_expired_sessions(self) -> None:
184
+ """Remove all expired sessions (maintenance cleanup)."""
185
+ now = datetime.utcnow()
186
+ expired = [
187
+ sid for sid, session in self._sessions.items() if session.is_expired()
188
+ ]
189
+ for sid in expired:
190
+ self._sessions.pop(sid)
191
+
192
+ def set_user_active(self, username: str, active: bool) -> None:
193
+ """Enable or disable a user account.
194
+
195
+ Args:
196
+ username: User name
197
+ active: Whether the user should be active
198
+ """
199
+ if username in self._user_access:
200
+ self._user_access[username].is_active = active
BirdNET Validator App/src/cache/__init__.py ADDED
File without changes
BirdNET Validator App/src/cache/ephemeral_cache_manager.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import hashlib
2
+ import tempfile
3
+ import time
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+
7
+
8
+ @dataclass(slots=True)
9
+ class CacheEntry:
10
+ key: str
11
+ path: Path
12
+ created_at: float
13
+ expires_at: float
14
+ size_bytes: int
15
+
16
+
17
+ class EphemeralCacheManager:
18
+ def __init__(
19
+ self,
20
+ cache_dir: str | None = None,
21
+ ttl_seconds: int = 300,
22
+ max_files: int = 256,
23
+ ) -> None:
24
+ self._base_dir = Path(cache_dir) if cache_dir else Path(tempfile.gettempdir()) / "birdnet-validator-cache"
25
+ self._base_dir.mkdir(parents=True, exist_ok=True)
26
+ self._ttl_seconds = ttl_seconds
27
+ self._max_files = max_files
28
+ self._entries: dict[str, CacheEntry] = {}
29
+
30
+ def get(self, key: str) -> Path | None:
31
+ self.cleanup_expired()
32
+ entry = self._entries.get(key)
33
+ if not entry:
34
+ return None
35
+ if not entry.path.exists():
36
+ self._entries.pop(key, None)
37
+ return None
38
+ return entry.path
39
+
40
+ def put_bytes(self, key: str, data: bytes, suffix: str = ".bin") -> Path:
41
+ self.cleanup_expired()
42
+ self._enforce_capacity()
43
+
44
+ digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
45
+ file_path = self._base_dir / f"{digest}{suffix}"
46
+ file_path.write_bytes(data)
47
+
48
+ now = time.time()
49
+ self._entries[key] = CacheEntry(
50
+ key=key,
51
+ path=file_path,
52
+ created_at=now,
53
+ expires_at=now + self._ttl_seconds,
54
+ size_bytes=len(data),
55
+ )
56
+ return file_path
57
+
58
+ def cleanup_key(self, key: str) -> None:
59
+ entry = self._entries.pop(key, None)
60
+ if not entry:
61
+ return
62
+ if entry.path.exists():
63
+ entry.path.unlink()
64
+
65
+ def cleanup_expired(self) -> None:
66
+ now = time.time()
67
+ expired_keys = [key for key, entry in self._entries.items() if entry.expires_at <= now or not entry.path.exists()]
68
+ for key in expired_keys:
69
+ self.cleanup_key(key)
70
+
71
+ def clear(self) -> None:
72
+ for key in list(self._entries.keys()):
73
+ self.cleanup_key(key)
74
+
75
+ def _enforce_capacity(self) -> None:
76
+ if len(self._entries) < self._max_files:
77
+ return
78
+
79
+ oldest_keys = sorted(self._entries.keys(), key=lambda key: self._entries[key].created_at)
80
+ overflow = (len(self._entries) - self._max_files) + 1
81
+ for key in oldest_keys[:overflow]:
82
+ self.cleanup_key(key)
BirdNET Validator App/src/domain/__init__.py ADDED
File without changes
BirdNET Validator App/src/domain/models.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from enum import Enum
2
+ from typing import Optional
3
+
4
+ from pydantic import BaseModel, Field
5
+
6
+
7
+ class Role(str, Enum):
8
+ admin = "admin"
9
+ validator = "validator"
10
+
11
+
12
+ class User(BaseModel):
13
+ username: str = Field(min_length=1)
14
+ role: Role
15
+ active: bool = True
16
+
17
+
18
+ class Project(BaseModel):
19
+ project_slug: str = Field(min_length=3)
20
+ name: str = Field(min_length=1)
21
+ dataset_repo_id: str = Field(min_length=3)
22
+ active: bool = True
23
+
24
+
25
+ class Detection(BaseModel):
26
+ detection_key: str = Field(min_length=16)
27
+ audio_id: str = Field(min_length=1)
28
+ scientific_name: str = Field(min_length=1)
29
+ confidence: float = Field(ge=0.0, le=1.0)
30
+ start_time: float = Field(ge=0.0)
31
+ end_time: float = Field(gt=0.0)
32
+
33
+
34
+ class Validation(BaseModel):
35
+ detection_key: str = Field(min_length=16)
36
+ status: str = Field(min_length=1)
37
+ corrected_species: Optional[str] = None
38
+ notes: str = ""
39
+ validator: str = Field(min_length=1)
40
+
41
+
42
+ class IndexManifest(BaseModel):
43
+ schema_version: str = "1.0.0"
44
+ project_slug: str
45
+ total_detections: int = Field(ge=0)
46
+ total_audio_files: int = Field(ge=0)
BirdNET Validator App/src/repositories/__init__.py ADDED
File without changes
BirdNET Validator App/src/repositories/append_only_validation_repository.py ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from datetime import UTC, datetime
3
+ from pathlib import Path
4
+ from uuid import uuid4
5
+
6
+ from src.domain.models import Validation
7
+
8
+
9
+ class OptimisticLockError(Exception):
10
+ def __init__(self, detection_key: str, expected_version: int, current_version: int) -> None:
11
+ self.detection_key = detection_key
12
+ self.expected_version = expected_version
13
+ self.current_version = current_version
14
+ super().__init__(
15
+ f"Optimistic lock conflict for detection_key={detection_key}: "
16
+ f"expected_version={expected_version}, current_version={current_version}"
17
+ )
18
+
19
+
20
+ class AppendOnlyValidationRepository:
21
+ def __init__(self, base_dir: str) -> None:
22
+ self._base_dir = Path(base_dir)
23
+
24
+ def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
25
+ project_dir = self._base_dir / project_slug / "validations"
26
+ project_dir.mkdir(parents=True, exist_ok=True)
27
+
28
+ current_file = project_dir / "current.json"
29
+ if current_file.exists():
30
+ current_payload = json.loads(current_file.read_text(encoding="utf-8"))
31
+ else:
32
+ current_payload = {}
33
+
34
+ current_item = current_payload.get(item.detection_key, {})
35
+ current_version = int(current_item.get("version", 0))
36
+ expected = expected_version if expected_version is not None else current_version
37
+ if expected != current_version:
38
+ raise OptimisticLockError(
39
+ detection_key=item.detection_key,
40
+ expected_version=expected,
41
+ current_version=current_version,
42
+ )
43
+
44
+ new_version = current_version + 1
45
+
46
+ event_date = datetime.now(UTC).strftime("%Y%m%d")
47
+ events_file = project_dir / f"events-{event_date}.jsonl"
48
+ event = {
49
+ "event_id": str(uuid4()),
50
+ "timestamp": datetime.now(UTC).isoformat(),
51
+ "project_slug": project_slug,
52
+ "expected_version": expected,
53
+ "previous_version": current_version,
54
+ "new_version": new_version,
55
+ **item.model_dump(),
56
+ }
57
+ with events_file.open("a", encoding="utf-8") as handle:
58
+ handle.write(json.dumps(event, ensure_ascii=True) + "\n")
59
+
60
+ current_payload[item.detection_key] = {
61
+ "status": item.status,
62
+ "corrected_species": item.corrected_species,
63
+ "notes": item.notes,
64
+ "validator": item.validator,
65
+ "updated_at": event["timestamp"],
66
+ "version": new_version,
67
+ }
68
+ current_file.write_text(json.dumps(current_payload, indent=2), encoding="utf-8")
69
+ return new_version
70
+
71
+ def list_events(self, project_slug: str) -> list[dict[str, object]]:
72
+ project_dir = self._base_dir / project_slug / "validations"
73
+ if not project_dir.exists():
74
+ return []
75
+
76
+ events: list[dict[str, object]] = []
77
+ for path in sorted(project_dir.glob("events-*.jsonl")):
78
+ with path.open("r", encoding="utf-8") as handle:
79
+ for line in handle:
80
+ payload = line.strip()
81
+ if payload:
82
+ events.append(json.loads(payload))
83
+ return events
84
+
85
+ def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]:
86
+ current_file = self._base_dir / project_slug / "validations" / "current.json"
87
+ if not current_file.exists():
88
+ return {}
89
+ return json.loads(current_file.read_text(encoding="utf-8"))
BirdNET Validator App/src/repositories/contracts.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Protocol
2
+
3
+ from src.domain.models import Detection, Project, User, Validation
4
+
5
+
6
+ class ProjectRepository(Protocol):
7
+ def get_project(self, project_slug: str) -> Project: ...
8
+
9
+
10
+ class DetectionRepository(Protocol):
11
+ def list_detections(
12
+ self,
13
+ project_slug: str,
14
+ page: int,
15
+ page_size: int,
16
+ scientific_name: str | None = None,
17
+ min_confidence: float | None = None,
18
+ max_confidence: float | None = None,
19
+ ) -> list[Detection]: ...
20
+
21
+ def count_detections(
22
+ self,
23
+ project_slug: str,
24
+ scientific_name: str | None = None,
25
+ min_confidence: float | None = None,
26
+ max_confidence: float | None = None,
27
+ ) -> int: ...
28
+
29
+
30
+ class ValidationRepository(Protocol):
31
+ def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int: ...
32
+
33
+
34
+ class AuthRepository(Protocol):
35
+ def authenticate(self, username: str) -> User: ...
BirdNET Validator App/src/repositories/in_memory_detection_repository.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import defaultdict
2
+ from typing import DefaultDict
3
+
4
+ from src.domain.models import Detection
5
+
6
+
7
+ class InMemoryDetectionRepository:
8
+ def __init__(self) -> None:
9
+ self._project_items: DefaultDict[str, list[Detection]] = defaultdict(list)
10
+
11
+ def seed(self, project_slug: str, detections: list[Detection]) -> None:
12
+ self._project_items[project_slug] = list(detections)
13
+
14
+ def list_detections(
15
+ self,
16
+ project_slug: str,
17
+ page: int,
18
+ page_size: int,
19
+ scientific_name: str | None = None,
20
+ min_confidence: float | None = None,
21
+ max_confidence: float | None = None,
22
+ ) -> list[Detection]:
23
+ if page < 1:
24
+ raise ValueError("page must be >= 1")
25
+ if page_size < 1:
26
+ raise ValueError("page_size must be >= 1")
27
+
28
+ filtered = self._apply_filters(
29
+ project_slug=project_slug,
30
+ scientific_name=scientific_name,
31
+ min_confidence=min_confidence,
32
+ max_confidence=max_confidence,
33
+ )
34
+
35
+ start = (page - 1) * page_size
36
+ end = start + page_size
37
+ return filtered[start:end]
38
+
39
+ def count_detections(
40
+ self,
41
+ project_slug: str,
42
+ scientific_name: str | None = None,
43
+ min_confidence: float | None = None,
44
+ max_confidence: float | None = None,
45
+ ) -> int:
46
+ return len(
47
+ self._apply_filters(
48
+ project_slug=project_slug,
49
+ scientific_name=scientific_name,
50
+ min_confidence=min_confidence,
51
+ max_confidence=max_confidence,
52
+ )
53
+ )
54
+
55
+ def _apply_filters(
56
+ self,
57
+ project_slug: str,
58
+ scientific_name: str | None,
59
+ min_confidence: float | None,
60
+ max_confidence: float | None,
61
+ ) -> list[Detection]:
62
+ items = self._project_items.get(project_slug, [])
63
+
64
+ def match(item: Detection) -> bool:
65
+ if scientific_name and item.scientific_name != scientific_name:
66
+ return False
67
+ if min_confidence is not None and item.confidence < min_confidence:
68
+ return False
69
+ if max_confidence is not None and item.confidence > max_confidence:
70
+ return False
71
+ return True
72
+
73
+ return [item for item in items if match(item)]
BirdNET Validator App/src/repositories/in_memory_validation_repository.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import defaultdict
2
+ from typing import DefaultDict
3
+
4
+ from src.domain.models import Validation
5
+ from src.repositories.append_only_validation_repository import OptimisticLockError
6
+
7
+
8
+ class InMemoryValidationRepository:
9
+ def __init__(self) -> None:
10
+ self._by_project: DefaultDict[str, list[Validation]] = defaultdict(list)
11
+ self._versions: DefaultDict[str, dict[str, int]] = defaultdict(dict)
12
+
13
+ def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
14
+ current_version = int(self._versions[project_slug].get(item.detection_key, 0))
15
+ expected = expected_version if expected_version is not None else current_version
16
+ if expected != current_version:
17
+ raise OptimisticLockError(item.detection_key, expected, current_version)
18
+
19
+ self._by_project[project_slug].append(item)
20
+ new_version = current_version + 1
21
+ self._versions[project_slug][item.detection_key] = new_version
22
+ return new_version
23
+
24
+ def list_validations(self, project_slug: str) -> list[Validation]:
25
+ return list(self._by_project.get(project_slug, []))
BirdNET Validator App/src/services/__init__.py ADDED
File without changes
BirdNET Validator App/src/services/audio_fetch_service.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from pathlib import Path
3
+
4
+ from huggingface_hub import hf_hub_download
5
+
6
+ from src.cache.ephemeral_cache_manager import EphemeralCacheManager
7
+
8
+
9
+ SUPPORTED_AUDIO_EXTENSIONS = (".wav", ".mp3", ".flac", ".ogg", ".m4a")
10
+
11
+
12
+ @dataclass(slots=True)
13
+ class AudioFetchResult:
14
+ cache_key: str
15
+ local_path: str
16
+ source: str
17
+
18
+
19
+ class AudioFetchService:
20
+ def __init__(self, cache_manager: EphemeralCacheManager) -> None:
21
+ self._cache = cache_manager
22
+
23
+ def fetch(self, dataset_repo: str, audio_id: str) -> AudioFetchResult:
24
+ cache_key = f"{dataset_repo}:{audio_id}"
25
+ cached_path = self._cache.get(cache_key)
26
+ if cached_path:
27
+ return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="cache")
28
+
29
+ target_filename, downloaded_path = self._resolve_remote_filename(dataset_repo=dataset_repo, audio_id=audio_id)
30
+ if downloaded_path is None:
31
+ downloaded = hf_hub_download(repo_id=dataset_repo, repo_type="dataset", filename=target_filename)
32
+ downloaded_path = Path(downloaded)
33
+
34
+ suffix = downloaded_path.suffix if downloaded_path.suffix else ".bin"
35
+ local_path = self._cache.put_bytes(cache_key, downloaded_path.read_bytes(), suffix=suffix)
36
+ return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="remote")
37
+
38
+ def cleanup_after_validation(self, cache_key: str) -> None:
39
+ self._cache.cleanup_key(cache_key)
40
+
41
+ def _resolve_remote_filename(self, dataset_repo: str, audio_id: str) -> tuple[str, Path | None]:
42
+ audio_path = Path(audio_id)
43
+ if audio_path.suffix:
44
+ return f"audio/{audio_id}", None
45
+
46
+ for extension in SUPPORTED_AUDIO_EXTENSIONS:
47
+ candidate = f"audio/{audio_id}{extension}"
48
+ try:
49
+ downloaded = hf_hub_download(repo_id=dataset_repo, repo_type="dataset", filename=candidate)
50
+ return candidate, Path(downloaded)
51
+ except Exception:
52
+ continue
53
+
54
+ raise FileNotFoundError(f"Unable to locate audio file for audio_id: {audio_id}")
BirdNET Validator App/src/services/detection_queue_service.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ from src.domain.models import Detection
4
+ from src.repositories.contracts import DetectionRepository
5
+
6
+
7
+ @dataclass(slots=True)
8
+ class DetectionPage:
9
+ project_slug: str
10
+ page: int
11
+ page_size: int
12
+ total_items: int
13
+ total_pages: int
14
+ has_next: bool
15
+ has_previous: bool
16
+ items: list[Detection]
17
+
18
+
19
+ class DetectionQueueService:
20
+ def __init__(self, repository: DetectionRepository) -> None:
21
+ self._repository = repository
22
+
23
+ def get_page(
24
+ self,
25
+ project_slug: str,
26
+ page: int,
27
+ page_size: int,
28
+ scientific_name: str | None = None,
29
+ min_confidence: float | None = None,
30
+ max_confidence: float | None = None,
31
+ ) -> DetectionPage:
32
+ if page < 1:
33
+ raise ValueError("page must be >= 1")
34
+ if page_size < 1:
35
+ raise ValueError("page_size must be >= 1")
36
+
37
+ total_items = self._repository.count_detections(
38
+ project_slug=project_slug,
39
+ scientific_name=scientific_name,
40
+ min_confidence=min_confidence,
41
+ max_confidence=max_confidence,
42
+ )
43
+ total_pages = max(1, ((total_items - 1) // page_size) + 1) if total_items else 1
44
+ page = min(page, total_pages)
45
+
46
+ items = self._repository.list_detections(
47
+ project_slug=project_slug,
48
+ page=page,
49
+ page_size=page_size,
50
+ scientific_name=scientific_name,
51
+ min_confidence=min_confidence,
52
+ max_confidence=max_confidence,
53
+ )
54
+
55
+ return DetectionPage(
56
+ project_slug=project_slug,
57
+ page=page,
58
+ page_size=page_size,
59
+ total_items=total_items,
60
+ total_pages=total_pages,
61
+ has_next=page < total_pages,
62
+ has_previous=page > 1,
63
+ items=items,
64
+ )
BirdNET Validator App/src/services/validation_service.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ from src.domain.models import Validation
4
+ from src.repositories.contracts import ValidationRepository
5
+
6
+
7
+ @dataclass(slots=True)
8
+ class ValidationWriteResult:
9
+ item: Validation
10
+ new_version: int
11
+
12
+
13
+ class ValidationService:
14
+ def __init__(self, repository: ValidationRepository) -> None:
15
+ self._repository = repository
16
+
17
+ def validate_detection(
18
+ self,
19
+ project_slug: str,
20
+ detection_key: str,
21
+ status: str,
22
+ validator: str,
23
+ notes: str = "",
24
+ corrected_species: str | None = None,
25
+ expected_version: int | None = None,
26
+ ) -> ValidationWriteResult:
27
+ item = Validation(
28
+ detection_key=detection_key,
29
+ status=status,
30
+ corrected_species=corrected_species,
31
+ notes=notes,
32
+ validator=validator,
33
+ )
34
+ new_version = self._repository.save_validation(
35
+ project_slug=project_slug,
36
+ item=item,
37
+ expected_version=expected_version,
38
+ )
39
+ return ValidationWriteResult(item=item, new_version=new_version)
BirdNET Validator App/src/telemetry/__init__.py ADDED
File without changes
BirdNET Validator App/src/ui/__init__.py ADDED
File without changes
BirdNET Validator App/src/ui/admin_panel.py ADDED
@@ -0,0 +1,281 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Gradio admin panel for managing projects and user assignments."""
2
+
3
+ from typing import List, Tuple
4
+
5
+ import gradio as gr
6
+
7
+ from src.auth.auth_service import AuthService, Session
8
+ from src.domain.models import Project, Role
9
+
10
+
11
+ class AdminPanelManager:
12
+ """Management backend for the admin panel."""
13
+
14
+ def __init__(self, auth_service: AuthService):
15
+ """Initialize admin panel manager.
16
+
17
+ Args:
18
+ auth_service: AuthService instance
19
+ """
20
+ self.auth_service = auth_service
21
+ self._projects: dict[str, Project] = {} # project_slug -> Project
22
+ self._user_project_assignments: dict[str, dict[str, str]] = {} # username -> {project_slug -> role}
23
+
24
+ def register_project(self, project: Project) -> bool:
25
+ """Register a new project (idempotent).
26
+
27
+ Args:
28
+ project: Project to register
29
+
30
+ Returns:
31
+ True if new, False if already existed
32
+ """
33
+ if project.project_slug in self._projects:
34
+ return False
35
+
36
+ self._projects[project.project_slug] = project
37
+ return True
38
+
39
+ def list_projects(self) -> List[dict]:
40
+ """List all projects as dictionaries for Gradio display.
41
+
42
+ Returns:
43
+ List of project dicts with slug, name, repo_id, active status
44
+ """
45
+ return [
46
+ {
47
+ "project_slug": p.project_slug,
48
+ "name": p.name,
49
+ "dataset_repo_id": p.dataset_repo_id,
50
+ "active": p.active,
51
+ }
52
+ for p in self._projects.values()
53
+ ]
54
+
55
+ def get_project(self, project_slug: str) -> Project | None:
56
+ """Get project by slug.
57
+
58
+ Args:
59
+ project_slug: Project slug
60
+
61
+ Returns:
62
+ Project if found, None otherwise
63
+ """
64
+ return self._projects.get(project_slug)
65
+
66
+ def list_users_for_project(self, project_slug: str) -> List[dict]:
67
+ """List all users assigned to a project.
68
+
69
+ Args:
70
+ project_slug: Project slug
71
+
72
+ Returns:
73
+ List of dicts with username and role
74
+ """
75
+ result = []
76
+ for username in self.auth_service._user_access.keys():
77
+ role = self.auth_service.get_user_role_for_project(username, project_slug)
78
+ if role is not None:
79
+ result.append({"username": username, "role": role.value})
80
+
81
+ return result
82
+
83
+ def assign_user_to_project(
84
+ self, username: str, project_slug: str, role: str
85
+ ) -> Tuple[bool, str]:
86
+ """Assign a user to a project with a role.
87
+
88
+ Args:
89
+ username: Username
90
+ project_slug: Project slug
91
+ role: "admin" or "validator"
92
+
93
+ Returns:
94
+ Tuple of (success, message)
95
+ """
96
+ if project_slug not in self._projects:
97
+ return False, f"Project '{project_slug}' not found"
98
+
99
+ if role not in ["admin", "validator"]:
100
+ return False, f"Invalid role: {role}"
101
+
102
+ # Get or create user's project access
103
+ if username not in self.auth_service._user_access:
104
+ self.auth_service.register_user_project_access(username, {})
105
+
106
+ access = self.auth_service._user_access[username]
107
+ access.project_slugs[project_slug] = Role(role)
108
+
109
+ return True, f"βœ… Assigned {username} to {project_slug} as {role}"
110
+
111
+ def remove_user_from_project(self, username: str, project_slug: str) -> Tuple[bool, str]:
112
+ """Remove a user's access to a project.
113
+
114
+ Args:
115
+ username: Username
116
+ project_slug: Project slug
117
+
118
+ Returns:
119
+ Tuple of (success, message)
120
+ """
121
+ if username not in self.auth_service._user_access:
122
+ return False, f"User '{username}' not found"
123
+
124
+ access = self.auth_service._user_access[username]
125
+ if project_slug not in access.project_slugs:
126
+ return False, f"User '{username}' is not assigned to project '{project_slug}'"
127
+
128
+ del access.project_slugs[project_slug]
129
+ return True, f"βœ… Removed {username} from {project_slug}"
130
+
131
+ def toggleproject_active(self, project_slug: str, active: bool) -> Tuple[bool, str]:
132
+ """Enable or disable a project.
133
+
134
+ Args:
135
+ project_slug: Project slug
136
+ active: Whether to activate or deactivate
137
+
138
+ Returns:
139
+ Tuple of (success, message)
140
+ """
141
+ if project_slug not in self._projects:
142
+ return False, f"Project '{project_slug}' not found"
143
+
144
+ self._projects[project_slug].active = active
145
+ status = "activated" if active else "deactivated"
146
+ return True, f"βœ… Project {project_slug} {status}"
147
+
148
+
149
+ def create_admin_panel(admin_manager: AdminPanelManager, current_session: Session) -> gr.Blocks:
150
+ """Create Gradio admin panel UI.
151
+
152
+ Args:
153
+ admin_manager: AdminPanelManager instance
154
+ current_session: Current user's session
155
+
156
+ Returns:
157
+ Gradio Blocks with admin panel tabs
158
+ """
159
+ if current_session.role != Role.admin:
160
+ with gr.Blocks() as restricted:
161
+ gr.Markdown("❌ **Access Denied**\n\nOnly administrators can access this panel.")
162
+ return restricted
163
+
164
+ with gr.Blocks(title="BirdNET Admin Panel") as admin_block:
165
+ gr.Markdown("# Admin Panel")
166
+ gr.Markdown(f"Logged in as: **{current_session.username}** (Admin)")
167
+
168
+ with gr.Tabs():
169
+ # Projects Tab
170
+ with gr.Tab("Projects"):
171
+ gr.Markdown("## Manage Projects")
172
+
173
+ with gr.Row():
174
+ with gr.Column(scale=2):
175
+ project_slug_input = gr.Textbox(
176
+ label="Project Slug",
177
+ placeholder="e.g., kenya-2024",
178
+ lines=1,
179
+ )
180
+ project_name_input = gr.Textbox(
181
+ label="Project Name",
182
+ placeholder="e.g., Kenya Survey 2024",
183
+ lines=1,
184
+ )
185
+
186
+ with gr.Column(scale=1):
187
+ repo_id_input = gr.Textbox(
188
+ label="HF Dataset Repo ID",
189
+ placeholder="e.g., org/dataset-name",
190
+ lines=1,
191
+ )
192
+
193
+ project_message = gr.Markdown()
194
+
195
+ # TODO: Implement create project button
196
+ # create_project_button = gr.Button("Create Project", variant="primary")
197
+
198
+ # Projects list
199
+ with gr.Row():
200
+ refresh_projects_button = gr.Button("Refresh Projects List")
201
+ projects_table = gr.Dataframe(
202
+ value=admin_manager.list_projects(),
203
+ headers=["project_slug", "name", "dataset_repo_id", "active"],
204
+ interactive=False,
205
+ )
206
+
207
+ refresh_projects_button.click(
208
+ fn=lambda: admin_manager.list_projects(),
209
+ outputs=[projects_table],
210
+ )
211
+
212
+ # Users Tab
213
+ with gr.Tab("Users"):
214
+ gr.Markdown("## Manage User Access")
215
+
216
+ with gr.Row():
217
+ username_input = gr.Textbox(
218
+ label="Username",
219
+ placeholder="e.g., validator_001",
220
+ lines=1,
221
+ )
222
+
223
+ project_select = gr.Dropdown(
224
+ choices=[p["project_slug"] for p in admin_manager.list_projects()],
225
+ label="Project",
226
+ )
227
+
228
+ role_select = gr.Dropdown(
229
+ choices=["admin", "validator"],
230
+ value="validator",
231
+ label="Role",
232
+ )
233
+
234
+ user_message = gr.Markdown()
235
+
236
+ assign_button = gr.Button("Assign User", variant="primary")
237
+ remove_button = gr.Button("Remove User", variant="stop")
238
+
239
+ def assign_user(username: str, project_slug: str, role: str) -> str:
240
+ success, msg = admin_manager.assign_user_to_project(username, project_slug, role)
241
+ return msg
242
+
243
+ def remove_user(username: str, project_slug: str) -> str:
244
+ success, msg = admin_manager.remove_user_from_project(username, project_slug)
245
+ return msg
246
+
247
+ assign_button.click(
248
+ fn=assign_user,
249
+ inputs=[username_input, project_select, role_select],
250
+ outputs=[user_message],
251
+ )
252
+
253
+ remove_button.click(
254
+ fn=remove_user,
255
+ inputs=[username_input, project_select],
256
+ outputs=[user_message],
257
+ )
258
+
259
+ # Users per project view
260
+ with gr.Row():
261
+ project_filter = gr.Dropdown(
262
+ choices=[p["project_slug"] for p in admin_manager.list_projects()],
263
+ label="View Users for Project",
264
+ )
265
+
266
+ users_table = gr.Dataframe(
267
+ value=[],
268
+ headers=["username", "role"],
269
+ interactive=False,
270
+ )
271
+
272
+ def update_users_table(project_slug: str):
273
+ return admin_manager.list_users_for_project(project_slug)
274
+
275
+ project_filter.change(
276
+ fn=update_users_table,
277
+ inputs=[project_filter],
278
+ outputs=[users_table],
279
+ )
280
+
281
+ return admin_block
BirdNET Validator App/src/ui/app_factory.py ADDED
@@ -0,0 +1,1542 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import tempfile
3
+ from datetime import date, datetime
4
+ from pathlib import Path
5
+ from typing import Protocol
6
+
7
+ from src.cache.ephemeral_cache_manager import EphemeralCacheManager
8
+ from src.domain.models import Detection, Project, Role
9
+ from src.repositories.append_only_validation_repository import AppendOnlyValidationRepository, OptimisticLockError
10
+ from src.repositories.in_memory_detection_repository import InMemoryDetectionRepository
11
+ from src.services.audio_fetch_service import AudioFetchService
12
+ from src.services.detection_queue_service import DetectionQueueService
13
+ from src.services.validation_service import ValidationService
14
+ from src.auth.auth_service import AuthService
15
+ from src.ui.login_page import create_login_page
16
+ from src.ui.project_selector import create_project_selector
17
+ from src.ui.admin_panel import AdminPanelManager, create_admin_panel
18
+
19
+
20
+ class _AudioFetchResultProtocol(Protocol):
21
+ cache_key: str
22
+ local_path: str
23
+ source: str
24
+
25
+
26
+ class _AudioServiceProtocol(Protocol):
27
+ def fetch(self, dataset_repo: str, audio_id: str) -> _AudioFetchResultProtocol: ...
28
+
29
+ def cleanup_after_validation(self, cache_key: str) -> None: ...
30
+
31
+
32
+ class _ValidationServiceProtocol(Protocol):
33
+ def validate_detection(
34
+ self,
35
+ project_slug: str,
36
+ detection_key: str,
37
+ status: str,
38
+ validator: str,
39
+ notes: str = "",
40
+ corrected_species: str | None = None,
41
+ expected_version: int | None = None,
42
+ ) -> object: ...
43
+
44
+
45
+ class _ValidationReadRepositoryProtocol(Protocol):
46
+ def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]: ...
47
+
48
+ def list_events(self, project_slug: str) -> list[dict[str, object]]: ...
49
+
50
+
51
+ class _QueueServiceProtocol(Protocol):
52
+ def get_page(
53
+ self,
54
+ project_slug: str,
55
+ page: int,
56
+ page_size: int,
57
+ scientific_name: str | None = None,
58
+ min_confidence: float | None = None,
59
+ max_confidence: float | None = None,
60
+ ) -> object: ...
61
+
62
+
63
+ def _seed_service() -> DetectionQueueService:
64
+ repo = InMemoryDetectionRepository()
65
+ demo_items = [
66
+ Detection(
67
+ detection_key="0000000000001001",
68
+ audio_id="audio_1001",
69
+ scientific_name="Cyanocorax cyanopogon",
70
+ confidence=0.93,
71
+ start_time=1.2,
72
+ end_time=2.5,
73
+ ),
74
+ Detection(
75
+ detection_key="0000000000001002",
76
+ audio_id="audio_1002",
77
+ scientific_name="Ramphastos toco",
78
+ confidence=0.88,
79
+ start_time=0.8,
80
+ end_time=2.1,
81
+ ),
82
+ Detection(
83
+ detection_key="0000000000001003",
84
+ audio_id="audio_1003",
85
+ scientific_name="Cyanocorax cyanopogon",
86
+ confidence=0.72,
87
+ start_time=3.1,
88
+ end_time=4.0,
89
+ ),
90
+ Detection(
91
+ detection_key="0000000000001004",
92
+ audio_id="audio_1004",
93
+ scientific_name="Psarocolius decumanus",
94
+ confidence=0.67,
95
+ start_time=5.0,
96
+ end_time=6.3,
97
+ ),
98
+ ]
99
+ repo.seed("demo-project", demo_items)
100
+ return DetectionQueueService(repo)
101
+
102
+
103
+ def _page_to_table(
104
+ service: _QueueServiceProtocol,
105
+ snapshot_reader: _ValidationReadRepositoryProtocol,
106
+ project_slug: str,
107
+ page: int,
108
+ scientific_name: str,
109
+ min_confidence: float,
110
+ validator_filter: str = "",
111
+ status_filter: str = "all",
112
+ updated_after: object = None,
113
+ conflict_detection_key: str = "",
114
+ show_conflicts_only: bool = False,
115
+ ):
116
+ filter_name = scientific_name.strip() if scientific_name.strip() else None
117
+ page_obj = service.get_page(
118
+ project_slug="demo-project",
119
+ page=page,
120
+ page_size=2,
121
+ scientific_name=filter_name,
122
+ min_confidence=min_confidence,
123
+ )
124
+
125
+ snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
126
+
127
+ normalized_status_filter = status_filter.strip().lower() if status_filter else "all"
128
+ normalized_validator_filter = validator_filter.strip().lower()
129
+ updated_after_date: date | None = None
130
+ if updated_after is not None:
131
+ if isinstance(updated_after, datetime):
132
+ updated_after_date = updated_after.date()
133
+ elif isinstance(updated_after, date):
134
+ updated_after_date = updated_after
135
+ elif isinstance(updated_after, (int, float)):
136
+ updated_after_date = datetime.fromtimestamp(float(updated_after)).date()
137
+ else:
138
+ updated_after_text = str(updated_after).strip()
139
+ if updated_after_text:
140
+ try:
141
+ updated_after_date = datetime.strptime(updated_after_text, "%Y-%m-%d").date()
142
+ except ValueError:
143
+ try:
144
+ updated_after_date = datetime.fromisoformat(updated_after_text.replace("Z", "+00:00")).date()
145
+ except ValueError:
146
+ updated_after_date = None
147
+
148
+ rows = [
149
+ [
150
+ item.detection_key,
151
+ item.audio_id,
152
+ item.scientific_name,
153
+ round(item.confidence, 3),
154
+ item.start_time,
155
+ item.end_time,
156
+ str(snapshot.get(item.detection_key, {}).get("status", "pending")),
157
+ int(snapshot.get(item.detection_key, {}).get("version", 0)),
158
+ "CONFLICT" if conflict_detection_key and item.detection_key == conflict_detection_key else "",
159
+ "HIGH" if conflict_detection_key and item.detection_key == conflict_detection_key else "",
160
+ ]
161
+ for item in page_obj.items
162
+ ]
163
+
164
+ if normalized_validator_filter:
165
+ rows = [
166
+ row
167
+ for row in rows
168
+ if normalized_validator_filter in str(snapshot.get(str(row[0]), {}).get("validator", "")).strip().lower()
169
+ ]
170
+
171
+ if normalized_status_filter and normalized_status_filter != "all":
172
+ rows = [row for row in rows if str(row[6]).strip().lower() == normalized_status_filter]
173
+
174
+ if updated_after_date:
175
+ filtered_rows: list[list[object]] = []
176
+ for row in rows:
177
+ snapshot_item = snapshot.get(str(row[0]), {})
178
+ updated_at_value = str(snapshot_item.get("updated_at", "")).strip()
179
+ if not updated_at_value:
180
+ continue
181
+ try:
182
+ item_date = datetime.fromisoformat(updated_at_value.replace("Z", "+00:00")).date()
183
+ if item_date >= updated_after_date:
184
+ filtered_rows.append(row)
185
+ except ValueError:
186
+ continue
187
+ rows = filtered_rows
188
+
189
+ if show_conflicts_only:
190
+ rows = [row for row in rows if str(row[8]) == "CONFLICT"]
191
+
192
+ status = f"Pagina {page_obj.page}/{page_obj.total_pages} | Total base: {page_obj.total_items} | Exibidos: {len(rows)}"
193
+ if show_conflicts_only:
194
+ status = f"{status} | Apenas conflitos: {len(rows)} item(ns)"
195
+ return rows, status, page_obj.page
196
+
197
+
198
+ def _build_validation_report(snapshot_reader: _ValidationReadRepositoryProtocol, project_slug: str) -> str:
199
+ snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
200
+ events = snapshot_reader.list_events(project_slug=project_slug)
201
+
202
+ counts: dict[str, int] = {}
203
+ for payload in snapshot.values():
204
+ status_value = str(payload.get("status", "unknown"))
205
+ counts[status_value] = counts.get(status_value, 0) + 1
206
+
207
+ parts = [
208
+ f"Projeto: {project_slug}",
209
+ f"Eventos append-only: {len(events)}",
210
+ f"Deteccoes com estado atual: {len(snapshot)}",
211
+ ]
212
+ if counts:
213
+ summary = ", ".join(f"{key}={value}" for key, value in sorted(counts.items()))
214
+ parts.append(f"Status atual: {summary}")
215
+ else:
216
+ parts.append("Status atual: sem validacoes")
217
+ return " | ".join(parts)
218
+
219
+
220
+ def _extract_audio_id(rows: object, selected_index: int) -> str:
221
+ normalized_rows: list[list[object]]
222
+
223
+ if hasattr(rows, "values"):
224
+ normalized_rows = [list(item) for item in rows.values.tolist()]
225
+ else:
226
+ normalized_rows = [list(item) for item in rows] if rows else []
227
+
228
+ if not normalized_rows:
229
+ raise ValueError("Nenhuma deteccao carregada na tabela")
230
+ if selected_index < 0 or selected_index >= len(normalized_rows):
231
+ raise ValueError("Selecione uma deteccao valida na tabela")
232
+
233
+ value = normalized_rows[selected_index][1]
234
+ audio_id = str(value).strip()
235
+ if not audio_id:
236
+ raise ValueError("audio_id invalido na deteccao selecionada")
237
+ return audio_id
238
+
239
+
240
+ def _extract_detection_key(rows: object, selected_index: int) -> str:
241
+ normalized_rows: list[list[object]]
242
+
243
+ if hasattr(rows, "values"):
244
+ normalized_rows = [list(item) for item in rows.values.tolist()]
245
+ else:
246
+ normalized_rows = [list(item) for item in rows] if rows else []
247
+
248
+ if not normalized_rows:
249
+ raise ValueError("Nenhuma deteccao carregada na tabela")
250
+ if selected_index < 0 or selected_index >= len(normalized_rows):
251
+ raise ValueError("Selecione uma deteccao valida na tabela")
252
+
253
+ value = normalized_rows[selected_index][0]
254
+ detection_key = str(value).strip()
255
+ if not detection_key:
256
+ raise ValueError("detection_key invalido na deteccao selecionada")
257
+ return detection_key
258
+
259
+
260
+ def _find_detection_row_index(rows: object, detection_key: str) -> int:
261
+ normalized_rows: list[list[object]]
262
+
263
+ if hasattr(rows, "values"):
264
+ normalized_rows = [list(item) for item in rows.values.tolist()]
265
+ else:
266
+ normalized_rows = [list(item) for item in rows] if rows else []
267
+
268
+ for index, row in enumerate(normalized_rows):
269
+ if str(row[0]).strip() == detection_key:
270
+ return index
271
+ return 0
272
+
273
+
274
+ def _extract_expected_version(rows: object, selected_index: int) -> int:
275
+ normalized_rows: list[list[object]]
276
+
277
+ if hasattr(rows, "values"):
278
+ normalized_rows = [list(item) for item in rows.values.tolist()]
279
+ else:
280
+ normalized_rows = [list(item) for item in rows] if rows else []
281
+
282
+ if not normalized_rows:
283
+ raise ValueError("Nenhuma deteccao carregada na tabela")
284
+ if selected_index < 0 or selected_index >= len(normalized_rows):
285
+ raise ValueError("Selecione uma deteccao valida na tabela")
286
+
287
+ value = normalized_rows[selected_index][7]
288
+ return int(value)
289
+
290
+
291
+ def _fetch_selected_audio(
292
+ audio_service: _AudioServiceProtocol,
293
+ dataset_repo: str,
294
+ rows: object,
295
+ selected_index: int,
296
+ previous_cache_key: str,
297
+ ) -> tuple[str | None, str, str]:
298
+ repo = dataset_repo.strip()
299
+ if not repo:
300
+ return None, "", "Informe dataset repo no formato owner/repo"
301
+
302
+ try:
303
+ audio_id = _extract_audio_id(rows=rows, selected_index=selected_index)
304
+ result = audio_service.fetch(dataset_repo=repo, audio_id=audio_id)
305
+ status = f"Audio carregado ({result.source}) para audio_id={audio_id}"
306
+ return result.local_path, result.cache_key, status
307
+ except Exception as exc:
308
+ if previous_cache_key:
309
+ return None, previous_cache_key, f"Falha ao carregar audio: {exc}"
310
+ return None, "", f"Falha ao carregar audio: {exc}"
311
+
312
+
313
+ def _cleanup_selected_audio(audio_service: _AudioServiceProtocol, cache_key: str) -> tuple[str, str | None]:
314
+ if not cache_key:
315
+ return "Nenhum audio em cache para limpar", None
316
+
317
+ audio_service.cleanup_after_validation(cache_key=cache_key)
318
+ return "Cache de audio limpo apos validacao", None
319
+
320
+
321
+ def _save_selected_validation(
322
+ validation_service: _ValidationServiceProtocol,
323
+ audio_service: _AudioServiceProtocol,
324
+ project_slug: str,
325
+ rows: object,
326
+ selected_index: int,
327
+ status_value: str,
328
+ validator: str,
329
+ notes: str,
330
+ cache_key: str,
331
+ ) -> tuple[str, str, str | None]:
332
+ validator_name = validator.strip()
333
+ if not validator_name:
334
+ return "Informe o nome do validador", cache_key, None
335
+
336
+ try:
337
+ detection_key = _extract_detection_key(rows=rows, selected_index=selected_index)
338
+ expected_version = _extract_expected_version(rows=rows, selected_index=selected_index)
339
+ _ = validation_service.validate_detection(
340
+ project_slug=project_slug,
341
+ detection_key=detection_key,
342
+ status=status_value,
343
+ validator=validator_name,
344
+ notes=notes.strip(),
345
+ expected_version=expected_version,
346
+ )
347
+ if cache_key:
348
+ audio_service.cleanup_after_validation(cache_key=cache_key)
349
+ return f"Validacao salva: {detection_key} -> {status_value}", "", None
350
+ except OptimisticLockError as exc:
351
+ return (
352
+ "Conflito de concorrencia: esta deteccao foi atualizada por outro validador "
353
+ f"(detection_key={exc.detection_key}, versao atual={exc.current_version}, esperada={exc.expected_version}). "
354
+ "Recarregue a tabela.",
355
+ cache_key,
356
+ None,
357
+ )
358
+ except Exception as exc:
359
+ return f"Falha ao salvar validacao: {exc}", cache_key, None
360
+
361
+
362
+ def _save_selected_validation_with_refresh(
363
+ validation_service: _ValidationServiceProtocol,
364
+ audio_service: _AudioServiceProtocol,
365
+ queue_service: _QueueServiceProtocol,
366
+ snapshot_reader: _ValidationReadRepositoryProtocol,
367
+ project_slug: str,
368
+ rows: object,
369
+ selected_index: int,
370
+ status_value: str,
371
+ validator: str,
372
+ notes: str,
373
+ cache_key: str,
374
+ page: int,
375
+ scientific_name: str,
376
+ min_confidence: float,
377
+ validator_filter: str,
378
+ status_filter: str,
379
+ updated_after: object,
380
+ show_conflicts_only: bool,
381
+ ) -> tuple[str, str, str | None, list[list[object]], int, int, str, str]:
382
+ selected_key = ""
383
+ try:
384
+ selected_key = _extract_detection_key(rows=rows, selected_index=selected_index)
385
+ except Exception:
386
+ selected_key = ""
387
+
388
+ save_status, updated_cache_key, audio_path = _save_selected_validation(
389
+ validation_service=validation_service,
390
+ audio_service=audio_service,
391
+ project_slug=project_slug,
392
+ rows=rows,
393
+ selected_index=selected_index,
394
+ status_value=status_value,
395
+ validator=validator,
396
+ notes=notes,
397
+ cache_key=cache_key,
398
+ )
399
+
400
+ refreshed_rows, page_status, refreshed_page = _page_to_table(
401
+ service=queue_service,
402
+ snapshot_reader=snapshot_reader,
403
+ project_slug=project_slug,
404
+ page=page,
405
+ scientific_name=scientific_name,
406
+ min_confidence=min_confidence,
407
+ validator_filter=validator_filter,
408
+ status_filter=status_filter,
409
+ updated_after=updated_after,
410
+ show_conflicts_only=show_conflicts_only,
411
+ )
412
+
413
+ if selected_key:
414
+ refreshed_index = _find_detection_row_index(refreshed_rows, selected_key)
415
+ else:
416
+ refreshed_index = 0
417
+
418
+ if "Conflito de concorrencia" in save_status:
419
+ conflict_key = selected_key
420
+ refreshed_rows, page_status, refreshed_page = _page_to_table(
421
+ service=queue_service,
422
+ snapshot_reader=snapshot_reader,
423
+ project_slug=project_slug,
424
+ page=refreshed_page,
425
+ scientific_name=scientific_name,
426
+ min_confidence=min_confidence,
427
+ validator_filter=validator_filter,
428
+ status_filter=status_filter,
429
+ updated_after=updated_after,
430
+ conflict_detection_key=conflict_key,
431
+ show_conflicts_only=show_conflicts_only,
432
+ )
433
+ refreshed_index = _find_detection_row_index(refreshed_rows, selected_key) if selected_key else 0
434
+ pending_status_value = status_value
435
+ status = f"{save_status} Tabela recarregada para resolver conflito."
436
+ else:
437
+ conflict_key = ""
438
+ pending_status_value = ""
439
+ status = f"{save_status} | {page_status}"
440
+
441
+ return (
442
+ status,
443
+ updated_cache_key,
444
+ audio_path,
445
+ refreshed_rows,
446
+ refreshed_page,
447
+ refreshed_index,
448
+ pending_status_value,
449
+ conflict_key,
450
+ )
451
+
452
+
453
+ def _reapply_last_conflict_validation_with_refresh(
454
+ validation_service: _ValidationServiceProtocol,
455
+ audio_service: _AudioServiceProtocol,
456
+ queue_service: _QueueServiceProtocol,
457
+ snapshot_reader: _ValidationReadRepositoryProtocol,
458
+ project_slug: str,
459
+ rows: object,
460
+ selected_index: int,
461
+ pending_status_value: str,
462
+ conflict_detection_key: str,
463
+ validator: str,
464
+ notes: str,
465
+ cache_key: str,
466
+ page: int,
467
+ scientific_name: str,
468
+ min_confidence: float,
469
+ validator_filter: str,
470
+ status_filter: str,
471
+ updated_after: object,
472
+ show_conflicts_only: bool,
473
+ ) -> tuple[str, str, str | None, list[list[object]], int, int, str, str]:
474
+ if not pending_status_value:
475
+ refreshed_rows, page_status, refreshed_page = _page_to_table(
476
+ service=queue_service,
477
+ snapshot_reader=snapshot_reader,
478
+ project_slug=project_slug,
479
+ page=page,
480
+ scientific_name=scientific_name,
481
+ min_confidence=min_confidence,
482
+ validator_filter=validator_filter,
483
+ status_filter=status_filter,
484
+ updated_after=updated_after,
485
+ show_conflicts_only=show_conflicts_only,
486
+ )
487
+ return (
488
+ f"Nenhuma validacao pendente para reaplicar | {page_status}",
489
+ cache_key,
490
+ None,
491
+ refreshed_rows,
492
+ refreshed_page,
493
+ selected_index,
494
+ "",
495
+ "",
496
+ )
497
+
498
+ target_index = _find_detection_row_index(rows, conflict_detection_key) if conflict_detection_key else selected_index
499
+ return _save_selected_validation_with_refresh(
500
+ validation_service=validation_service,
501
+ audio_service=audio_service,
502
+ queue_service=queue_service,
503
+ snapshot_reader=snapshot_reader,
504
+ project_slug=project_slug,
505
+ rows=rows,
506
+ selected_index=target_index,
507
+ status_value=pending_status_value,
508
+ validator=validator,
509
+ notes=notes,
510
+ cache_key=cache_key,
511
+ page=page,
512
+ scientific_name=scientific_name,
513
+ min_confidence=min_confidence,
514
+ validator_filter=validator_filter,
515
+ status_filter=status_filter,
516
+ updated_after=updated_after,
517
+ show_conflicts_only=show_conflicts_only,
518
+ )
519
+
520
+
521
+ def _batch_validate_conflicts(
522
+ validation_service: _ValidationServiceProtocol,
523
+ audio_service: _AudioServiceProtocol,
524
+ queue_service: _QueueServiceProtocol,
525
+ snapshot_reader: _ValidationReadRepositoryProtocol,
526
+ project_slug: str,
527
+ rows: object,
528
+ status_value: str,
529
+ validator: str,
530
+ notes: str,
531
+ cache_key: str,
532
+ page: int,
533
+ scientific_name: str,
534
+ min_confidence: float,
535
+ validator_filter: str,
536
+ status_filter: str,
537
+ updated_after: object,
538
+ ) -> tuple[str, str, str | None, list[list[object]], int]:
539
+ """Apply the same validation status to all visible conflicts in the table."""
540
+ validator_name = validator.strip()
541
+ if not validator_name:
542
+ return "Informe o nome do validador", "", None, [], page
543
+
544
+ normalized_rows: list[list[object]]
545
+ if hasattr(rows, "values"):
546
+ normalized_rows = [list(item) for item in rows.values.tolist()]
547
+ else:
548
+ normalized_rows = [list(item) for item in rows] if rows else []
549
+
550
+ if not normalized_rows:
551
+ return "Nenhuma deteccao com conflito para validar", "", None, [], page
552
+
553
+ conflict_rows = [row for row in normalized_rows if str(row[8]) == "CONFLICT"]
554
+ if not conflict_rows:
555
+ return "Nenhuma deteccao com conflito identificada na tabela", "", None, normalized_rows, page
556
+
557
+ success_count = 0
558
+ failure_count = 0
559
+ conflict_count = 0
560
+
561
+ for row in conflict_rows:
562
+ try:
563
+ detection_key = str(row[0]).strip()
564
+ expected_version = int(row[7])
565
+
566
+ _ = validation_service.validate_detection(
567
+ project_slug=project_slug,
568
+ detection_key=detection_key,
569
+ status=status_value,
570
+ validator=validator_name,
571
+ notes=notes.strip(),
572
+ expected_version=expected_version,
573
+ )
574
+ success_count += 1
575
+ if cache_key:
576
+ audio_service.cleanup_after_validation(cache_key=cache_key)
577
+ except OptimisticLockError:
578
+ conflict_count += 1
579
+ except Exception:
580
+ failure_count += 1
581
+
582
+ refreshed_rows, page_status, refreshed_page = _page_to_table(
583
+ service=queue_service,
584
+ snapshot_reader=snapshot_reader,
585
+ project_slug=project_slug,
586
+ page=page,
587
+ scientific_name=scientific_name,
588
+ min_confidence=min_confidence,
589
+ validator_filter=validator_filter,
590
+ status_filter=status_filter,
591
+ updated_after=updated_after,
592
+ show_conflicts_only=False,
593
+ )
594
+
595
+ summary = f"Processados {len(conflict_rows)} conflitos: {success_count} sucesso, {conflict_count} novos conflitos, {failure_count} falhas"
596
+ status = f"{summary} | {page_status}"
597
+
598
+ return status, "", None, refreshed_rows, refreshed_page
599
+
600
+
601
+ def _batch_reapply_all_pending(
602
+ validation_service: _ValidationServiceProtocol,
603
+ audio_service: _AudioServiceProtocol,
604
+ queue_service: _QueueServiceProtocol,
605
+ snapshot_reader: _ValidationReadRepositoryProtocol,
606
+ project_slug: str,
607
+ rows: object,
608
+ pending_statuses: dict[str, str],
609
+ validator: str,
610
+ notes: str,
611
+ cache_key: str,
612
+ page: int,
613
+ scientific_name: str,
614
+ min_confidence: float,
615
+ validator_filter: str,
616
+ status_filter: str,
617
+ updated_after: object,
618
+ ) -> tuple[str, str, str | None, list[list[object]], int]:
619
+ """Reapply all pending validations (stored conflicts) with current version."""
620
+ if not pending_statuses:
621
+ return "Nenhuma validacao pendente para reaplicar", "", None, [], page
622
+
623
+ validator_name = validator.strip()
624
+ if not validator_name:
625
+ return "Informe o nome do validador", "", None, [], page
626
+
627
+ success_count = 0
628
+ conflict_count = 0
629
+ failure_count = 0
630
+
631
+ snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
632
+
633
+ for detection_key, status_value in pending_statuses.items():
634
+ try:
635
+ current_version = int(snapshot.get(detection_key, {}).get("version", 0))
636
+
637
+ _ = validation_service.validate_detection(
638
+ project_slug=project_slug,
639
+ detection_key=detection_key,
640
+ status=status_value,
641
+ validator=validator_name,
642
+ notes=notes.strip(),
643
+ expected_version=current_version,
644
+ )
645
+ success_count += 1
646
+ if cache_key:
647
+ audio_service.cleanup_after_validation(cache_key=cache_key)
648
+ except OptimisticLockError:
649
+ conflict_count += 1
650
+ except Exception:
651
+ failure_count += 1
652
+
653
+ refreshed_rows, page_status, refreshed_page = _page_to_table(
654
+ service=queue_service,
655
+ snapshot_reader=snapshot_reader,
656
+ project_slug=project_slug,
657
+ page=page,
658
+ scientific_name=scientific_name,
659
+ min_confidence=min_confidence,
660
+ validator_filter=validator_filter,
661
+ status_filter=status_filter,
662
+ updated_after=updated_after,
663
+ show_conflicts_only=False,
664
+ )
665
+
666
+ summary = f"Reaplicadas {len(pending_statuses)} validacoes: {success_count} sucesso, {conflict_count} novos conflitos, {failure_count} falhas"
667
+ status = f"{summary} | {page_status}"
668
+
669
+ return status, "", None, refreshed_rows, refreshed_page
670
+
671
+
672
+ def build_demo_app(project_slug: str = "demo-project") -> gr.Blocks:
673
+ """Build the demo validation app for a given project.
674
+
675
+ Args:
676
+ project_slug: Project identifier (default: demo-project)
677
+
678
+ Returns:
679
+ Gradio Blocks with validation interface
680
+ """
681
+ service = _seed_service()
682
+ audio_service = AudioFetchService(EphemeralCacheManager(ttl_seconds=300, max_files=128))
683
+ validation_base_dir = str(Path(tempfile.gettempdir()) / "birdnet-validator-validations")
684
+ validation_repository = AppendOnlyValidationRepository(base_dir=validation_base_dir)
685
+ validation_service = ValidationService(validation_repository)
686
+
687
+ with gr.Blocks(title="BirdNET Validator HF") as demo:
688
+ gr.Markdown("# BirdNET Validator HF")
689
+ gr.Markdown("Sprint 2: fila paginada + audio sob demanda com cache efemero.")
690
+
691
+ dataset_repo = gr.Textbox(label="Dataset repo", value="SEU_USUARIO/birdnet-projeto-dataset")
692
+
693
+ with gr.Row():
694
+ species_filter = gr.Textbox(label="Filtro especie", placeholder="Ex: Cyanocorax cyanopogon")
695
+ min_confidence = gr.Slider(label="Confianca minima", minimum=0.0, maximum=1.0, step=0.01, value=0.0)
696
+ show_conflicts_only = gr.Checkbox(label="Mostrar apenas conflitos", value=False)
697
+
698
+ with gr.Row():
699
+ validator_filter = gr.Textbox(label="Filtro validador", placeholder="Ex: validator-demo")
700
+ validation_status_filter = gr.Dropdown(
701
+ label="Filtro status",
702
+ choices=["all", "pending", "positive", "negative", "uncertain", "skip"],
703
+ value="all",
704
+ )
705
+ updated_after_filter = gr.DateTime(label="Atualizado a partir de", include_time=False, type="string")
706
+
707
+ with gr.Row():
708
+ prev_btn = gr.Button("Pagina anterior")
709
+ next_btn = gr.Button("Proxima pagina")
710
+ refresh_btn = gr.Button("Aplicar filtros")
711
+
712
+ page_state = gr.State(value=1)
713
+ table = gr.Dataframe(
714
+ headers=[
715
+ "detection_key",
716
+ "audio_id",
717
+ "scientific_name",
718
+ "confidence",
719
+ "start_time",
720
+ "end_time",
721
+ "validation_status",
722
+ "version",
723
+ "conflict_flag",
724
+ "conflict_severity",
725
+ ],
726
+ label="Deteccoes",
727
+ interactive=False,
728
+ )
729
+ selected_index = gr.Number(label="Linha selecionada", value=0, precision=0)
730
+
731
+ with gr.Row():
732
+ load_audio_btn = gr.Button("Carregar audio selecionado")
733
+ clear_audio_btn = gr.Button("Limpar cache apos validacao")
734
+
735
+ with gr.Row():
736
+ validator_name = gr.Textbox(label="Validador", value="validator-demo")
737
+ validation_notes = gr.Textbox(label="Notas", placeholder="Opcional")
738
+
739
+ with gr.Row():
740
+ approve_btn = gr.Button("Validar positivo")
741
+ reject_btn = gr.Button("Validar negativo")
742
+ uncertain_btn = gr.Button("Indeterminado")
743
+ skip_btn = gr.Button("Pular")
744
+ reapply_btn = gr.Button("Reaplicar validacao apos conflito")
745
+
746
+ with gr.Row():
747
+ batch_approve_conflicts_btn = gr.Button("Aprovar todos os conflitos")
748
+ batch_reject_conflicts_btn = gr.Button("Rejeitar todos os conflitos")
749
+
750
+ report_btn = gr.Button("Gerar relatorio de validacoes")
751
+
752
+ audio_player = gr.Audio(label="Audio sob demanda", type="filepath")
753
+ cache_key_state = gr.State(value="")
754
+ pending_status_state = gr.State(value="")
755
+ conflict_detection_key_state = gr.State(value="")
756
+ status = gr.Textbox(label="Status", interactive=False)
757
+ report_box = gr.Textbox(label="Relatorio", interactive=False)
758
+
759
+ # Keyboard shortcuts: 1=positive, 2=negative, 3=uncertain, 4=skip, R=reapply
760
+ keyboard_shortcuts_info = gr.HTML(
761
+ value="<div style='font-size: 12px; color: #666; padding: 8px; background-color: #f5f5f5; border-radius: 4px; margin-bottom: 10px;'>"
762
+ "<strong>Atalhos de teclado:</strong> 1=Positivo | 2=Negativo | 3=Indeterminado | 4=Pular | R=Reaplicar"
763
+ "</div>"
764
+ "<script>"
765
+ "document.addEventListener('keydown', function(event) {"
766
+ " if (event.target.tagName === 'INPUT' || event.target.tagName === 'TEXTAREA') return;"
767
+ " const key = event.key.toLowerCase();"
768
+ " let buttonText = null;"
769
+ " if (key === '1') buttonText = 'Validar positivo';"
770
+ " else if (key === '2') buttonText = 'Validar negativo';"
771
+ " else if (key === '3') buttonText = 'Indeterminado';"
772
+ " else if (key === '4') buttonText = 'Pular';"
773
+ " else if (key === 'r') buttonText = 'Reaplicar validacao apos conflito';"
774
+ " if (buttonText) {"
775
+ " event.preventDefault();"
776
+ " const buttons = document.querySelectorAll('button');"
777
+ " for (const btn of buttons) {"
778
+ " if (btn.textContent.includes(buttonText)) {"
779
+ " btn.click();"
780
+ " break;"
781
+ " }"
782
+ " }"
783
+ " }"
784
+ "});"
785
+ "</script>"
786
+ )
787
+
788
+ def refresh(
789
+ page: int,
790
+ species: str,
791
+ confidence: float,
792
+ validator_filter_value: str,
793
+ status_filter_value: str,
794
+ updated_after_value: object,
795
+ only_conflicts: bool,
796
+ ):
797
+ return _page_to_table(
798
+ service=service,
799
+ snapshot_reader=validation_repository,
800
+ project_slug=project_slug,
801
+ page=page,
802
+ scientific_name=species,
803
+ min_confidence=confidence,
804
+ validator_filter=validator_filter_value,
805
+ status_filter=status_filter_value,
806
+ updated_after=updated_after_value,
807
+ show_conflicts_only=only_conflicts,
808
+ )
809
+
810
+ def go_next(
811
+ page: int,
812
+ species: str,
813
+ confidence: float,
814
+ validator_filter_value: str,
815
+ status_filter_value: str,
816
+ updated_after_value: object,
817
+ only_conflicts: bool,
818
+ ):
819
+ return refresh(
820
+ page + 1,
821
+ species,
822
+ confidence,
823
+ validator_filter_value,
824
+ status_filter_value,
825
+ updated_after_value,
826
+ only_conflicts,
827
+ )
828
+
829
+ def go_prev(
830
+ page: int,
831
+ species: str,
832
+ confidence: float,
833
+ validator_filter_value: str,
834
+ status_filter_value: str,
835
+ updated_after_value: object,
836
+ only_conflicts: bool,
837
+ ):
838
+ return refresh(
839
+ max(1, page - 1),
840
+ species,
841
+ confidence,
842
+ validator_filter_value,
843
+ status_filter_value,
844
+ updated_after_value,
845
+ only_conflicts,
846
+ )
847
+
848
+ def on_select(evt: gr.SelectData):
849
+ if isinstance(evt.index, tuple):
850
+ return int(evt.index[0])
851
+ if isinstance(evt.index, int):
852
+ return int(evt.index)
853
+ return 0
854
+
855
+ demo.load(
856
+ fn=refresh,
857
+ inputs=[
858
+ page_state,
859
+ species_filter,
860
+ min_confidence,
861
+ validator_filter,
862
+ validation_status_filter,
863
+ updated_after_filter,
864
+ show_conflicts_only,
865
+ ],
866
+ outputs=[table, status, page_state],
867
+ )
868
+ refresh_btn.click(
869
+ fn=lambda species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: refresh(
870
+ 1,
871
+ species,
872
+ confidence,
873
+ validator_filter_value,
874
+ status_filter_value,
875
+ updated_after_value,
876
+ only_conflicts,
877
+ ),
878
+ inputs=[
879
+ species_filter,
880
+ min_confidence,
881
+ validator_filter,
882
+ validation_status_filter,
883
+ updated_after_filter,
884
+ show_conflicts_only,
885
+ ],
886
+ outputs=[table, status, page_state],
887
+ )
888
+ next_btn.click(
889
+ fn=go_next,
890
+ inputs=[
891
+ page_state,
892
+ species_filter,
893
+ min_confidence,
894
+ validator_filter,
895
+ validation_status_filter,
896
+ updated_after_filter,
897
+ show_conflicts_only,
898
+ ],
899
+ outputs=[table, status, page_state],
900
+ )
901
+ prev_btn.click(
902
+ fn=go_prev,
903
+ inputs=[
904
+ page_state,
905
+ species_filter,
906
+ min_confidence,
907
+ validator_filter,
908
+ validation_status_filter,
909
+ updated_after_filter,
910
+ show_conflicts_only,
911
+ ],
912
+ outputs=[table, status, page_state],
913
+ )
914
+ table.select(fn=on_select, inputs=None, outputs=[selected_index])
915
+ load_audio_btn.click(
916
+ fn=lambda repo, rows, idx, cache_key: _fetch_selected_audio(
917
+ audio_service=audio_service,
918
+ dataset_repo=repo,
919
+ rows=rows,
920
+ selected_index=int(idx),
921
+ previous_cache_key=cache_key,
922
+ ),
923
+ inputs=[dataset_repo, table, selected_index, cache_key_state],
924
+ outputs=[audio_player, cache_key_state, status],
925
+ )
926
+ clear_audio_btn.click(
927
+ fn=lambda cache_key: _cleanup_selected_audio(audio_service=audio_service, cache_key=cache_key),
928
+ inputs=[cache_key_state],
929
+ outputs=[status, audio_player],
930
+ )
931
+ approve_btn.click(
932
+ fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
933
+ validation_service=validation_service,
934
+ audio_service=audio_service,
935
+ queue_service=service,
936
+ snapshot_reader=validation_repository,
937
+ project_slug=project_slug,
938
+ rows=rows,
939
+ selected_index=int(idx),
940
+ status_value="positive",
941
+ validator=name,
942
+ notes=notes,
943
+ cache_key=cache_key,
944
+ page=int(page),
945
+ scientific_name=species,
946
+ min_confidence=float(confidence),
947
+ validator_filter=validator_filter_value,
948
+ status_filter=status_filter_value,
949
+ updated_after=updated_after_value,
950
+ show_conflicts_only=bool(only_conflicts),
951
+ ),
952
+ inputs=[
953
+ table,
954
+ selected_index,
955
+ validator_name,
956
+ validation_notes,
957
+ cache_key_state,
958
+ page_state,
959
+ species_filter,
960
+ min_confidence,
961
+ validator_filter,
962
+ validation_status_filter,
963
+ updated_after_filter,
964
+ show_conflicts_only,
965
+ ],
966
+ outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
967
+ )
968
+ reject_btn.click(
969
+ fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
970
+ validation_service=validation_service,
971
+ audio_service=audio_service,
972
+ queue_service=service,
973
+ snapshot_reader=validation_repository,
974
+ project_slug=project_slug,
975
+ rows=rows,
976
+ selected_index=int(idx),
977
+ status_value="negative",
978
+ validator=name,
979
+ notes=notes,
980
+ cache_key=cache_key,
981
+ page=int(page),
982
+ scientific_name=species,
983
+ min_confidence=float(confidence),
984
+ validator_filter=validator_filter_value,
985
+ status_filter=status_filter_value,
986
+ updated_after=updated_after_value,
987
+ show_conflicts_only=bool(only_conflicts),
988
+ ),
989
+ inputs=[
990
+ table,
991
+ selected_index,
992
+ validator_name,
993
+ validation_notes,
994
+ cache_key_state,
995
+ page_state,
996
+ species_filter,
997
+ min_confidence,
998
+ validator_filter,
999
+ validation_status_filter,
1000
+ updated_after_filter,
1001
+ show_conflicts_only,
1002
+ ],
1003
+ outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
1004
+ )
1005
+ uncertain_btn.click(
1006
+ fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
1007
+ validation_service=validation_service,
1008
+ audio_service=audio_service,
1009
+ queue_service=service,
1010
+ snapshot_reader=validation_repository,
1011
+ project_slug=project_slug,
1012
+ rows=rows,
1013
+ selected_index=int(idx),
1014
+ status_value="uncertain",
1015
+ validator=name,
1016
+ notes=notes,
1017
+ cache_key=cache_key,
1018
+ page=int(page),
1019
+ scientific_name=species,
1020
+ min_confidence=float(confidence),
1021
+ validator_filter=validator_filter_value,
1022
+ status_filter=status_filter_value,
1023
+ updated_after=updated_after_value,
1024
+ show_conflicts_only=bool(only_conflicts),
1025
+ ),
1026
+ inputs=[
1027
+ table,
1028
+ selected_index,
1029
+ validator_name,
1030
+ validation_notes,
1031
+ cache_key_state,
1032
+ page_state,
1033
+ species_filter,
1034
+ min_confidence,
1035
+ validator_filter,
1036
+ validation_status_filter,
1037
+ updated_after_filter,
1038
+ show_conflicts_only,
1039
+ ],
1040
+ outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
1041
+ )
1042
+ skip_btn.click(
1043
+ fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
1044
+ validation_service=validation_service,
1045
+ audio_service=audio_service,
1046
+ queue_service=service,
1047
+ snapshot_reader=validation_repository,
1048
+ project_slug=project_slug,
1049
+ rows=rows,
1050
+ selected_index=int(idx),
1051
+ status_value="skip",
1052
+ validator=name,
1053
+ notes=notes,
1054
+ cache_key=cache_key,
1055
+ page=int(page),
1056
+ scientific_name=species,
1057
+ min_confidence=float(confidence),
1058
+ validator_filter=validator_filter_value,
1059
+ status_filter=status_filter_value,
1060
+ updated_after=updated_after_value,
1061
+ show_conflicts_only=bool(only_conflicts),
1062
+ ),
1063
+ inputs=[
1064
+ table,
1065
+ selected_index,
1066
+ validator_name,
1067
+ validation_notes,
1068
+ cache_key_state,
1069
+ page_state,
1070
+ species_filter,
1071
+ min_confidence,
1072
+ validator_filter,
1073
+ validation_status_filter,
1074
+ updated_after_filter,
1075
+ show_conflicts_only,
1076
+ ],
1077
+ outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
1078
+ )
1079
+ reapply_btn.click(
1080
+ fn=lambda rows, idx, pending_status, conflict_key, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _reapply_last_conflict_validation_with_refresh(
1081
+ validation_service=validation_service,
1082
+ audio_service=audio_service,
1083
+ queue_service=service,
1084
+ snapshot_reader=validation_repository,
1085
+ project_slug=project_slug,
1086
+ rows=rows,
1087
+ selected_index=int(idx),
1088
+ pending_status_value=pending_status,
1089
+ conflict_detection_key=conflict_key,
1090
+ validator=name,
1091
+ notes=notes,
1092
+ cache_key=cache_key,
1093
+ page=int(page),
1094
+ scientific_name=species,
1095
+ min_confidence=float(confidence),
1096
+ validator_filter=validator_filter_value,
1097
+ status_filter=status_filter_value,
1098
+ updated_after=updated_after_value,
1099
+ show_conflicts_only=bool(only_conflicts),
1100
+ ),
1101
+ inputs=[
1102
+ table,
1103
+ selected_index,
1104
+ pending_status_state,
1105
+ conflict_detection_key_state,
1106
+ validator_name,
1107
+ validation_notes,
1108
+ cache_key_state,
1109
+ page_state,
1110
+ species_filter,
1111
+ min_confidence,
1112
+ validator_filter,
1113
+ validation_status_filter,
1114
+ updated_after_filter,
1115
+ show_conflicts_only,
1116
+ ],
1117
+ outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
1118
+ )
1119
+ batch_approve_conflicts_btn.click(
1120
+ fn=lambda rows, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value: _batch_validate_conflicts(
1121
+ validation_service=validation_service,
1122
+ audio_service=audio_service,
1123
+ queue_service=service,
1124
+ snapshot_reader=validation_repository,
1125
+ project_slug=project_slug,
1126
+ rows=rows,
1127
+ status_value="positive",
1128
+ validator=name,
1129
+ notes=notes,
1130
+ cache_key=cache_key,
1131
+ page=int(page),
1132
+ scientific_name=species,
1133
+ min_confidence=float(confidence),
1134
+ validator_filter=validator_filter_value,
1135
+ status_filter=status_filter_value,
1136
+ updated_after=updated_after_value,
1137
+ ),
1138
+ inputs=[
1139
+ table,
1140
+ validator_name,
1141
+ validation_notes,
1142
+ cache_key_state,
1143
+ page_state,
1144
+ species_filter,
1145
+ min_confidence,
1146
+ validator_filter,
1147
+ validation_status_filter,
1148
+ updated_after_filter,
1149
+ ],
1150
+ outputs=[status, cache_key_state, audio_player, table, page_state],
1151
+ )
1152
+ batch_reject_conflicts_btn.click(
1153
+ fn=lambda rows, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value: _batch_validate_conflicts(
1154
+ validation_service=validation_service,
1155
+ audio_service=audio_service,
1156
+ queue_service=service,
1157
+ snapshot_reader=validation_repository,
1158
+ project_slug=project_slug,
1159
+ rows=rows,
1160
+ status_value="negative",
1161
+ validator=name,
1162
+ notes=notes,
1163
+ cache_key=cache_key,
1164
+ page=int(page),
1165
+ scientific_name=species,
1166
+ min_confidence=float(confidence),
1167
+ validator_filter=validator_filter_value,
1168
+ status_filter=status_filter_value,
1169
+ updated_after=updated_after_value,
1170
+ ),
1171
+ inputs=[
1172
+ table,
1173
+ validator_name,
1174
+ validation_notes,
1175
+ cache_key_state,
1176
+ page_state,
1177
+ species_filter,
1178
+ min_confidence,
1179
+ validator_filter,
1180
+ validation_status_filter,
1181
+ updated_after_filter,
1182
+ ],
1183
+ outputs=[status, cache_key_state, audio_player, table, page_state],
1184
+ )
1185
+ report_btn.click(
1186
+ fn=lambda: _build_validation_report(validation_repository, project_slug),
1187
+ inputs=None,
1188
+ outputs=[report_box],
1189
+ )
1190
+
1191
+ return demo
1192
+
1193
+
1194
+ def create_app() -> gr.Blocks:
1195
+ """Build the BirdNET Validator app with multi-project auth integration.
1196
+
1197
+ Returns multi-tab interface with:
1198
+ - Login tab for user authentication
1199
+ - Project selection for authorized projects
1200
+ - Admin panel for project/user management (admin only)
1201
+ - Validation interface for selected project
1202
+
1203
+ Returns:
1204
+ Gradio Blocks with full auth-integrated app
1205
+ """
1206
+ # Initialize auth service
1207
+ auth_service = AuthService(session_ttl_minutes=120)
1208
+
1209
+ # Setup demo users (in production: database/LDAP/OAuth)
1210
+ auth_service.register_user_project_access(
1211
+ "admin_user",
1212
+ {"kenya-2024": Role.admin, "nairobi-2023": Role.admin},
1213
+ )
1214
+ auth_service.register_user_project_access(
1215
+ "validator_demo",
1216
+ {"demo-project": Role.validator, "kenya-2024": Role.validator},
1217
+ )
1218
+ auth_service.register_user_project_access(
1219
+ "validator_other",
1220
+ {"nairobi-2023": Role.validator},
1221
+ )
1222
+
1223
+ # Initialize admin panel manager
1224
+ admin_manager = AdminPanelManager(auth_service)
1225
+
1226
+ # Register projects
1227
+ admin_manager.register_project(
1228
+ Project(
1229
+ project_slug="kenya-2024",
1230
+ name="Kenya Survey 2024",
1231
+ dataset_repo_id="birdnet/kenya-2024-dataset",
1232
+ active=True,
1233
+ )
1234
+ )
1235
+ admin_manager.register_project(
1236
+ Project(
1237
+ project_slug="nairobi-2023",
1238
+ name="Nairobi Survey 2023",
1239
+ dataset_repo_id="birdnet/nairobi-2023-dataset",
1240
+ active=True,
1241
+ )
1242
+ )
1243
+ admin_manager.register_project(
1244
+ Project(
1245
+ project_slug="demo-project",
1246
+ name="Demo Project",
1247
+ dataset_repo_id="birdnet/demo-dataset",
1248
+ active=True,
1249
+ )
1250
+ )
1251
+
1252
+ with gr.Blocks(title="BirdNET Validator HF - Multi-Project") as wrapper:
1253
+ gr.Markdown("# BirdNET Validator HF - Sprint 4: SeguranΓ§a Multi-Projeto")
1254
+ gr.Markdown("**VersΓ£o com autenticaΓ§Γ£o, autorizaΓ§Γ£o por projeto, e painel admin**")
1255
+
1256
+ # Session state
1257
+ session_state = gr.State(value=None)
1258
+ selected_project_state = gr.State(value=None)
1259
+
1260
+ def _project_rows() -> list[list[object]]:
1261
+ projects = admin_manager.list_projects()
1262
+ return [
1263
+ [
1264
+ p["project_slug"],
1265
+ p["name"],
1266
+ p["dataset_repo_id"],
1267
+ "yes" if bool(p["active"]) else "no",
1268
+ ]
1269
+ for p in projects
1270
+ ]
1271
+
1272
+ def _project_slugs() -> list[str]:
1273
+ return [p["project_slug"] for p in admin_manager.list_projects()]
1274
+
1275
+ with gr.Tabs():
1276
+ # ===== TAB 1: Login =====
1277
+ with gr.Tab("πŸ” Login", id="login_tab"):
1278
+ login_block, username_input, session_output, login_button, error_message = (
1279
+ create_login_page(auth_service)
1280
+ )
1281
+
1282
+ # Store session ID when login succeeds
1283
+ def handle_login_success(session_id: str):
1284
+ """Process successful login and store session."""
1285
+ if session_id:
1286
+ return auth_service.get_session(session_id)
1287
+ return None
1288
+
1289
+ session_output.change(
1290
+ fn=handle_login_success,
1291
+ inputs=[session_output],
1292
+ outputs=[session_state],
1293
+ )
1294
+
1295
+ # ===== TAB 2: Admin Panel =====
1296
+ with gr.Tab("βš™οΈ Admin", id="admin_tab"):
1297
+ admin_info = gr.Markdown(value="⚠️ Faça login primeiro")
1298
+
1299
+ def create_admin_display(session):
1300
+ """Show admin panel or access denied message."""
1301
+ if session is None:
1302
+ return (
1303
+ "❌ **Not Logged In** β€” Please login first in the Login tab.",
1304
+ gr.update(visible=False),
1305
+ )
1306
+ if session.role.value != "admin":
1307
+ return (
1308
+ "❌ **Access Denied** β€” Only administrators can access this panel.",
1309
+ gr.update(visible=False),
1310
+ )
1311
+ return (
1312
+ f"βœ… **Admin Panel** β€” Welcome {session.username}! Click below to manage projects and users.",
1313
+ gr.update(visible=True),
1314
+ )
1315
+
1316
+ with gr.Group(visible=False) as admin_controls:
1317
+ gr.Markdown("#### Projetos Cadastrados")
1318
+
1319
+ with gr.Row():
1320
+ create_project_slug = gr.Textbox(
1321
+ label="Novo Project Slug",
1322
+ placeholder="ex: amazonas-2026",
1323
+ )
1324
+ create_project_name = gr.Textbox(
1325
+ label="Nome do Projeto",
1326
+ placeholder="ex: Amazonas Survey 2026",
1327
+ )
1328
+ create_project_repo = gr.Textbox(
1329
+ label="HF Dataset Repo ID",
1330
+ placeholder="ex: birdnet/amazonas-2026-dataset",
1331
+ )
1332
+
1333
+ create_project_message = gr.Markdown()
1334
+
1335
+ def create_project(session, slug: str, name: str, repo_id: str):
1336
+ if session is None or session.role.value != "admin":
1337
+ return "❌ Access denied. Apenas admin pode criar projetos.", gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
1338
+
1339
+ slug = (slug or "").strip()
1340
+ name = (name or "").strip()
1341
+ repo_id = (repo_id or "").strip()
1342
+ if not slug or not name or not repo_id:
1343
+ return "⚠️ Preencha slug, nome e repo id.", gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
1344
+
1345
+ created = admin_manager.register_project(
1346
+ Project(
1347
+ project_slug=slug,
1348
+ name=name,
1349
+ dataset_repo_id=repo_id,
1350
+ active=True,
1351
+ )
1352
+ )
1353
+ if not created:
1354
+ return (
1355
+ f"⚠️ Projeto '{slug}' jÑ existe.",
1356
+ _project_rows(),
1357
+ gr.update(choices=_project_slugs()),
1358
+ gr.update(),
1359
+ gr.update(),
1360
+ gr.update(),
1361
+ )
1362
+
1363
+ return (
1364
+ f"βœ… Projeto '{slug}' criado com sucesso.",
1365
+ _project_rows(),
1366
+ gr.update(choices=_project_slugs(), value=slug),
1367
+ gr.update(value=""),
1368
+ gr.update(value=""),
1369
+ gr.update(value=""),
1370
+ )
1371
+
1372
+ create_project_btn = gr.Button("βž• Criar Projeto", variant="primary")
1373
+ projects_table = gr.Dataframe(
1374
+ value=_project_rows(),
1375
+ headers=["project_slug", "name", "dataset_repo_id", "active"],
1376
+ interactive=False,
1377
+ )
1378
+ refresh_projects_btn = gr.Button("πŸ”„ Atualizar Lista")
1379
+
1380
+ def refresh_projects(session):
1381
+ if session is None or session.role.value != "admin":
1382
+ return []
1383
+ return _project_rows()
1384
+
1385
+ refresh_projects_btn.click(
1386
+ fn=refresh_projects,
1387
+ inputs=[session_state],
1388
+ outputs=[projects_table],
1389
+ )
1390
+
1391
+ with gr.Group(visible=False) as admin_users_controls:
1392
+ gr.Markdown("#### Assignar UsuΓ‘rio a Projeto")
1393
+ with gr.Row():
1394
+ admin_username = gr.Textbox(
1395
+ label="UsuΓ‘rio", placeholder="validator_001"
1396
+ )
1397
+ admin_project = gr.Dropdown(
1398
+ choices=_project_slugs(),
1399
+ label="Projeto",
1400
+ )
1401
+ admin_role = gr.Dropdown(
1402
+ choices=["admin", "validator"],
1403
+ value="validator",
1404
+ label="Role",
1405
+ )
1406
+
1407
+ admin_message = gr.Markdown()
1408
+
1409
+ def assign_user(session, username: str, project: str, role: str):
1410
+ if session is None or session.role.value != "admin":
1411
+ return "❌ Access denied. Apenas admin pode atribuir usuÑrios.", gr.update(), gr.update(), gr.update()
1412
+ success, msg = admin_manager.assign_user_to_project(
1413
+ username, project, role
1414
+ )
1415
+ if success:
1416
+ return msg, gr.update(value=""), gr.update(value=None), gr.update(value="validator")
1417
+ return msg, gr.update(), gr.update(), gr.update()
1418
+
1419
+ assign_btn = gr.Button("βœ… Assignar", variant="primary")
1420
+ assign_btn.click(
1421
+ fn=assign_user,
1422
+ inputs=[session_state, admin_username, admin_project, admin_role],
1423
+ outputs=[admin_message, admin_username, admin_project, admin_role],
1424
+ )
1425
+
1426
+ create_project_btn.click(
1427
+ fn=create_project,
1428
+ inputs=[session_state, create_project_slug, create_project_name, create_project_repo],
1429
+ outputs=[
1430
+ create_project_message,
1431
+ projects_table,
1432
+ admin_project,
1433
+ create_project_slug,
1434
+ create_project_name,
1435
+ create_project_repo,
1436
+ ],
1437
+ )
1438
+
1439
+ session_state.change(
1440
+ fn=create_admin_display,
1441
+ inputs=[session_state],
1442
+ outputs=[admin_info, admin_controls],
1443
+ )
1444
+
1445
+ session_state.change(
1446
+ fn=lambda s: gr.update(visible=bool(s is not None and s.role.value == "admin")),
1447
+ inputs=[session_state],
1448
+ outputs=[admin_users_controls],
1449
+ )
1450
+
1451
+ # ===== TAB 3: Project Selection =====
1452
+ with gr.Tab("πŸ“ Selecionar Projeto", id="project_tab"):
1453
+ project_info_display = gr.Markdown(
1454
+ value="⚠️ Faça login primeiro na aba **Login**"
1455
+ )
1456
+ project_selector = gr.Dropdown(
1457
+ choices=[],
1458
+ label="Projeto Autorizado",
1459
+ interactive=False,
1460
+ )
1461
+
1462
+ def update_project_selector(session):
1463
+ """Update project dropdown when user logs in."""
1464
+ if session is None:
1465
+ return (
1466
+ gr.Dropdown(choices=[], value=None, interactive=False),
1467
+ "❌ Not logged in. Please login first.",
1468
+ None,
1469
+ )
1470
+
1471
+ projects = session.authorized_projects
1472
+ if not projects:
1473
+ return (
1474
+ gr.Dropdown(choices=[], value=None, interactive=False),
1475
+ "⚠️ **No Projects Assigned**\n\nYou don't have access to any projects yet. Contact an administrator.",
1476
+ None,
1477
+ )
1478
+
1479
+ selected = projects[0]
1480
+ role = auth_service.get_user_role_for_project(session.username, selected)
1481
+ role_label = role.value.upper() if role else "UNKNOWN"
1482
+ return (
1483
+ gr.Dropdown(choices=projects, value=selected, interactive=True),
1484
+ f"πŸ“ **Project:** {selected} | **Your Role:** {role_label}",
1485
+ selected,
1486
+ )
1487
+
1488
+ session_state.change(
1489
+ fn=update_project_selector,
1490
+ inputs=[session_state],
1491
+ outputs=[project_selector, project_info_display, selected_project_state],
1492
+ )
1493
+
1494
+ def update_selected_project(selected: str, session):
1495
+ """Update state when project is selected."""
1496
+ if session and selected:
1497
+ return selected
1498
+ return None
1499
+
1500
+ project_selector.change(
1501
+ fn=update_selected_project,
1502
+ inputs=[project_selector, session_state],
1503
+ outputs=[selected_project_state],
1504
+ )
1505
+
1506
+ # ===== TAB 4: Validation =====
1507
+ with gr.Tab("βœ“ ValidaΓ§Γ£o", id="validation_tab"):
1508
+ validation_status = gr.Markdown(
1509
+ value="ℹ️ FaΓ§a login e selecione um projeto para comeΓ§ar"
1510
+ )
1511
+
1512
+ def get_validation_status(session, selected_project):
1513
+ """Show status message based on login/project state."""
1514
+ if session is None:
1515
+ return "❌ **NΓ£o autenticado** β€” FaΓ§a login na aba **Login** primeiro"
1516
+ if selected_project is None:
1517
+ return f"⚠️ **Projeto nΓ£o selecionado** β€” Selecione um projeto na aba **Selecionar Projeto**"
1518
+ return f"βœ… **Pronto para validar** β€” Projeto: **{selected_project}** | UsuΓ‘rio: **{session.username}**\n\n(ValidaΓ§Γ£o serΓ‘ carregada abaixo)"
1519
+
1520
+ session_state.change(
1521
+ fn=lambda s, p: get_validation_status(s, p),
1522
+ inputs=[session_state, selected_project_state],
1523
+ outputs=[validation_status],
1524
+ )
1525
+
1526
+ selected_project_state.change(
1527
+ fn=lambda s, p: get_validation_status(s, p),
1528
+ inputs=[session_state, selected_project_state],
1529
+ outputs=[validation_status],
1530
+ )
1531
+
1532
+ # Note: In a real implementation, we would load build_demo_app(selected_project) here
1533
+ # For now, we show a placeholder since Gradio tabs don't support dynamic nested Blocks
1534
+ gr.Markdown("---")
1535
+ validation_placeholder = gr.Textbox(
1536
+ label="ValidaΓ§Γ£o",
1537
+ value="πŸ“‹ Interface de validaΓ§Γ£o serΓ‘ carregada aqui para o projeto selecionado",
1538
+ interactive=False,
1539
+ lines=10,
1540
+ )
1541
+
1542
+ return wrapper
BirdNET Validator App/src/ui/login_page.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Gradio login page component for multi-project authorization."""
2
+
3
+ from typing import Optional, Tuple
4
+
5
+ import gradio as gr
6
+
7
+ from src.auth.auth_service import AuthService, Session
8
+
9
+
10
+ def create_login_page(auth_service: AuthService) -> Tuple[gr.Blocks, gr.Textbox, gr.Textbox, gr.Button, gr.Markdown]:
11
+ """Create a Gradio login page with username input and session tracking.
12
+
13
+ Args:
14
+ auth_service: AuthService instance for login validation
15
+
16
+ Returns:
17
+ Tuple of (login_block, username_input, session_output, login_button, error_message)
18
+ """
19
+
20
+ with gr.Blocks(title="BirdNET Validator - Login") as login_block:
21
+ gr.Markdown("# BirdNET Validation Platform")
22
+ gr.Markdown("Login to access multi-project validation workflows")
23
+
24
+ username_input = gr.Textbox(
25
+ label="Username",
26
+ placeholder="Enter your username",
27
+ lines=1,
28
+ )
29
+
30
+ error_message = gr.Markdown()
31
+
32
+ login_button = gr.Button("Login", variant="primary", scale=1)
33
+
34
+ session_output = gr.Textbox(
35
+ label="Session ID",
36
+ interactive=False,
37
+ visible=False,
38
+ )
39
+
40
+ def perform_login(username: str) -> Tuple[str, str]:
41
+ """Attempt login and return session ID or error message.
42
+
43
+ Args:
44
+ username: Username to authenticate
45
+
46
+ Returns:
47
+ Tuple of (session_id, error_message)
48
+ """
49
+ if not username or not username.strip():
50
+ return "", "❌ Please enter a username"
51
+
52
+ username = username.strip()
53
+ session = auth_service.login(username)
54
+
55
+ if session is None:
56
+ return "", f"❌ User '{username}' not found or inactive"
57
+
58
+ return session.session_id, f"βœ… Welcome, {username}! (Admin)" if session.role.value == "admin" else f"βœ… Welcome, {username}! (Validator)"
59
+
60
+ login_button.click(
61
+ fn=perform_login,
62
+ inputs=[username_input],
63
+ outputs=[session_output, error_message],
64
+ )
65
+
66
+ return login_block, username_input, session_output, login_button, error_message
BirdNET Validator App/src/ui/project_selector.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Project selector component for switching between authorized projects."""
2
+
3
+ from typing import List, Tuple
4
+
5
+ import gradio as gr
6
+
7
+ from src.auth.auth_service import AuthService, Session
8
+
9
+
10
+ def create_project_selector(
11
+ auth_service: AuthService, current_session: Session
12
+ ) -> Tuple[gr.Dropdown, gr.Markdown]:
13
+ """Create a project selector dropdown for the current user.
14
+
15
+ Args:
16
+ auth_service: AuthService instance
17
+ current_session: Current user's session
18
+
19
+ Returns:
20
+ Tuple of (project_dropdown, project_info_display)
21
+ """
22
+ authorized_projects = current_session.authorized_projects
23
+
24
+ if not authorized_projects:
25
+ with gr.Group():
26
+ gr.Markdown("⚠️ **No Projects Assigned**\n\nYou don't have access to any projects yet. Contact an administrator.")
27
+
28
+ return (
29
+ gr.Dropdown(choices=[], label="Project", interactive=False),
30
+ gr.Markdown(""),
31
+ )
32
+
33
+ with gr.Group():
34
+ project_dropdown = gr.Dropdown(
35
+ choices=authorized_projects,
36
+ value=authorized_projects[0],
37
+ label="Select Project",
38
+ interactive=True,
39
+ )
40
+
41
+ project_info = gr.Markdown()
42
+
43
+ def update_project_info(selected_project: str) -> str:
44
+ """Display info about the selected project.
45
+
46
+ Args:
47
+ selected_project: Project slug
48
+
49
+ Returns:
50
+ Markdown formatted project info
51
+ """
52
+ role = auth_service.get_user_role_for_project(
53
+ current_session.username, selected_project
54
+ )
55
+ if role:
56
+ return f"πŸ“ **Project:** {selected_project} | **Your Role:** {role.value.upper()}"
57
+ return f"πŸ“ **Project:** {selected_project}"
58
+
59
+ # Initial display
60
+ initial_project = authorized_projects[0]
61
+ role = auth_service.get_user_role_for_project(
62
+ current_session.username, initial_project
63
+ )
64
+ initial_info = (
65
+ f"πŸ“ **Project:** {initial_project} | **Your Role:** {role.value.upper()}"
66
+ if role
67
+ else f"πŸ“ **Project:** {initial_project}"
68
+ )
69
+ project_info.value = initial_info
70
+
71
+ project_dropdown.change(
72
+ fn=update_project_info,
73
+ inputs=[project_dropdown],
74
+ outputs=[project_info],
75
+ )
76
+
77
+ return project_dropdown, project_info
BirdNET Validator App/src/ui/upload_app_factory.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from pathlib import Path
3
+ from typing import Any
4
+
5
+ import gradio as gr
6
+ from huggingface_hub import HfApi
7
+
8
+ from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
9
+
10
+
11
+ def _as_pretty_json(payload: dict[str, Any]) -> str:
12
+ return json.dumps(payload, indent=2, ensure_ascii=False)
13
+
14
+
15
+ def _build_api(token: str) -> HfApi:
16
+ clean_token = (token or "").strip()
17
+ if clean_token:
18
+ return HfApi(token=clean_token)
19
+ return HfApi()
20
+
21
+
22
+ def build_upload_app() -> gr.Blocks:
23
+ with gr.Blocks(title="BirdNET Segments Uploader") as demo:
24
+ gr.Markdown("# BirdNET Segments Uploader")
25
+ gr.Markdown(
26
+ "Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
27
+ "Recomendado: executar dry-run antes do upload real."
28
+ )
29
+
30
+ with gr.Row():
31
+ project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
32
+ dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
33
+
34
+ detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
35
+ segments_root = gr.Textbox(
36
+ label="Pasta raiz de segmentos",
37
+ placeholder=r"ex: C:\dados\BirdNET Segments",
38
+ )
39
+
40
+ with gr.Accordion("Configuracao avancada", open=False):
41
+ hf_token = gr.Textbox(
42
+ label="HF Token (opcional)",
43
+ type="password",
44
+ placeholder="Se vazio, usa sessao ja autenticada no ambiente",
45
+ )
46
+ with gr.Row():
47
+ batch_size = gr.Number(label="Batch size", value=200, precision=0)
48
+ shard_size = gr.Number(label="Shard size", value=10000, precision=0)
49
+ with gr.Row():
50
+ max_retries = gr.Number(label="Max retries", value=3, precision=0)
51
+ retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
52
+ resume_state_file = gr.Textbox(
53
+ label="Resume state file",
54
+ value=".ingest-segments-state.json",
55
+ )
56
+ report_file = gr.Textbox(
57
+ label="Report file local (opcional)",
58
+ placeholder="ex: .ingest-run-report.json",
59
+ )
60
+
61
+ with gr.Row():
62
+ dry_run_button = gr.Button("Dry-run", variant="secondary")
63
+ upload_button = gr.Button("Upload real", variant="primary")
64
+
65
+ status = gr.Markdown(value="Pronto")
66
+ result_json = gr.Code(label="Resultado JSON", language="json")
67
+
68
+ def run_dry_run(
69
+ project: str,
70
+ repo: str,
71
+ csv_path: str | None,
72
+ segments_path: str,
73
+ report_path: str,
74
+ ) -> tuple[str, str]:
75
+ project = (project or "").strip()
76
+ if not project:
77
+ return "❌ Informe o project slug", "{}"
78
+
79
+ if not csv_path:
80
+ return "❌ Selecione o CSV de deteccoes", "{}"
81
+
82
+ if not segments_path or not Path(segments_path).exists():
83
+ return "❌ Pasta de segmentos invalida", "{}"
84
+
85
+ try:
86
+ result = run_ingest_segments_dry_run(
87
+ project_slug=project,
88
+ detections_csv=csv_path,
89
+ segments_root=segments_path,
90
+ )
91
+ if report_path.strip():
92
+ Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
93
+ summary = (
94
+ "βœ… Dry-run finalizado | "
95
+ f"Matched: {result['matched_rows']} | "
96
+ f"Unmatched: {result['unmatched_rows']} | "
97
+ f"Ambiguous: {result['ambiguous_rows']}"
98
+ )
99
+ return summary, _as_pretty_json(result)
100
+ except Exception as exc:
101
+ return f"❌ Dry-run falhou: {exc}", "{}"
102
+
103
+ def run_upload(
104
+ project: str,
105
+ repo: str,
106
+ csv_path: str | None,
107
+ segments_path: str,
108
+ token: str,
109
+ batch: float,
110
+ shard: float,
111
+ retries: float,
112
+ backoff: float,
113
+ resume_file: str,
114
+ report_path: str,
115
+ ) -> tuple[str, str]:
116
+ project = (project or "").strip()
117
+ repo = (repo or "").strip()
118
+ if not project:
119
+ return "❌ Informe o project slug", "{}"
120
+ if not repo or "/" not in repo:
121
+ return "❌ Informe dataset repo no formato owner/repo", "{}"
122
+ if not csv_path:
123
+ return "❌ Selecione o CSV de deteccoes", "{}"
124
+ if not segments_path or not Path(segments_path).exists():
125
+ return "❌ Pasta de segmentos invalida", "{}"
126
+
127
+ try:
128
+ api = _build_api(token)
129
+ result = ingest_segments_to_hf(
130
+ api=api,
131
+ project_slug=project,
132
+ dataset_repo=repo,
133
+ detections_csv=csv_path,
134
+ segments_root=segments_path,
135
+ batch_size=int(batch),
136
+ shard_size=int(shard),
137
+ max_retries=int(retries),
138
+ retry_backoff_seconds=float(backoff),
139
+ resume_state_file=resume_file,
140
+ )
141
+ if report_path.strip():
142
+ Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
143
+ summary = (
144
+ "βœ… Upload finalizado | "
145
+ f"Uploaded now: {result['uploaded_audio_now']} | "
146
+ f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
147
+ f"Failed uploads: {result['failed_uploads']}"
148
+ )
149
+ return summary, _as_pretty_json(result)
150
+ except Exception as exc:
151
+ return f"❌ Upload falhou: {exc}", "{}"
152
+
153
+ dry_run_button.click(
154
+ fn=run_dry_run,
155
+ inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
156
+ outputs=[status, result_json],
157
+ )
158
+
159
+ upload_button.click(
160
+ fn=run_upload,
161
+ inputs=[
162
+ project_slug,
163
+ dataset_repo,
164
+ detections_csv,
165
+ segments_root,
166
+ hf_token,
167
+ batch_size,
168
+ shard_size,
169
+ max_retries,
170
+ retry_backoff_seconds,
171
+ resume_state_file,
172
+ report_file,
173
+ ],
174
+ outputs=[status, result_json],
175
+ )
176
+
177
+ return demo