Spaces:
Sleeping
Sleeping
Commit Β·
f5bcf37
1
Parent(s): 837d92e
chore: bootstrap BirdNET uploader standalone repo
Browse files- BirdNET Uploader App/README.md +84 -0
- BirdNET Uploader App/app.py +9 -0
- BirdNET Uploader App/cli/hf_dataset_cli.py +969 -0
- BirdNET Uploader App/pyproject.toml +9 -0
- BirdNET Uploader App/requirements.txt +7 -0
- BirdNET Uploader App/src/__init__.py +0 -0
- BirdNET Uploader App/src/ui/__init__.py +0 -0
- BirdNET Uploader App/src/ui/upload_app_factory.py +177 -0
- BirdNET Validator App/README.md +83 -0
- BirdNET Validator App/app.py +6 -0
- BirdNET Validator App/pyproject.toml +9 -0
- BirdNET Validator App/requirements.txt +7 -0
- BirdNET Validator App/src/__init__.py +0 -0
- BirdNET Validator App/src/auth/__init__.py +0 -0
- BirdNET Validator App/src/auth/auth_service.py +200 -0
- BirdNET Validator App/src/cache/__init__.py +0 -0
- BirdNET Validator App/src/cache/ephemeral_cache_manager.py +82 -0
- BirdNET Validator App/src/domain/__init__.py +0 -0
- BirdNET Validator App/src/domain/models.py +46 -0
- BirdNET Validator App/src/repositories/__init__.py +0 -0
- BirdNET Validator App/src/repositories/append_only_validation_repository.py +89 -0
- BirdNET Validator App/src/repositories/contracts.py +35 -0
- BirdNET Validator App/src/repositories/in_memory_detection_repository.py +73 -0
- BirdNET Validator App/src/repositories/in_memory_validation_repository.py +25 -0
- BirdNET Validator App/src/services/__init__.py +0 -0
- BirdNET Validator App/src/services/audio_fetch_service.py +54 -0
- BirdNET Validator App/src/services/detection_queue_service.py +64 -0
- BirdNET Validator App/src/services/validation_service.py +39 -0
- BirdNET Validator App/src/telemetry/__init__.py +0 -0
- BirdNET Validator App/src/ui/__init__.py +0 -0
- BirdNET Validator App/src/ui/admin_panel.py +281 -0
- BirdNET Validator App/src/ui/app_factory.py +1542 -0
- BirdNET Validator App/src/ui/login_page.py +66 -0
- BirdNET Validator App/src/ui/project_selector.py +77 -0
- BirdNET Validator App/src/ui/upload_app_factory.py +177 -0
BirdNET Uploader App/README.md
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BirdNET Uploader App
|
| 2 |
+
|
| 3 |
+
Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
|
| 4 |
+
- Multi-projeto
|
| 5 |
+
- Audio sob demanda
|
| 6 |
+
- Persistencia de validacoes
|
| 7 |
+
- Escalabilidade para datasets grandes
|
| 8 |
+
|
| 9 |
+
## Desenvolvimento local
|
| 10 |
+
1. Criar ambiente virtual Python 3.11+
|
| 11 |
+
2. Instalar dependencias:
|
| 12 |
+
pip install -r requirements.txt
|
| 13 |
+
3. Rodar app:
|
| 14 |
+
python app.py
|
| 15 |
+
|
| 16 |
+
## Objetivo desta pasta
|
| 17 |
+
Contem apenas os arquivos necessarios para o app de upload/ingestao.
|
| 18 |
+
Use esta pasta para criar/publicar o repositorio do uploader de forma independente.
|
| 19 |
+
|
| 20 |
+
## Fluxo recomendado (separado)
|
| 21 |
+
1. Uploader app (porta 7862 por padrao):
|
| 22 |
+
- Rodar dry-run com CSV + pasta de segmentos
|
| 23 |
+
- Rodar upload real para owner/repo do projeto
|
| 24 |
+
2. Validator app (porta 7860):
|
| 25 |
+
- Validar deteccoes do projeto com dados ja publicados
|
| 26 |
+
|
| 27 |
+
Essa separacao evita impacto de performance na validacao durante uploads grandes.
|
| 28 |
+
|
| 29 |
+
## CLI de dataset (Sprint 1)
|
| 30 |
+
Comandos principais:
|
| 31 |
+
|
| 32 |
+
1. Criar estrutura do projeto no dataset Hugging Face:
|
| 33 |
+
python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 34 |
+
|
| 35 |
+
2. Gerar manifesto e shards iniciais de indice:
|
| 36 |
+
python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
|
| 37 |
+
|
| 38 |
+
3. Fazer upload de audio em lotes com retry e resume:
|
| 39 |
+
python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
|
| 40 |
+
|
| 41 |
+
4. Verificar consistencia basica do projeto:
|
| 42 |
+
python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 43 |
+
|
| 44 |
+
5. Ingestao BirdNET por projeto (dry-run):
|
| 45 |
+
python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
|
| 46 |
+
|
| 47 |
+
6. Ingestao BirdNET por projeto (upload real):
|
| 48 |
+
python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
|
| 49 |
+
|
| 50 |
+
Estrutura base criada por projeto/dataset:
|
| 51 |
+
- audio/
|
| 52 |
+
- index/
|
| 53 |
+
- index/shards/
|
| 54 |
+
- validations/
|
| 55 |
+
- audit/
|
| 56 |
+
- manifest.json
|
| 57 |
+
|
| 58 |
+
## Estrutura
|
| 59 |
+
- src/domain: modelos de dominio
|
| 60 |
+
- src/repositories: contratos de persistencia
|
| 61 |
+
- src/services: servicos de aplicacao
|
| 62 |
+
- src/auth: autenticacao e autorizacao
|
| 63 |
+
- src/ui: montagem da interface
|
| 64 |
+
- src/cache: cache efemero local
|
| 65 |
+
- cli: comandos de ingestao/publicacao
|
| 66 |
+
- tests: testes unitarios e integracao
|
| 67 |
+
|
| 68 |
+
## Estado atual de implementacao
|
| 69 |
+
- Sprint 1: CLI create-project, sync-audio, build-index, verify-project
|
| 70 |
+
- Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
|
| 71 |
+
- Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
|
| 72 |
+
- Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
|
| 73 |
+
- Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
|
| 74 |
+
- Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
|
| 75 |
+
- Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
|
| 76 |
+
- Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
|
| 77 |
+
- Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
|
| 78 |
+
- Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
|
| 79 |
+
- Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
|
| 80 |
+
- Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
|
| 81 |
+
- Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
|
| 82 |
+
- Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
|
| 83 |
+
- Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
|
| 84 |
+
- Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
|
BirdNET Uploader App/app.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
from src.ui.upload_app_factory import build_upload_app
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
if __name__ == "__main__":
|
| 7 |
+
app = build_upload_app()
|
| 8 |
+
port = int(os.getenv("BIRDNET_UPLOADER_PORT", "7862"))
|
| 9 |
+
app.launch(server_name="0.0.0.0", server_port=port)
|
BirdNET Uploader App/cli/hf_dataset_cli.py
ADDED
|
@@ -0,0 +1,969 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import hashlib
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
import re
|
| 6 |
+
import tempfile
|
| 7 |
+
import time
|
| 8 |
+
from dataclasses import asdict, dataclass
|
| 9 |
+
from datetime import UTC, datetime
|
| 10 |
+
from io import BytesIO
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Any
|
| 13 |
+
|
| 14 |
+
import pandas as pd
|
| 15 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
SCHEMA_VERSION = "1.0.0"
|
| 19 |
+
REQUIRED_PROJECT_PREFIXES = (
|
| 20 |
+
"audio/",
|
| 21 |
+
"index/",
|
| 22 |
+
"index/shards/",
|
| 23 |
+
"validations/",
|
| 24 |
+
"audit/",
|
| 25 |
+
)
|
| 26 |
+
REQUIRED_PLACEHOLDER_FILES = (
|
| 27 |
+
"audio/.gitkeep",
|
| 28 |
+
"index/.gitkeep",
|
| 29 |
+
"index/shards/.gitkeep",
|
| 30 |
+
"validations/.gitkeep",
|
| 31 |
+
"audit/.gitkeep",
|
| 32 |
+
)
|
| 33 |
+
REQUIRED_DETECTIONS_COLUMNS = (
|
| 34 |
+
"detection_key",
|
| 35 |
+
"audio_id",
|
| 36 |
+
"scientific_name",
|
| 37 |
+
"confidence",
|
| 38 |
+
"start_time",
|
| 39 |
+
"end_time",
|
| 40 |
+
)
|
| 41 |
+
SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
|
| 42 |
+
REQUIRED_BIRDNET_COLUMNS = (
|
| 43 |
+
"source_file",
|
| 44 |
+
"scientific_name",
|
| 45 |
+
"confidence",
|
| 46 |
+
"start_time",
|
| 47 |
+
"end_time",
|
| 48 |
+
)
|
| 49 |
+
SEGMENT_FILENAME_PATTERN = re.compile(
|
| 50 |
+
r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@dataclass(slots=True)
|
| 55 |
+
class ShardMetadata:
|
| 56 |
+
path: str
|
| 57 |
+
rows: int
|
| 58 |
+
sha256: str
|
| 59 |
+
size_bytes: int
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@dataclass(slots=True)
|
| 63 |
+
class SegmentFileRecord:
|
| 64 |
+
scientific_name: str
|
| 65 |
+
source_stem: str
|
| 66 |
+
start_time: float
|
| 67 |
+
end_time: float
|
| 68 |
+
confidence_pct: int
|
| 69 |
+
absolute_path: Path
|
| 70 |
+
relative_path: str
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _utcnow_iso() -> str:
|
| 74 |
+
return datetime.now(UTC).replace(microsecond=0).isoformat()
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
|
| 78 |
+
api.upload_file(
|
| 79 |
+
path_or_fileobj=BytesIO(content.encode("utf-8")),
|
| 80 |
+
path_in_repo=path_in_repo,
|
| 81 |
+
repo_id=dataset_repo,
|
| 82 |
+
repo_type="dataset",
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
|
| 87 |
+
api.upload_file(
|
| 88 |
+
path_or_fileobj=BytesIO(b""),
|
| 89 |
+
path_in_repo=path_in_repo,
|
| 90 |
+
repo_id=dataset_repo,
|
| 91 |
+
repo_type="dataset",
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def ensure_project_dataset_structure(
|
| 96 |
+
api: HfApi,
|
| 97 |
+
project_slug: str,
|
| 98 |
+
dataset_repo: str,
|
| 99 |
+
create_private_repo: bool,
|
| 100 |
+
) -> dict[str, Any]:
|
| 101 |
+
api.create_repo(
|
| 102 |
+
repo_id=dataset_repo,
|
| 103 |
+
repo_type="dataset",
|
| 104 |
+
private=create_private_repo,
|
| 105 |
+
exist_ok=True,
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 109 |
+
created_paths: list[str] = []
|
| 110 |
+
|
| 111 |
+
for placeholder in REQUIRED_PLACEHOLDER_FILES:
|
| 112 |
+
if placeholder not in files:
|
| 113 |
+
_upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
|
| 114 |
+
created_paths.append(placeholder)
|
| 115 |
+
|
| 116 |
+
manifest_path = "manifest.json"
|
| 117 |
+
if manifest_path not in files:
|
| 118 |
+
manifest = {
|
| 119 |
+
"schema_version": SCHEMA_VERSION,
|
| 120 |
+
"project_slug": project_slug,
|
| 121 |
+
"dataset_repo_id": dataset_repo,
|
| 122 |
+
"created_at": _utcnow_iso(),
|
| 123 |
+
"updated_at": _utcnow_iso(),
|
| 124 |
+
"index": {
|
| 125 |
+
"total_detections": 0,
|
| 126 |
+
"total_audio_files": 0,
|
| 127 |
+
"shard_size": 0,
|
| 128 |
+
"shards": [],
|
| 129 |
+
},
|
| 130 |
+
}
|
| 131 |
+
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
|
| 132 |
+
created_paths.append(manifest_path)
|
| 133 |
+
|
| 134 |
+
return {
|
| 135 |
+
"dataset_repo": dataset_repo,
|
| 136 |
+
"project_slug": project_slug,
|
| 137 |
+
"created_paths": created_paths,
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def load_detections_table(detections_file: str) -> pd.DataFrame:
|
| 142 |
+
input_path = Path(detections_file)
|
| 143 |
+
if not input_path.exists():
|
| 144 |
+
raise FileNotFoundError(f"Detections file not found: {detections_file}")
|
| 145 |
+
|
| 146 |
+
suffix = input_path.suffix.lower()
|
| 147 |
+
if suffix == ".csv":
|
| 148 |
+
frame = pd.read_csv(input_path)
|
| 149 |
+
elif suffix in {".jsonl", ".ndjson"}:
|
| 150 |
+
frame = pd.read_json(input_path, lines=True)
|
| 151 |
+
elif suffix == ".parquet":
|
| 152 |
+
frame = pd.read_parquet(input_path)
|
| 153 |
+
else:
|
| 154 |
+
raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
|
| 155 |
+
|
| 156 |
+
missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
|
| 157 |
+
if missing_columns:
|
| 158 |
+
raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
|
| 159 |
+
|
| 160 |
+
if frame.empty:
|
| 161 |
+
return frame
|
| 162 |
+
|
| 163 |
+
deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
|
| 164 |
+
return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
|
| 168 |
+
input_path = Path(detections_csv)
|
| 169 |
+
if not input_path.exists():
|
| 170 |
+
raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
|
| 171 |
+
|
| 172 |
+
frame = pd.read_csv(input_path)
|
| 173 |
+
missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
|
| 174 |
+
if missing_columns:
|
| 175 |
+
raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
|
| 176 |
+
|
| 177 |
+
if frame.empty:
|
| 178 |
+
return frame
|
| 179 |
+
|
| 180 |
+
normalized = frame.copy()
|
| 181 |
+
normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
|
| 182 |
+
normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
|
| 183 |
+
normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
|
| 184 |
+
normalized["species_key"] = normalized["scientific_name"].str.casefold()
|
| 185 |
+
normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
|
| 186 |
+
normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
|
| 187 |
+
normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
|
| 188 |
+
|
| 189 |
+
normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
|
| 190 |
+
return normalized
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
|
| 194 |
+
payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
|
| 195 |
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
|
| 199 |
+
stem = Path(file_name).stem
|
| 200 |
+
match = SEGMENT_FILENAME_PATTERN.match(stem)
|
| 201 |
+
if not match:
|
| 202 |
+
return None
|
| 203 |
+
|
| 204 |
+
source_stem = match.group("source_stem")
|
| 205 |
+
start_time = float(match.group("start"))
|
| 206 |
+
end_time = float(match.group("end"))
|
| 207 |
+
confidence_pct = int(match.group("confidence_pct"))
|
| 208 |
+
return source_stem, start_time, end_time, confidence_pct
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
|
| 212 |
+
root = Path(segments_root)
|
| 213 |
+
if not root.exists() or not root.is_dir():
|
| 214 |
+
raise FileNotFoundError(f"Segments root not found: {segments_root}")
|
| 215 |
+
|
| 216 |
+
records: list[SegmentFileRecord] = []
|
| 217 |
+
for absolute_path in root.rglob("*"):
|
| 218 |
+
if not absolute_path.is_file():
|
| 219 |
+
continue
|
| 220 |
+
if absolute_path.suffix.lower() != ".wav":
|
| 221 |
+
continue
|
| 222 |
+
|
| 223 |
+
rel = absolute_path.relative_to(root)
|
| 224 |
+
scientific_name = rel.parts[0] if rel.parts else ""
|
| 225 |
+
parsed = parse_segment_filename(absolute_path.name)
|
| 226 |
+
if not parsed:
|
| 227 |
+
continue
|
| 228 |
+
source_stem, start_time, end_time, confidence_pct = parsed
|
| 229 |
+
records.append(
|
| 230 |
+
SegmentFileRecord(
|
| 231 |
+
scientific_name=scientific_name,
|
| 232 |
+
source_stem=source_stem,
|
| 233 |
+
start_time=start_time,
|
| 234 |
+
end_time=end_time,
|
| 235 |
+
confidence_pct=confidence_pct,
|
| 236 |
+
absolute_path=absolute_path,
|
| 237 |
+
relative_path=rel.as_posix(),
|
| 238 |
+
)
|
| 239 |
+
)
|
| 240 |
+
|
| 241 |
+
return records
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 245 |
+
frame = load_birdnet_detections_csv(detections_csv=detections_csv)
|
| 246 |
+
segment_records = discover_segment_records(segments_root=segments_root)
|
| 247 |
+
|
| 248 |
+
by_species: dict[str, list[SegmentFileRecord]] = {}
|
| 249 |
+
for record in segment_records:
|
| 250 |
+
by_species.setdefault(record.scientific_name.casefold(), []).append(record)
|
| 251 |
+
|
| 252 |
+
matched_rows: list[dict[str, Any]] = []
|
| 253 |
+
unmatched_rows: list[dict[str, Any]] = []
|
| 254 |
+
ambiguous_rows: list[dict[str, Any]] = []
|
| 255 |
+
|
| 256 |
+
for row in frame.to_dict(orient="records"):
|
| 257 |
+
species_key = str(row.get("species_key", "")).casefold()
|
| 258 |
+
source_stem = str(row.get("source_stem", "")).strip()
|
| 259 |
+
start_time = float(row["start_time"])
|
| 260 |
+
end_time = float(row["end_time"])
|
| 261 |
+
confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
|
| 262 |
+
|
| 263 |
+
species_candidates = by_species.get(species_key, [])
|
| 264 |
+
exact_matches = [
|
| 265 |
+
record
|
| 266 |
+
for record in species_candidates
|
| 267 |
+
if record.source_stem == source_stem
|
| 268 |
+
and abs(record.start_time - start_time) < 1e-6
|
| 269 |
+
and abs(record.end_time - end_time) < 1e-6
|
| 270 |
+
]
|
| 271 |
+
|
| 272 |
+
if not exact_matches:
|
| 273 |
+
unmatched_rows.append(
|
| 274 |
+
{
|
| 275 |
+
"source_file": row.get("source_file", ""),
|
| 276 |
+
"scientific_name": row.get("scientific_name", ""),
|
| 277 |
+
"start_time": start_time,
|
| 278 |
+
"end_time": end_time,
|
| 279 |
+
}
|
| 280 |
+
)
|
| 281 |
+
continue
|
| 282 |
+
|
| 283 |
+
selected = exact_matches[0]
|
| 284 |
+
if len(exact_matches) > 1:
|
| 285 |
+
expected_pct = int(round(confidence * 100))
|
| 286 |
+
exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
|
| 287 |
+
selected = exact_matches[0]
|
| 288 |
+
ambiguous_rows.append(
|
| 289 |
+
{
|
| 290 |
+
"source_file": row.get("source_file", ""),
|
| 291 |
+
"scientific_name": row.get("scientific_name", ""),
|
| 292 |
+
"start_time": start_time,
|
| 293 |
+
"end_time": end_time,
|
| 294 |
+
"candidates": len(exact_matches),
|
| 295 |
+
}
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
matched_rows.append(
|
| 299 |
+
{
|
| 300 |
+
"project_slug": project_slug,
|
| 301 |
+
"detection_key": _build_detection_key(
|
| 302 |
+
project_slug=project_slug,
|
| 303 |
+
source_file=str(row.get("source_file", "")),
|
| 304 |
+
scientific_name=str(row.get("scientific_name", "")),
|
| 305 |
+
start_time=start_time,
|
| 306 |
+
end_time=end_time,
|
| 307 |
+
),
|
| 308 |
+
"audio_id": source_stem,
|
| 309 |
+
"segment_filename": selected.absolute_path.name,
|
| 310 |
+
"segment_relpath": selected.relative_path,
|
| 311 |
+
"scientific_name": row.get("scientific_name", ""),
|
| 312 |
+
"common_name": row.get("common_name", ""),
|
| 313 |
+
"confidence": confidence,
|
| 314 |
+
"start_time": start_time,
|
| 315 |
+
"end_time": end_time,
|
| 316 |
+
"exact_start": row.get("exact_start", start_time),
|
| 317 |
+
"exact_end": row.get("exact_end", end_time),
|
| 318 |
+
"locality": row.get("locality", ""),
|
| 319 |
+
"point": row.get("point", ""),
|
| 320 |
+
"date_folder": row.get("date_folder", ""),
|
| 321 |
+
"min_freq": row.get("min_freq", None),
|
| 322 |
+
"max_freq": row.get("max_freq", None),
|
| 323 |
+
"box_source": row.get("box_source", ""),
|
| 324 |
+
"label": row.get("label", ""),
|
| 325 |
+
"source_file": row.get("source_file", ""),
|
| 326 |
+
"schema_version": SCHEMA_VERSION,
|
| 327 |
+
"ingested_at": _utcnow_iso(),
|
| 328 |
+
}
|
| 329 |
+
)
|
| 330 |
+
|
| 331 |
+
return {
|
| 332 |
+
"frame": frame,
|
| 333 |
+
"segment_records_total": len(segment_records),
|
| 334 |
+
"matched_rows": matched_rows,
|
| 335 |
+
"unmatched_rows": unmatched_rows,
|
| 336 |
+
"ambiguous_rows": ambiguous_rows,
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 341 |
+
match_result = _match_birdnet_rows(
|
| 342 |
+
project_slug=project_slug,
|
| 343 |
+
detections_csv=detections_csv,
|
| 344 |
+
segments_root=segments_root,
|
| 345 |
+
)
|
| 346 |
+
frame = match_result["frame"]
|
| 347 |
+
matched_rows = match_result["matched_rows"]
|
| 348 |
+
unmatched_rows = match_result["unmatched_rows"]
|
| 349 |
+
ambiguous_rows = match_result["ambiguous_rows"]
|
| 350 |
+
|
| 351 |
+
unique_audio = {row["audio_id"] for row in matched_rows}
|
| 352 |
+
result = {
|
| 353 |
+
"mode": "dry-run",
|
| 354 |
+
"project_slug": project_slug,
|
| 355 |
+
"detections_csv": detections_csv,
|
| 356 |
+
"segments_root": segments_root,
|
| 357 |
+
"csv_rows_total": int(len(frame)),
|
| 358 |
+
"segments_found_total": int(match_result["segment_records_total"]),
|
| 359 |
+
"matched_rows": len(matched_rows),
|
| 360 |
+
"unmatched_rows": len(unmatched_rows),
|
| 361 |
+
"ambiguous_rows": len(ambiguous_rows),
|
| 362 |
+
"unique_audio_ids": len(unique_audio),
|
| 363 |
+
"sample_unmatched": unmatched_rows[:20],
|
| 364 |
+
"sample_ambiguous": ambiguous_rows[:20],
|
| 365 |
+
}
|
| 366 |
+
return result
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
def _file_sha256(file_path: Path) -> str:
|
| 370 |
+
hasher = hashlib.sha256()
|
| 371 |
+
with file_path.open("rb") as handle:
|
| 372 |
+
for chunk in iter(lambda: handle.read(65536), b""):
|
| 373 |
+
hasher.update(chunk)
|
| 374 |
+
return hasher.hexdigest()
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
|
| 378 |
+
if shard_size <= 0:
|
| 379 |
+
raise ValueError("shard_size must be greater than zero")
|
| 380 |
+
|
| 381 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 382 |
+
shards: list[ShardMetadata] = []
|
| 383 |
+
|
| 384 |
+
if frame.empty:
|
| 385 |
+
return shards
|
| 386 |
+
|
| 387 |
+
total_rows = len(frame)
|
| 388 |
+
total_shards = math.ceil(total_rows / shard_size)
|
| 389 |
+
|
| 390 |
+
for index in range(total_shards):
|
| 391 |
+
start = index * shard_size
|
| 392 |
+
end = min(start + shard_size, total_rows)
|
| 393 |
+
shard_frame = frame.iloc[start:end]
|
| 394 |
+
shard_filename = f"shard-{index:05d}.parquet"
|
| 395 |
+
local_path = output_dir / shard_filename
|
| 396 |
+
shard_frame.to_parquet(local_path, index=False)
|
| 397 |
+
|
| 398 |
+
shards.append(
|
| 399 |
+
ShardMetadata(
|
| 400 |
+
path=f"index/shards/{shard_filename}",
|
| 401 |
+
rows=len(shard_frame),
|
| 402 |
+
sha256=_file_sha256(local_path),
|
| 403 |
+
size_bytes=local_path.stat().st_size,
|
| 404 |
+
)
|
| 405 |
+
)
|
| 406 |
+
|
| 407 |
+
return shards
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
def build_manifest_payload(
|
| 411 |
+
project_slug: str,
|
| 412 |
+
dataset_repo: str,
|
| 413 |
+
frame: pd.DataFrame,
|
| 414 |
+
shard_size: int,
|
| 415 |
+
shard_metadata: list[ShardMetadata],
|
| 416 |
+
) -> dict[str, Any]:
|
| 417 |
+
now = _utcnow_iso()
|
| 418 |
+
return {
|
| 419 |
+
"schema_version": SCHEMA_VERSION,
|
| 420 |
+
"project_slug": project_slug,
|
| 421 |
+
"dataset_repo_id": dataset_repo,
|
| 422 |
+
"updated_at": now,
|
| 423 |
+
"index": {
|
| 424 |
+
"total_detections": len(frame),
|
| 425 |
+
"total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
|
| 426 |
+
"shard_size": shard_size,
|
| 427 |
+
"shards": [asdict(item) for item in shard_metadata],
|
| 428 |
+
},
|
| 429 |
+
}
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
def discover_audio_files(local_audio_dir: str) -> list[Path]:
|
| 433 |
+
audio_root = Path(local_audio_dir)
|
| 434 |
+
if not audio_root.exists() or not audio_root.is_dir():
|
| 435 |
+
raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
|
| 436 |
+
|
| 437 |
+
files = [
|
| 438 |
+
path
|
| 439 |
+
for path in audio_root.rglob("*")
|
| 440 |
+
if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
|
| 441 |
+
]
|
| 442 |
+
return sorted(files)
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
|
| 446 |
+
if chunk_size <= 0:
|
| 447 |
+
raise ValueError("chunk_size must be greater than zero")
|
| 448 |
+
return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
def _load_resume_state(state_file: Path) -> dict[str, Any]:
|
| 452 |
+
if not state_file.exists():
|
| 453 |
+
return {"uploaded": [], "failed": []}
|
| 454 |
+
|
| 455 |
+
payload = json.loads(state_file.read_text(encoding="utf-8"))
|
| 456 |
+
uploaded = payload.get("uploaded", [])
|
| 457 |
+
failed = payload.get("failed", [])
|
| 458 |
+
return {
|
| 459 |
+
"uploaded": uploaded if isinstance(uploaded, list) else [],
|
| 460 |
+
"failed": failed if isinstance(failed, list) else [],
|
| 461 |
+
}
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
|
| 465 |
+
state_file.parent.mkdir(parents=True, exist_ok=True)
|
| 466 |
+
payload = {
|
| 467 |
+
"updated_at": _utcnow_iso(),
|
| 468 |
+
"uploaded": sorted(uploaded),
|
| 469 |
+
"failed": sorted(failed),
|
| 470 |
+
}
|
| 471 |
+
state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
def _upload_audio_with_retry(
|
| 475 |
+
api: HfApi,
|
| 476 |
+
dataset_repo: str,
|
| 477 |
+
local_path: Path,
|
| 478 |
+
path_in_repo: str,
|
| 479 |
+
max_retries: int,
|
| 480 |
+
retry_backoff_seconds: float,
|
| 481 |
+
) -> None:
|
| 482 |
+
attempts = 0
|
| 483 |
+
while True:
|
| 484 |
+
attempts += 1
|
| 485 |
+
try:
|
| 486 |
+
api.upload_file(
|
| 487 |
+
path_or_fileobj=str(local_path),
|
| 488 |
+
path_in_repo=path_in_repo,
|
| 489 |
+
repo_id=dataset_repo,
|
| 490 |
+
repo_type="dataset",
|
| 491 |
+
)
|
| 492 |
+
return
|
| 493 |
+
except Exception:
|
| 494 |
+
if attempts > max_retries:
|
| 495 |
+
raise
|
| 496 |
+
time.sleep(retry_backoff_seconds * attempts)
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
def ingest_segments_to_hf(
|
| 500 |
+
api: HfApi,
|
| 501 |
+
project_slug: str,
|
| 502 |
+
dataset_repo: str,
|
| 503 |
+
detections_csv: str,
|
| 504 |
+
segments_root: str,
|
| 505 |
+
batch_size: int,
|
| 506 |
+
shard_size: int,
|
| 507 |
+
max_retries: int,
|
| 508 |
+
retry_backoff_seconds: float,
|
| 509 |
+
resume_state_file: str,
|
| 510 |
+
) -> dict[str, Any]:
|
| 511 |
+
ensure_project_dataset_structure(
|
| 512 |
+
api=api,
|
| 513 |
+
project_slug=project_slug,
|
| 514 |
+
dataset_repo=dataset_repo,
|
| 515 |
+
create_private_repo=False,
|
| 516 |
+
)
|
| 517 |
+
|
| 518 |
+
match_result = _match_birdnet_rows(
|
| 519 |
+
project_slug=project_slug,
|
| 520 |
+
detections_csv=detections_csv,
|
| 521 |
+
segments_root=segments_root,
|
| 522 |
+
)
|
| 523 |
+
frame = match_result["frame"]
|
| 524 |
+
matched_rows = match_result["matched_rows"]
|
| 525 |
+
unmatched_rows = match_result["unmatched_rows"]
|
| 526 |
+
ambiguous_rows = match_result["ambiguous_rows"]
|
| 527 |
+
|
| 528 |
+
# Deduplicate by remote path so retries/resume work on a stable key.
|
| 529 |
+
path_to_local: dict[str, Path] = {}
|
| 530 |
+
for row in matched_rows:
|
| 531 |
+
repo_path = f"audio/segments/{row['segment_relpath']}"
|
| 532 |
+
if repo_path in path_to_local:
|
| 533 |
+
continue
|
| 534 |
+
path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
|
| 535 |
+
|
| 536 |
+
state_path = Path(resume_state_file)
|
| 537 |
+
state_payload = _load_resume_state(state_file=state_path)
|
| 538 |
+
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 539 |
+
failed_state = set(str(item) for item in state_payload["failed"])
|
| 540 |
+
|
| 541 |
+
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 542 |
+
|
| 543 |
+
pending: list[tuple[Path, str]] = []
|
| 544 |
+
skipped_existing = 0
|
| 545 |
+
for repo_path, local_path in path_to_local.items():
|
| 546 |
+
if repo_path in remote_files:
|
| 547 |
+
uploaded_state.add(repo_path)
|
| 548 |
+
skipped_existing += 1
|
| 549 |
+
continue
|
| 550 |
+
if repo_path in uploaded_state:
|
| 551 |
+
continue
|
| 552 |
+
pending.append((local_path, repo_path))
|
| 553 |
+
|
| 554 |
+
uploaded_now: set[str] = set()
|
| 555 |
+
failed_now: set[str] = set()
|
| 556 |
+
|
| 557 |
+
for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
|
| 558 |
+
batch_uploaded = 0
|
| 559 |
+
batch_failed = 0
|
| 560 |
+
for local_path, repo_path in batch:
|
| 561 |
+
try:
|
| 562 |
+
_upload_audio_with_retry(
|
| 563 |
+
api=api,
|
| 564 |
+
dataset_repo=dataset_repo,
|
| 565 |
+
local_path=local_path,
|
| 566 |
+
path_in_repo=repo_path,
|
| 567 |
+
max_retries=max_retries,
|
| 568 |
+
retry_backoff_seconds=retry_backoff_seconds,
|
| 569 |
+
)
|
| 570 |
+
uploaded_state.add(repo_path)
|
| 571 |
+
failed_state.discard(repo_path)
|
| 572 |
+
uploaded_now.add(repo_path)
|
| 573 |
+
batch_uploaded += 1
|
| 574 |
+
except Exception:
|
| 575 |
+
failed_state.add(repo_path)
|
| 576 |
+
failed_now.add(repo_path)
|
| 577 |
+
batch_failed += 1
|
| 578 |
+
|
| 579 |
+
print(
|
| 580 |
+
json.dumps(
|
| 581 |
+
{
|
| 582 |
+
"event": "ingest-segments-audio-batch",
|
| 583 |
+
"batch_index": batch_index,
|
| 584 |
+
"batch_size": len(batch),
|
| 585 |
+
"uploaded": batch_uploaded,
|
| 586 |
+
"failed": batch_failed,
|
| 587 |
+
}
|
| 588 |
+
)
|
| 589 |
+
)
|
| 590 |
+
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
|
| 591 |
+
|
| 592 |
+
index_frame = pd.DataFrame(matched_rows)
|
| 593 |
+
if not index_frame.empty:
|
| 594 |
+
index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
|
| 595 |
+
|
| 596 |
+
with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
|
| 597 |
+
temp_dir = Path(temp_dir_str)
|
| 598 |
+
shard_dir = temp_dir / "index" / "shards"
|
| 599 |
+
shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
|
| 600 |
+
|
| 601 |
+
for item in shard_metadata:
|
| 602 |
+
local_path = temp_dir / item.path
|
| 603 |
+
api.upload_file(
|
| 604 |
+
path_or_fileobj=str(local_path),
|
| 605 |
+
path_in_repo=item.path,
|
| 606 |
+
repo_id=dataset_repo,
|
| 607 |
+
repo_type="dataset",
|
| 608 |
+
)
|
| 609 |
+
|
| 610 |
+
manifest = build_manifest_payload(
|
| 611 |
+
project_slug=project_slug,
|
| 612 |
+
dataset_repo=dataset_repo,
|
| 613 |
+
frame=index_frame,
|
| 614 |
+
shard_size=shard_size,
|
| 615 |
+
shard_metadata=shard_metadata,
|
| 616 |
+
)
|
| 617 |
+
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 618 |
+
|
| 619 |
+
run_report = {
|
| 620 |
+
"mode": "execute",
|
| 621 |
+
"project_slug": project_slug,
|
| 622 |
+
"dataset_repo": dataset_repo,
|
| 623 |
+
"detections_csv": detections_csv,
|
| 624 |
+
"segments_root": segments_root,
|
| 625 |
+
"csv_rows_total": int(len(frame)),
|
| 626 |
+
"segments_found_total": int(match_result["segment_records_total"]),
|
| 627 |
+
"matched_rows": len(matched_rows),
|
| 628 |
+
"unmatched_rows": len(unmatched_rows),
|
| 629 |
+
"ambiguous_rows": len(ambiguous_rows),
|
| 630 |
+
"unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
|
| 631 |
+
"pending_audio_uploads": len(pending),
|
| 632 |
+
"uploaded_audio_now": len(uploaded_now),
|
| 633 |
+
"uploaded_audio_skipped_existing": skipped_existing,
|
| 634 |
+
"failed_uploads": len(failed_now),
|
| 635 |
+
"resume_state_file": str(state_path),
|
| 636 |
+
"index_rows_written": len(index_frame),
|
| 637 |
+
"shards_written": len(shard_metadata),
|
| 638 |
+
"sample_unmatched": unmatched_rows[:20],
|
| 639 |
+
"sample_ambiguous": ambiguous_rows[:20],
|
| 640 |
+
}
|
| 641 |
+
|
| 642 |
+
audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
|
| 643 |
+
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
|
| 644 |
+
return run_report
|
| 645 |
+
|
| 646 |
+
|
| 647 |
+
def sync_audio_batches(
|
| 648 |
+
api: HfApi,
|
| 649 |
+
project_slug: str,
|
| 650 |
+
dataset_repo: str,
|
| 651 |
+
local_audio_dir: str,
|
| 652 |
+
batch_size: int,
|
| 653 |
+
max_retries: int,
|
| 654 |
+
retry_backoff_seconds: float,
|
| 655 |
+
resume_state_file: str,
|
| 656 |
+
) -> dict[str, Any]:
|
| 657 |
+
ensure_project_dataset_structure(
|
| 658 |
+
api=api,
|
| 659 |
+
project_slug=project_slug,
|
| 660 |
+
dataset_repo=dataset_repo,
|
| 661 |
+
create_private_repo=False,
|
| 662 |
+
)
|
| 663 |
+
|
| 664 |
+
all_files = discover_audio_files(local_audio_dir=local_audio_dir)
|
| 665 |
+
state_path = Path(resume_state_file)
|
| 666 |
+
state_payload = _load_resume_state(state_file=state_path)
|
| 667 |
+
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 668 |
+
|
| 669 |
+
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 670 |
+
|
| 671 |
+
pending: list[tuple[Path, str]] = []
|
| 672 |
+
for path in all_files:
|
| 673 |
+
rel = path.relative_to(Path(local_audio_dir)).as_posix()
|
| 674 |
+
repo_path = f"audio/{rel}"
|
| 675 |
+
|
| 676 |
+
if repo_path in remote_files:
|
| 677 |
+
uploaded_state.add(repo_path)
|
| 678 |
+
continue
|
| 679 |
+
|
| 680 |
+
if repo_path in uploaded_state:
|
| 681 |
+
continue
|
| 682 |
+
|
| 683 |
+
pending.append((path, repo_path))
|
| 684 |
+
|
| 685 |
+
pending_paths = [item[0] for item in pending]
|
| 686 |
+
by_local_to_repo = {local: repo for local, repo in pending}
|
| 687 |
+
batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
|
| 688 |
+
|
| 689 |
+
uploaded_now: set[str] = set()
|
| 690 |
+
failed: set[str] = set()
|
| 691 |
+
|
| 692 |
+
for batch_index, batch in enumerate(batches, start=1):
|
| 693 |
+
batch_uploaded = 0
|
| 694 |
+
batch_failed = 0
|
| 695 |
+
|
| 696 |
+
for local_path in batch:
|
| 697 |
+
repo_path = by_local_to_repo[local_path]
|
| 698 |
+
try:
|
| 699 |
+
_upload_audio_with_retry(
|
| 700 |
+
api=api,
|
| 701 |
+
dataset_repo=dataset_repo,
|
| 702 |
+
local_path=local_path,
|
| 703 |
+
path_in_repo=repo_path,
|
| 704 |
+
max_retries=max_retries,
|
| 705 |
+
retry_backoff_seconds=retry_backoff_seconds,
|
| 706 |
+
)
|
| 707 |
+
uploaded_now.add(repo_path)
|
| 708 |
+
uploaded_state.add(repo_path)
|
| 709 |
+
batch_uploaded += 1
|
| 710 |
+
except Exception:
|
| 711 |
+
failed.add(repo_path)
|
| 712 |
+
batch_failed += 1
|
| 713 |
+
|
| 714 |
+
print(
|
| 715 |
+
json.dumps(
|
| 716 |
+
{
|
| 717 |
+
"event": "sync-audio-batch",
|
| 718 |
+
"batch_index": batch_index,
|
| 719 |
+
"batch_size": len(batch),
|
| 720 |
+
"uploaded": batch_uploaded,
|
| 721 |
+
"failed": batch_failed,
|
| 722 |
+
}
|
| 723 |
+
)
|
| 724 |
+
)
|
| 725 |
+
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
|
| 726 |
+
|
| 727 |
+
return {
|
| 728 |
+
"project_slug": project_slug,
|
| 729 |
+
"dataset_repo": dataset_repo,
|
| 730 |
+
"total_local_audio_files": len(all_files),
|
| 731 |
+
"pending_uploads": len(pending),
|
| 732 |
+
"uploaded_now": len(uploaded_now),
|
| 733 |
+
"failed": len(failed),
|
| 734 |
+
"resume_state_file": str(state_path),
|
| 735 |
+
}
|
| 736 |
+
|
| 737 |
+
|
| 738 |
+
def build_and_upload_index(
|
| 739 |
+
api: HfApi,
|
| 740 |
+
project_slug: str,
|
| 741 |
+
dataset_repo: str,
|
| 742 |
+
detections_file: str,
|
| 743 |
+
shard_size: int,
|
| 744 |
+
) -> dict[str, Any]:
|
| 745 |
+
ensure_project_dataset_structure(
|
| 746 |
+
api=api,
|
| 747 |
+
project_slug=project_slug,
|
| 748 |
+
dataset_repo=dataset_repo,
|
| 749 |
+
create_private_repo=False,
|
| 750 |
+
)
|
| 751 |
+
|
| 752 |
+
frame = load_detections_table(detections_file=detections_file)
|
| 753 |
+
|
| 754 |
+
with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
|
| 755 |
+
temp_dir = Path(temp_dir_str)
|
| 756 |
+
shard_dir = temp_dir / "index" / "shards"
|
| 757 |
+
metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
|
| 758 |
+
|
| 759 |
+
for item in metadata:
|
| 760 |
+
local_path = temp_dir / item.path
|
| 761 |
+
api.upload_file(
|
| 762 |
+
path_or_fileobj=str(local_path),
|
| 763 |
+
path_in_repo=item.path,
|
| 764 |
+
repo_id=dataset_repo,
|
| 765 |
+
repo_type="dataset",
|
| 766 |
+
)
|
| 767 |
+
|
| 768 |
+
manifest = build_manifest_payload(
|
| 769 |
+
project_slug=project_slug,
|
| 770 |
+
dataset_repo=dataset_repo,
|
| 771 |
+
frame=frame,
|
| 772 |
+
shard_size=shard_size,
|
| 773 |
+
shard_metadata=metadata,
|
| 774 |
+
)
|
| 775 |
+
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 776 |
+
|
| 777 |
+
return {
|
| 778 |
+
"project_slug": project_slug,
|
| 779 |
+
"dataset_repo": dataset_repo,
|
| 780 |
+
"total_detections": manifest["index"]["total_detections"],
|
| 781 |
+
"total_audio_files": manifest["index"]["total_audio_files"],
|
| 782 |
+
"total_shards": len(metadata),
|
| 783 |
+
}
|
| 784 |
+
|
| 785 |
+
|
| 786 |
+
def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
|
| 787 |
+
errors: list[str] = []
|
| 788 |
+
|
| 789 |
+
for prefix in REQUIRED_PROJECT_PREFIXES:
|
| 790 |
+
if not any(path.startswith(prefix) for path in repo_files):
|
| 791 |
+
errors.append(f"Missing prefix in dataset repo: {prefix}")
|
| 792 |
+
|
| 793 |
+
if manifest_payload.get("schema_version") != SCHEMA_VERSION:
|
| 794 |
+
errors.append(
|
| 795 |
+
f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
|
| 796 |
+
)
|
| 797 |
+
|
| 798 |
+
if manifest_payload.get("project_slug") != project_slug:
|
| 799 |
+
errors.append(
|
| 800 |
+
f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
|
| 801 |
+
)
|
| 802 |
+
|
| 803 |
+
index_section = manifest_payload.get("index", {})
|
| 804 |
+
shards = index_section.get("shards", [])
|
| 805 |
+
if not isinstance(shards, list):
|
| 806 |
+
errors.append("Manifest index.shards must be a list")
|
| 807 |
+
return errors
|
| 808 |
+
|
| 809 |
+
for shard in shards:
|
| 810 |
+
shard_path = shard.get("path")
|
| 811 |
+
if not shard_path or not isinstance(shard_path, str):
|
| 812 |
+
errors.append("Shard entry missing path")
|
| 813 |
+
continue
|
| 814 |
+
if shard_path not in repo_files:
|
| 815 |
+
errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
|
| 816 |
+
|
| 817 |
+
return errors
|
| 818 |
+
|
| 819 |
+
|
| 820 |
+
def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
|
| 821 |
+
repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 822 |
+
if "manifest.json" not in repo_files:
|
| 823 |
+
return {
|
| 824 |
+
"ok": False,
|
| 825 |
+
"errors": ["manifest.json not found in dataset repository"],
|
| 826 |
+
}
|
| 827 |
+
|
| 828 |
+
manifest_local = hf_hub_download(
|
| 829 |
+
repo_id=dataset_repo,
|
| 830 |
+
repo_type="dataset",
|
| 831 |
+
filename="manifest.json",
|
| 832 |
+
)
|
| 833 |
+
manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
|
| 834 |
+
errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
|
| 835 |
+
|
| 836 |
+
return {
|
| 837 |
+
"ok": len(errors) == 0,
|
| 838 |
+
"errors": errors,
|
| 839 |
+
"total_files": len(repo_files),
|
| 840 |
+
"total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
|
| 841 |
+
}
|
| 842 |
+
|
| 843 |
+
|
| 844 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 845 |
+
parser = argparse.ArgumentParser(description="HF dataset project CLI")
|
| 846 |
+
sub = parser.add_subparsers(dest="command", required=True)
|
| 847 |
+
|
| 848 |
+
create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
|
| 849 |
+
create_project.add_argument("--project-slug", required=True)
|
| 850 |
+
create_project.add_argument("--dataset-repo", required=True)
|
| 851 |
+
create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
|
| 852 |
+
|
| 853 |
+
build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
|
| 854 |
+
build_index.add_argument("--project-slug", required=True)
|
| 855 |
+
build_index.add_argument("--dataset-repo", required=True)
|
| 856 |
+
build_index.add_argument("--detections-file", required=True)
|
| 857 |
+
build_index.add_argument("--shard-size", type=int, default=10000)
|
| 858 |
+
|
| 859 |
+
sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
|
| 860 |
+
sync_audio.add_argument("--project-slug", required=True)
|
| 861 |
+
sync_audio.add_argument("--dataset-repo", required=True)
|
| 862 |
+
sync_audio.add_argument("--local-audio-dir", required=True)
|
| 863 |
+
sync_audio.add_argument("--batch-size", type=int, default=100)
|
| 864 |
+
sync_audio.add_argument("--max-retries", type=int, default=3)
|
| 865 |
+
sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 866 |
+
sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
|
| 867 |
+
|
| 868 |
+
verify_project = sub.add_parser("verify-project", help="Verify project integrity")
|
| 869 |
+
verify_project.add_argument("--project-slug", required=True)
|
| 870 |
+
verify_project.add_argument("--dataset-repo", required=True)
|
| 871 |
+
|
| 872 |
+
ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
|
| 873 |
+
ingest_segments.add_argument("--project-slug", required=True)
|
| 874 |
+
ingest_segments.add_argument("--dataset-repo", required=True)
|
| 875 |
+
ingest_segments.add_argument("--detections-csv", required=True)
|
| 876 |
+
ingest_segments.add_argument("--segments-root", required=True)
|
| 877 |
+
ingest_segments.add_argument("--batch-size", type=int, default=200)
|
| 878 |
+
ingest_segments.add_argument("--shard-size", type=int, default=10000)
|
| 879 |
+
ingest_segments.add_argument("--max-retries", type=int, default=3)
|
| 880 |
+
ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 881 |
+
ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
|
| 882 |
+
ingest_segments.add_argument("--dry-run", action="store_true")
|
| 883 |
+
ingest_segments.add_argument("--report-file", default="")
|
| 884 |
+
|
| 885 |
+
return parser
|
| 886 |
+
|
| 887 |
+
|
| 888 |
+
def main() -> int:
|
| 889 |
+
parser = build_parser()
|
| 890 |
+
args = parser.parse_args()
|
| 891 |
+
api = HfApi()
|
| 892 |
+
|
| 893 |
+
if args.command == "create-project":
|
| 894 |
+
result = ensure_project_dataset_structure(
|
| 895 |
+
api=api,
|
| 896 |
+
project_slug=args.project_slug,
|
| 897 |
+
dataset_repo=args.dataset_repo,
|
| 898 |
+
create_private_repo=bool(args.private),
|
| 899 |
+
)
|
| 900 |
+
print(json.dumps(result, indent=2))
|
| 901 |
+
return 0
|
| 902 |
+
|
| 903 |
+
if args.command == "build-index":
|
| 904 |
+
result = build_and_upload_index(
|
| 905 |
+
api=api,
|
| 906 |
+
project_slug=args.project_slug,
|
| 907 |
+
dataset_repo=args.dataset_repo,
|
| 908 |
+
detections_file=args.detections_file,
|
| 909 |
+
shard_size=args.shard_size,
|
| 910 |
+
)
|
| 911 |
+
print(json.dumps(result, indent=2))
|
| 912 |
+
return 0
|
| 913 |
+
|
| 914 |
+
if args.command == "sync-audio":
|
| 915 |
+
result = sync_audio_batches(
|
| 916 |
+
api=api,
|
| 917 |
+
project_slug=args.project_slug,
|
| 918 |
+
dataset_repo=args.dataset_repo,
|
| 919 |
+
local_audio_dir=args.local_audio_dir,
|
| 920 |
+
batch_size=args.batch_size,
|
| 921 |
+
max_retries=args.max_retries,
|
| 922 |
+
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 923 |
+
resume_state_file=args.resume_state_file,
|
| 924 |
+
)
|
| 925 |
+
print(json.dumps(result, indent=2))
|
| 926 |
+
return 0 if result["failed"] == 0 else 1
|
| 927 |
+
|
| 928 |
+
if args.command == "verify-project":
|
| 929 |
+
result = verify_project(
|
| 930 |
+
api=api,
|
| 931 |
+
project_slug=args.project_slug,
|
| 932 |
+
dataset_repo=args.dataset_repo,
|
| 933 |
+
)
|
| 934 |
+
print(json.dumps(result, indent=2))
|
| 935 |
+
return 0 if result["ok"] else 1
|
| 936 |
+
|
| 937 |
+
if args.command == "ingest-segments":
|
| 938 |
+
if args.dry_run:
|
| 939 |
+
result = run_ingest_segments_dry_run(
|
| 940 |
+
project_slug=args.project_slug,
|
| 941 |
+
detections_csv=args.detections_csv,
|
| 942 |
+
segments_root=args.segments_root,
|
| 943 |
+
)
|
| 944 |
+
else:
|
| 945 |
+
result = ingest_segments_to_hf(
|
| 946 |
+
api=api,
|
| 947 |
+
project_slug=args.project_slug,
|
| 948 |
+
dataset_repo=args.dataset_repo,
|
| 949 |
+
detections_csv=args.detections_csv,
|
| 950 |
+
segments_root=args.segments_root,
|
| 951 |
+
batch_size=args.batch_size,
|
| 952 |
+
shard_size=args.shard_size,
|
| 953 |
+
max_retries=args.max_retries,
|
| 954 |
+
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 955 |
+
resume_state_file=args.resume_state_file,
|
| 956 |
+
)
|
| 957 |
+
|
| 958 |
+
if args.report_file:
|
| 959 |
+
Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
| 960 |
+
|
| 961 |
+
print(json.dumps(result, indent=2))
|
| 962 |
+
return 0 if result.get("failed_uploads", 0) == 0 else 1
|
| 963 |
+
|
| 964 |
+
parser.error("Unknown command")
|
| 965 |
+
return 2
|
| 966 |
+
|
| 967 |
+
|
| 968 |
+
if __name__ == "__main__":
|
| 969 |
+
raise SystemExit(main())
|
BirdNET Uploader App/pyproject.toml
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "BirdNET-validator-App"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
description = "BirdNET detection validator for Hugging Face Spaces"
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = ">=3.11"
|
| 7 |
+
|
| 8 |
+
[tool.pytest.ini_options]
|
| 9 |
+
testpaths = ["tests"]
|
BirdNET Uploader App/requirements.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio>=5.0.0
|
| 2 |
+
huggingface_hub>=0.30.0
|
| 3 |
+
pydantic>=2.8.0
|
| 4 |
+
pandas>=2.2.0
|
| 5 |
+
pyarrow>=17.0.0
|
| 6 |
+
soundfile>=0.12.1
|
| 7 |
+
pytest>=8.3.0
|
BirdNET Uploader App/src/__init__.py
ADDED
|
File without changes
|
BirdNET Uploader App/src/ui/__init__.py
ADDED
|
File without changes
|
BirdNET Uploader App/src/ui/upload_app_factory.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
from huggingface_hub import HfApi
|
| 7 |
+
|
| 8 |
+
from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _as_pretty_json(payload: dict[str, Any]) -> str:
|
| 12 |
+
return json.dumps(payload, indent=2, ensure_ascii=False)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _build_api(token: str) -> HfApi:
|
| 16 |
+
clean_token = (token or "").strip()
|
| 17 |
+
if clean_token:
|
| 18 |
+
return HfApi(token=clean_token)
|
| 19 |
+
return HfApi()
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def build_upload_app() -> gr.Blocks:
|
| 23 |
+
with gr.Blocks(title="BirdNET Segments Uploader") as demo:
|
| 24 |
+
gr.Markdown("# BirdNET Segments Uploader")
|
| 25 |
+
gr.Markdown(
|
| 26 |
+
"Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
|
| 27 |
+
"Recomendado: executar dry-run antes do upload real."
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
with gr.Row():
|
| 31 |
+
project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
|
| 32 |
+
dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
|
| 33 |
+
|
| 34 |
+
detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
|
| 35 |
+
segments_root = gr.Textbox(
|
| 36 |
+
label="Pasta raiz de segmentos",
|
| 37 |
+
placeholder=r"ex: C:\dados\BirdNET Segments",
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
with gr.Accordion("Configuracao avancada", open=False):
|
| 41 |
+
hf_token = gr.Textbox(
|
| 42 |
+
label="HF Token (opcional)",
|
| 43 |
+
type="password",
|
| 44 |
+
placeholder="Se vazio, usa sessao ja autenticada no ambiente",
|
| 45 |
+
)
|
| 46 |
+
with gr.Row():
|
| 47 |
+
batch_size = gr.Number(label="Batch size", value=200, precision=0)
|
| 48 |
+
shard_size = gr.Number(label="Shard size", value=10000, precision=0)
|
| 49 |
+
with gr.Row():
|
| 50 |
+
max_retries = gr.Number(label="Max retries", value=3, precision=0)
|
| 51 |
+
retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
|
| 52 |
+
resume_state_file = gr.Textbox(
|
| 53 |
+
label="Resume state file",
|
| 54 |
+
value=".ingest-segments-state.json",
|
| 55 |
+
)
|
| 56 |
+
report_file = gr.Textbox(
|
| 57 |
+
label="Report file local (opcional)",
|
| 58 |
+
placeholder="ex: .ingest-run-report.json",
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
with gr.Row():
|
| 62 |
+
dry_run_button = gr.Button("Dry-run", variant="secondary")
|
| 63 |
+
upload_button = gr.Button("Upload real", variant="primary")
|
| 64 |
+
|
| 65 |
+
status = gr.Markdown(value="Pronto")
|
| 66 |
+
result_json = gr.Code(label="Resultado JSON", language="json")
|
| 67 |
+
|
| 68 |
+
def run_dry_run(
|
| 69 |
+
project: str,
|
| 70 |
+
repo: str,
|
| 71 |
+
csv_path: str | None,
|
| 72 |
+
segments_path: str,
|
| 73 |
+
report_path: str,
|
| 74 |
+
) -> tuple[str, str]:
|
| 75 |
+
project = (project or "").strip()
|
| 76 |
+
if not project:
|
| 77 |
+
return "β Informe o project slug", "{}"
|
| 78 |
+
|
| 79 |
+
if not csv_path:
|
| 80 |
+
return "β Selecione o CSV de deteccoes", "{}"
|
| 81 |
+
|
| 82 |
+
if not segments_path or not Path(segments_path).exists():
|
| 83 |
+
return "β Pasta de segmentos invalida", "{}"
|
| 84 |
+
|
| 85 |
+
try:
|
| 86 |
+
result = run_ingest_segments_dry_run(
|
| 87 |
+
project_slug=project,
|
| 88 |
+
detections_csv=csv_path,
|
| 89 |
+
segments_root=segments_path,
|
| 90 |
+
)
|
| 91 |
+
if report_path.strip():
|
| 92 |
+
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 93 |
+
summary = (
|
| 94 |
+
"β
Dry-run finalizado | "
|
| 95 |
+
f"Matched: {result['matched_rows']} | "
|
| 96 |
+
f"Unmatched: {result['unmatched_rows']} | "
|
| 97 |
+
f"Ambiguous: {result['ambiguous_rows']}"
|
| 98 |
+
)
|
| 99 |
+
return summary, _as_pretty_json(result)
|
| 100 |
+
except Exception as exc:
|
| 101 |
+
return f"β Dry-run falhou: {exc}", "{}"
|
| 102 |
+
|
| 103 |
+
def run_upload(
|
| 104 |
+
project: str,
|
| 105 |
+
repo: str,
|
| 106 |
+
csv_path: str | None,
|
| 107 |
+
segments_path: str,
|
| 108 |
+
token: str,
|
| 109 |
+
batch: float,
|
| 110 |
+
shard: float,
|
| 111 |
+
retries: float,
|
| 112 |
+
backoff: float,
|
| 113 |
+
resume_file: str,
|
| 114 |
+
report_path: str,
|
| 115 |
+
) -> tuple[str, str]:
|
| 116 |
+
project = (project or "").strip()
|
| 117 |
+
repo = (repo or "").strip()
|
| 118 |
+
if not project:
|
| 119 |
+
return "β Informe o project slug", "{}"
|
| 120 |
+
if not repo or "/" not in repo:
|
| 121 |
+
return "β Informe dataset repo no formato owner/repo", "{}"
|
| 122 |
+
if not csv_path:
|
| 123 |
+
return "β Selecione o CSV de deteccoes", "{}"
|
| 124 |
+
if not segments_path or not Path(segments_path).exists():
|
| 125 |
+
return "β Pasta de segmentos invalida", "{}"
|
| 126 |
+
|
| 127 |
+
try:
|
| 128 |
+
api = _build_api(token)
|
| 129 |
+
result = ingest_segments_to_hf(
|
| 130 |
+
api=api,
|
| 131 |
+
project_slug=project,
|
| 132 |
+
dataset_repo=repo,
|
| 133 |
+
detections_csv=csv_path,
|
| 134 |
+
segments_root=segments_path,
|
| 135 |
+
batch_size=int(batch),
|
| 136 |
+
shard_size=int(shard),
|
| 137 |
+
max_retries=int(retries),
|
| 138 |
+
retry_backoff_seconds=float(backoff),
|
| 139 |
+
resume_state_file=resume_file,
|
| 140 |
+
)
|
| 141 |
+
if report_path.strip():
|
| 142 |
+
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 143 |
+
summary = (
|
| 144 |
+
"β
Upload finalizado | "
|
| 145 |
+
f"Uploaded now: {result['uploaded_audio_now']} | "
|
| 146 |
+
f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
|
| 147 |
+
f"Failed uploads: {result['failed_uploads']}"
|
| 148 |
+
)
|
| 149 |
+
return summary, _as_pretty_json(result)
|
| 150 |
+
except Exception as exc:
|
| 151 |
+
return f"β Upload falhou: {exc}", "{}"
|
| 152 |
+
|
| 153 |
+
dry_run_button.click(
|
| 154 |
+
fn=run_dry_run,
|
| 155 |
+
inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
|
| 156 |
+
outputs=[status, result_json],
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
upload_button.click(
|
| 160 |
+
fn=run_upload,
|
| 161 |
+
inputs=[
|
| 162 |
+
project_slug,
|
| 163 |
+
dataset_repo,
|
| 164 |
+
detections_csv,
|
| 165 |
+
segments_root,
|
| 166 |
+
hf_token,
|
| 167 |
+
batch_size,
|
| 168 |
+
shard_size,
|
| 169 |
+
max_retries,
|
| 170 |
+
retry_backoff_seconds,
|
| 171 |
+
resume_state_file,
|
| 172 |
+
report_file,
|
| 173 |
+
],
|
| 174 |
+
outputs=[status, result_json],
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
return demo
|
BirdNET Validator App/README.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BirdNET Validator App
|
| 2 |
+
|
| 3 |
+
Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
|
| 4 |
+
- Multi-projeto
|
| 5 |
+
- Audio sob demanda
|
| 6 |
+
- Persistencia de validacoes
|
| 7 |
+
- Escalabilidade para datasets grandes
|
| 8 |
+
|
| 9 |
+
## Desenvolvimento local
|
| 10 |
+
1. Criar ambiente virtual Python 3.11+
|
| 11 |
+
2. Instalar dependencias:
|
| 12 |
+
pip install -r requirements.txt
|
| 13 |
+
3. Rodar app:
|
| 14 |
+
python app.py
|
| 15 |
+
|
| 16 |
+
## Objetivo desta pasta
|
| 17 |
+
Contem os arquivos do app de validacao para evolucao/publicacao independente.
|
| 18 |
+
|
| 19 |
+
## Fluxo recomendado (separado)
|
| 20 |
+
1. Uploader app (porta 7862 por padrao):
|
| 21 |
+
- Rodar dry-run com CSV + pasta de segmentos
|
| 22 |
+
- Rodar upload real para owner/repo do projeto
|
| 23 |
+
2. Validator app (porta 7860):
|
| 24 |
+
- Validar deteccoes do projeto com dados ja publicados
|
| 25 |
+
|
| 26 |
+
Essa separacao evita impacto de performance na validacao durante uploads grandes.
|
| 27 |
+
|
| 28 |
+
## CLI de dataset (Sprint 1)
|
| 29 |
+
Comandos principais:
|
| 30 |
+
|
| 31 |
+
1. Criar estrutura do projeto no dataset Hugging Face:
|
| 32 |
+
python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 33 |
+
|
| 34 |
+
2. Gerar manifesto e shards iniciais de indice:
|
| 35 |
+
python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
|
| 36 |
+
|
| 37 |
+
3. Fazer upload de audio em lotes com retry e resume:
|
| 38 |
+
python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
|
| 39 |
+
|
| 40 |
+
4. Verificar consistencia basica do projeto:
|
| 41 |
+
python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 42 |
+
|
| 43 |
+
5. Ingestao BirdNET por projeto (dry-run):
|
| 44 |
+
python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
|
| 45 |
+
|
| 46 |
+
6. Ingestao BirdNET por projeto (upload real):
|
| 47 |
+
python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
|
| 48 |
+
|
| 49 |
+
Estrutura base criada por projeto/dataset:
|
| 50 |
+
- audio/
|
| 51 |
+
- index/
|
| 52 |
+
- index/shards/
|
| 53 |
+
- validations/
|
| 54 |
+
- audit/
|
| 55 |
+
- manifest.json
|
| 56 |
+
|
| 57 |
+
## Estrutura
|
| 58 |
+
- src/domain: modelos de dominio
|
| 59 |
+
- src/repositories: contratos de persistencia
|
| 60 |
+
- src/services: servicos de aplicacao
|
| 61 |
+
- src/auth: autenticacao e autorizacao
|
| 62 |
+
- src/ui: montagem da interface
|
| 63 |
+
- src/cache: cache efemero local
|
| 64 |
+
- cli: comandos de ingestao/publicacao
|
| 65 |
+
- tests: testes unitarios e integracao
|
| 66 |
+
|
| 67 |
+
## Estado atual de implementacao
|
| 68 |
+
- Sprint 1: CLI create-project, sync-audio, build-index, verify-project
|
| 69 |
+
- Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
|
| 70 |
+
- Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
|
| 71 |
+
- Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
|
| 72 |
+
- Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
|
| 73 |
+
- Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
|
| 74 |
+
- Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
|
| 75 |
+
- Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
|
| 76 |
+
- Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
|
| 77 |
+
- Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
|
| 78 |
+
- Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
|
| 79 |
+
- Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
|
| 80 |
+
- Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
|
| 81 |
+
- Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
|
| 82 |
+
- Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
|
| 83 |
+
- Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
|
BirdNET Validator App/app.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from src.ui.app_factory import create_app
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
if __name__ == "__main__":
|
| 5 |
+
app = create_app()
|
| 6 |
+
app.launch(server_name="0.0.0.0", server_port=7860)
|
BirdNET Validator App/pyproject.toml
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "BirdNET-validator-App"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
description = "BirdNET detection validator for Hugging Face Spaces"
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = ">=3.11"
|
| 7 |
+
|
| 8 |
+
[tool.pytest.ini_options]
|
| 9 |
+
testpaths = ["tests"]
|
BirdNET Validator App/requirements.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio>=5.0.0
|
| 2 |
+
huggingface_hub>=0.30.0
|
| 3 |
+
pydantic>=2.8.0
|
| 4 |
+
pandas>=2.2.0
|
| 5 |
+
pyarrow>=17.0.0
|
| 6 |
+
soundfile>=0.12.1
|
| 7 |
+
pytest>=8.3.0
|
BirdNET Validator App/src/__init__.py
ADDED
|
File without changes
|
BirdNET Validator App/src/auth/__init__.py
ADDED
|
File without changes
|
BirdNET Validator App/src/auth/auth_service.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AuthService: User authentication, session management, and project ACL."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass, field
|
| 4 |
+
from datetime import datetime, timedelta
|
| 5 |
+
from typing import Dict, List, Optional
|
| 6 |
+
import uuid
|
| 7 |
+
|
| 8 |
+
from src.domain.models import Role, User
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class UserProjectAccess:
|
| 13 |
+
"""Maps a user to projects with specific roles."""
|
| 14 |
+
|
| 15 |
+
username: str
|
| 16 |
+
project_slugs: Dict[str, Role] # Maps project_slug -> role (admin or validator)
|
| 17 |
+
is_active: bool = True
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@dataclass
|
| 21 |
+
class Session:
|
| 22 |
+
"""Represents an authenticated user session."""
|
| 23 |
+
|
| 24 |
+
session_id: str
|
| 25 |
+
username: str
|
| 26 |
+
role: Role
|
| 27 |
+
authorized_projects: List[str] # Projects this user can access
|
| 28 |
+
created_at: datetime
|
| 29 |
+
last_activity: datetime
|
| 30 |
+
expires_at: datetime
|
| 31 |
+
|
| 32 |
+
def is_expired(self) -> bool:
|
| 33 |
+
"""Check if session has expired."""
|
| 34 |
+
return datetime.utcnow() > self.expires_at
|
| 35 |
+
|
| 36 |
+
def update_activity(self, ttl_minutes: int = 120) -> None:
|
| 37 |
+
"""Update last activity timestamp and extend expiration."""
|
| 38 |
+
self.last_activity = datetime.utcnow()
|
| 39 |
+
self.expires_at = datetime.utcnow() + timedelta(minutes=ttl_minutes)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class AuthService:
|
| 43 |
+
"""Manages user authentication, sessions, and project access control."""
|
| 44 |
+
|
| 45 |
+
def __init__(self, session_ttl_minutes: int = 120):
|
| 46 |
+
"""Initialize AuthService with session TTL.
|
| 47 |
+
|
| 48 |
+
Args:
|
| 49 |
+
session_ttl_minutes: Session time-to-live in minutes (default: 2 hours)
|
| 50 |
+
"""
|
| 51 |
+
self.session_ttl_minutes = session_ttl_minutes
|
| 52 |
+
self._sessions: Dict[str, Session] = {} # session_id -> Session
|
| 53 |
+
self._user_access: Dict[str, UserProjectAccess] = {} # username -> UserProjectAccess
|
| 54 |
+
|
| 55 |
+
def register_user_project_access(
|
| 56 |
+
self, username: str, project_access: Dict[str, Role]
|
| 57 |
+
) -> None:
|
| 58 |
+
"""Register or update a user's access to projects.
|
| 59 |
+
|
| 60 |
+
Args:
|
| 61 |
+
username: User name
|
| 62 |
+
project_access: Dict mapping project_slug to Role (admin or validator)
|
| 63 |
+
"""
|
| 64 |
+
self._user_access[username] = UserProjectAccess(
|
| 65 |
+
username=username,
|
| 66 |
+
project_slugs=project_access,
|
| 67 |
+
is_active=True,
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
def login(self, username: str) -> Optional[Session]:
|
| 71 |
+
"""Authenticate a user and create a new session.
|
| 72 |
+
|
| 73 |
+
Args:
|
| 74 |
+
username: User name to authenticate
|
| 75 |
+
|
| 76 |
+
Returns:
|
| 77 |
+
Session if user exists and is active, None otherwise
|
| 78 |
+
"""
|
| 79 |
+
if username not in self._user_access:
|
| 80 |
+
return None
|
| 81 |
+
|
| 82 |
+
access = self._user_access[username]
|
| 83 |
+
if not access.is_active:
|
| 84 |
+
return None
|
| 85 |
+
|
| 86 |
+
# Determine highest role across all projects (admin > validator)
|
| 87 |
+
role = Role.validator
|
| 88 |
+
for proj_role in access.project_slugs.values():
|
| 89 |
+
if proj_role == Role.admin:
|
| 90 |
+
role = Role.admin
|
| 91 |
+
break
|
| 92 |
+
|
| 93 |
+
session_id = str(uuid.uuid4())
|
| 94 |
+
now = datetime.utcnow()
|
| 95 |
+
session = Session(
|
| 96 |
+
session_id=session_id,
|
| 97 |
+
username=username,
|
| 98 |
+
role=role,
|
| 99 |
+
authorized_projects=list(access.project_slugs.keys()),
|
| 100 |
+
created_at=now,
|
| 101 |
+
last_activity=now,
|
| 102 |
+
expires_at=now + timedelta(minutes=self.session_ttl_minutes),
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
self._sessions[session_id] = session
|
| 106 |
+
return session
|
| 107 |
+
|
| 108 |
+
def get_session(self, session_id: str) -> Optional[Session]:
|
| 109 |
+
"""Retrieve an active session by ID.
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
session_id: Session ID
|
| 113 |
+
|
| 114 |
+
Returns:
|
| 115 |
+
Session if valid and not expired, None otherwise
|
| 116 |
+
"""
|
| 117 |
+
session = self._sessions.get(session_id)
|
| 118 |
+
if session is None:
|
| 119 |
+
return None
|
| 120 |
+
|
| 121 |
+
if session.is_expired():
|
| 122 |
+
self._sessions.pop(session_id, None)
|
| 123 |
+
return None
|
| 124 |
+
|
| 125 |
+
session.update_activity(self.session_ttl_minutes)
|
| 126 |
+
return session
|
| 127 |
+
|
| 128 |
+
def logout(self, session_id: str) -> None:
|
| 129 |
+
"""Delete a session.
|
| 130 |
+
|
| 131 |
+
Args:
|
| 132 |
+
session_id: Session ID to invalidate
|
| 133 |
+
"""
|
| 134 |
+
self._sessions.pop(session_id, None)
|
| 135 |
+
|
| 136 |
+
def is_user_authorized_for_project(self, username: str, project_slug: str) -> bool:
|
| 137 |
+
"""Check if a user has access to a specific project.
|
| 138 |
+
|
| 139 |
+
Args:
|
| 140 |
+
username: User name
|
| 141 |
+
project_slug: Project slug
|
| 142 |
+
|
| 143 |
+
Returns:
|
| 144 |
+
True if user has access (admin or validator), False otherwise
|
| 145 |
+
"""
|
| 146 |
+
access = self._user_access.get(username)
|
| 147 |
+
if access is None or not access.is_active:
|
| 148 |
+
return False
|
| 149 |
+
|
| 150 |
+
return project_slug in access.project_slugs
|
| 151 |
+
|
| 152 |
+
def get_user_role_for_project(self, username: str, project_slug: str) -> Optional[Role]:
|
| 153 |
+
"""Get the specific role a user has for a project.
|
| 154 |
+
|
| 155 |
+
Args:
|
| 156 |
+
username: User name
|
| 157 |
+
project_slug: Project slug
|
| 158 |
+
|
| 159 |
+
Returns:
|
| 160 |
+
Role if user has access, None otherwise
|
| 161 |
+
"""
|
| 162 |
+
access = self._user_access.get(username)
|
| 163 |
+
if access is None or not access.is_active:
|
| 164 |
+
return None
|
| 165 |
+
|
| 166 |
+
return access.project_slugs.get(project_slug)
|
| 167 |
+
|
| 168 |
+
def list_user_projects(self, username: str) -> List[str]:
|
| 169 |
+
"""List all projects a user has access to.
|
| 170 |
+
|
| 171 |
+
Args:
|
| 172 |
+
username: User name
|
| 173 |
+
|
| 174 |
+
Returns:
|
| 175 |
+
List of project slugs
|
| 176 |
+
"""
|
| 177 |
+
access = self._user_access.get(username)
|
| 178 |
+
if access is None or not access.is_active:
|
| 179 |
+
return []
|
| 180 |
+
|
| 181 |
+
return list(access.project_slugs.keys())
|
| 182 |
+
|
| 183 |
+
def cleanup_expired_sessions(self) -> None:
|
| 184 |
+
"""Remove all expired sessions (maintenance cleanup)."""
|
| 185 |
+
now = datetime.utcnow()
|
| 186 |
+
expired = [
|
| 187 |
+
sid for sid, session in self._sessions.items() if session.is_expired()
|
| 188 |
+
]
|
| 189 |
+
for sid in expired:
|
| 190 |
+
self._sessions.pop(sid)
|
| 191 |
+
|
| 192 |
+
def set_user_active(self, username: str, active: bool) -> None:
|
| 193 |
+
"""Enable or disable a user account.
|
| 194 |
+
|
| 195 |
+
Args:
|
| 196 |
+
username: User name
|
| 197 |
+
active: Whether the user should be active
|
| 198 |
+
"""
|
| 199 |
+
if username in self._user_access:
|
| 200 |
+
self._user_access[username].is_active = active
|
BirdNET Validator App/src/cache/__init__.py
ADDED
|
File without changes
|
BirdNET Validator App/src/cache/ephemeral_cache_manager.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import tempfile
|
| 3 |
+
import time
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
@dataclass(slots=True)
|
| 9 |
+
class CacheEntry:
|
| 10 |
+
key: str
|
| 11 |
+
path: Path
|
| 12 |
+
created_at: float
|
| 13 |
+
expires_at: float
|
| 14 |
+
size_bytes: int
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class EphemeralCacheManager:
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
cache_dir: str | None = None,
|
| 21 |
+
ttl_seconds: int = 300,
|
| 22 |
+
max_files: int = 256,
|
| 23 |
+
) -> None:
|
| 24 |
+
self._base_dir = Path(cache_dir) if cache_dir else Path(tempfile.gettempdir()) / "birdnet-validator-cache"
|
| 25 |
+
self._base_dir.mkdir(parents=True, exist_ok=True)
|
| 26 |
+
self._ttl_seconds = ttl_seconds
|
| 27 |
+
self._max_files = max_files
|
| 28 |
+
self._entries: dict[str, CacheEntry] = {}
|
| 29 |
+
|
| 30 |
+
def get(self, key: str) -> Path | None:
|
| 31 |
+
self.cleanup_expired()
|
| 32 |
+
entry = self._entries.get(key)
|
| 33 |
+
if not entry:
|
| 34 |
+
return None
|
| 35 |
+
if not entry.path.exists():
|
| 36 |
+
self._entries.pop(key, None)
|
| 37 |
+
return None
|
| 38 |
+
return entry.path
|
| 39 |
+
|
| 40 |
+
def put_bytes(self, key: str, data: bytes, suffix: str = ".bin") -> Path:
|
| 41 |
+
self.cleanup_expired()
|
| 42 |
+
self._enforce_capacity()
|
| 43 |
+
|
| 44 |
+
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
|
| 45 |
+
file_path = self._base_dir / f"{digest}{suffix}"
|
| 46 |
+
file_path.write_bytes(data)
|
| 47 |
+
|
| 48 |
+
now = time.time()
|
| 49 |
+
self._entries[key] = CacheEntry(
|
| 50 |
+
key=key,
|
| 51 |
+
path=file_path,
|
| 52 |
+
created_at=now,
|
| 53 |
+
expires_at=now + self._ttl_seconds,
|
| 54 |
+
size_bytes=len(data),
|
| 55 |
+
)
|
| 56 |
+
return file_path
|
| 57 |
+
|
| 58 |
+
def cleanup_key(self, key: str) -> None:
|
| 59 |
+
entry = self._entries.pop(key, None)
|
| 60 |
+
if not entry:
|
| 61 |
+
return
|
| 62 |
+
if entry.path.exists():
|
| 63 |
+
entry.path.unlink()
|
| 64 |
+
|
| 65 |
+
def cleanup_expired(self) -> None:
|
| 66 |
+
now = time.time()
|
| 67 |
+
expired_keys = [key for key, entry in self._entries.items() if entry.expires_at <= now or not entry.path.exists()]
|
| 68 |
+
for key in expired_keys:
|
| 69 |
+
self.cleanup_key(key)
|
| 70 |
+
|
| 71 |
+
def clear(self) -> None:
|
| 72 |
+
for key in list(self._entries.keys()):
|
| 73 |
+
self.cleanup_key(key)
|
| 74 |
+
|
| 75 |
+
def _enforce_capacity(self) -> None:
|
| 76 |
+
if len(self._entries) < self._max_files:
|
| 77 |
+
return
|
| 78 |
+
|
| 79 |
+
oldest_keys = sorted(self._entries.keys(), key=lambda key: self._entries[key].created_at)
|
| 80 |
+
overflow = (len(self._entries) - self._max_files) + 1
|
| 81 |
+
for key in oldest_keys[:overflow]:
|
| 82 |
+
self.cleanup_key(key)
|
BirdNET Validator App/src/domain/__init__.py
ADDED
|
File without changes
|
BirdNET Validator App/src/domain/models.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from enum import Enum
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
from pydantic import BaseModel, Field
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class Role(str, Enum):
|
| 8 |
+
admin = "admin"
|
| 9 |
+
validator = "validator"
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class User(BaseModel):
|
| 13 |
+
username: str = Field(min_length=1)
|
| 14 |
+
role: Role
|
| 15 |
+
active: bool = True
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class Project(BaseModel):
|
| 19 |
+
project_slug: str = Field(min_length=3)
|
| 20 |
+
name: str = Field(min_length=1)
|
| 21 |
+
dataset_repo_id: str = Field(min_length=3)
|
| 22 |
+
active: bool = True
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class Detection(BaseModel):
|
| 26 |
+
detection_key: str = Field(min_length=16)
|
| 27 |
+
audio_id: str = Field(min_length=1)
|
| 28 |
+
scientific_name: str = Field(min_length=1)
|
| 29 |
+
confidence: float = Field(ge=0.0, le=1.0)
|
| 30 |
+
start_time: float = Field(ge=0.0)
|
| 31 |
+
end_time: float = Field(gt=0.0)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class Validation(BaseModel):
|
| 35 |
+
detection_key: str = Field(min_length=16)
|
| 36 |
+
status: str = Field(min_length=1)
|
| 37 |
+
corrected_species: Optional[str] = None
|
| 38 |
+
notes: str = ""
|
| 39 |
+
validator: str = Field(min_length=1)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class IndexManifest(BaseModel):
|
| 43 |
+
schema_version: str = "1.0.0"
|
| 44 |
+
project_slug: str
|
| 45 |
+
total_detections: int = Field(ge=0)
|
| 46 |
+
total_audio_files: int = Field(ge=0)
|
BirdNET Validator App/src/repositories/__init__.py
ADDED
|
File without changes
|
BirdNET Validator App/src/repositories/append_only_validation_repository.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from datetime import UTC, datetime
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from uuid import uuid4
|
| 5 |
+
|
| 6 |
+
from src.domain.models import Validation
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class OptimisticLockError(Exception):
|
| 10 |
+
def __init__(self, detection_key: str, expected_version: int, current_version: int) -> None:
|
| 11 |
+
self.detection_key = detection_key
|
| 12 |
+
self.expected_version = expected_version
|
| 13 |
+
self.current_version = current_version
|
| 14 |
+
super().__init__(
|
| 15 |
+
f"Optimistic lock conflict for detection_key={detection_key}: "
|
| 16 |
+
f"expected_version={expected_version}, current_version={current_version}"
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class AppendOnlyValidationRepository:
|
| 21 |
+
def __init__(self, base_dir: str) -> None:
|
| 22 |
+
self._base_dir = Path(base_dir)
|
| 23 |
+
|
| 24 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 25 |
+
project_dir = self._base_dir / project_slug / "validations"
|
| 26 |
+
project_dir.mkdir(parents=True, exist_ok=True)
|
| 27 |
+
|
| 28 |
+
current_file = project_dir / "current.json"
|
| 29 |
+
if current_file.exists():
|
| 30 |
+
current_payload = json.loads(current_file.read_text(encoding="utf-8"))
|
| 31 |
+
else:
|
| 32 |
+
current_payload = {}
|
| 33 |
+
|
| 34 |
+
current_item = current_payload.get(item.detection_key, {})
|
| 35 |
+
current_version = int(current_item.get("version", 0))
|
| 36 |
+
expected = expected_version if expected_version is not None else current_version
|
| 37 |
+
if expected != current_version:
|
| 38 |
+
raise OptimisticLockError(
|
| 39 |
+
detection_key=item.detection_key,
|
| 40 |
+
expected_version=expected,
|
| 41 |
+
current_version=current_version,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
new_version = current_version + 1
|
| 45 |
+
|
| 46 |
+
event_date = datetime.now(UTC).strftime("%Y%m%d")
|
| 47 |
+
events_file = project_dir / f"events-{event_date}.jsonl"
|
| 48 |
+
event = {
|
| 49 |
+
"event_id": str(uuid4()),
|
| 50 |
+
"timestamp": datetime.now(UTC).isoformat(),
|
| 51 |
+
"project_slug": project_slug,
|
| 52 |
+
"expected_version": expected,
|
| 53 |
+
"previous_version": current_version,
|
| 54 |
+
"new_version": new_version,
|
| 55 |
+
**item.model_dump(),
|
| 56 |
+
}
|
| 57 |
+
with events_file.open("a", encoding="utf-8") as handle:
|
| 58 |
+
handle.write(json.dumps(event, ensure_ascii=True) + "\n")
|
| 59 |
+
|
| 60 |
+
current_payload[item.detection_key] = {
|
| 61 |
+
"status": item.status,
|
| 62 |
+
"corrected_species": item.corrected_species,
|
| 63 |
+
"notes": item.notes,
|
| 64 |
+
"validator": item.validator,
|
| 65 |
+
"updated_at": event["timestamp"],
|
| 66 |
+
"version": new_version,
|
| 67 |
+
}
|
| 68 |
+
current_file.write_text(json.dumps(current_payload, indent=2), encoding="utf-8")
|
| 69 |
+
return new_version
|
| 70 |
+
|
| 71 |
+
def list_events(self, project_slug: str) -> list[dict[str, object]]:
|
| 72 |
+
project_dir = self._base_dir / project_slug / "validations"
|
| 73 |
+
if not project_dir.exists():
|
| 74 |
+
return []
|
| 75 |
+
|
| 76 |
+
events: list[dict[str, object]] = []
|
| 77 |
+
for path in sorted(project_dir.glob("events-*.jsonl")):
|
| 78 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 79 |
+
for line in handle:
|
| 80 |
+
payload = line.strip()
|
| 81 |
+
if payload:
|
| 82 |
+
events.append(json.loads(payload))
|
| 83 |
+
return events
|
| 84 |
+
|
| 85 |
+
def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]:
|
| 86 |
+
current_file = self._base_dir / project_slug / "validations" / "current.json"
|
| 87 |
+
if not current_file.exists():
|
| 88 |
+
return {}
|
| 89 |
+
return json.loads(current_file.read_text(encoding="utf-8"))
|
BirdNET Validator App/src/repositories/contracts.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Protocol
|
| 2 |
+
|
| 3 |
+
from src.domain.models import Detection, Project, User, Validation
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class ProjectRepository(Protocol):
|
| 7 |
+
def get_project(self, project_slug: str) -> Project: ...
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class DetectionRepository(Protocol):
|
| 11 |
+
def list_detections(
|
| 12 |
+
self,
|
| 13 |
+
project_slug: str,
|
| 14 |
+
page: int,
|
| 15 |
+
page_size: int,
|
| 16 |
+
scientific_name: str | None = None,
|
| 17 |
+
min_confidence: float | None = None,
|
| 18 |
+
max_confidence: float | None = None,
|
| 19 |
+
) -> list[Detection]: ...
|
| 20 |
+
|
| 21 |
+
def count_detections(
|
| 22 |
+
self,
|
| 23 |
+
project_slug: str,
|
| 24 |
+
scientific_name: str | None = None,
|
| 25 |
+
min_confidence: float | None = None,
|
| 26 |
+
max_confidence: float | None = None,
|
| 27 |
+
) -> int: ...
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class ValidationRepository(Protocol):
|
| 31 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int: ...
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class AuthRepository(Protocol):
|
| 35 |
+
def authenticate(self, username: str) -> User: ...
|
BirdNET Validator App/src/repositories/in_memory_detection_repository.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import defaultdict
|
| 2 |
+
from typing import DefaultDict
|
| 3 |
+
|
| 4 |
+
from src.domain.models import Detection
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class InMemoryDetectionRepository:
|
| 8 |
+
def __init__(self) -> None:
|
| 9 |
+
self._project_items: DefaultDict[str, list[Detection]] = defaultdict(list)
|
| 10 |
+
|
| 11 |
+
def seed(self, project_slug: str, detections: list[Detection]) -> None:
|
| 12 |
+
self._project_items[project_slug] = list(detections)
|
| 13 |
+
|
| 14 |
+
def list_detections(
|
| 15 |
+
self,
|
| 16 |
+
project_slug: str,
|
| 17 |
+
page: int,
|
| 18 |
+
page_size: int,
|
| 19 |
+
scientific_name: str | None = None,
|
| 20 |
+
min_confidence: float | None = None,
|
| 21 |
+
max_confidence: float | None = None,
|
| 22 |
+
) -> list[Detection]:
|
| 23 |
+
if page < 1:
|
| 24 |
+
raise ValueError("page must be >= 1")
|
| 25 |
+
if page_size < 1:
|
| 26 |
+
raise ValueError("page_size must be >= 1")
|
| 27 |
+
|
| 28 |
+
filtered = self._apply_filters(
|
| 29 |
+
project_slug=project_slug,
|
| 30 |
+
scientific_name=scientific_name,
|
| 31 |
+
min_confidence=min_confidence,
|
| 32 |
+
max_confidence=max_confidence,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
start = (page - 1) * page_size
|
| 36 |
+
end = start + page_size
|
| 37 |
+
return filtered[start:end]
|
| 38 |
+
|
| 39 |
+
def count_detections(
|
| 40 |
+
self,
|
| 41 |
+
project_slug: str,
|
| 42 |
+
scientific_name: str | None = None,
|
| 43 |
+
min_confidence: float | None = None,
|
| 44 |
+
max_confidence: float | None = None,
|
| 45 |
+
) -> int:
|
| 46 |
+
return len(
|
| 47 |
+
self._apply_filters(
|
| 48 |
+
project_slug=project_slug,
|
| 49 |
+
scientific_name=scientific_name,
|
| 50 |
+
min_confidence=min_confidence,
|
| 51 |
+
max_confidence=max_confidence,
|
| 52 |
+
)
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
def _apply_filters(
|
| 56 |
+
self,
|
| 57 |
+
project_slug: str,
|
| 58 |
+
scientific_name: str | None,
|
| 59 |
+
min_confidence: float | None,
|
| 60 |
+
max_confidence: float | None,
|
| 61 |
+
) -> list[Detection]:
|
| 62 |
+
items = self._project_items.get(project_slug, [])
|
| 63 |
+
|
| 64 |
+
def match(item: Detection) -> bool:
|
| 65 |
+
if scientific_name and item.scientific_name != scientific_name:
|
| 66 |
+
return False
|
| 67 |
+
if min_confidence is not None and item.confidence < min_confidence:
|
| 68 |
+
return False
|
| 69 |
+
if max_confidence is not None and item.confidence > max_confidence:
|
| 70 |
+
return False
|
| 71 |
+
return True
|
| 72 |
+
|
| 73 |
+
return [item for item in items if match(item)]
|
BirdNET Validator App/src/repositories/in_memory_validation_repository.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import defaultdict
|
| 2 |
+
from typing import DefaultDict
|
| 3 |
+
|
| 4 |
+
from src.domain.models import Validation
|
| 5 |
+
from src.repositories.append_only_validation_repository import OptimisticLockError
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class InMemoryValidationRepository:
|
| 9 |
+
def __init__(self) -> None:
|
| 10 |
+
self._by_project: DefaultDict[str, list[Validation]] = defaultdict(list)
|
| 11 |
+
self._versions: DefaultDict[str, dict[str, int]] = defaultdict(dict)
|
| 12 |
+
|
| 13 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 14 |
+
current_version = int(self._versions[project_slug].get(item.detection_key, 0))
|
| 15 |
+
expected = expected_version if expected_version is not None else current_version
|
| 16 |
+
if expected != current_version:
|
| 17 |
+
raise OptimisticLockError(item.detection_key, expected, current_version)
|
| 18 |
+
|
| 19 |
+
self._by_project[project_slug].append(item)
|
| 20 |
+
new_version = current_version + 1
|
| 21 |
+
self._versions[project_slug][item.detection_key] = new_version
|
| 22 |
+
return new_version
|
| 23 |
+
|
| 24 |
+
def list_validations(self, project_slug: str) -> list[Validation]:
|
| 25 |
+
return list(self._by_project.get(project_slug, []))
|
BirdNET Validator App/src/services/__init__.py
ADDED
|
File without changes
|
BirdNET Validator App/src/services/audio_fetch_service.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
from huggingface_hub import hf_hub_download
|
| 5 |
+
|
| 6 |
+
from src.cache.ephemeral_cache_manager import EphemeralCacheManager
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
SUPPORTED_AUDIO_EXTENSIONS = (".wav", ".mp3", ".flac", ".ogg", ".m4a")
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass(slots=True)
|
| 13 |
+
class AudioFetchResult:
|
| 14 |
+
cache_key: str
|
| 15 |
+
local_path: str
|
| 16 |
+
source: str
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class AudioFetchService:
|
| 20 |
+
def __init__(self, cache_manager: EphemeralCacheManager) -> None:
|
| 21 |
+
self._cache = cache_manager
|
| 22 |
+
|
| 23 |
+
def fetch(self, dataset_repo: str, audio_id: str) -> AudioFetchResult:
|
| 24 |
+
cache_key = f"{dataset_repo}:{audio_id}"
|
| 25 |
+
cached_path = self._cache.get(cache_key)
|
| 26 |
+
if cached_path:
|
| 27 |
+
return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="cache")
|
| 28 |
+
|
| 29 |
+
target_filename, downloaded_path = self._resolve_remote_filename(dataset_repo=dataset_repo, audio_id=audio_id)
|
| 30 |
+
if downloaded_path is None:
|
| 31 |
+
downloaded = hf_hub_download(repo_id=dataset_repo, repo_type="dataset", filename=target_filename)
|
| 32 |
+
downloaded_path = Path(downloaded)
|
| 33 |
+
|
| 34 |
+
suffix = downloaded_path.suffix if downloaded_path.suffix else ".bin"
|
| 35 |
+
local_path = self._cache.put_bytes(cache_key, downloaded_path.read_bytes(), suffix=suffix)
|
| 36 |
+
return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="remote")
|
| 37 |
+
|
| 38 |
+
def cleanup_after_validation(self, cache_key: str) -> None:
|
| 39 |
+
self._cache.cleanup_key(cache_key)
|
| 40 |
+
|
| 41 |
+
def _resolve_remote_filename(self, dataset_repo: str, audio_id: str) -> tuple[str, Path | None]:
|
| 42 |
+
audio_path = Path(audio_id)
|
| 43 |
+
if audio_path.suffix:
|
| 44 |
+
return f"audio/{audio_id}", None
|
| 45 |
+
|
| 46 |
+
for extension in SUPPORTED_AUDIO_EXTENSIONS:
|
| 47 |
+
candidate = f"audio/{audio_id}{extension}"
|
| 48 |
+
try:
|
| 49 |
+
downloaded = hf_hub_download(repo_id=dataset_repo, repo_type="dataset", filename=candidate)
|
| 50 |
+
return candidate, Path(downloaded)
|
| 51 |
+
except Exception:
|
| 52 |
+
continue
|
| 53 |
+
|
| 54 |
+
raise FileNotFoundError(f"Unable to locate audio file for audio_id: {audio_id}")
|
BirdNET Validator App/src/services/detection_queue_service.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
|
| 3 |
+
from src.domain.models import Detection
|
| 4 |
+
from src.repositories.contracts import DetectionRepository
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass(slots=True)
|
| 8 |
+
class DetectionPage:
|
| 9 |
+
project_slug: str
|
| 10 |
+
page: int
|
| 11 |
+
page_size: int
|
| 12 |
+
total_items: int
|
| 13 |
+
total_pages: int
|
| 14 |
+
has_next: bool
|
| 15 |
+
has_previous: bool
|
| 16 |
+
items: list[Detection]
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class DetectionQueueService:
|
| 20 |
+
def __init__(self, repository: DetectionRepository) -> None:
|
| 21 |
+
self._repository = repository
|
| 22 |
+
|
| 23 |
+
def get_page(
|
| 24 |
+
self,
|
| 25 |
+
project_slug: str,
|
| 26 |
+
page: int,
|
| 27 |
+
page_size: int,
|
| 28 |
+
scientific_name: str | None = None,
|
| 29 |
+
min_confidence: float | None = None,
|
| 30 |
+
max_confidence: float | None = None,
|
| 31 |
+
) -> DetectionPage:
|
| 32 |
+
if page < 1:
|
| 33 |
+
raise ValueError("page must be >= 1")
|
| 34 |
+
if page_size < 1:
|
| 35 |
+
raise ValueError("page_size must be >= 1")
|
| 36 |
+
|
| 37 |
+
total_items = self._repository.count_detections(
|
| 38 |
+
project_slug=project_slug,
|
| 39 |
+
scientific_name=scientific_name,
|
| 40 |
+
min_confidence=min_confidence,
|
| 41 |
+
max_confidence=max_confidence,
|
| 42 |
+
)
|
| 43 |
+
total_pages = max(1, ((total_items - 1) // page_size) + 1) if total_items else 1
|
| 44 |
+
page = min(page, total_pages)
|
| 45 |
+
|
| 46 |
+
items = self._repository.list_detections(
|
| 47 |
+
project_slug=project_slug,
|
| 48 |
+
page=page,
|
| 49 |
+
page_size=page_size,
|
| 50 |
+
scientific_name=scientific_name,
|
| 51 |
+
min_confidence=min_confidence,
|
| 52 |
+
max_confidence=max_confidence,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
return DetectionPage(
|
| 56 |
+
project_slug=project_slug,
|
| 57 |
+
page=page,
|
| 58 |
+
page_size=page_size,
|
| 59 |
+
total_items=total_items,
|
| 60 |
+
total_pages=total_pages,
|
| 61 |
+
has_next=page < total_pages,
|
| 62 |
+
has_previous=page > 1,
|
| 63 |
+
items=items,
|
| 64 |
+
)
|
BirdNET Validator App/src/services/validation_service.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
|
| 3 |
+
from src.domain.models import Validation
|
| 4 |
+
from src.repositories.contracts import ValidationRepository
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass(slots=True)
|
| 8 |
+
class ValidationWriteResult:
|
| 9 |
+
item: Validation
|
| 10 |
+
new_version: int
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ValidationService:
|
| 14 |
+
def __init__(self, repository: ValidationRepository) -> None:
|
| 15 |
+
self._repository = repository
|
| 16 |
+
|
| 17 |
+
def validate_detection(
|
| 18 |
+
self,
|
| 19 |
+
project_slug: str,
|
| 20 |
+
detection_key: str,
|
| 21 |
+
status: str,
|
| 22 |
+
validator: str,
|
| 23 |
+
notes: str = "",
|
| 24 |
+
corrected_species: str | None = None,
|
| 25 |
+
expected_version: int | None = None,
|
| 26 |
+
) -> ValidationWriteResult:
|
| 27 |
+
item = Validation(
|
| 28 |
+
detection_key=detection_key,
|
| 29 |
+
status=status,
|
| 30 |
+
corrected_species=corrected_species,
|
| 31 |
+
notes=notes,
|
| 32 |
+
validator=validator,
|
| 33 |
+
)
|
| 34 |
+
new_version = self._repository.save_validation(
|
| 35 |
+
project_slug=project_slug,
|
| 36 |
+
item=item,
|
| 37 |
+
expected_version=expected_version,
|
| 38 |
+
)
|
| 39 |
+
return ValidationWriteResult(item=item, new_version=new_version)
|
BirdNET Validator App/src/telemetry/__init__.py
ADDED
|
File without changes
|
BirdNET Validator App/src/ui/__init__.py
ADDED
|
File without changes
|
BirdNET Validator App/src/ui/admin_panel.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Gradio admin panel for managing projects and user assignments."""
|
| 2 |
+
|
| 3 |
+
from typing import List, Tuple
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
|
| 7 |
+
from src.auth.auth_service import AuthService, Session
|
| 8 |
+
from src.domain.models import Project, Role
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class AdminPanelManager:
|
| 12 |
+
"""Management backend for the admin panel."""
|
| 13 |
+
|
| 14 |
+
def __init__(self, auth_service: AuthService):
|
| 15 |
+
"""Initialize admin panel manager.
|
| 16 |
+
|
| 17 |
+
Args:
|
| 18 |
+
auth_service: AuthService instance
|
| 19 |
+
"""
|
| 20 |
+
self.auth_service = auth_service
|
| 21 |
+
self._projects: dict[str, Project] = {} # project_slug -> Project
|
| 22 |
+
self._user_project_assignments: dict[str, dict[str, str]] = {} # username -> {project_slug -> role}
|
| 23 |
+
|
| 24 |
+
def register_project(self, project: Project) -> bool:
|
| 25 |
+
"""Register a new project (idempotent).
|
| 26 |
+
|
| 27 |
+
Args:
|
| 28 |
+
project: Project to register
|
| 29 |
+
|
| 30 |
+
Returns:
|
| 31 |
+
True if new, False if already existed
|
| 32 |
+
"""
|
| 33 |
+
if project.project_slug in self._projects:
|
| 34 |
+
return False
|
| 35 |
+
|
| 36 |
+
self._projects[project.project_slug] = project
|
| 37 |
+
return True
|
| 38 |
+
|
| 39 |
+
def list_projects(self) -> List[dict]:
|
| 40 |
+
"""List all projects as dictionaries for Gradio display.
|
| 41 |
+
|
| 42 |
+
Returns:
|
| 43 |
+
List of project dicts with slug, name, repo_id, active status
|
| 44 |
+
"""
|
| 45 |
+
return [
|
| 46 |
+
{
|
| 47 |
+
"project_slug": p.project_slug,
|
| 48 |
+
"name": p.name,
|
| 49 |
+
"dataset_repo_id": p.dataset_repo_id,
|
| 50 |
+
"active": p.active,
|
| 51 |
+
}
|
| 52 |
+
for p in self._projects.values()
|
| 53 |
+
]
|
| 54 |
+
|
| 55 |
+
def get_project(self, project_slug: str) -> Project | None:
|
| 56 |
+
"""Get project by slug.
|
| 57 |
+
|
| 58 |
+
Args:
|
| 59 |
+
project_slug: Project slug
|
| 60 |
+
|
| 61 |
+
Returns:
|
| 62 |
+
Project if found, None otherwise
|
| 63 |
+
"""
|
| 64 |
+
return self._projects.get(project_slug)
|
| 65 |
+
|
| 66 |
+
def list_users_for_project(self, project_slug: str) -> List[dict]:
|
| 67 |
+
"""List all users assigned to a project.
|
| 68 |
+
|
| 69 |
+
Args:
|
| 70 |
+
project_slug: Project slug
|
| 71 |
+
|
| 72 |
+
Returns:
|
| 73 |
+
List of dicts with username and role
|
| 74 |
+
"""
|
| 75 |
+
result = []
|
| 76 |
+
for username in self.auth_service._user_access.keys():
|
| 77 |
+
role = self.auth_service.get_user_role_for_project(username, project_slug)
|
| 78 |
+
if role is not None:
|
| 79 |
+
result.append({"username": username, "role": role.value})
|
| 80 |
+
|
| 81 |
+
return result
|
| 82 |
+
|
| 83 |
+
def assign_user_to_project(
|
| 84 |
+
self, username: str, project_slug: str, role: str
|
| 85 |
+
) -> Tuple[bool, str]:
|
| 86 |
+
"""Assign a user to a project with a role.
|
| 87 |
+
|
| 88 |
+
Args:
|
| 89 |
+
username: Username
|
| 90 |
+
project_slug: Project slug
|
| 91 |
+
role: "admin" or "validator"
|
| 92 |
+
|
| 93 |
+
Returns:
|
| 94 |
+
Tuple of (success, message)
|
| 95 |
+
"""
|
| 96 |
+
if project_slug not in self._projects:
|
| 97 |
+
return False, f"Project '{project_slug}' not found"
|
| 98 |
+
|
| 99 |
+
if role not in ["admin", "validator"]:
|
| 100 |
+
return False, f"Invalid role: {role}"
|
| 101 |
+
|
| 102 |
+
# Get or create user's project access
|
| 103 |
+
if username not in self.auth_service._user_access:
|
| 104 |
+
self.auth_service.register_user_project_access(username, {})
|
| 105 |
+
|
| 106 |
+
access = self.auth_service._user_access[username]
|
| 107 |
+
access.project_slugs[project_slug] = Role(role)
|
| 108 |
+
|
| 109 |
+
return True, f"β
Assigned {username} to {project_slug} as {role}"
|
| 110 |
+
|
| 111 |
+
def remove_user_from_project(self, username: str, project_slug: str) -> Tuple[bool, str]:
|
| 112 |
+
"""Remove a user's access to a project.
|
| 113 |
+
|
| 114 |
+
Args:
|
| 115 |
+
username: Username
|
| 116 |
+
project_slug: Project slug
|
| 117 |
+
|
| 118 |
+
Returns:
|
| 119 |
+
Tuple of (success, message)
|
| 120 |
+
"""
|
| 121 |
+
if username not in self.auth_service._user_access:
|
| 122 |
+
return False, f"User '{username}' not found"
|
| 123 |
+
|
| 124 |
+
access = self.auth_service._user_access[username]
|
| 125 |
+
if project_slug not in access.project_slugs:
|
| 126 |
+
return False, f"User '{username}' is not assigned to project '{project_slug}'"
|
| 127 |
+
|
| 128 |
+
del access.project_slugs[project_slug]
|
| 129 |
+
return True, f"β
Removed {username} from {project_slug}"
|
| 130 |
+
|
| 131 |
+
def toggleproject_active(self, project_slug: str, active: bool) -> Tuple[bool, str]:
|
| 132 |
+
"""Enable or disable a project.
|
| 133 |
+
|
| 134 |
+
Args:
|
| 135 |
+
project_slug: Project slug
|
| 136 |
+
active: Whether to activate or deactivate
|
| 137 |
+
|
| 138 |
+
Returns:
|
| 139 |
+
Tuple of (success, message)
|
| 140 |
+
"""
|
| 141 |
+
if project_slug not in self._projects:
|
| 142 |
+
return False, f"Project '{project_slug}' not found"
|
| 143 |
+
|
| 144 |
+
self._projects[project_slug].active = active
|
| 145 |
+
status = "activated" if active else "deactivated"
|
| 146 |
+
return True, f"β
Project {project_slug} {status}"
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def create_admin_panel(admin_manager: AdminPanelManager, current_session: Session) -> gr.Blocks:
|
| 150 |
+
"""Create Gradio admin panel UI.
|
| 151 |
+
|
| 152 |
+
Args:
|
| 153 |
+
admin_manager: AdminPanelManager instance
|
| 154 |
+
current_session: Current user's session
|
| 155 |
+
|
| 156 |
+
Returns:
|
| 157 |
+
Gradio Blocks with admin panel tabs
|
| 158 |
+
"""
|
| 159 |
+
if current_session.role != Role.admin:
|
| 160 |
+
with gr.Blocks() as restricted:
|
| 161 |
+
gr.Markdown("β **Access Denied**\n\nOnly administrators can access this panel.")
|
| 162 |
+
return restricted
|
| 163 |
+
|
| 164 |
+
with gr.Blocks(title="BirdNET Admin Panel") as admin_block:
|
| 165 |
+
gr.Markdown("# Admin Panel")
|
| 166 |
+
gr.Markdown(f"Logged in as: **{current_session.username}** (Admin)")
|
| 167 |
+
|
| 168 |
+
with gr.Tabs():
|
| 169 |
+
# Projects Tab
|
| 170 |
+
with gr.Tab("Projects"):
|
| 171 |
+
gr.Markdown("## Manage Projects")
|
| 172 |
+
|
| 173 |
+
with gr.Row():
|
| 174 |
+
with gr.Column(scale=2):
|
| 175 |
+
project_slug_input = gr.Textbox(
|
| 176 |
+
label="Project Slug",
|
| 177 |
+
placeholder="e.g., kenya-2024",
|
| 178 |
+
lines=1,
|
| 179 |
+
)
|
| 180 |
+
project_name_input = gr.Textbox(
|
| 181 |
+
label="Project Name",
|
| 182 |
+
placeholder="e.g., Kenya Survey 2024",
|
| 183 |
+
lines=1,
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
with gr.Column(scale=1):
|
| 187 |
+
repo_id_input = gr.Textbox(
|
| 188 |
+
label="HF Dataset Repo ID",
|
| 189 |
+
placeholder="e.g., org/dataset-name",
|
| 190 |
+
lines=1,
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
project_message = gr.Markdown()
|
| 194 |
+
|
| 195 |
+
# TODO: Implement create project button
|
| 196 |
+
# create_project_button = gr.Button("Create Project", variant="primary")
|
| 197 |
+
|
| 198 |
+
# Projects list
|
| 199 |
+
with gr.Row():
|
| 200 |
+
refresh_projects_button = gr.Button("Refresh Projects List")
|
| 201 |
+
projects_table = gr.Dataframe(
|
| 202 |
+
value=admin_manager.list_projects(),
|
| 203 |
+
headers=["project_slug", "name", "dataset_repo_id", "active"],
|
| 204 |
+
interactive=False,
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
refresh_projects_button.click(
|
| 208 |
+
fn=lambda: admin_manager.list_projects(),
|
| 209 |
+
outputs=[projects_table],
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
# Users Tab
|
| 213 |
+
with gr.Tab("Users"):
|
| 214 |
+
gr.Markdown("## Manage User Access")
|
| 215 |
+
|
| 216 |
+
with gr.Row():
|
| 217 |
+
username_input = gr.Textbox(
|
| 218 |
+
label="Username",
|
| 219 |
+
placeholder="e.g., validator_001",
|
| 220 |
+
lines=1,
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
project_select = gr.Dropdown(
|
| 224 |
+
choices=[p["project_slug"] for p in admin_manager.list_projects()],
|
| 225 |
+
label="Project",
|
| 226 |
+
)
|
| 227 |
+
|
| 228 |
+
role_select = gr.Dropdown(
|
| 229 |
+
choices=["admin", "validator"],
|
| 230 |
+
value="validator",
|
| 231 |
+
label="Role",
|
| 232 |
+
)
|
| 233 |
+
|
| 234 |
+
user_message = gr.Markdown()
|
| 235 |
+
|
| 236 |
+
assign_button = gr.Button("Assign User", variant="primary")
|
| 237 |
+
remove_button = gr.Button("Remove User", variant="stop")
|
| 238 |
+
|
| 239 |
+
def assign_user(username: str, project_slug: str, role: str) -> str:
|
| 240 |
+
success, msg = admin_manager.assign_user_to_project(username, project_slug, role)
|
| 241 |
+
return msg
|
| 242 |
+
|
| 243 |
+
def remove_user(username: str, project_slug: str) -> str:
|
| 244 |
+
success, msg = admin_manager.remove_user_from_project(username, project_slug)
|
| 245 |
+
return msg
|
| 246 |
+
|
| 247 |
+
assign_button.click(
|
| 248 |
+
fn=assign_user,
|
| 249 |
+
inputs=[username_input, project_select, role_select],
|
| 250 |
+
outputs=[user_message],
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
remove_button.click(
|
| 254 |
+
fn=remove_user,
|
| 255 |
+
inputs=[username_input, project_select],
|
| 256 |
+
outputs=[user_message],
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
# Users per project view
|
| 260 |
+
with gr.Row():
|
| 261 |
+
project_filter = gr.Dropdown(
|
| 262 |
+
choices=[p["project_slug"] for p in admin_manager.list_projects()],
|
| 263 |
+
label="View Users for Project",
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
users_table = gr.Dataframe(
|
| 267 |
+
value=[],
|
| 268 |
+
headers=["username", "role"],
|
| 269 |
+
interactive=False,
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
def update_users_table(project_slug: str):
|
| 273 |
+
return admin_manager.list_users_for_project(project_slug)
|
| 274 |
+
|
| 275 |
+
project_filter.change(
|
| 276 |
+
fn=update_users_table,
|
| 277 |
+
inputs=[project_filter],
|
| 278 |
+
outputs=[users_table],
|
| 279 |
+
)
|
| 280 |
+
|
| 281 |
+
return admin_block
|
BirdNET Validator App/src/ui/app_factory.py
ADDED
|
@@ -0,0 +1,1542 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
import tempfile
|
| 3 |
+
from datetime import date, datetime
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Protocol
|
| 6 |
+
|
| 7 |
+
from src.cache.ephemeral_cache_manager import EphemeralCacheManager
|
| 8 |
+
from src.domain.models import Detection, Project, Role
|
| 9 |
+
from src.repositories.append_only_validation_repository import AppendOnlyValidationRepository, OptimisticLockError
|
| 10 |
+
from src.repositories.in_memory_detection_repository import InMemoryDetectionRepository
|
| 11 |
+
from src.services.audio_fetch_service import AudioFetchService
|
| 12 |
+
from src.services.detection_queue_service import DetectionQueueService
|
| 13 |
+
from src.services.validation_service import ValidationService
|
| 14 |
+
from src.auth.auth_service import AuthService
|
| 15 |
+
from src.ui.login_page import create_login_page
|
| 16 |
+
from src.ui.project_selector import create_project_selector
|
| 17 |
+
from src.ui.admin_panel import AdminPanelManager, create_admin_panel
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class _AudioFetchResultProtocol(Protocol):
|
| 21 |
+
cache_key: str
|
| 22 |
+
local_path: str
|
| 23 |
+
source: str
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class _AudioServiceProtocol(Protocol):
|
| 27 |
+
def fetch(self, dataset_repo: str, audio_id: str) -> _AudioFetchResultProtocol: ...
|
| 28 |
+
|
| 29 |
+
def cleanup_after_validation(self, cache_key: str) -> None: ...
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class _ValidationServiceProtocol(Protocol):
|
| 33 |
+
def validate_detection(
|
| 34 |
+
self,
|
| 35 |
+
project_slug: str,
|
| 36 |
+
detection_key: str,
|
| 37 |
+
status: str,
|
| 38 |
+
validator: str,
|
| 39 |
+
notes: str = "",
|
| 40 |
+
corrected_species: str | None = None,
|
| 41 |
+
expected_version: int | None = None,
|
| 42 |
+
) -> object: ...
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class _ValidationReadRepositoryProtocol(Protocol):
|
| 46 |
+
def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]: ...
|
| 47 |
+
|
| 48 |
+
def list_events(self, project_slug: str) -> list[dict[str, object]]: ...
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class _QueueServiceProtocol(Protocol):
|
| 52 |
+
def get_page(
|
| 53 |
+
self,
|
| 54 |
+
project_slug: str,
|
| 55 |
+
page: int,
|
| 56 |
+
page_size: int,
|
| 57 |
+
scientific_name: str | None = None,
|
| 58 |
+
min_confidence: float | None = None,
|
| 59 |
+
max_confidence: float | None = None,
|
| 60 |
+
) -> object: ...
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _seed_service() -> DetectionQueueService:
|
| 64 |
+
repo = InMemoryDetectionRepository()
|
| 65 |
+
demo_items = [
|
| 66 |
+
Detection(
|
| 67 |
+
detection_key="0000000000001001",
|
| 68 |
+
audio_id="audio_1001",
|
| 69 |
+
scientific_name="Cyanocorax cyanopogon",
|
| 70 |
+
confidence=0.93,
|
| 71 |
+
start_time=1.2,
|
| 72 |
+
end_time=2.5,
|
| 73 |
+
),
|
| 74 |
+
Detection(
|
| 75 |
+
detection_key="0000000000001002",
|
| 76 |
+
audio_id="audio_1002",
|
| 77 |
+
scientific_name="Ramphastos toco",
|
| 78 |
+
confidence=0.88,
|
| 79 |
+
start_time=0.8,
|
| 80 |
+
end_time=2.1,
|
| 81 |
+
),
|
| 82 |
+
Detection(
|
| 83 |
+
detection_key="0000000000001003",
|
| 84 |
+
audio_id="audio_1003",
|
| 85 |
+
scientific_name="Cyanocorax cyanopogon",
|
| 86 |
+
confidence=0.72,
|
| 87 |
+
start_time=3.1,
|
| 88 |
+
end_time=4.0,
|
| 89 |
+
),
|
| 90 |
+
Detection(
|
| 91 |
+
detection_key="0000000000001004",
|
| 92 |
+
audio_id="audio_1004",
|
| 93 |
+
scientific_name="Psarocolius decumanus",
|
| 94 |
+
confidence=0.67,
|
| 95 |
+
start_time=5.0,
|
| 96 |
+
end_time=6.3,
|
| 97 |
+
),
|
| 98 |
+
]
|
| 99 |
+
repo.seed("demo-project", demo_items)
|
| 100 |
+
return DetectionQueueService(repo)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _page_to_table(
|
| 104 |
+
service: _QueueServiceProtocol,
|
| 105 |
+
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 106 |
+
project_slug: str,
|
| 107 |
+
page: int,
|
| 108 |
+
scientific_name: str,
|
| 109 |
+
min_confidence: float,
|
| 110 |
+
validator_filter: str = "",
|
| 111 |
+
status_filter: str = "all",
|
| 112 |
+
updated_after: object = None,
|
| 113 |
+
conflict_detection_key: str = "",
|
| 114 |
+
show_conflicts_only: bool = False,
|
| 115 |
+
):
|
| 116 |
+
filter_name = scientific_name.strip() if scientific_name.strip() else None
|
| 117 |
+
page_obj = service.get_page(
|
| 118 |
+
project_slug="demo-project",
|
| 119 |
+
page=page,
|
| 120 |
+
page_size=2,
|
| 121 |
+
scientific_name=filter_name,
|
| 122 |
+
min_confidence=min_confidence,
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
|
| 126 |
+
|
| 127 |
+
normalized_status_filter = status_filter.strip().lower() if status_filter else "all"
|
| 128 |
+
normalized_validator_filter = validator_filter.strip().lower()
|
| 129 |
+
updated_after_date: date | None = None
|
| 130 |
+
if updated_after is not None:
|
| 131 |
+
if isinstance(updated_after, datetime):
|
| 132 |
+
updated_after_date = updated_after.date()
|
| 133 |
+
elif isinstance(updated_after, date):
|
| 134 |
+
updated_after_date = updated_after
|
| 135 |
+
elif isinstance(updated_after, (int, float)):
|
| 136 |
+
updated_after_date = datetime.fromtimestamp(float(updated_after)).date()
|
| 137 |
+
else:
|
| 138 |
+
updated_after_text = str(updated_after).strip()
|
| 139 |
+
if updated_after_text:
|
| 140 |
+
try:
|
| 141 |
+
updated_after_date = datetime.strptime(updated_after_text, "%Y-%m-%d").date()
|
| 142 |
+
except ValueError:
|
| 143 |
+
try:
|
| 144 |
+
updated_after_date = datetime.fromisoformat(updated_after_text.replace("Z", "+00:00")).date()
|
| 145 |
+
except ValueError:
|
| 146 |
+
updated_after_date = None
|
| 147 |
+
|
| 148 |
+
rows = [
|
| 149 |
+
[
|
| 150 |
+
item.detection_key,
|
| 151 |
+
item.audio_id,
|
| 152 |
+
item.scientific_name,
|
| 153 |
+
round(item.confidence, 3),
|
| 154 |
+
item.start_time,
|
| 155 |
+
item.end_time,
|
| 156 |
+
str(snapshot.get(item.detection_key, {}).get("status", "pending")),
|
| 157 |
+
int(snapshot.get(item.detection_key, {}).get("version", 0)),
|
| 158 |
+
"CONFLICT" if conflict_detection_key and item.detection_key == conflict_detection_key else "",
|
| 159 |
+
"HIGH" if conflict_detection_key and item.detection_key == conflict_detection_key else "",
|
| 160 |
+
]
|
| 161 |
+
for item in page_obj.items
|
| 162 |
+
]
|
| 163 |
+
|
| 164 |
+
if normalized_validator_filter:
|
| 165 |
+
rows = [
|
| 166 |
+
row
|
| 167 |
+
for row in rows
|
| 168 |
+
if normalized_validator_filter in str(snapshot.get(str(row[0]), {}).get("validator", "")).strip().lower()
|
| 169 |
+
]
|
| 170 |
+
|
| 171 |
+
if normalized_status_filter and normalized_status_filter != "all":
|
| 172 |
+
rows = [row for row in rows if str(row[6]).strip().lower() == normalized_status_filter]
|
| 173 |
+
|
| 174 |
+
if updated_after_date:
|
| 175 |
+
filtered_rows: list[list[object]] = []
|
| 176 |
+
for row in rows:
|
| 177 |
+
snapshot_item = snapshot.get(str(row[0]), {})
|
| 178 |
+
updated_at_value = str(snapshot_item.get("updated_at", "")).strip()
|
| 179 |
+
if not updated_at_value:
|
| 180 |
+
continue
|
| 181 |
+
try:
|
| 182 |
+
item_date = datetime.fromisoformat(updated_at_value.replace("Z", "+00:00")).date()
|
| 183 |
+
if item_date >= updated_after_date:
|
| 184 |
+
filtered_rows.append(row)
|
| 185 |
+
except ValueError:
|
| 186 |
+
continue
|
| 187 |
+
rows = filtered_rows
|
| 188 |
+
|
| 189 |
+
if show_conflicts_only:
|
| 190 |
+
rows = [row for row in rows if str(row[8]) == "CONFLICT"]
|
| 191 |
+
|
| 192 |
+
status = f"Pagina {page_obj.page}/{page_obj.total_pages} | Total base: {page_obj.total_items} | Exibidos: {len(rows)}"
|
| 193 |
+
if show_conflicts_only:
|
| 194 |
+
status = f"{status} | Apenas conflitos: {len(rows)} item(ns)"
|
| 195 |
+
return rows, status, page_obj.page
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _build_validation_report(snapshot_reader: _ValidationReadRepositoryProtocol, project_slug: str) -> str:
|
| 199 |
+
snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
|
| 200 |
+
events = snapshot_reader.list_events(project_slug=project_slug)
|
| 201 |
+
|
| 202 |
+
counts: dict[str, int] = {}
|
| 203 |
+
for payload in snapshot.values():
|
| 204 |
+
status_value = str(payload.get("status", "unknown"))
|
| 205 |
+
counts[status_value] = counts.get(status_value, 0) + 1
|
| 206 |
+
|
| 207 |
+
parts = [
|
| 208 |
+
f"Projeto: {project_slug}",
|
| 209 |
+
f"Eventos append-only: {len(events)}",
|
| 210 |
+
f"Deteccoes com estado atual: {len(snapshot)}",
|
| 211 |
+
]
|
| 212 |
+
if counts:
|
| 213 |
+
summary = ", ".join(f"{key}={value}" for key, value in sorted(counts.items()))
|
| 214 |
+
parts.append(f"Status atual: {summary}")
|
| 215 |
+
else:
|
| 216 |
+
parts.append("Status atual: sem validacoes")
|
| 217 |
+
return " | ".join(parts)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _extract_audio_id(rows: object, selected_index: int) -> str:
|
| 221 |
+
normalized_rows: list[list[object]]
|
| 222 |
+
|
| 223 |
+
if hasattr(rows, "values"):
|
| 224 |
+
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 225 |
+
else:
|
| 226 |
+
normalized_rows = [list(item) for item in rows] if rows else []
|
| 227 |
+
|
| 228 |
+
if not normalized_rows:
|
| 229 |
+
raise ValueError("Nenhuma deteccao carregada na tabela")
|
| 230 |
+
if selected_index < 0 or selected_index >= len(normalized_rows):
|
| 231 |
+
raise ValueError("Selecione uma deteccao valida na tabela")
|
| 232 |
+
|
| 233 |
+
value = normalized_rows[selected_index][1]
|
| 234 |
+
audio_id = str(value).strip()
|
| 235 |
+
if not audio_id:
|
| 236 |
+
raise ValueError("audio_id invalido na deteccao selecionada")
|
| 237 |
+
return audio_id
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _extract_detection_key(rows: object, selected_index: int) -> str:
|
| 241 |
+
normalized_rows: list[list[object]]
|
| 242 |
+
|
| 243 |
+
if hasattr(rows, "values"):
|
| 244 |
+
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 245 |
+
else:
|
| 246 |
+
normalized_rows = [list(item) for item in rows] if rows else []
|
| 247 |
+
|
| 248 |
+
if not normalized_rows:
|
| 249 |
+
raise ValueError("Nenhuma deteccao carregada na tabela")
|
| 250 |
+
if selected_index < 0 or selected_index >= len(normalized_rows):
|
| 251 |
+
raise ValueError("Selecione uma deteccao valida na tabela")
|
| 252 |
+
|
| 253 |
+
value = normalized_rows[selected_index][0]
|
| 254 |
+
detection_key = str(value).strip()
|
| 255 |
+
if not detection_key:
|
| 256 |
+
raise ValueError("detection_key invalido na deteccao selecionada")
|
| 257 |
+
return detection_key
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _find_detection_row_index(rows: object, detection_key: str) -> int:
|
| 261 |
+
normalized_rows: list[list[object]]
|
| 262 |
+
|
| 263 |
+
if hasattr(rows, "values"):
|
| 264 |
+
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 265 |
+
else:
|
| 266 |
+
normalized_rows = [list(item) for item in rows] if rows else []
|
| 267 |
+
|
| 268 |
+
for index, row in enumerate(normalized_rows):
|
| 269 |
+
if str(row[0]).strip() == detection_key:
|
| 270 |
+
return index
|
| 271 |
+
return 0
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def _extract_expected_version(rows: object, selected_index: int) -> int:
|
| 275 |
+
normalized_rows: list[list[object]]
|
| 276 |
+
|
| 277 |
+
if hasattr(rows, "values"):
|
| 278 |
+
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 279 |
+
else:
|
| 280 |
+
normalized_rows = [list(item) for item in rows] if rows else []
|
| 281 |
+
|
| 282 |
+
if not normalized_rows:
|
| 283 |
+
raise ValueError("Nenhuma deteccao carregada na tabela")
|
| 284 |
+
if selected_index < 0 or selected_index >= len(normalized_rows):
|
| 285 |
+
raise ValueError("Selecione uma deteccao valida na tabela")
|
| 286 |
+
|
| 287 |
+
value = normalized_rows[selected_index][7]
|
| 288 |
+
return int(value)
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def _fetch_selected_audio(
|
| 292 |
+
audio_service: _AudioServiceProtocol,
|
| 293 |
+
dataset_repo: str,
|
| 294 |
+
rows: object,
|
| 295 |
+
selected_index: int,
|
| 296 |
+
previous_cache_key: str,
|
| 297 |
+
) -> tuple[str | None, str, str]:
|
| 298 |
+
repo = dataset_repo.strip()
|
| 299 |
+
if not repo:
|
| 300 |
+
return None, "", "Informe dataset repo no formato owner/repo"
|
| 301 |
+
|
| 302 |
+
try:
|
| 303 |
+
audio_id = _extract_audio_id(rows=rows, selected_index=selected_index)
|
| 304 |
+
result = audio_service.fetch(dataset_repo=repo, audio_id=audio_id)
|
| 305 |
+
status = f"Audio carregado ({result.source}) para audio_id={audio_id}"
|
| 306 |
+
return result.local_path, result.cache_key, status
|
| 307 |
+
except Exception as exc:
|
| 308 |
+
if previous_cache_key:
|
| 309 |
+
return None, previous_cache_key, f"Falha ao carregar audio: {exc}"
|
| 310 |
+
return None, "", f"Falha ao carregar audio: {exc}"
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _cleanup_selected_audio(audio_service: _AudioServiceProtocol, cache_key: str) -> tuple[str, str | None]:
|
| 314 |
+
if not cache_key:
|
| 315 |
+
return "Nenhum audio em cache para limpar", None
|
| 316 |
+
|
| 317 |
+
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 318 |
+
return "Cache de audio limpo apos validacao", None
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def _save_selected_validation(
|
| 322 |
+
validation_service: _ValidationServiceProtocol,
|
| 323 |
+
audio_service: _AudioServiceProtocol,
|
| 324 |
+
project_slug: str,
|
| 325 |
+
rows: object,
|
| 326 |
+
selected_index: int,
|
| 327 |
+
status_value: str,
|
| 328 |
+
validator: str,
|
| 329 |
+
notes: str,
|
| 330 |
+
cache_key: str,
|
| 331 |
+
) -> tuple[str, str, str | None]:
|
| 332 |
+
validator_name = validator.strip()
|
| 333 |
+
if not validator_name:
|
| 334 |
+
return "Informe o nome do validador", cache_key, None
|
| 335 |
+
|
| 336 |
+
try:
|
| 337 |
+
detection_key = _extract_detection_key(rows=rows, selected_index=selected_index)
|
| 338 |
+
expected_version = _extract_expected_version(rows=rows, selected_index=selected_index)
|
| 339 |
+
_ = validation_service.validate_detection(
|
| 340 |
+
project_slug=project_slug,
|
| 341 |
+
detection_key=detection_key,
|
| 342 |
+
status=status_value,
|
| 343 |
+
validator=validator_name,
|
| 344 |
+
notes=notes.strip(),
|
| 345 |
+
expected_version=expected_version,
|
| 346 |
+
)
|
| 347 |
+
if cache_key:
|
| 348 |
+
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 349 |
+
return f"Validacao salva: {detection_key} -> {status_value}", "", None
|
| 350 |
+
except OptimisticLockError as exc:
|
| 351 |
+
return (
|
| 352 |
+
"Conflito de concorrencia: esta deteccao foi atualizada por outro validador "
|
| 353 |
+
f"(detection_key={exc.detection_key}, versao atual={exc.current_version}, esperada={exc.expected_version}). "
|
| 354 |
+
"Recarregue a tabela.",
|
| 355 |
+
cache_key,
|
| 356 |
+
None,
|
| 357 |
+
)
|
| 358 |
+
except Exception as exc:
|
| 359 |
+
return f"Falha ao salvar validacao: {exc}", cache_key, None
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _save_selected_validation_with_refresh(
|
| 363 |
+
validation_service: _ValidationServiceProtocol,
|
| 364 |
+
audio_service: _AudioServiceProtocol,
|
| 365 |
+
queue_service: _QueueServiceProtocol,
|
| 366 |
+
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 367 |
+
project_slug: str,
|
| 368 |
+
rows: object,
|
| 369 |
+
selected_index: int,
|
| 370 |
+
status_value: str,
|
| 371 |
+
validator: str,
|
| 372 |
+
notes: str,
|
| 373 |
+
cache_key: str,
|
| 374 |
+
page: int,
|
| 375 |
+
scientific_name: str,
|
| 376 |
+
min_confidence: float,
|
| 377 |
+
validator_filter: str,
|
| 378 |
+
status_filter: str,
|
| 379 |
+
updated_after: object,
|
| 380 |
+
show_conflicts_only: bool,
|
| 381 |
+
) -> tuple[str, str, str | None, list[list[object]], int, int, str, str]:
|
| 382 |
+
selected_key = ""
|
| 383 |
+
try:
|
| 384 |
+
selected_key = _extract_detection_key(rows=rows, selected_index=selected_index)
|
| 385 |
+
except Exception:
|
| 386 |
+
selected_key = ""
|
| 387 |
+
|
| 388 |
+
save_status, updated_cache_key, audio_path = _save_selected_validation(
|
| 389 |
+
validation_service=validation_service,
|
| 390 |
+
audio_service=audio_service,
|
| 391 |
+
project_slug=project_slug,
|
| 392 |
+
rows=rows,
|
| 393 |
+
selected_index=selected_index,
|
| 394 |
+
status_value=status_value,
|
| 395 |
+
validator=validator,
|
| 396 |
+
notes=notes,
|
| 397 |
+
cache_key=cache_key,
|
| 398 |
+
)
|
| 399 |
+
|
| 400 |
+
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 401 |
+
service=queue_service,
|
| 402 |
+
snapshot_reader=snapshot_reader,
|
| 403 |
+
project_slug=project_slug,
|
| 404 |
+
page=page,
|
| 405 |
+
scientific_name=scientific_name,
|
| 406 |
+
min_confidence=min_confidence,
|
| 407 |
+
validator_filter=validator_filter,
|
| 408 |
+
status_filter=status_filter,
|
| 409 |
+
updated_after=updated_after,
|
| 410 |
+
show_conflicts_only=show_conflicts_only,
|
| 411 |
+
)
|
| 412 |
+
|
| 413 |
+
if selected_key:
|
| 414 |
+
refreshed_index = _find_detection_row_index(refreshed_rows, selected_key)
|
| 415 |
+
else:
|
| 416 |
+
refreshed_index = 0
|
| 417 |
+
|
| 418 |
+
if "Conflito de concorrencia" in save_status:
|
| 419 |
+
conflict_key = selected_key
|
| 420 |
+
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 421 |
+
service=queue_service,
|
| 422 |
+
snapshot_reader=snapshot_reader,
|
| 423 |
+
project_slug=project_slug,
|
| 424 |
+
page=refreshed_page,
|
| 425 |
+
scientific_name=scientific_name,
|
| 426 |
+
min_confidence=min_confidence,
|
| 427 |
+
validator_filter=validator_filter,
|
| 428 |
+
status_filter=status_filter,
|
| 429 |
+
updated_after=updated_after,
|
| 430 |
+
conflict_detection_key=conflict_key,
|
| 431 |
+
show_conflicts_only=show_conflicts_only,
|
| 432 |
+
)
|
| 433 |
+
refreshed_index = _find_detection_row_index(refreshed_rows, selected_key) if selected_key else 0
|
| 434 |
+
pending_status_value = status_value
|
| 435 |
+
status = f"{save_status} Tabela recarregada para resolver conflito."
|
| 436 |
+
else:
|
| 437 |
+
conflict_key = ""
|
| 438 |
+
pending_status_value = ""
|
| 439 |
+
status = f"{save_status} | {page_status}"
|
| 440 |
+
|
| 441 |
+
return (
|
| 442 |
+
status,
|
| 443 |
+
updated_cache_key,
|
| 444 |
+
audio_path,
|
| 445 |
+
refreshed_rows,
|
| 446 |
+
refreshed_page,
|
| 447 |
+
refreshed_index,
|
| 448 |
+
pending_status_value,
|
| 449 |
+
conflict_key,
|
| 450 |
+
)
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
def _reapply_last_conflict_validation_with_refresh(
|
| 454 |
+
validation_service: _ValidationServiceProtocol,
|
| 455 |
+
audio_service: _AudioServiceProtocol,
|
| 456 |
+
queue_service: _QueueServiceProtocol,
|
| 457 |
+
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 458 |
+
project_slug: str,
|
| 459 |
+
rows: object,
|
| 460 |
+
selected_index: int,
|
| 461 |
+
pending_status_value: str,
|
| 462 |
+
conflict_detection_key: str,
|
| 463 |
+
validator: str,
|
| 464 |
+
notes: str,
|
| 465 |
+
cache_key: str,
|
| 466 |
+
page: int,
|
| 467 |
+
scientific_name: str,
|
| 468 |
+
min_confidence: float,
|
| 469 |
+
validator_filter: str,
|
| 470 |
+
status_filter: str,
|
| 471 |
+
updated_after: object,
|
| 472 |
+
show_conflicts_only: bool,
|
| 473 |
+
) -> tuple[str, str, str | None, list[list[object]], int, int, str, str]:
|
| 474 |
+
if not pending_status_value:
|
| 475 |
+
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 476 |
+
service=queue_service,
|
| 477 |
+
snapshot_reader=snapshot_reader,
|
| 478 |
+
project_slug=project_slug,
|
| 479 |
+
page=page,
|
| 480 |
+
scientific_name=scientific_name,
|
| 481 |
+
min_confidence=min_confidence,
|
| 482 |
+
validator_filter=validator_filter,
|
| 483 |
+
status_filter=status_filter,
|
| 484 |
+
updated_after=updated_after,
|
| 485 |
+
show_conflicts_only=show_conflicts_only,
|
| 486 |
+
)
|
| 487 |
+
return (
|
| 488 |
+
f"Nenhuma validacao pendente para reaplicar | {page_status}",
|
| 489 |
+
cache_key,
|
| 490 |
+
None,
|
| 491 |
+
refreshed_rows,
|
| 492 |
+
refreshed_page,
|
| 493 |
+
selected_index,
|
| 494 |
+
"",
|
| 495 |
+
"",
|
| 496 |
+
)
|
| 497 |
+
|
| 498 |
+
target_index = _find_detection_row_index(rows, conflict_detection_key) if conflict_detection_key else selected_index
|
| 499 |
+
return _save_selected_validation_with_refresh(
|
| 500 |
+
validation_service=validation_service,
|
| 501 |
+
audio_service=audio_service,
|
| 502 |
+
queue_service=queue_service,
|
| 503 |
+
snapshot_reader=snapshot_reader,
|
| 504 |
+
project_slug=project_slug,
|
| 505 |
+
rows=rows,
|
| 506 |
+
selected_index=target_index,
|
| 507 |
+
status_value=pending_status_value,
|
| 508 |
+
validator=validator,
|
| 509 |
+
notes=notes,
|
| 510 |
+
cache_key=cache_key,
|
| 511 |
+
page=page,
|
| 512 |
+
scientific_name=scientific_name,
|
| 513 |
+
min_confidence=min_confidence,
|
| 514 |
+
validator_filter=validator_filter,
|
| 515 |
+
status_filter=status_filter,
|
| 516 |
+
updated_after=updated_after,
|
| 517 |
+
show_conflicts_only=show_conflicts_only,
|
| 518 |
+
)
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def _batch_validate_conflicts(
|
| 522 |
+
validation_service: _ValidationServiceProtocol,
|
| 523 |
+
audio_service: _AudioServiceProtocol,
|
| 524 |
+
queue_service: _QueueServiceProtocol,
|
| 525 |
+
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 526 |
+
project_slug: str,
|
| 527 |
+
rows: object,
|
| 528 |
+
status_value: str,
|
| 529 |
+
validator: str,
|
| 530 |
+
notes: str,
|
| 531 |
+
cache_key: str,
|
| 532 |
+
page: int,
|
| 533 |
+
scientific_name: str,
|
| 534 |
+
min_confidence: float,
|
| 535 |
+
validator_filter: str,
|
| 536 |
+
status_filter: str,
|
| 537 |
+
updated_after: object,
|
| 538 |
+
) -> tuple[str, str, str | None, list[list[object]], int]:
|
| 539 |
+
"""Apply the same validation status to all visible conflicts in the table."""
|
| 540 |
+
validator_name = validator.strip()
|
| 541 |
+
if not validator_name:
|
| 542 |
+
return "Informe o nome do validador", "", None, [], page
|
| 543 |
+
|
| 544 |
+
normalized_rows: list[list[object]]
|
| 545 |
+
if hasattr(rows, "values"):
|
| 546 |
+
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 547 |
+
else:
|
| 548 |
+
normalized_rows = [list(item) for item in rows] if rows else []
|
| 549 |
+
|
| 550 |
+
if not normalized_rows:
|
| 551 |
+
return "Nenhuma deteccao com conflito para validar", "", None, [], page
|
| 552 |
+
|
| 553 |
+
conflict_rows = [row for row in normalized_rows if str(row[8]) == "CONFLICT"]
|
| 554 |
+
if not conflict_rows:
|
| 555 |
+
return "Nenhuma deteccao com conflito identificada na tabela", "", None, normalized_rows, page
|
| 556 |
+
|
| 557 |
+
success_count = 0
|
| 558 |
+
failure_count = 0
|
| 559 |
+
conflict_count = 0
|
| 560 |
+
|
| 561 |
+
for row in conflict_rows:
|
| 562 |
+
try:
|
| 563 |
+
detection_key = str(row[0]).strip()
|
| 564 |
+
expected_version = int(row[7])
|
| 565 |
+
|
| 566 |
+
_ = validation_service.validate_detection(
|
| 567 |
+
project_slug=project_slug,
|
| 568 |
+
detection_key=detection_key,
|
| 569 |
+
status=status_value,
|
| 570 |
+
validator=validator_name,
|
| 571 |
+
notes=notes.strip(),
|
| 572 |
+
expected_version=expected_version,
|
| 573 |
+
)
|
| 574 |
+
success_count += 1
|
| 575 |
+
if cache_key:
|
| 576 |
+
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 577 |
+
except OptimisticLockError:
|
| 578 |
+
conflict_count += 1
|
| 579 |
+
except Exception:
|
| 580 |
+
failure_count += 1
|
| 581 |
+
|
| 582 |
+
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 583 |
+
service=queue_service,
|
| 584 |
+
snapshot_reader=snapshot_reader,
|
| 585 |
+
project_slug=project_slug,
|
| 586 |
+
page=page,
|
| 587 |
+
scientific_name=scientific_name,
|
| 588 |
+
min_confidence=min_confidence,
|
| 589 |
+
validator_filter=validator_filter,
|
| 590 |
+
status_filter=status_filter,
|
| 591 |
+
updated_after=updated_after,
|
| 592 |
+
show_conflicts_only=False,
|
| 593 |
+
)
|
| 594 |
+
|
| 595 |
+
summary = f"Processados {len(conflict_rows)} conflitos: {success_count} sucesso, {conflict_count} novos conflitos, {failure_count} falhas"
|
| 596 |
+
status = f"{summary} | {page_status}"
|
| 597 |
+
|
| 598 |
+
return status, "", None, refreshed_rows, refreshed_page
|
| 599 |
+
|
| 600 |
+
|
| 601 |
+
def _batch_reapply_all_pending(
|
| 602 |
+
validation_service: _ValidationServiceProtocol,
|
| 603 |
+
audio_service: _AudioServiceProtocol,
|
| 604 |
+
queue_service: _QueueServiceProtocol,
|
| 605 |
+
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 606 |
+
project_slug: str,
|
| 607 |
+
rows: object,
|
| 608 |
+
pending_statuses: dict[str, str],
|
| 609 |
+
validator: str,
|
| 610 |
+
notes: str,
|
| 611 |
+
cache_key: str,
|
| 612 |
+
page: int,
|
| 613 |
+
scientific_name: str,
|
| 614 |
+
min_confidence: float,
|
| 615 |
+
validator_filter: str,
|
| 616 |
+
status_filter: str,
|
| 617 |
+
updated_after: object,
|
| 618 |
+
) -> tuple[str, str, str | None, list[list[object]], int]:
|
| 619 |
+
"""Reapply all pending validations (stored conflicts) with current version."""
|
| 620 |
+
if not pending_statuses:
|
| 621 |
+
return "Nenhuma validacao pendente para reaplicar", "", None, [], page
|
| 622 |
+
|
| 623 |
+
validator_name = validator.strip()
|
| 624 |
+
if not validator_name:
|
| 625 |
+
return "Informe o nome do validador", "", None, [], page
|
| 626 |
+
|
| 627 |
+
success_count = 0
|
| 628 |
+
conflict_count = 0
|
| 629 |
+
failure_count = 0
|
| 630 |
+
|
| 631 |
+
snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
|
| 632 |
+
|
| 633 |
+
for detection_key, status_value in pending_statuses.items():
|
| 634 |
+
try:
|
| 635 |
+
current_version = int(snapshot.get(detection_key, {}).get("version", 0))
|
| 636 |
+
|
| 637 |
+
_ = validation_service.validate_detection(
|
| 638 |
+
project_slug=project_slug,
|
| 639 |
+
detection_key=detection_key,
|
| 640 |
+
status=status_value,
|
| 641 |
+
validator=validator_name,
|
| 642 |
+
notes=notes.strip(),
|
| 643 |
+
expected_version=current_version,
|
| 644 |
+
)
|
| 645 |
+
success_count += 1
|
| 646 |
+
if cache_key:
|
| 647 |
+
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 648 |
+
except OptimisticLockError:
|
| 649 |
+
conflict_count += 1
|
| 650 |
+
except Exception:
|
| 651 |
+
failure_count += 1
|
| 652 |
+
|
| 653 |
+
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 654 |
+
service=queue_service,
|
| 655 |
+
snapshot_reader=snapshot_reader,
|
| 656 |
+
project_slug=project_slug,
|
| 657 |
+
page=page,
|
| 658 |
+
scientific_name=scientific_name,
|
| 659 |
+
min_confidence=min_confidence,
|
| 660 |
+
validator_filter=validator_filter,
|
| 661 |
+
status_filter=status_filter,
|
| 662 |
+
updated_after=updated_after,
|
| 663 |
+
show_conflicts_only=False,
|
| 664 |
+
)
|
| 665 |
+
|
| 666 |
+
summary = f"Reaplicadas {len(pending_statuses)} validacoes: {success_count} sucesso, {conflict_count} novos conflitos, {failure_count} falhas"
|
| 667 |
+
status = f"{summary} | {page_status}"
|
| 668 |
+
|
| 669 |
+
return status, "", None, refreshed_rows, refreshed_page
|
| 670 |
+
|
| 671 |
+
|
| 672 |
+
def build_demo_app(project_slug: str = "demo-project") -> gr.Blocks:
|
| 673 |
+
"""Build the demo validation app for a given project.
|
| 674 |
+
|
| 675 |
+
Args:
|
| 676 |
+
project_slug: Project identifier (default: demo-project)
|
| 677 |
+
|
| 678 |
+
Returns:
|
| 679 |
+
Gradio Blocks with validation interface
|
| 680 |
+
"""
|
| 681 |
+
service = _seed_service()
|
| 682 |
+
audio_service = AudioFetchService(EphemeralCacheManager(ttl_seconds=300, max_files=128))
|
| 683 |
+
validation_base_dir = str(Path(tempfile.gettempdir()) / "birdnet-validator-validations")
|
| 684 |
+
validation_repository = AppendOnlyValidationRepository(base_dir=validation_base_dir)
|
| 685 |
+
validation_service = ValidationService(validation_repository)
|
| 686 |
+
|
| 687 |
+
with gr.Blocks(title="BirdNET Validator HF") as demo:
|
| 688 |
+
gr.Markdown("# BirdNET Validator HF")
|
| 689 |
+
gr.Markdown("Sprint 2: fila paginada + audio sob demanda com cache efemero.")
|
| 690 |
+
|
| 691 |
+
dataset_repo = gr.Textbox(label="Dataset repo", value="SEU_USUARIO/birdnet-projeto-dataset")
|
| 692 |
+
|
| 693 |
+
with gr.Row():
|
| 694 |
+
species_filter = gr.Textbox(label="Filtro especie", placeholder="Ex: Cyanocorax cyanopogon")
|
| 695 |
+
min_confidence = gr.Slider(label="Confianca minima", minimum=0.0, maximum=1.0, step=0.01, value=0.0)
|
| 696 |
+
show_conflicts_only = gr.Checkbox(label="Mostrar apenas conflitos", value=False)
|
| 697 |
+
|
| 698 |
+
with gr.Row():
|
| 699 |
+
validator_filter = gr.Textbox(label="Filtro validador", placeholder="Ex: validator-demo")
|
| 700 |
+
validation_status_filter = gr.Dropdown(
|
| 701 |
+
label="Filtro status",
|
| 702 |
+
choices=["all", "pending", "positive", "negative", "uncertain", "skip"],
|
| 703 |
+
value="all",
|
| 704 |
+
)
|
| 705 |
+
updated_after_filter = gr.DateTime(label="Atualizado a partir de", include_time=False, type="string")
|
| 706 |
+
|
| 707 |
+
with gr.Row():
|
| 708 |
+
prev_btn = gr.Button("Pagina anterior")
|
| 709 |
+
next_btn = gr.Button("Proxima pagina")
|
| 710 |
+
refresh_btn = gr.Button("Aplicar filtros")
|
| 711 |
+
|
| 712 |
+
page_state = gr.State(value=1)
|
| 713 |
+
table = gr.Dataframe(
|
| 714 |
+
headers=[
|
| 715 |
+
"detection_key",
|
| 716 |
+
"audio_id",
|
| 717 |
+
"scientific_name",
|
| 718 |
+
"confidence",
|
| 719 |
+
"start_time",
|
| 720 |
+
"end_time",
|
| 721 |
+
"validation_status",
|
| 722 |
+
"version",
|
| 723 |
+
"conflict_flag",
|
| 724 |
+
"conflict_severity",
|
| 725 |
+
],
|
| 726 |
+
label="Deteccoes",
|
| 727 |
+
interactive=False,
|
| 728 |
+
)
|
| 729 |
+
selected_index = gr.Number(label="Linha selecionada", value=0, precision=0)
|
| 730 |
+
|
| 731 |
+
with gr.Row():
|
| 732 |
+
load_audio_btn = gr.Button("Carregar audio selecionado")
|
| 733 |
+
clear_audio_btn = gr.Button("Limpar cache apos validacao")
|
| 734 |
+
|
| 735 |
+
with gr.Row():
|
| 736 |
+
validator_name = gr.Textbox(label="Validador", value="validator-demo")
|
| 737 |
+
validation_notes = gr.Textbox(label="Notas", placeholder="Opcional")
|
| 738 |
+
|
| 739 |
+
with gr.Row():
|
| 740 |
+
approve_btn = gr.Button("Validar positivo")
|
| 741 |
+
reject_btn = gr.Button("Validar negativo")
|
| 742 |
+
uncertain_btn = gr.Button("Indeterminado")
|
| 743 |
+
skip_btn = gr.Button("Pular")
|
| 744 |
+
reapply_btn = gr.Button("Reaplicar validacao apos conflito")
|
| 745 |
+
|
| 746 |
+
with gr.Row():
|
| 747 |
+
batch_approve_conflicts_btn = gr.Button("Aprovar todos os conflitos")
|
| 748 |
+
batch_reject_conflicts_btn = gr.Button("Rejeitar todos os conflitos")
|
| 749 |
+
|
| 750 |
+
report_btn = gr.Button("Gerar relatorio de validacoes")
|
| 751 |
+
|
| 752 |
+
audio_player = gr.Audio(label="Audio sob demanda", type="filepath")
|
| 753 |
+
cache_key_state = gr.State(value="")
|
| 754 |
+
pending_status_state = gr.State(value="")
|
| 755 |
+
conflict_detection_key_state = gr.State(value="")
|
| 756 |
+
status = gr.Textbox(label="Status", interactive=False)
|
| 757 |
+
report_box = gr.Textbox(label="Relatorio", interactive=False)
|
| 758 |
+
|
| 759 |
+
# Keyboard shortcuts: 1=positive, 2=negative, 3=uncertain, 4=skip, R=reapply
|
| 760 |
+
keyboard_shortcuts_info = gr.HTML(
|
| 761 |
+
value="<div style='font-size: 12px; color: #666; padding: 8px; background-color: #f5f5f5; border-radius: 4px; margin-bottom: 10px;'>"
|
| 762 |
+
"<strong>Atalhos de teclado:</strong> 1=Positivo | 2=Negativo | 3=Indeterminado | 4=Pular | R=Reaplicar"
|
| 763 |
+
"</div>"
|
| 764 |
+
"<script>"
|
| 765 |
+
"document.addEventListener('keydown', function(event) {"
|
| 766 |
+
" if (event.target.tagName === 'INPUT' || event.target.tagName === 'TEXTAREA') return;"
|
| 767 |
+
" const key = event.key.toLowerCase();"
|
| 768 |
+
" let buttonText = null;"
|
| 769 |
+
" if (key === '1') buttonText = 'Validar positivo';"
|
| 770 |
+
" else if (key === '2') buttonText = 'Validar negativo';"
|
| 771 |
+
" else if (key === '3') buttonText = 'Indeterminado';"
|
| 772 |
+
" else if (key === '4') buttonText = 'Pular';"
|
| 773 |
+
" else if (key === 'r') buttonText = 'Reaplicar validacao apos conflito';"
|
| 774 |
+
" if (buttonText) {"
|
| 775 |
+
" event.preventDefault();"
|
| 776 |
+
" const buttons = document.querySelectorAll('button');"
|
| 777 |
+
" for (const btn of buttons) {"
|
| 778 |
+
" if (btn.textContent.includes(buttonText)) {"
|
| 779 |
+
" btn.click();"
|
| 780 |
+
" break;"
|
| 781 |
+
" }"
|
| 782 |
+
" }"
|
| 783 |
+
" }"
|
| 784 |
+
"});"
|
| 785 |
+
"</script>"
|
| 786 |
+
)
|
| 787 |
+
|
| 788 |
+
def refresh(
|
| 789 |
+
page: int,
|
| 790 |
+
species: str,
|
| 791 |
+
confidence: float,
|
| 792 |
+
validator_filter_value: str,
|
| 793 |
+
status_filter_value: str,
|
| 794 |
+
updated_after_value: object,
|
| 795 |
+
only_conflicts: bool,
|
| 796 |
+
):
|
| 797 |
+
return _page_to_table(
|
| 798 |
+
service=service,
|
| 799 |
+
snapshot_reader=validation_repository,
|
| 800 |
+
project_slug=project_slug,
|
| 801 |
+
page=page,
|
| 802 |
+
scientific_name=species,
|
| 803 |
+
min_confidence=confidence,
|
| 804 |
+
validator_filter=validator_filter_value,
|
| 805 |
+
status_filter=status_filter_value,
|
| 806 |
+
updated_after=updated_after_value,
|
| 807 |
+
show_conflicts_only=only_conflicts,
|
| 808 |
+
)
|
| 809 |
+
|
| 810 |
+
def go_next(
|
| 811 |
+
page: int,
|
| 812 |
+
species: str,
|
| 813 |
+
confidence: float,
|
| 814 |
+
validator_filter_value: str,
|
| 815 |
+
status_filter_value: str,
|
| 816 |
+
updated_after_value: object,
|
| 817 |
+
only_conflicts: bool,
|
| 818 |
+
):
|
| 819 |
+
return refresh(
|
| 820 |
+
page + 1,
|
| 821 |
+
species,
|
| 822 |
+
confidence,
|
| 823 |
+
validator_filter_value,
|
| 824 |
+
status_filter_value,
|
| 825 |
+
updated_after_value,
|
| 826 |
+
only_conflicts,
|
| 827 |
+
)
|
| 828 |
+
|
| 829 |
+
def go_prev(
|
| 830 |
+
page: int,
|
| 831 |
+
species: str,
|
| 832 |
+
confidence: float,
|
| 833 |
+
validator_filter_value: str,
|
| 834 |
+
status_filter_value: str,
|
| 835 |
+
updated_after_value: object,
|
| 836 |
+
only_conflicts: bool,
|
| 837 |
+
):
|
| 838 |
+
return refresh(
|
| 839 |
+
max(1, page - 1),
|
| 840 |
+
species,
|
| 841 |
+
confidence,
|
| 842 |
+
validator_filter_value,
|
| 843 |
+
status_filter_value,
|
| 844 |
+
updated_after_value,
|
| 845 |
+
only_conflicts,
|
| 846 |
+
)
|
| 847 |
+
|
| 848 |
+
def on_select(evt: gr.SelectData):
|
| 849 |
+
if isinstance(evt.index, tuple):
|
| 850 |
+
return int(evt.index[0])
|
| 851 |
+
if isinstance(evt.index, int):
|
| 852 |
+
return int(evt.index)
|
| 853 |
+
return 0
|
| 854 |
+
|
| 855 |
+
demo.load(
|
| 856 |
+
fn=refresh,
|
| 857 |
+
inputs=[
|
| 858 |
+
page_state,
|
| 859 |
+
species_filter,
|
| 860 |
+
min_confidence,
|
| 861 |
+
validator_filter,
|
| 862 |
+
validation_status_filter,
|
| 863 |
+
updated_after_filter,
|
| 864 |
+
show_conflicts_only,
|
| 865 |
+
],
|
| 866 |
+
outputs=[table, status, page_state],
|
| 867 |
+
)
|
| 868 |
+
refresh_btn.click(
|
| 869 |
+
fn=lambda species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: refresh(
|
| 870 |
+
1,
|
| 871 |
+
species,
|
| 872 |
+
confidence,
|
| 873 |
+
validator_filter_value,
|
| 874 |
+
status_filter_value,
|
| 875 |
+
updated_after_value,
|
| 876 |
+
only_conflicts,
|
| 877 |
+
),
|
| 878 |
+
inputs=[
|
| 879 |
+
species_filter,
|
| 880 |
+
min_confidence,
|
| 881 |
+
validator_filter,
|
| 882 |
+
validation_status_filter,
|
| 883 |
+
updated_after_filter,
|
| 884 |
+
show_conflicts_only,
|
| 885 |
+
],
|
| 886 |
+
outputs=[table, status, page_state],
|
| 887 |
+
)
|
| 888 |
+
next_btn.click(
|
| 889 |
+
fn=go_next,
|
| 890 |
+
inputs=[
|
| 891 |
+
page_state,
|
| 892 |
+
species_filter,
|
| 893 |
+
min_confidence,
|
| 894 |
+
validator_filter,
|
| 895 |
+
validation_status_filter,
|
| 896 |
+
updated_after_filter,
|
| 897 |
+
show_conflicts_only,
|
| 898 |
+
],
|
| 899 |
+
outputs=[table, status, page_state],
|
| 900 |
+
)
|
| 901 |
+
prev_btn.click(
|
| 902 |
+
fn=go_prev,
|
| 903 |
+
inputs=[
|
| 904 |
+
page_state,
|
| 905 |
+
species_filter,
|
| 906 |
+
min_confidence,
|
| 907 |
+
validator_filter,
|
| 908 |
+
validation_status_filter,
|
| 909 |
+
updated_after_filter,
|
| 910 |
+
show_conflicts_only,
|
| 911 |
+
],
|
| 912 |
+
outputs=[table, status, page_state],
|
| 913 |
+
)
|
| 914 |
+
table.select(fn=on_select, inputs=None, outputs=[selected_index])
|
| 915 |
+
load_audio_btn.click(
|
| 916 |
+
fn=lambda repo, rows, idx, cache_key: _fetch_selected_audio(
|
| 917 |
+
audio_service=audio_service,
|
| 918 |
+
dataset_repo=repo,
|
| 919 |
+
rows=rows,
|
| 920 |
+
selected_index=int(idx),
|
| 921 |
+
previous_cache_key=cache_key,
|
| 922 |
+
),
|
| 923 |
+
inputs=[dataset_repo, table, selected_index, cache_key_state],
|
| 924 |
+
outputs=[audio_player, cache_key_state, status],
|
| 925 |
+
)
|
| 926 |
+
clear_audio_btn.click(
|
| 927 |
+
fn=lambda cache_key: _cleanup_selected_audio(audio_service=audio_service, cache_key=cache_key),
|
| 928 |
+
inputs=[cache_key_state],
|
| 929 |
+
outputs=[status, audio_player],
|
| 930 |
+
)
|
| 931 |
+
approve_btn.click(
|
| 932 |
+
fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
|
| 933 |
+
validation_service=validation_service,
|
| 934 |
+
audio_service=audio_service,
|
| 935 |
+
queue_service=service,
|
| 936 |
+
snapshot_reader=validation_repository,
|
| 937 |
+
project_slug=project_slug,
|
| 938 |
+
rows=rows,
|
| 939 |
+
selected_index=int(idx),
|
| 940 |
+
status_value="positive",
|
| 941 |
+
validator=name,
|
| 942 |
+
notes=notes,
|
| 943 |
+
cache_key=cache_key,
|
| 944 |
+
page=int(page),
|
| 945 |
+
scientific_name=species,
|
| 946 |
+
min_confidence=float(confidence),
|
| 947 |
+
validator_filter=validator_filter_value,
|
| 948 |
+
status_filter=status_filter_value,
|
| 949 |
+
updated_after=updated_after_value,
|
| 950 |
+
show_conflicts_only=bool(only_conflicts),
|
| 951 |
+
),
|
| 952 |
+
inputs=[
|
| 953 |
+
table,
|
| 954 |
+
selected_index,
|
| 955 |
+
validator_name,
|
| 956 |
+
validation_notes,
|
| 957 |
+
cache_key_state,
|
| 958 |
+
page_state,
|
| 959 |
+
species_filter,
|
| 960 |
+
min_confidence,
|
| 961 |
+
validator_filter,
|
| 962 |
+
validation_status_filter,
|
| 963 |
+
updated_after_filter,
|
| 964 |
+
show_conflicts_only,
|
| 965 |
+
],
|
| 966 |
+
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 967 |
+
)
|
| 968 |
+
reject_btn.click(
|
| 969 |
+
fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
|
| 970 |
+
validation_service=validation_service,
|
| 971 |
+
audio_service=audio_service,
|
| 972 |
+
queue_service=service,
|
| 973 |
+
snapshot_reader=validation_repository,
|
| 974 |
+
project_slug=project_slug,
|
| 975 |
+
rows=rows,
|
| 976 |
+
selected_index=int(idx),
|
| 977 |
+
status_value="negative",
|
| 978 |
+
validator=name,
|
| 979 |
+
notes=notes,
|
| 980 |
+
cache_key=cache_key,
|
| 981 |
+
page=int(page),
|
| 982 |
+
scientific_name=species,
|
| 983 |
+
min_confidence=float(confidence),
|
| 984 |
+
validator_filter=validator_filter_value,
|
| 985 |
+
status_filter=status_filter_value,
|
| 986 |
+
updated_after=updated_after_value,
|
| 987 |
+
show_conflicts_only=bool(only_conflicts),
|
| 988 |
+
),
|
| 989 |
+
inputs=[
|
| 990 |
+
table,
|
| 991 |
+
selected_index,
|
| 992 |
+
validator_name,
|
| 993 |
+
validation_notes,
|
| 994 |
+
cache_key_state,
|
| 995 |
+
page_state,
|
| 996 |
+
species_filter,
|
| 997 |
+
min_confidence,
|
| 998 |
+
validator_filter,
|
| 999 |
+
validation_status_filter,
|
| 1000 |
+
updated_after_filter,
|
| 1001 |
+
show_conflicts_only,
|
| 1002 |
+
],
|
| 1003 |
+
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 1004 |
+
)
|
| 1005 |
+
uncertain_btn.click(
|
| 1006 |
+
fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
|
| 1007 |
+
validation_service=validation_service,
|
| 1008 |
+
audio_service=audio_service,
|
| 1009 |
+
queue_service=service,
|
| 1010 |
+
snapshot_reader=validation_repository,
|
| 1011 |
+
project_slug=project_slug,
|
| 1012 |
+
rows=rows,
|
| 1013 |
+
selected_index=int(idx),
|
| 1014 |
+
status_value="uncertain",
|
| 1015 |
+
validator=name,
|
| 1016 |
+
notes=notes,
|
| 1017 |
+
cache_key=cache_key,
|
| 1018 |
+
page=int(page),
|
| 1019 |
+
scientific_name=species,
|
| 1020 |
+
min_confidence=float(confidence),
|
| 1021 |
+
validator_filter=validator_filter_value,
|
| 1022 |
+
status_filter=status_filter_value,
|
| 1023 |
+
updated_after=updated_after_value,
|
| 1024 |
+
show_conflicts_only=bool(only_conflicts),
|
| 1025 |
+
),
|
| 1026 |
+
inputs=[
|
| 1027 |
+
table,
|
| 1028 |
+
selected_index,
|
| 1029 |
+
validator_name,
|
| 1030 |
+
validation_notes,
|
| 1031 |
+
cache_key_state,
|
| 1032 |
+
page_state,
|
| 1033 |
+
species_filter,
|
| 1034 |
+
min_confidence,
|
| 1035 |
+
validator_filter,
|
| 1036 |
+
validation_status_filter,
|
| 1037 |
+
updated_after_filter,
|
| 1038 |
+
show_conflicts_only,
|
| 1039 |
+
],
|
| 1040 |
+
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 1041 |
+
)
|
| 1042 |
+
skip_btn.click(
|
| 1043 |
+
fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
|
| 1044 |
+
validation_service=validation_service,
|
| 1045 |
+
audio_service=audio_service,
|
| 1046 |
+
queue_service=service,
|
| 1047 |
+
snapshot_reader=validation_repository,
|
| 1048 |
+
project_slug=project_slug,
|
| 1049 |
+
rows=rows,
|
| 1050 |
+
selected_index=int(idx),
|
| 1051 |
+
status_value="skip",
|
| 1052 |
+
validator=name,
|
| 1053 |
+
notes=notes,
|
| 1054 |
+
cache_key=cache_key,
|
| 1055 |
+
page=int(page),
|
| 1056 |
+
scientific_name=species,
|
| 1057 |
+
min_confidence=float(confidence),
|
| 1058 |
+
validator_filter=validator_filter_value,
|
| 1059 |
+
status_filter=status_filter_value,
|
| 1060 |
+
updated_after=updated_after_value,
|
| 1061 |
+
show_conflicts_only=bool(only_conflicts),
|
| 1062 |
+
),
|
| 1063 |
+
inputs=[
|
| 1064 |
+
table,
|
| 1065 |
+
selected_index,
|
| 1066 |
+
validator_name,
|
| 1067 |
+
validation_notes,
|
| 1068 |
+
cache_key_state,
|
| 1069 |
+
page_state,
|
| 1070 |
+
species_filter,
|
| 1071 |
+
min_confidence,
|
| 1072 |
+
validator_filter,
|
| 1073 |
+
validation_status_filter,
|
| 1074 |
+
updated_after_filter,
|
| 1075 |
+
show_conflicts_only,
|
| 1076 |
+
],
|
| 1077 |
+
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 1078 |
+
)
|
| 1079 |
+
reapply_btn.click(
|
| 1080 |
+
fn=lambda rows, idx, pending_status, conflict_key, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _reapply_last_conflict_validation_with_refresh(
|
| 1081 |
+
validation_service=validation_service,
|
| 1082 |
+
audio_service=audio_service,
|
| 1083 |
+
queue_service=service,
|
| 1084 |
+
snapshot_reader=validation_repository,
|
| 1085 |
+
project_slug=project_slug,
|
| 1086 |
+
rows=rows,
|
| 1087 |
+
selected_index=int(idx),
|
| 1088 |
+
pending_status_value=pending_status,
|
| 1089 |
+
conflict_detection_key=conflict_key,
|
| 1090 |
+
validator=name,
|
| 1091 |
+
notes=notes,
|
| 1092 |
+
cache_key=cache_key,
|
| 1093 |
+
page=int(page),
|
| 1094 |
+
scientific_name=species,
|
| 1095 |
+
min_confidence=float(confidence),
|
| 1096 |
+
validator_filter=validator_filter_value,
|
| 1097 |
+
status_filter=status_filter_value,
|
| 1098 |
+
updated_after=updated_after_value,
|
| 1099 |
+
show_conflicts_only=bool(only_conflicts),
|
| 1100 |
+
),
|
| 1101 |
+
inputs=[
|
| 1102 |
+
table,
|
| 1103 |
+
selected_index,
|
| 1104 |
+
pending_status_state,
|
| 1105 |
+
conflict_detection_key_state,
|
| 1106 |
+
validator_name,
|
| 1107 |
+
validation_notes,
|
| 1108 |
+
cache_key_state,
|
| 1109 |
+
page_state,
|
| 1110 |
+
species_filter,
|
| 1111 |
+
min_confidence,
|
| 1112 |
+
validator_filter,
|
| 1113 |
+
validation_status_filter,
|
| 1114 |
+
updated_after_filter,
|
| 1115 |
+
show_conflicts_only,
|
| 1116 |
+
],
|
| 1117 |
+
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 1118 |
+
)
|
| 1119 |
+
batch_approve_conflicts_btn.click(
|
| 1120 |
+
fn=lambda rows, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value: _batch_validate_conflicts(
|
| 1121 |
+
validation_service=validation_service,
|
| 1122 |
+
audio_service=audio_service,
|
| 1123 |
+
queue_service=service,
|
| 1124 |
+
snapshot_reader=validation_repository,
|
| 1125 |
+
project_slug=project_slug,
|
| 1126 |
+
rows=rows,
|
| 1127 |
+
status_value="positive",
|
| 1128 |
+
validator=name,
|
| 1129 |
+
notes=notes,
|
| 1130 |
+
cache_key=cache_key,
|
| 1131 |
+
page=int(page),
|
| 1132 |
+
scientific_name=species,
|
| 1133 |
+
min_confidence=float(confidence),
|
| 1134 |
+
validator_filter=validator_filter_value,
|
| 1135 |
+
status_filter=status_filter_value,
|
| 1136 |
+
updated_after=updated_after_value,
|
| 1137 |
+
),
|
| 1138 |
+
inputs=[
|
| 1139 |
+
table,
|
| 1140 |
+
validator_name,
|
| 1141 |
+
validation_notes,
|
| 1142 |
+
cache_key_state,
|
| 1143 |
+
page_state,
|
| 1144 |
+
species_filter,
|
| 1145 |
+
min_confidence,
|
| 1146 |
+
validator_filter,
|
| 1147 |
+
validation_status_filter,
|
| 1148 |
+
updated_after_filter,
|
| 1149 |
+
],
|
| 1150 |
+
outputs=[status, cache_key_state, audio_player, table, page_state],
|
| 1151 |
+
)
|
| 1152 |
+
batch_reject_conflicts_btn.click(
|
| 1153 |
+
fn=lambda rows, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value: _batch_validate_conflicts(
|
| 1154 |
+
validation_service=validation_service,
|
| 1155 |
+
audio_service=audio_service,
|
| 1156 |
+
queue_service=service,
|
| 1157 |
+
snapshot_reader=validation_repository,
|
| 1158 |
+
project_slug=project_slug,
|
| 1159 |
+
rows=rows,
|
| 1160 |
+
status_value="negative",
|
| 1161 |
+
validator=name,
|
| 1162 |
+
notes=notes,
|
| 1163 |
+
cache_key=cache_key,
|
| 1164 |
+
page=int(page),
|
| 1165 |
+
scientific_name=species,
|
| 1166 |
+
min_confidence=float(confidence),
|
| 1167 |
+
validator_filter=validator_filter_value,
|
| 1168 |
+
status_filter=status_filter_value,
|
| 1169 |
+
updated_after=updated_after_value,
|
| 1170 |
+
),
|
| 1171 |
+
inputs=[
|
| 1172 |
+
table,
|
| 1173 |
+
validator_name,
|
| 1174 |
+
validation_notes,
|
| 1175 |
+
cache_key_state,
|
| 1176 |
+
page_state,
|
| 1177 |
+
species_filter,
|
| 1178 |
+
min_confidence,
|
| 1179 |
+
validator_filter,
|
| 1180 |
+
validation_status_filter,
|
| 1181 |
+
updated_after_filter,
|
| 1182 |
+
],
|
| 1183 |
+
outputs=[status, cache_key_state, audio_player, table, page_state],
|
| 1184 |
+
)
|
| 1185 |
+
report_btn.click(
|
| 1186 |
+
fn=lambda: _build_validation_report(validation_repository, project_slug),
|
| 1187 |
+
inputs=None,
|
| 1188 |
+
outputs=[report_box],
|
| 1189 |
+
)
|
| 1190 |
+
|
| 1191 |
+
return demo
|
| 1192 |
+
|
| 1193 |
+
|
| 1194 |
+
def create_app() -> gr.Blocks:
|
| 1195 |
+
"""Build the BirdNET Validator app with multi-project auth integration.
|
| 1196 |
+
|
| 1197 |
+
Returns multi-tab interface with:
|
| 1198 |
+
- Login tab for user authentication
|
| 1199 |
+
- Project selection for authorized projects
|
| 1200 |
+
- Admin panel for project/user management (admin only)
|
| 1201 |
+
- Validation interface for selected project
|
| 1202 |
+
|
| 1203 |
+
Returns:
|
| 1204 |
+
Gradio Blocks with full auth-integrated app
|
| 1205 |
+
"""
|
| 1206 |
+
# Initialize auth service
|
| 1207 |
+
auth_service = AuthService(session_ttl_minutes=120)
|
| 1208 |
+
|
| 1209 |
+
# Setup demo users (in production: database/LDAP/OAuth)
|
| 1210 |
+
auth_service.register_user_project_access(
|
| 1211 |
+
"admin_user",
|
| 1212 |
+
{"kenya-2024": Role.admin, "nairobi-2023": Role.admin},
|
| 1213 |
+
)
|
| 1214 |
+
auth_service.register_user_project_access(
|
| 1215 |
+
"validator_demo",
|
| 1216 |
+
{"demo-project": Role.validator, "kenya-2024": Role.validator},
|
| 1217 |
+
)
|
| 1218 |
+
auth_service.register_user_project_access(
|
| 1219 |
+
"validator_other",
|
| 1220 |
+
{"nairobi-2023": Role.validator},
|
| 1221 |
+
)
|
| 1222 |
+
|
| 1223 |
+
# Initialize admin panel manager
|
| 1224 |
+
admin_manager = AdminPanelManager(auth_service)
|
| 1225 |
+
|
| 1226 |
+
# Register projects
|
| 1227 |
+
admin_manager.register_project(
|
| 1228 |
+
Project(
|
| 1229 |
+
project_slug="kenya-2024",
|
| 1230 |
+
name="Kenya Survey 2024",
|
| 1231 |
+
dataset_repo_id="birdnet/kenya-2024-dataset",
|
| 1232 |
+
active=True,
|
| 1233 |
+
)
|
| 1234 |
+
)
|
| 1235 |
+
admin_manager.register_project(
|
| 1236 |
+
Project(
|
| 1237 |
+
project_slug="nairobi-2023",
|
| 1238 |
+
name="Nairobi Survey 2023",
|
| 1239 |
+
dataset_repo_id="birdnet/nairobi-2023-dataset",
|
| 1240 |
+
active=True,
|
| 1241 |
+
)
|
| 1242 |
+
)
|
| 1243 |
+
admin_manager.register_project(
|
| 1244 |
+
Project(
|
| 1245 |
+
project_slug="demo-project",
|
| 1246 |
+
name="Demo Project",
|
| 1247 |
+
dataset_repo_id="birdnet/demo-dataset",
|
| 1248 |
+
active=True,
|
| 1249 |
+
)
|
| 1250 |
+
)
|
| 1251 |
+
|
| 1252 |
+
with gr.Blocks(title="BirdNET Validator HF - Multi-Project") as wrapper:
|
| 1253 |
+
gr.Markdown("# BirdNET Validator HF - Sprint 4: SeguranΓ§a Multi-Projeto")
|
| 1254 |
+
gr.Markdown("**VersΓ£o com autenticaΓ§Γ£o, autorizaΓ§Γ£o por projeto, e painel admin**")
|
| 1255 |
+
|
| 1256 |
+
# Session state
|
| 1257 |
+
session_state = gr.State(value=None)
|
| 1258 |
+
selected_project_state = gr.State(value=None)
|
| 1259 |
+
|
| 1260 |
+
def _project_rows() -> list[list[object]]:
|
| 1261 |
+
projects = admin_manager.list_projects()
|
| 1262 |
+
return [
|
| 1263 |
+
[
|
| 1264 |
+
p["project_slug"],
|
| 1265 |
+
p["name"],
|
| 1266 |
+
p["dataset_repo_id"],
|
| 1267 |
+
"yes" if bool(p["active"]) else "no",
|
| 1268 |
+
]
|
| 1269 |
+
for p in projects
|
| 1270 |
+
]
|
| 1271 |
+
|
| 1272 |
+
def _project_slugs() -> list[str]:
|
| 1273 |
+
return [p["project_slug"] for p in admin_manager.list_projects()]
|
| 1274 |
+
|
| 1275 |
+
with gr.Tabs():
|
| 1276 |
+
# ===== TAB 1: Login =====
|
| 1277 |
+
with gr.Tab("π Login", id="login_tab"):
|
| 1278 |
+
login_block, username_input, session_output, login_button, error_message = (
|
| 1279 |
+
create_login_page(auth_service)
|
| 1280 |
+
)
|
| 1281 |
+
|
| 1282 |
+
# Store session ID when login succeeds
|
| 1283 |
+
def handle_login_success(session_id: str):
|
| 1284 |
+
"""Process successful login and store session."""
|
| 1285 |
+
if session_id:
|
| 1286 |
+
return auth_service.get_session(session_id)
|
| 1287 |
+
return None
|
| 1288 |
+
|
| 1289 |
+
session_output.change(
|
| 1290 |
+
fn=handle_login_success,
|
| 1291 |
+
inputs=[session_output],
|
| 1292 |
+
outputs=[session_state],
|
| 1293 |
+
)
|
| 1294 |
+
|
| 1295 |
+
# ===== TAB 2: Admin Panel =====
|
| 1296 |
+
with gr.Tab("βοΈ Admin", id="admin_tab"):
|
| 1297 |
+
admin_info = gr.Markdown(value="β οΈ FaΓ§a login primeiro")
|
| 1298 |
+
|
| 1299 |
+
def create_admin_display(session):
|
| 1300 |
+
"""Show admin panel or access denied message."""
|
| 1301 |
+
if session is None:
|
| 1302 |
+
return (
|
| 1303 |
+
"β **Not Logged In** β Please login first in the Login tab.",
|
| 1304 |
+
gr.update(visible=False),
|
| 1305 |
+
)
|
| 1306 |
+
if session.role.value != "admin":
|
| 1307 |
+
return (
|
| 1308 |
+
"β **Access Denied** β Only administrators can access this panel.",
|
| 1309 |
+
gr.update(visible=False),
|
| 1310 |
+
)
|
| 1311 |
+
return (
|
| 1312 |
+
f"β
**Admin Panel** β Welcome {session.username}! Click below to manage projects and users.",
|
| 1313 |
+
gr.update(visible=True),
|
| 1314 |
+
)
|
| 1315 |
+
|
| 1316 |
+
with gr.Group(visible=False) as admin_controls:
|
| 1317 |
+
gr.Markdown("#### Projetos Cadastrados")
|
| 1318 |
+
|
| 1319 |
+
with gr.Row():
|
| 1320 |
+
create_project_slug = gr.Textbox(
|
| 1321 |
+
label="Novo Project Slug",
|
| 1322 |
+
placeholder="ex: amazonas-2026",
|
| 1323 |
+
)
|
| 1324 |
+
create_project_name = gr.Textbox(
|
| 1325 |
+
label="Nome do Projeto",
|
| 1326 |
+
placeholder="ex: Amazonas Survey 2026",
|
| 1327 |
+
)
|
| 1328 |
+
create_project_repo = gr.Textbox(
|
| 1329 |
+
label="HF Dataset Repo ID",
|
| 1330 |
+
placeholder="ex: birdnet/amazonas-2026-dataset",
|
| 1331 |
+
)
|
| 1332 |
+
|
| 1333 |
+
create_project_message = gr.Markdown()
|
| 1334 |
+
|
| 1335 |
+
def create_project(session, slug: str, name: str, repo_id: str):
|
| 1336 |
+
if session is None or session.role.value != "admin":
|
| 1337 |
+
return "β Access denied. Apenas admin pode criar projetos.", gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
|
| 1338 |
+
|
| 1339 |
+
slug = (slug or "").strip()
|
| 1340 |
+
name = (name or "").strip()
|
| 1341 |
+
repo_id = (repo_id or "").strip()
|
| 1342 |
+
if not slug or not name or not repo_id:
|
| 1343 |
+
return "β οΈ Preencha slug, nome e repo id.", gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
|
| 1344 |
+
|
| 1345 |
+
created = admin_manager.register_project(
|
| 1346 |
+
Project(
|
| 1347 |
+
project_slug=slug,
|
| 1348 |
+
name=name,
|
| 1349 |
+
dataset_repo_id=repo_id,
|
| 1350 |
+
active=True,
|
| 1351 |
+
)
|
| 1352 |
+
)
|
| 1353 |
+
if not created:
|
| 1354 |
+
return (
|
| 1355 |
+
f"β οΈ Projeto '{slug}' jΓ‘ existe.",
|
| 1356 |
+
_project_rows(),
|
| 1357 |
+
gr.update(choices=_project_slugs()),
|
| 1358 |
+
gr.update(),
|
| 1359 |
+
gr.update(),
|
| 1360 |
+
gr.update(),
|
| 1361 |
+
)
|
| 1362 |
+
|
| 1363 |
+
return (
|
| 1364 |
+
f"β
Projeto '{slug}' criado com sucesso.",
|
| 1365 |
+
_project_rows(),
|
| 1366 |
+
gr.update(choices=_project_slugs(), value=slug),
|
| 1367 |
+
gr.update(value=""),
|
| 1368 |
+
gr.update(value=""),
|
| 1369 |
+
gr.update(value=""),
|
| 1370 |
+
)
|
| 1371 |
+
|
| 1372 |
+
create_project_btn = gr.Button("β Criar Projeto", variant="primary")
|
| 1373 |
+
projects_table = gr.Dataframe(
|
| 1374 |
+
value=_project_rows(),
|
| 1375 |
+
headers=["project_slug", "name", "dataset_repo_id", "active"],
|
| 1376 |
+
interactive=False,
|
| 1377 |
+
)
|
| 1378 |
+
refresh_projects_btn = gr.Button("π Atualizar Lista")
|
| 1379 |
+
|
| 1380 |
+
def refresh_projects(session):
|
| 1381 |
+
if session is None or session.role.value != "admin":
|
| 1382 |
+
return []
|
| 1383 |
+
return _project_rows()
|
| 1384 |
+
|
| 1385 |
+
refresh_projects_btn.click(
|
| 1386 |
+
fn=refresh_projects,
|
| 1387 |
+
inputs=[session_state],
|
| 1388 |
+
outputs=[projects_table],
|
| 1389 |
+
)
|
| 1390 |
+
|
| 1391 |
+
with gr.Group(visible=False) as admin_users_controls:
|
| 1392 |
+
gr.Markdown("#### Assignar UsuΓ‘rio a Projeto")
|
| 1393 |
+
with gr.Row():
|
| 1394 |
+
admin_username = gr.Textbox(
|
| 1395 |
+
label="UsuΓ‘rio", placeholder="validator_001"
|
| 1396 |
+
)
|
| 1397 |
+
admin_project = gr.Dropdown(
|
| 1398 |
+
choices=_project_slugs(),
|
| 1399 |
+
label="Projeto",
|
| 1400 |
+
)
|
| 1401 |
+
admin_role = gr.Dropdown(
|
| 1402 |
+
choices=["admin", "validator"],
|
| 1403 |
+
value="validator",
|
| 1404 |
+
label="Role",
|
| 1405 |
+
)
|
| 1406 |
+
|
| 1407 |
+
admin_message = gr.Markdown()
|
| 1408 |
+
|
| 1409 |
+
def assign_user(session, username: str, project: str, role: str):
|
| 1410 |
+
if session is None or session.role.value != "admin":
|
| 1411 |
+
return "β Access denied. Apenas admin pode atribuir usuΓ‘rios.", gr.update(), gr.update(), gr.update()
|
| 1412 |
+
success, msg = admin_manager.assign_user_to_project(
|
| 1413 |
+
username, project, role
|
| 1414 |
+
)
|
| 1415 |
+
if success:
|
| 1416 |
+
return msg, gr.update(value=""), gr.update(value=None), gr.update(value="validator")
|
| 1417 |
+
return msg, gr.update(), gr.update(), gr.update()
|
| 1418 |
+
|
| 1419 |
+
assign_btn = gr.Button("β
Assignar", variant="primary")
|
| 1420 |
+
assign_btn.click(
|
| 1421 |
+
fn=assign_user,
|
| 1422 |
+
inputs=[session_state, admin_username, admin_project, admin_role],
|
| 1423 |
+
outputs=[admin_message, admin_username, admin_project, admin_role],
|
| 1424 |
+
)
|
| 1425 |
+
|
| 1426 |
+
create_project_btn.click(
|
| 1427 |
+
fn=create_project,
|
| 1428 |
+
inputs=[session_state, create_project_slug, create_project_name, create_project_repo],
|
| 1429 |
+
outputs=[
|
| 1430 |
+
create_project_message,
|
| 1431 |
+
projects_table,
|
| 1432 |
+
admin_project,
|
| 1433 |
+
create_project_slug,
|
| 1434 |
+
create_project_name,
|
| 1435 |
+
create_project_repo,
|
| 1436 |
+
],
|
| 1437 |
+
)
|
| 1438 |
+
|
| 1439 |
+
session_state.change(
|
| 1440 |
+
fn=create_admin_display,
|
| 1441 |
+
inputs=[session_state],
|
| 1442 |
+
outputs=[admin_info, admin_controls],
|
| 1443 |
+
)
|
| 1444 |
+
|
| 1445 |
+
session_state.change(
|
| 1446 |
+
fn=lambda s: gr.update(visible=bool(s is not None and s.role.value == "admin")),
|
| 1447 |
+
inputs=[session_state],
|
| 1448 |
+
outputs=[admin_users_controls],
|
| 1449 |
+
)
|
| 1450 |
+
|
| 1451 |
+
# ===== TAB 3: Project Selection =====
|
| 1452 |
+
with gr.Tab("π Selecionar Projeto", id="project_tab"):
|
| 1453 |
+
project_info_display = gr.Markdown(
|
| 1454 |
+
value="β οΈ FaΓ§a login primeiro na aba **Login**"
|
| 1455 |
+
)
|
| 1456 |
+
project_selector = gr.Dropdown(
|
| 1457 |
+
choices=[],
|
| 1458 |
+
label="Projeto Autorizado",
|
| 1459 |
+
interactive=False,
|
| 1460 |
+
)
|
| 1461 |
+
|
| 1462 |
+
def update_project_selector(session):
|
| 1463 |
+
"""Update project dropdown when user logs in."""
|
| 1464 |
+
if session is None:
|
| 1465 |
+
return (
|
| 1466 |
+
gr.Dropdown(choices=[], value=None, interactive=False),
|
| 1467 |
+
"β Not logged in. Please login first.",
|
| 1468 |
+
None,
|
| 1469 |
+
)
|
| 1470 |
+
|
| 1471 |
+
projects = session.authorized_projects
|
| 1472 |
+
if not projects:
|
| 1473 |
+
return (
|
| 1474 |
+
gr.Dropdown(choices=[], value=None, interactive=False),
|
| 1475 |
+
"β οΈ **No Projects Assigned**\n\nYou don't have access to any projects yet. Contact an administrator.",
|
| 1476 |
+
None,
|
| 1477 |
+
)
|
| 1478 |
+
|
| 1479 |
+
selected = projects[0]
|
| 1480 |
+
role = auth_service.get_user_role_for_project(session.username, selected)
|
| 1481 |
+
role_label = role.value.upper() if role else "UNKNOWN"
|
| 1482 |
+
return (
|
| 1483 |
+
gr.Dropdown(choices=projects, value=selected, interactive=True),
|
| 1484 |
+
f"π **Project:** {selected} | **Your Role:** {role_label}",
|
| 1485 |
+
selected,
|
| 1486 |
+
)
|
| 1487 |
+
|
| 1488 |
+
session_state.change(
|
| 1489 |
+
fn=update_project_selector,
|
| 1490 |
+
inputs=[session_state],
|
| 1491 |
+
outputs=[project_selector, project_info_display, selected_project_state],
|
| 1492 |
+
)
|
| 1493 |
+
|
| 1494 |
+
def update_selected_project(selected: str, session):
|
| 1495 |
+
"""Update state when project is selected."""
|
| 1496 |
+
if session and selected:
|
| 1497 |
+
return selected
|
| 1498 |
+
return None
|
| 1499 |
+
|
| 1500 |
+
project_selector.change(
|
| 1501 |
+
fn=update_selected_project,
|
| 1502 |
+
inputs=[project_selector, session_state],
|
| 1503 |
+
outputs=[selected_project_state],
|
| 1504 |
+
)
|
| 1505 |
+
|
| 1506 |
+
# ===== TAB 4: Validation =====
|
| 1507 |
+
with gr.Tab("β ValidaΓ§Γ£o", id="validation_tab"):
|
| 1508 |
+
validation_status = gr.Markdown(
|
| 1509 |
+
value="βΉοΈ FaΓ§a login e selecione um projeto para comeΓ§ar"
|
| 1510 |
+
)
|
| 1511 |
+
|
| 1512 |
+
def get_validation_status(session, selected_project):
|
| 1513 |
+
"""Show status message based on login/project state."""
|
| 1514 |
+
if session is None:
|
| 1515 |
+
return "β **NΓ£o autenticado** β FaΓ§a login na aba **Login** primeiro"
|
| 1516 |
+
if selected_project is None:
|
| 1517 |
+
return f"β οΈ **Projeto nΓ£o selecionado** β Selecione um projeto na aba **Selecionar Projeto**"
|
| 1518 |
+
return f"β
**Pronto para validar** β Projeto: **{selected_project}** | UsuΓ‘rio: **{session.username}**\n\n(ValidaΓ§Γ£o serΓ‘ carregada abaixo)"
|
| 1519 |
+
|
| 1520 |
+
session_state.change(
|
| 1521 |
+
fn=lambda s, p: get_validation_status(s, p),
|
| 1522 |
+
inputs=[session_state, selected_project_state],
|
| 1523 |
+
outputs=[validation_status],
|
| 1524 |
+
)
|
| 1525 |
+
|
| 1526 |
+
selected_project_state.change(
|
| 1527 |
+
fn=lambda s, p: get_validation_status(s, p),
|
| 1528 |
+
inputs=[session_state, selected_project_state],
|
| 1529 |
+
outputs=[validation_status],
|
| 1530 |
+
)
|
| 1531 |
+
|
| 1532 |
+
# Note: In a real implementation, we would load build_demo_app(selected_project) here
|
| 1533 |
+
# For now, we show a placeholder since Gradio tabs don't support dynamic nested Blocks
|
| 1534 |
+
gr.Markdown("---")
|
| 1535 |
+
validation_placeholder = gr.Textbox(
|
| 1536 |
+
label="ValidaΓ§Γ£o",
|
| 1537 |
+
value="π Interface de validaΓ§Γ£o serΓ‘ carregada aqui para o projeto selecionado",
|
| 1538 |
+
interactive=False,
|
| 1539 |
+
lines=10,
|
| 1540 |
+
)
|
| 1541 |
+
|
| 1542 |
+
return wrapper
|
BirdNET Validator App/src/ui/login_page.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Gradio login page component for multi-project authorization."""
|
| 2 |
+
|
| 3 |
+
from typing import Optional, Tuple
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
|
| 7 |
+
from src.auth.auth_service import AuthService, Session
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def create_login_page(auth_service: AuthService) -> Tuple[gr.Blocks, gr.Textbox, gr.Textbox, gr.Button, gr.Markdown]:
|
| 11 |
+
"""Create a Gradio login page with username input and session tracking.
|
| 12 |
+
|
| 13 |
+
Args:
|
| 14 |
+
auth_service: AuthService instance for login validation
|
| 15 |
+
|
| 16 |
+
Returns:
|
| 17 |
+
Tuple of (login_block, username_input, session_output, login_button, error_message)
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
with gr.Blocks(title="BirdNET Validator - Login") as login_block:
|
| 21 |
+
gr.Markdown("# BirdNET Validation Platform")
|
| 22 |
+
gr.Markdown("Login to access multi-project validation workflows")
|
| 23 |
+
|
| 24 |
+
username_input = gr.Textbox(
|
| 25 |
+
label="Username",
|
| 26 |
+
placeholder="Enter your username",
|
| 27 |
+
lines=1,
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
error_message = gr.Markdown()
|
| 31 |
+
|
| 32 |
+
login_button = gr.Button("Login", variant="primary", scale=1)
|
| 33 |
+
|
| 34 |
+
session_output = gr.Textbox(
|
| 35 |
+
label="Session ID",
|
| 36 |
+
interactive=False,
|
| 37 |
+
visible=False,
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
def perform_login(username: str) -> Tuple[str, str]:
|
| 41 |
+
"""Attempt login and return session ID or error message.
|
| 42 |
+
|
| 43 |
+
Args:
|
| 44 |
+
username: Username to authenticate
|
| 45 |
+
|
| 46 |
+
Returns:
|
| 47 |
+
Tuple of (session_id, error_message)
|
| 48 |
+
"""
|
| 49 |
+
if not username or not username.strip():
|
| 50 |
+
return "", "β Please enter a username"
|
| 51 |
+
|
| 52 |
+
username = username.strip()
|
| 53 |
+
session = auth_service.login(username)
|
| 54 |
+
|
| 55 |
+
if session is None:
|
| 56 |
+
return "", f"β User '{username}' not found or inactive"
|
| 57 |
+
|
| 58 |
+
return session.session_id, f"β
Welcome, {username}! (Admin)" if session.role.value == "admin" else f"β
Welcome, {username}! (Validator)"
|
| 59 |
+
|
| 60 |
+
login_button.click(
|
| 61 |
+
fn=perform_login,
|
| 62 |
+
inputs=[username_input],
|
| 63 |
+
outputs=[session_output, error_message],
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
return login_block, username_input, session_output, login_button, error_message
|
BirdNET Validator App/src/ui/project_selector.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Project selector component for switching between authorized projects."""
|
| 2 |
+
|
| 3 |
+
from typing import List, Tuple
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
|
| 7 |
+
from src.auth.auth_service import AuthService, Session
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def create_project_selector(
|
| 11 |
+
auth_service: AuthService, current_session: Session
|
| 12 |
+
) -> Tuple[gr.Dropdown, gr.Markdown]:
|
| 13 |
+
"""Create a project selector dropdown for the current user.
|
| 14 |
+
|
| 15 |
+
Args:
|
| 16 |
+
auth_service: AuthService instance
|
| 17 |
+
current_session: Current user's session
|
| 18 |
+
|
| 19 |
+
Returns:
|
| 20 |
+
Tuple of (project_dropdown, project_info_display)
|
| 21 |
+
"""
|
| 22 |
+
authorized_projects = current_session.authorized_projects
|
| 23 |
+
|
| 24 |
+
if not authorized_projects:
|
| 25 |
+
with gr.Group():
|
| 26 |
+
gr.Markdown("β οΈ **No Projects Assigned**\n\nYou don't have access to any projects yet. Contact an administrator.")
|
| 27 |
+
|
| 28 |
+
return (
|
| 29 |
+
gr.Dropdown(choices=[], label="Project", interactive=False),
|
| 30 |
+
gr.Markdown(""),
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
with gr.Group():
|
| 34 |
+
project_dropdown = gr.Dropdown(
|
| 35 |
+
choices=authorized_projects,
|
| 36 |
+
value=authorized_projects[0],
|
| 37 |
+
label="Select Project",
|
| 38 |
+
interactive=True,
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
project_info = gr.Markdown()
|
| 42 |
+
|
| 43 |
+
def update_project_info(selected_project: str) -> str:
|
| 44 |
+
"""Display info about the selected project.
|
| 45 |
+
|
| 46 |
+
Args:
|
| 47 |
+
selected_project: Project slug
|
| 48 |
+
|
| 49 |
+
Returns:
|
| 50 |
+
Markdown formatted project info
|
| 51 |
+
"""
|
| 52 |
+
role = auth_service.get_user_role_for_project(
|
| 53 |
+
current_session.username, selected_project
|
| 54 |
+
)
|
| 55 |
+
if role:
|
| 56 |
+
return f"π **Project:** {selected_project} | **Your Role:** {role.value.upper()}"
|
| 57 |
+
return f"π **Project:** {selected_project}"
|
| 58 |
+
|
| 59 |
+
# Initial display
|
| 60 |
+
initial_project = authorized_projects[0]
|
| 61 |
+
role = auth_service.get_user_role_for_project(
|
| 62 |
+
current_session.username, initial_project
|
| 63 |
+
)
|
| 64 |
+
initial_info = (
|
| 65 |
+
f"π **Project:** {initial_project} | **Your Role:** {role.value.upper()}"
|
| 66 |
+
if role
|
| 67 |
+
else f"π **Project:** {initial_project}"
|
| 68 |
+
)
|
| 69 |
+
project_info.value = initial_info
|
| 70 |
+
|
| 71 |
+
project_dropdown.change(
|
| 72 |
+
fn=update_project_info,
|
| 73 |
+
inputs=[project_dropdown],
|
| 74 |
+
outputs=[project_info],
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
return project_dropdown, project_info
|
BirdNET Validator App/src/ui/upload_app_factory.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
from huggingface_hub import HfApi
|
| 7 |
+
|
| 8 |
+
from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _as_pretty_json(payload: dict[str, Any]) -> str:
|
| 12 |
+
return json.dumps(payload, indent=2, ensure_ascii=False)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _build_api(token: str) -> HfApi:
|
| 16 |
+
clean_token = (token or "").strip()
|
| 17 |
+
if clean_token:
|
| 18 |
+
return HfApi(token=clean_token)
|
| 19 |
+
return HfApi()
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def build_upload_app() -> gr.Blocks:
|
| 23 |
+
with gr.Blocks(title="BirdNET Segments Uploader") as demo:
|
| 24 |
+
gr.Markdown("# BirdNET Segments Uploader")
|
| 25 |
+
gr.Markdown(
|
| 26 |
+
"Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
|
| 27 |
+
"Recomendado: executar dry-run antes do upload real."
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
with gr.Row():
|
| 31 |
+
project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
|
| 32 |
+
dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
|
| 33 |
+
|
| 34 |
+
detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
|
| 35 |
+
segments_root = gr.Textbox(
|
| 36 |
+
label="Pasta raiz de segmentos",
|
| 37 |
+
placeholder=r"ex: C:\dados\BirdNET Segments",
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
with gr.Accordion("Configuracao avancada", open=False):
|
| 41 |
+
hf_token = gr.Textbox(
|
| 42 |
+
label="HF Token (opcional)",
|
| 43 |
+
type="password",
|
| 44 |
+
placeholder="Se vazio, usa sessao ja autenticada no ambiente",
|
| 45 |
+
)
|
| 46 |
+
with gr.Row():
|
| 47 |
+
batch_size = gr.Number(label="Batch size", value=200, precision=0)
|
| 48 |
+
shard_size = gr.Number(label="Shard size", value=10000, precision=0)
|
| 49 |
+
with gr.Row():
|
| 50 |
+
max_retries = gr.Number(label="Max retries", value=3, precision=0)
|
| 51 |
+
retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
|
| 52 |
+
resume_state_file = gr.Textbox(
|
| 53 |
+
label="Resume state file",
|
| 54 |
+
value=".ingest-segments-state.json",
|
| 55 |
+
)
|
| 56 |
+
report_file = gr.Textbox(
|
| 57 |
+
label="Report file local (opcional)",
|
| 58 |
+
placeholder="ex: .ingest-run-report.json",
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
with gr.Row():
|
| 62 |
+
dry_run_button = gr.Button("Dry-run", variant="secondary")
|
| 63 |
+
upload_button = gr.Button("Upload real", variant="primary")
|
| 64 |
+
|
| 65 |
+
status = gr.Markdown(value="Pronto")
|
| 66 |
+
result_json = gr.Code(label="Resultado JSON", language="json")
|
| 67 |
+
|
| 68 |
+
def run_dry_run(
|
| 69 |
+
project: str,
|
| 70 |
+
repo: str,
|
| 71 |
+
csv_path: str | None,
|
| 72 |
+
segments_path: str,
|
| 73 |
+
report_path: str,
|
| 74 |
+
) -> tuple[str, str]:
|
| 75 |
+
project = (project or "").strip()
|
| 76 |
+
if not project:
|
| 77 |
+
return "β Informe o project slug", "{}"
|
| 78 |
+
|
| 79 |
+
if not csv_path:
|
| 80 |
+
return "β Selecione o CSV de deteccoes", "{}"
|
| 81 |
+
|
| 82 |
+
if not segments_path or not Path(segments_path).exists():
|
| 83 |
+
return "β Pasta de segmentos invalida", "{}"
|
| 84 |
+
|
| 85 |
+
try:
|
| 86 |
+
result = run_ingest_segments_dry_run(
|
| 87 |
+
project_slug=project,
|
| 88 |
+
detections_csv=csv_path,
|
| 89 |
+
segments_root=segments_path,
|
| 90 |
+
)
|
| 91 |
+
if report_path.strip():
|
| 92 |
+
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 93 |
+
summary = (
|
| 94 |
+
"β
Dry-run finalizado | "
|
| 95 |
+
f"Matched: {result['matched_rows']} | "
|
| 96 |
+
f"Unmatched: {result['unmatched_rows']} | "
|
| 97 |
+
f"Ambiguous: {result['ambiguous_rows']}"
|
| 98 |
+
)
|
| 99 |
+
return summary, _as_pretty_json(result)
|
| 100 |
+
except Exception as exc:
|
| 101 |
+
return f"β Dry-run falhou: {exc}", "{}"
|
| 102 |
+
|
| 103 |
+
def run_upload(
|
| 104 |
+
project: str,
|
| 105 |
+
repo: str,
|
| 106 |
+
csv_path: str | None,
|
| 107 |
+
segments_path: str,
|
| 108 |
+
token: str,
|
| 109 |
+
batch: float,
|
| 110 |
+
shard: float,
|
| 111 |
+
retries: float,
|
| 112 |
+
backoff: float,
|
| 113 |
+
resume_file: str,
|
| 114 |
+
report_path: str,
|
| 115 |
+
) -> tuple[str, str]:
|
| 116 |
+
project = (project or "").strip()
|
| 117 |
+
repo = (repo or "").strip()
|
| 118 |
+
if not project:
|
| 119 |
+
return "β Informe o project slug", "{}"
|
| 120 |
+
if not repo or "/" not in repo:
|
| 121 |
+
return "β Informe dataset repo no formato owner/repo", "{}"
|
| 122 |
+
if not csv_path:
|
| 123 |
+
return "β Selecione o CSV de deteccoes", "{}"
|
| 124 |
+
if not segments_path or not Path(segments_path).exists():
|
| 125 |
+
return "β Pasta de segmentos invalida", "{}"
|
| 126 |
+
|
| 127 |
+
try:
|
| 128 |
+
api = _build_api(token)
|
| 129 |
+
result = ingest_segments_to_hf(
|
| 130 |
+
api=api,
|
| 131 |
+
project_slug=project,
|
| 132 |
+
dataset_repo=repo,
|
| 133 |
+
detections_csv=csv_path,
|
| 134 |
+
segments_root=segments_path,
|
| 135 |
+
batch_size=int(batch),
|
| 136 |
+
shard_size=int(shard),
|
| 137 |
+
max_retries=int(retries),
|
| 138 |
+
retry_backoff_seconds=float(backoff),
|
| 139 |
+
resume_state_file=resume_file,
|
| 140 |
+
)
|
| 141 |
+
if report_path.strip():
|
| 142 |
+
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 143 |
+
summary = (
|
| 144 |
+
"β
Upload finalizado | "
|
| 145 |
+
f"Uploaded now: {result['uploaded_audio_now']} | "
|
| 146 |
+
f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
|
| 147 |
+
f"Failed uploads: {result['failed_uploads']}"
|
| 148 |
+
)
|
| 149 |
+
return summary, _as_pretty_json(result)
|
| 150 |
+
except Exception as exc:
|
| 151 |
+
return f"β Upload falhou: {exc}", "{}"
|
| 152 |
+
|
| 153 |
+
dry_run_button.click(
|
| 154 |
+
fn=run_dry_run,
|
| 155 |
+
inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
|
| 156 |
+
outputs=[status, result_json],
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
upload_button.click(
|
| 160 |
+
fn=run_upload,
|
| 161 |
+
inputs=[
|
| 162 |
+
project_slug,
|
| 163 |
+
dataset_repo,
|
| 164 |
+
detections_csv,
|
| 165 |
+
segments_root,
|
| 166 |
+
hf_token,
|
| 167 |
+
batch_size,
|
| 168 |
+
shard_size,
|
| 169 |
+
max_retries,
|
| 170 |
+
retry_backoff_seconds,
|
| 171 |
+
resume_state_file,
|
| 172 |
+
report_file,
|
| 173 |
+
],
|
| 174 |
+
outputs=[status, result_json],
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
return demo
|