Spaces:
Sleeping
Sleeping
Commit ·
c85843b
1
Parent(s): f5bcf37
refactor: focus repository on validator app only
Browse files- .ingest-dry-run-report.json +0 -14
- BirdNET Uploader App/README.md +0 -84
- BirdNET Uploader App/app.py +0 -9
- BirdNET Uploader App/cli/hf_dataset_cli.py +0 -969
- BirdNET Uploader App/pyproject.toml +0 -9
- BirdNET Uploader App/requirements.txt +0 -7
- BirdNET Uploader App/src/__init__.py +0 -0
- BirdNET Uploader App/src/ui/__init__.py +0 -0
- BirdNET Uploader App/src/ui/upload_app_factory.py +0 -177
- BirdNET Validator App/README.md +0 -83
- BirdNET Validator App/app.py +0 -6
- BirdNET Validator App/pyproject.toml +0 -9
- BirdNET Validator App/requirements.txt +0 -7
- BirdNET Validator App/src/__init__.py +0 -0
- BirdNET Validator App/src/auth/__init__.py +0 -0
- BirdNET Validator App/src/auth/auth_service.py +0 -200
- BirdNET Validator App/src/cache/__init__.py +0 -0
- BirdNET Validator App/src/cache/ephemeral_cache_manager.py +0 -82
- BirdNET Validator App/src/domain/__init__.py +0 -0
- BirdNET Validator App/src/domain/models.py +0 -46
- BirdNET Validator App/src/repositories/__init__.py +0 -0
- BirdNET Validator App/src/repositories/append_only_validation_repository.py +0 -89
- BirdNET Validator App/src/repositories/contracts.py +0 -35
- BirdNET Validator App/src/repositories/in_memory_detection_repository.py +0 -73
- BirdNET Validator App/src/repositories/in_memory_validation_repository.py +0 -25
- BirdNET Validator App/src/services/__init__.py +0 -0
- BirdNET Validator App/src/services/audio_fetch_service.py +0 -54
- BirdNET Validator App/src/services/detection_queue_service.py +0 -64
- BirdNET Validator App/src/services/validation_service.py +0 -39
- BirdNET Validator App/src/telemetry/__init__.py +0 -0
- BirdNET Validator App/src/ui/__init__.py +0 -0
- BirdNET Validator App/src/ui/admin_panel.py +0 -281
- BirdNET Validator App/src/ui/app_factory.py +0 -1542
- BirdNET Validator App/src/ui/login_page.py +0 -66
- BirdNET Validator App/src/ui/project_selector.py +0 -77
- BirdNET Validator App/src/ui/upload_app_factory.py +0 -177
- README.md +64 -82
- app_uploader.py +0 -9
- cli/hf_dataset_cli.py +0 -969
- docs/HF_SEGMENTS_INGESTION_PLAN.md +0 -194
- pyproject.toml +1 -1
- requirements.txt +0 -3
- src/ui/upload_app_factory.py +0 -177
- tests/unit/test_hf_dataset_cli.py +0 -231
- tests/unit/test_hf_ingest_segments_cli.py +0 -127
.ingest-dry-run-report.json
DELETED
|
@@ -1,14 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"mode": "dry-run",
|
| 3 |
-
"project_slug": "ppbio-aiuaba",
|
| 4 |
-
"detections_csv": "c:\\Users\\jonat\\OneDrive - MSFT\\Doutorado PPGEtno\\BirdNET Analyzer\\00_ALL_DETECTIONS.csv",
|
| 5 |
-
"segments_root": "c:\\Users\\jonat\\OneDrive - MSFT\\Doutorado PPGEtno\\BirdNET Segments",
|
| 6 |
-
"csv_rows_total": 146561,
|
| 7 |
-
"segments_found_total": 146561,
|
| 8 |
-
"matched_rows": 146561,
|
| 9 |
-
"unmatched_rows": 0,
|
| 10 |
-
"ambiguous_rows": 0,
|
| 11 |
-
"unique_audio_ids": 31265,
|
| 12 |
-
"sample_unmatched": [],
|
| 13 |
-
"sample_ambiguous": []
|
| 14 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Uploader App/README.md
DELETED
|
@@ -1,84 +0,0 @@
|
|
| 1 |
-
# BirdNET Uploader App
|
| 2 |
-
|
| 3 |
-
Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
|
| 4 |
-
- Multi-projeto
|
| 5 |
-
- Audio sob demanda
|
| 6 |
-
- Persistencia de validacoes
|
| 7 |
-
- Escalabilidade para datasets grandes
|
| 8 |
-
|
| 9 |
-
## Desenvolvimento local
|
| 10 |
-
1. Criar ambiente virtual Python 3.11+
|
| 11 |
-
2. Instalar dependencias:
|
| 12 |
-
pip install -r requirements.txt
|
| 13 |
-
3. Rodar app:
|
| 14 |
-
python app.py
|
| 15 |
-
|
| 16 |
-
## Objetivo desta pasta
|
| 17 |
-
Contem apenas os arquivos necessarios para o app de upload/ingestao.
|
| 18 |
-
Use esta pasta para criar/publicar o repositorio do uploader de forma independente.
|
| 19 |
-
|
| 20 |
-
## Fluxo recomendado (separado)
|
| 21 |
-
1. Uploader app (porta 7862 por padrao):
|
| 22 |
-
- Rodar dry-run com CSV + pasta de segmentos
|
| 23 |
-
- Rodar upload real para owner/repo do projeto
|
| 24 |
-
2. Validator app (porta 7860):
|
| 25 |
-
- Validar deteccoes do projeto com dados ja publicados
|
| 26 |
-
|
| 27 |
-
Essa separacao evita impacto de performance na validacao durante uploads grandes.
|
| 28 |
-
|
| 29 |
-
## CLI de dataset (Sprint 1)
|
| 30 |
-
Comandos principais:
|
| 31 |
-
|
| 32 |
-
1. Criar estrutura do projeto no dataset Hugging Face:
|
| 33 |
-
python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 34 |
-
|
| 35 |
-
2. Gerar manifesto e shards iniciais de indice:
|
| 36 |
-
python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
|
| 37 |
-
|
| 38 |
-
3. Fazer upload de audio em lotes com retry e resume:
|
| 39 |
-
python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
|
| 40 |
-
|
| 41 |
-
4. Verificar consistencia basica do projeto:
|
| 42 |
-
python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 43 |
-
|
| 44 |
-
5. Ingestao BirdNET por projeto (dry-run):
|
| 45 |
-
python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
|
| 46 |
-
|
| 47 |
-
6. Ingestao BirdNET por projeto (upload real):
|
| 48 |
-
python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
|
| 49 |
-
|
| 50 |
-
Estrutura base criada por projeto/dataset:
|
| 51 |
-
- audio/
|
| 52 |
-
- index/
|
| 53 |
-
- index/shards/
|
| 54 |
-
- validations/
|
| 55 |
-
- audit/
|
| 56 |
-
- manifest.json
|
| 57 |
-
|
| 58 |
-
## Estrutura
|
| 59 |
-
- src/domain: modelos de dominio
|
| 60 |
-
- src/repositories: contratos de persistencia
|
| 61 |
-
- src/services: servicos de aplicacao
|
| 62 |
-
- src/auth: autenticacao e autorizacao
|
| 63 |
-
- src/ui: montagem da interface
|
| 64 |
-
- src/cache: cache efemero local
|
| 65 |
-
- cli: comandos de ingestao/publicacao
|
| 66 |
-
- tests: testes unitarios e integracao
|
| 67 |
-
|
| 68 |
-
## Estado atual de implementacao
|
| 69 |
-
- Sprint 1: CLI create-project, sync-audio, build-index, verify-project
|
| 70 |
-
- Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
|
| 71 |
-
- Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
|
| 72 |
-
- Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
|
| 73 |
-
- Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
|
| 74 |
-
- Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
|
| 75 |
-
- Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
|
| 76 |
-
- Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
|
| 77 |
-
- Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
|
| 78 |
-
- Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
|
| 79 |
-
- Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
|
| 80 |
-
- Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
|
| 81 |
-
- Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
|
| 82 |
-
- Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
|
| 83 |
-
- Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
|
| 84 |
-
- Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Uploader App/app.py
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
|
| 3 |
-
from src.ui.upload_app_factory import build_upload_app
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
if __name__ == "__main__":
|
| 7 |
-
app = build_upload_app()
|
| 8 |
-
port = int(os.getenv("BIRDNET_UPLOADER_PORT", "7862"))
|
| 9 |
-
app.launch(server_name="0.0.0.0", server_port=port)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Uploader App/cli/hf_dataset_cli.py
DELETED
|
@@ -1,969 +0,0 @@
|
|
| 1 |
-
import argparse
|
| 2 |
-
import hashlib
|
| 3 |
-
import json
|
| 4 |
-
import math
|
| 5 |
-
import re
|
| 6 |
-
import tempfile
|
| 7 |
-
import time
|
| 8 |
-
from dataclasses import asdict, dataclass
|
| 9 |
-
from datetime import UTC, datetime
|
| 10 |
-
from io import BytesIO
|
| 11 |
-
from pathlib import Path
|
| 12 |
-
from typing import Any
|
| 13 |
-
|
| 14 |
-
import pandas as pd
|
| 15 |
-
from huggingface_hub import HfApi, hf_hub_download
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
SCHEMA_VERSION = "1.0.0"
|
| 19 |
-
REQUIRED_PROJECT_PREFIXES = (
|
| 20 |
-
"audio/",
|
| 21 |
-
"index/",
|
| 22 |
-
"index/shards/",
|
| 23 |
-
"validations/",
|
| 24 |
-
"audit/",
|
| 25 |
-
)
|
| 26 |
-
REQUIRED_PLACEHOLDER_FILES = (
|
| 27 |
-
"audio/.gitkeep",
|
| 28 |
-
"index/.gitkeep",
|
| 29 |
-
"index/shards/.gitkeep",
|
| 30 |
-
"validations/.gitkeep",
|
| 31 |
-
"audit/.gitkeep",
|
| 32 |
-
)
|
| 33 |
-
REQUIRED_DETECTIONS_COLUMNS = (
|
| 34 |
-
"detection_key",
|
| 35 |
-
"audio_id",
|
| 36 |
-
"scientific_name",
|
| 37 |
-
"confidence",
|
| 38 |
-
"start_time",
|
| 39 |
-
"end_time",
|
| 40 |
-
)
|
| 41 |
-
SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
|
| 42 |
-
REQUIRED_BIRDNET_COLUMNS = (
|
| 43 |
-
"source_file",
|
| 44 |
-
"scientific_name",
|
| 45 |
-
"confidence",
|
| 46 |
-
"start_time",
|
| 47 |
-
"end_time",
|
| 48 |
-
)
|
| 49 |
-
SEGMENT_FILENAME_PATTERN = re.compile(
|
| 50 |
-
r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
|
| 51 |
-
)
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
@dataclass(slots=True)
|
| 55 |
-
class ShardMetadata:
|
| 56 |
-
path: str
|
| 57 |
-
rows: int
|
| 58 |
-
sha256: str
|
| 59 |
-
size_bytes: int
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
@dataclass(slots=True)
|
| 63 |
-
class SegmentFileRecord:
|
| 64 |
-
scientific_name: str
|
| 65 |
-
source_stem: str
|
| 66 |
-
start_time: float
|
| 67 |
-
end_time: float
|
| 68 |
-
confidence_pct: int
|
| 69 |
-
absolute_path: Path
|
| 70 |
-
relative_path: str
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
def _utcnow_iso() -> str:
|
| 74 |
-
return datetime.now(UTC).replace(microsecond=0).isoformat()
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
|
| 78 |
-
api.upload_file(
|
| 79 |
-
path_or_fileobj=BytesIO(content.encode("utf-8")),
|
| 80 |
-
path_in_repo=path_in_repo,
|
| 81 |
-
repo_id=dataset_repo,
|
| 82 |
-
repo_type="dataset",
|
| 83 |
-
)
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
|
| 87 |
-
api.upload_file(
|
| 88 |
-
path_or_fileobj=BytesIO(b""),
|
| 89 |
-
path_in_repo=path_in_repo,
|
| 90 |
-
repo_id=dataset_repo,
|
| 91 |
-
repo_type="dataset",
|
| 92 |
-
)
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
def ensure_project_dataset_structure(
|
| 96 |
-
api: HfApi,
|
| 97 |
-
project_slug: str,
|
| 98 |
-
dataset_repo: str,
|
| 99 |
-
create_private_repo: bool,
|
| 100 |
-
) -> dict[str, Any]:
|
| 101 |
-
api.create_repo(
|
| 102 |
-
repo_id=dataset_repo,
|
| 103 |
-
repo_type="dataset",
|
| 104 |
-
private=create_private_repo,
|
| 105 |
-
exist_ok=True,
|
| 106 |
-
)
|
| 107 |
-
|
| 108 |
-
files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 109 |
-
created_paths: list[str] = []
|
| 110 |
-
|
| 111 |
-
for placeholder in REQUIRED_PLACEHOLDER_FILES:
|
| 112 |
-
if placeholder not in files:
|
| 113 |
-
_upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
|
| 114 |
-
created_paths.append(placeholder)
|
| 115 |
-
|
| 116 |
-
manifest_path = "manifest.json"
|
| 117 |
-
if manifest_path not in files:
|
| 118 |
-
manifest = {
|
| 119 |
-
"schema_version": SCHEMA_VERSION,
|
| 120 |
-
"project_slug": project_slug,
|
| 121 |
-
"dataset_repo_id": dataset_repo,
|
| 122 |
-
"created_at": _utcnow_iso(),
|
| 123 |
-
"updated_at": _utcnow_iso(),
|
| 124 |
-
"index": {
|
| 125 |
-
"total_detections": 0,
|
| 126 |
-
"total_audio_files": 0,
|
| 127 |
-
"shard_size": 0,
|
| 128 |
-
"shards": [],
|
| 129 |
-
},
|
| 130 |
-
}
|
| 131 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
|
| 132 |
-
created_paths.append(manifest_path)
|
| 133 |
-
|
| 134 |
-
return {
|
| 135 |
-
"dataset_repo": dataset_repo,
|
| 136 |
-
"project_slug": project_slug,
|
| 137 |
-
"created_paths": created_paths,
|
| 138 |
-
}
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
def load_detections_table(detections_file: str) -> pd.DataFrame:
|
| 142 |
-
input_path = Path(detections_file)
|
| 143 |
-
if not input_path.exists():
|
| 144 |
-
raise FileNotFoundError(f"Detections file not found: {detections_file}")
|
| 145 |
-
|
| 146 |
-
suffix = input_path.suffix.lower()
|
| 147 |
-
if suffix == ".csv":
|
| 148 |
-
frame = pd.read_csv(input_path)
|
| 149 |
-
elif suffix in {".jsonl", ".ndjson"}:
|
| 150 |
-
frame = pd.read_json(input_path, lines=True)
|
| 151 |
-
elif suffix == ".parquet":
|
| 152 |
-
frame = pd.read_parquet(input_path)
|
| 153 |
-
else:
|
| 154 |
-
raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
|
| 155 |
-
|
| 156 |
-
missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
|
| 157 |
-
if missing_columns:
|
| 158 |
-
raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
|
| 159 |
-
|
| 160 |
-
if frame.empty:
|
| 161 |
-
return frame
|
| 162 |
-
|
| 163 |
-
deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
|
| 164 |
-
return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
|
| 168 |
-
input_path = Path(detections_csv)
|
| 169 |
-
if not input_path.exists():
|
| 170 |
-
raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
|
| 171 |
-
|
| 172 |
-
frame = pd.read_csv(input_path)
|
| 173 |
-
missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
|
| 174 |
-
if missing_columns:
|
| 175 |
-
raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
|
| 176 |
-
|
| 177 |
-
if frame.empty:
|
| 178 |
-
return frame
|
| 179 |
-
|
| 180 |
-
normalized = frame.copy()
|
| 181 |
-
normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
|
| 182 |
-
normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
|
| 183 |
-
normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
|
| 184 |
-
normalized["species_key"] = normalized["scientific_name"].str.casefold()
|
| 185 |
-
normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
|
| 186 |
-
normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
|
| 187 |
-
normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
|
| 188 |
-
|
| 189 |
-
normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
|
| 190 |
-
return normalized
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
|
| 194 |
-
payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
|
| 195 |
-
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
|
| 199 |
-
stem = Path(file_name).stem
|
| 200 |
-
match = SEGMENT_FILENAME_PATTERN.match(stem)
|
| 201 |
-
if not match:
|
| 202 |
-
return None
|
| 203 |
-
|
| 204 |
-
source_stem = match.group("source_stem")
|
| 205 |
-
start_time = float(match.group("start"))
|
| 206 |
-
end_time = float(match.group("end"))
|
| 207 |
-
confidence_pct = int(match.group("confidence_pct"))
|
| 208 |
-
return source_stem, start_time, end_time, confidence_pct
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
|
| 212 |
-
root = Path(segments_root)
|
| 213 |
-
if not root.exists() or not root.is_dir():
|
| 214 |
-
raise FileNotFoundError(f"Segments root not found: {segments_root}")
|
| 215 |
-
|
| 216 |
-
records: list[SegmentFileRecord] = []
|
| 217 |
-
for absolute_path in root.rglob("*"):
|
| 218 |
-
if not absolute_path.is_file():
|
| 219 |
-
continue
|
| 220 |
-
if absolute_path.suffix.lower() != ".wav":
|
| 221 |
-
continue
|
| 222 |
-
|
| 223 |
-
rel = absolute_path.relative_to(root)
|
| 224 |
-
scientific_name = rel.parts[0] if rel.parts else ""
|
| 225 |
-
parsed = parse_segment_filename(absolute_path.name)
|
| 226 |
-
if not parsed:
|
| 227 |
-
continue
|
| 228 |
-
source_stem, start_time, end_time, confidence_pct = parsed
|
| 229 |
-
records.append(
|
| 230 |
-
SegmentFileRecord(
|
| 231 |
-
scientific_name=scientific_name,
|
| 232 |
-
source_stem=source_stem,
|
| 233 |
-
start_time=start_time,
|
| 234 |
-
end_time=end_time,
|
| 235 |
-
confidence_pct=confidence_pct,
|
| 236 |
-
absolute_path=absolute_path,
|
| 237 |
-
relative_path=rel.as_posix(),
|
| 238 |
-
)
|
| 239 |
-
)
|
| 240 |
-
|
| 241 |
-
return records
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 245 |
-
frame = load_birdnet_detections_csv(detections_csv=detections_csv)
|
| 246 |
-
segment_records = discover_segment_records(segments_root=segments_root)
|
| 247 |
-
|
| 248 |
-
by_species: dict[str, list[SegmentFileRecord]] = {}
|
| 249 |
-
for record in segment_records:
|
| 250 |
-
by_species.setdefault(record.scientific_name.casefold(), []).append(record)
|
| 251 |
-
|
| 252 |
-
matched_rows: list[dict[str, Any]] = []
|
| 253 |
-
unmatched_rows: list[dict[str, Any]] = []
|
| 254 |
-
ambiguous_rows: list[dict[str, Any]] = []
|
| 255 |
-
|
| 256 |
-
for row in frame.to_dict(orient="records"):
|
| 257 |
-
species_key = str(row.get("species_key", "")).casefold()
|
| 258 |
-
source_stem = str(row.get("source_stem", "")).strip()
|
| 259 |
-
start_time = float(row["start_time"])
|
| 260 |
-
end_time = float(row["end_time"])
|
| 261 |
-
confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
|
| 262 |
-
|
| 263 |
-
species_candidates = by_species.get(species_key, [])
|
| 264 |
-
exact_matches = [
|
| 265 |
-
record
|
| 266 |
-
for record in species_candidates
|
| 267 |
-
if record.source_stem == source_stem
|
| 268 |
-
and abs(record.start_time - start_time) < 1e-6
|
| 269 |
-
and abs(record.end_time - end_time) < 1e-6
|
| 270 |
-
]
|
| 271 |
-
|
| 272 |
-
if not exact_matches:
|
| 273 |
-
unmatched_rows.append(
|
| 274 |
-
{
|
| 275 |
-
"source_file": row.get("source_file", ""),
|
| 276 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 277 |
-
"start_time": start_time,
|
| 278 |
-
"end_time": end_time,
|
| 279 |
-
}
|
| 280 |
-
)
|
| 281 |
-
continue
|
| 282 |
-
|
| 283 |
-
selected = exact_matches[0]
|
| 284 |
-
if len(exact_matches) > 1:
|
| 285 |
-
expected_pct = int(round(confidence * 100))
|
| 286 |
-
exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
|
| 287 |
-
selected = exact_matches[0]
|
| 288 |
-
ambiguous_rows.append(
|
| 289 |
-
{
|
| 290 |
-
"source_file": row.get("source_file", ""),
|
| 291 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 292 |
-
"start_time": start_time,
|
| 293 |
-
"end_time": end_time,
|
| 294 |
-
"candidates": len(exact_matches),
|
| 295 |
-
}
|
| 296 |
-
)
|
| 297 |
-
|
| 298 |
-
matched_rows.append(
|
| 299 |
-
{
|
| 300 |
-
"project_slug": project_slug,
|
| 301 |
-
"detection_key": _build_detection_key(
|
| 302 |
-
project_slug=project_slug,
|
| 303 |
-
source_file=str(row.get("source_file", "")),
|
| 304 |
-
scientific_name=str(row.get("scientific_name", "")),
|
| 305 |
-
start_time=start_time,
|
| 306 |
-
end_time=end_time,
|
| 307 |
-
),
|
| 308 |
-
"audio_id": source_stem,
|
| 309 |
-
"segment_filename": selected.absolute_path.name,
|
| 310 |
-
"segment_relpath": selected.relative_path,
|
| 311 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 312 |
-
"common_name": row.get("common_name", ""),
|
| 313 |
-
"confidence": confidence,
|
| 314 |
-
"start_time": start_time,
|
| 315 |
-
"end_time": end_time,
|
| 316 |
-
"exact_start": row.get("exact_start", start_time),
|
| 317 |
-
"exact_end": row.get("exact_end", end_time),
|
| 318 |
-
"locality": row.get("locality", ""),
|
| 319 |
-
"point": row.get("point", ""),
|
| 320 |
-
"date_folder": row.get("date_folder", ""),
|
| 321 |
-
"min_freq": row.get("min_freq", None),
|
| 322 |
-
"max_freq": row.get("max_freq", None),
|
| 323 |
-
"box_source": row.get("box_source", ""),
|
| 324 |
-
"label": row.get("label", ""),
|
| 325 |
-
"source_file": row.get("source_file", ""),
|
| 326 |
-
"schema_version": SCHEMA_VERSION,
|
| 327 |
-
"ingested_at": _utcnow_iso(),
|
| 328 |
-
}
|
| 329 |
-
)
|
| 330 |
-
|
| 331 |
-
return {
|
| 332 |
-
"frame": frame,
|
| 333 |
-
"segment_records_total": len(segment_records),
|
| 334 |
-
"matched_rows": matched_rows,
|
| 335 |
-
"unmatched_rows": unmatched_rows,
|
| 336 |
-
"ambiguous_rows": ambiguous_rows,
|
| 337 |
-
}
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 341 |
-
match_result = _match_birdnet_rows(
|
| 342 |
-
project_slug=project_slug,
|
| 343 |
-
detections_csv=detections_csv,
|
| 344 |
-
segments_root=segments_root,
|
| 345 |
-
)
|
| 346 |
-
frame = match_result["frame"]
|
| 347 |
-
matched_rows = match_result["matched_rows"]
|
| 348 |
-
unmatched_rows = match_result["unmatched_rows"]
|
| 349 |
-
ambiguous_rows = match_result["ambiguous_rows"]
|
| 350 |
-
|
| 351 |
-
unique_audio = {row["audio_id"] for row in matched_rows}
|
| 352 |
-
result = {
|
| 353 |
-
"mode": "dry-run",
|
| 354 |
-
"project_slug": project_slug,
|
| 355 |
-
"detections_csv": detections_csv,
|
| 356 |
-
"segments_root": segments_root,
|
| 357 |
-
"csv_rows_total": int(len(frame)),
|
| 358 |
-
"segments_found_total": int(match_result["segment_records_total"]),
|
| 359 |
-
"matched_rows": len(matched_rows),
|
| 360 |
-
"unmatched_rows": len(unmatched_rows),
|
| 361 |
-
"ambiguous_rows": len(ambiguous_rows),
|
| 362 |
-
"unique_audio_ids": len(unique_audio),
|
| 363 |
-
"sample_unmatched": unmatched_rows[:20],
|
| 364 |
-
"sample_ambiguous": ambiguous_rows[:20],
|
| 365 |
-
}
|
| 366 |
-
return result
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
def _file_sha256(file_path: Path) -> str:
|
| 370 |
-
hasher = hashlib.sha256()
|
| 371 |
-
with file_path.open("rb") as handle:
|
| 372 |
-
for chunk in iter(lambda: handle.read(65536), b""):
|
| 373 |
-
hasher.update(chunk)
|
| 374 |
-
return hasher.hexdigest()
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
|
| 378 |
-
if shard_size <= 0:
|
| 379 |
-
raise ValueError("shard_size must be greater than zero")
|
| 380 |
-
|
| 381 |
-
output_dir.mkdir(parents=True, exist_ok=True)
|
| 382 |
-
shards: list[ShardMetadata] = []
|
| 383 |
-
|
| 384 |
-
if frame.empty:
|
| 385 |
-
return shards
|
| 386 |
-
|
| 387 |
-
total_rows = len(frame)
|
| 388 |
-
total_shards = math.ceil(total_rows / shard_size)
|
| 389 |
-
|
| 390 |
-
for index in range(total_shards):
|
| 391 |
-
start = index * shard_size
|
| 392 |
-
end = min(start + shard_size, total_rows)
|
| 393 |
-
shard_frame = frame.iloc[start:end]
|
| 394 |
-
shard_filename = f"shard-{index:05d}.parquet"
|
| 395 |
-
local_path = output_dir / shard_filename
|
| 396 |
-
shard_frame.to_parquet(local_path, index=False)
|
| 397 |
-
|
| 398 |
-
shards.append(
|
| 399 |
-
ShardMetadata(
|
| 400 |
-
path=f"index/shards/{shard_filename}",
|
| 401 |
-
rows=len(shard_frame),
|
| 402 |
-
sha256=_file_sha256(local_path),
|
| 403 |
-
size_bytes=local_path.stat().st_size,
|
| 404 |
-
)
|
| 405 |
-
)
|
| 406 |
-
|
| 407 |
-
return shards
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
def build_manifest_payload(
|
| 411 |
-
project_slug: str,
|
| 412 |
-
dataset_repo: str,
|
| 413 |
-
frame: pd.DataFrame,
|
| 414 |
-
shard_size: int,
|
| 415 |
-
shard_metadata: list[ShardMetadata],
|
| 416 |
-
) -> dict[str, Any]:
|
| 417 |
-
now = _utcnow_iso()
|
| 418 |
-
return {
|
| 419 |
-
"schema_version": SCHEMA_VERSION,
|
| 420 |
-
"project_slug": project_slug,
|
| 421 |
-
"dataset_repo_id": dataset_repo,
|
| 422 |
-
"updated_at": now,
|
| 423 |
-
"index": {
|
| 424 |
-
"total_detections": len(frame),
|
| 425 |
-
"total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
|
| 426 |
-
"shard_size": shard_size,
|
| 427 |
-
"shards": [asdict(item) for item in shard_metadata],
|
| 428 |
-
},
|
| 429 |
-
}
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
def discover_audio_files(local_audio_dir: str) -> list[Path]:
|
| 433 |
-
audio_root = Path(local_audio_dir)
|
| 434 |
-
if not audio_root.exists() or not audio_root.is_dir():
|
| 435 |
-
raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
|
| 436 |
-
|
| 437 |
-
files = [
|
| 438 |
-
path
|
| 439 |
-
for path in audio_root.rglob("*")
|
| 440 |
-
if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
|
| 441 |
-
]
|
| 442 |
-
return sorted(files)
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
|
| 446 |
-
if chunk_size <= 0:
|
| 447 |
-
raise ValueError("chunk_size must be greater than zero")
|
| 448 |
-
return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
def _load_resume_state(state_file: Path) -> dict[str, Any]:
|
| 452 |
-
if not state_file.exists():
|
| 453 |
-
return {"uploaded": [], "failed": []}
|
| 454 |
-
|
| 455 |
-
payload = json.loads(state_file.read_text(encoding="utf-8"))
|
| 456 |
-
uploaded = payload.get("uploaded", [])
|
| 457 |
-
failed = payload.get("failed", [])
|
| 458 |
-
return {
|
| 459 |
-
"uploaded": uploaded if isinstance(uploaded, list) else [],
|
| 460 |
-
"failed": failed if isinstance(failed, list) else [],
|
| 461 |
-
}
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
|
| 465 |
-
state_file.parent.mkdir(parents=True, exist_ok=True)
|
| 466 |
-
payload = {
|
| 467 |
-
"updated_at": _utcnow_iso(),
|
| 468 |
-
"uploaded": sorted(uploaded),
|
| 469 |
-
"failed": sorted(failed),
|
| 470 |
-
}
|
| 471 |
-
state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
def _upload_audio_with_retry(
|
| 475 |
-
api: HfApi,
|
| 476 |
-
dataset_repo: str,
|
| 477 |
-
local_path: Path,
|
| 478 |
-
path_in_repo: str,
|
| 479 |
-
max_retries: int,
|
| 480 |
-
retry_backoff_seconds: float,
|
| 481 |
-
) -> None:
|
| 482 |
-
attempts = 0
|
| 483 |
-
while True:
|
| 484 |
-
attempts += 1
|
| 485 |
-
try:
|
| 486 |
-
api.upload_file(
|
| 487 |
-
path_or_fileobj=str(local_path),
|
| 488 |
-
path_in_repo=path_in_repo,
|
| 489 |
-
repo_id=dataset_repo,
|
| 490 |
-
repo_type="dataset",
|
| 491 |
-
)
|
| 492 |
-
return
|
| 493 |
-
except Exception:
|
| 494 |
-
if attempts > max_retries:
|
| 495 |
-
raise
|
| 496 |
-
time.sleep(retry_backoff_seconds * attempts)
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
def ingest_segments_to_hf(
|
| 500 |
-
api: HfApi,
|
| 501 |
-
project_slug: str,
|
| 502 |
-
dataset_repo: str,
|
| 503 |
-
detections_csv: str,
|
| 504 |
-
segments_root: str,
|
| 505 |
-
batch_size: int,
|
| 506 |
-
shard_size: int,
|
| 507 |
-
max_retries: int,
|
| 508 |
-
retry_backoff_seconds: float,
|
| 509 |
-
resume_state_file: str,
|
| 510 |
-
) -> dict[str, Any]:
|
| 511 |
-
ensure_project_dataset_structure(
|
| 512 |
-
api=api,
|
| 513 |
-
project_slug=project_slug,
|
| 514 |
-
dataset_repo=dataset_repo,
|
| 515 |
-
create_private_repo=False,
|
| 516 |
-
)
|
| 517 |
-
|
| 518 |
-
match_result = _match_birdnet_rows(
|
| 519 |
-
project_slug=project_slug,
|
| 520 |
-
detections_csv=detections_csv,
|
| 521 |
-
segments_root=segments_root,
|
| 522 |
-
)
|
| 523 |
-
frame = match_result["frame"]
|
| 524 |
-
matched_rows = match_result["matched_rows"]
|
| 525 |
-
unmatched_rows = match_result["unmatched_rows"]
|
| 526 |
-
ambiguous_rows = match_result["ambiguous_rows"]
|
| 527 |
-
|
| 528 |
-
# Deduplicate by remote path so retries/resume work on a stable key.
|
| 529 |
-
path_to_local: dict[str, Path] = {}
|
| 530 |
-
for row in matched_rows:
|
| 531 |
-
repo_path = f"audio/segments/{row['segment_relpath']}"
|
| 532 |
-
if repo_path in path_to_local:
|
| 533 |
-
continue
|
| 534 |
-
path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
|
| 535 |
-
|
| 536 |
-
state_path = Path(resume_state_file)
|
| 537 |
-
state_payload = _load_resume_state(state_file=state_path)
|
| 538 |
-
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 539 |
-
failed_state = set(str(item) for item in state_payload["failed"])
|
| 540 |
-
|
| 541 |
-
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 542 |
-
|
| 543 |
-
pending: list[tuple[Path, str]] = []
|
| 544 |
-
skipped_existing = 0
|
| 545 |
-
for repo_path, local_path in path_to_local.items():
|
| 546 |
-
if repo_path in remote_files:
|
| 547 |
-
uploaded_state.add(repo_path)
|
| 548 |
-
skipped_existing += 1
|
| 549 |
-
continue
|
| 550 |
-
if repo_path in uploaded_state:
|
| 551 |
-
continue
|
| 552 |
-
pending.append((local_path, repo_path))
|
| 553 |
-
|
| 554 |
-
uploaded_now: set[str] = set()
|
| 555 |
-
failed_now: set[str] = set()
|
| 556 |
-
|
| 557 |
-
for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
|
| 558 |
-
batch_uploaded = 0
|
| 559 |
-
batch_failed = 0
|
| 560 |
-
for local_path, repo_path in batch:
|
| 561 |
-
try:
|
| 562 |
-
_upload_audio_with_retry(
|
| 563 |
-
api=api,
|
| 564 |
-
dataset_repo=dataset_repo,
|
| 565 |
-
local_path=local_path,
|
| 566 |
-
path_in_repo=repo_path,
|
| 567 |
-
max_retries=max_retries,
|
| 568 |
-
retry_backoff_seconds=retry_backoff_seconds,
|
| 569 |
-
)
|
| 570 |
-
uploaded_state.add(repo_path)
|
| 571 |
-
failed_state.discard(repo_path)
|
| 572 |
-
uploaded_now.add(repo_path)
|
| 573 |
-
batch_uploaded += 1
|
| 574 |
-
except Exception:
|
| 575 |
-
failed_state.add(repo_path)
|
| 576 |
-
failed_now.add(repo_path)
|
| 577 |
-
batch_failed += 1
|
| 578 |
-
|
| 579 |
-
print(
|
| 580 |
-
json.dumps(
|
| 581 |
-
{
|
| 582 |
-
"event": "ingest-segments-audio-batch",
|
| 583 |
-
"batch_index": batch_index,
|
| 584 |
-
"batch_size": len(batch),
|
| 585 |
-
"uploaded": batch_uploaded,
|
| 586 |
-
"failed": batch_failed,
|
| 587 |
-
}
|
| 588 |
-
)
|
| 589 |
-
)
|
| 590 |
-
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
|
| 591 |
-
|
| 592 |
-
index_frame = pd.DataFrame(matched_rows)
|
| 593 |
-
if not index_frame.empty:
|
| 594 |
-
index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
|
| 595 |
-
|
| 596 |
-
with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
|
| 597 |
-
temp_dir = Path(temp_dir_str)
|
| 598 |
-
shard_dir = temp_dir / "index" / "shards"
|
| 599 |
-
shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
|
| 600 |
-
|
| 601 |
-
for item in shard_metadata:
|
| 602 |
-
local_path = temp_dir / item.path
|
| 603 |
-
api.upload_file(
|
| 604 |
-
path_or_fileobj=str(local_path),
|
| 605 |
-
path_in_repo=item.path,
|
| 606 |
-
repo_id=dataset_repo,
|
| 607 |
-
repo_type="dataset",
|
| 608 |
-
)
|
| 609 |
-
|
| 610 |
-
manifest = build_manifest_payload(
|
| 611 |
-
project_slug=project_slug,
|
| 612 |
-
dataset_repo=dataset_repo,
|
| 613 |
-
frame=index_frame,
|
| 614 |
-
shard_size=shard_size,
|
| 615 |
-
shard_metadata=shard_metadata,
|
| 616 |
-
)
|
| 617 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 618 |
-
|
| 619 |
-
run_report = {
|
| 620 |
-
"mode": "execute",
|
| 621 |
-
"project_slug": project_slug,
|
| 622 |
-
"dataset_repo": dataset_repo,
|
| 623 |
-
"detections_csv": detections_csv,
|
| 624 |
-
"segments_root": segments_root,
|
| 625 |
-
"csv_rows_total": int(len(frame)),
|
| 626 |
-
"segments_found_total": int(match_result["segment_records_total"]),
|
| 627 |
-
"matched_rows": len(matched_rows),
|
| 628 |
-
"unmatched_rows": len(unmatched_rows),
|
| 629 |
-
"ambiguous_rows": len(ambiguous_rows),
|
| 630 |
-
"unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
|
| 631 |
-
"pending_audio_uploads": len(pending),
|
| 632 |
-
"uploaded_audio_now": len(uploaded_now),
|
| 633 |
-
"uploaded_audio_skipped_existing": skipped_existing,
|
| 634 |
-
"failed_uploads": len(failed_now),
|
| 635 |
-
"resume_state_file": str(state_path),
|
| 636 |
-
"index_rows_written": len(index_frame),
|
| 637 |
-
"shards_written": len(shard_metadata),
|
| 638 |
-
"sample_unmatched": unmatched_rows[:20],
|
| 639 |
-
"sample_ambiguous": ambiguous_rows[:20],
|
| 640 |
-
}
|
| 641 |
-
|
| 642 |
-
audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
|
| 643 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
|
| 644 |
-
return run_report
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
def sync_audio_batches(
|
| 648 |
-
api: HfApi,
|
| 649 |
-
project_slug: str,
|
| 650 |
-
dataset_repo: str,
|
| 651 |
-
local_audio_dir: str,
|
| 652 |
-
batch_size: int,
|
| 653 |
-
max_retries: int,
|
| 654 |
-
retry_backoff_seconds: float,
|
| 655 |
-
resume_state_file: str,
|
| 656 |
-
) -> dict[str, Any]:
|
| 657 |
-
ensure_project_dataset_structure(
|
| 658 |
-
api=api,
|
| 659 |
-
project_slug=project_slug,
|
| 660 |
-
dataset_repo=dataset_repo,
|
| 661 |
-
create_private_repo=False,
|
| 662 |
-
)
|
| 663 |
-
|
| 664 |
-
all_files = discover_audio_files(local_audio_dir=local_audio_dir)
|
| 665 |
-
state_path = Path(resume_state_file)
|
| 666 |
-
state_payload = _load_resume_state(state_file=state_path)
|
| 667 |
-
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 668 |
-
|
| 669 |
-
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 670 |
-
|
| 671 |
-
pending: list[tuple[Path, str]] = []
|
| 672 |
-
for path in all_files:
|
| 673 |
-
rel = path.relative_to(Path(local_audio_dir)).as_posix()
|
| 674 |
-
repo_path = f"audio/{rel}"
|
| 675 |
-
|
| 676 |
-
if repo_path in remote_files:
|
| 677 |
-
uploaded_state.add(repo_path)
|
| 678 |
-
continue
|
| 679 |
-
|
| 680 |
-
if repo_path in uploaded_state:
|
| 681 |
-
continue
|
| 682 |
-
|
| 683 |
-
pending.append((path, repo_path))
|
| 684 |
-
|
| 685 |
-
pending_paths = [item[0] for item in pending]
|
| 686 |
-
by_local_to_repo = {local: repo for local, repo in pending}
|
| 687 |
-
batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
|
| 688 |
-
|
| 689 |
-
uploaded_now: set[str] = set()
|
| 690 |
-
failed: set[str] = set()
|
| 691 |
-
|
| 692 |
-
for batch_index, batch in enumerate(batches, start=1):
|
| 693 |
-
batch_uploaded = 0
|
| 694 |
-
batch_failed = 0
|
| 695 |
-
|
| 696 |
-
for local_path in batch:
|
| 697 |
-
repo_path = by_local_to_repo[local_path]
|
| 698 |
-
try:
|
| 699 |
-
_upload_audio_with_retry(
|
| 700 |
-
api=api,
|
| 701 |
-
dataset_repo=dataset_repo,
|
| 702 |
-
local_path=local_path,
|
| 703 |
-
path_in_repo=repo_path,
|
| 704 |
-
max_retries=max_retries,
|
| 705 |
-
retry_backoff_seconds=retry_backoff_seconds,
|
| 706 |
-
)
|
| 707 |
-
uploaded_now.add(repo_path)
|
| 708 |
-
uploaded_state.add(repo_path)
|
| 709 |
-
batch_uploaded += 1
|
| 710 |
-
except Exception:
|
| 711 |
-
failed.add(repo_path)
|
| 712 |
-
batch_failed += 1
|
| 713 |
-
|
| 714 |
-
print(
|
| 715 |
-
json.dumps(
|
| 716 |
-
{
|
| 717 |
-
"event": "sync-audio-batch",
|
| 718 |
-
"batch_index": batch_index,
|
| 719 |
-
"batch_size": len(batch),
|
| 720 |
-
"uploaded": batch_uploaded,
|
| 721 |
-
"failed": batch_failed,
|
| 722 |
-
}
|
| 723 |
-
)
|
| 724 |
-
)
|
| 725 |
-
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
|
| 726 |
-
|
| 727 |
-
return {
|
| 728 |
-
"project_slug": project_slug,
|
| 729 |
-
"dataset_repo": dataset_repo,
|
| 730 |
-
"total_local_audio_files": len(all_files),
|
| 731 |
-
"pending_uploads": len(pending),
|
| 732 |
-
"uploaded_now": len(uploaded_now),
|
| 733 |
-
"failed": len(failed),
|
| 734 |
-
"resume_state_file": str(state_path),
|
| 735 |
-
}
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
def build_and_upload_index(
|
| 739 |
-
api: HfApi,
|
| 740 |
-
project_slug: str,
|
| 741 |
-
dataset_repo: str,
|
| 742 |
-
detections_file: str,
|
| 743 |
-
shard_size: int,
|
| 744 |
-
) -> dict[str, Any]:
|
| 745 |
-
ensure_project_dataset_structure(
|
| 746 |
-
api=api,
|
| 747 |
-
project_slug=project_slug,
|
| 748 |
-
dataset_repo=dataset_repo,
|
| 749 |
-
create_private_repo=False,
|
| 750 |
-
)
|
| 751 |
-
|
| 752 |
-
frame = load_detections_table(detections_file=detections_file)
|
| 753 |
-
|
| 754 |
-
with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
|
| 755 |
-
temp_dir = Path(temp_dir_str)
|
| 756 |
-
shard_dir = temp_dir / "index" / "shards"
|
| 757 |
-
metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
|
| 758 |
-
|
| 759 |
-
for item in metadata:
|
| 760 |
-
local_path = temp_dir / item.path
|
| 761 |
-
api.upload_file(
|
| 762 |
-
path_or_fileobj=str(local_path),
|
| 763 |
-
path_in_repo=item.path,
|
| 764 |
-
repo_id=dataset_repo,
|
| 765 |
-
repo_type="dataset",
|
| 766 |
-
)
|
| 767 |
-
|
| 768 |
-
manifest = build_manifest_payload(
|
| 769 |
-
project_slug=project_slug,
|
| 770 |
-
dataset_repo=dataset_repo,
|
| 771 |
-
frame=frame,
|
| 772 |
-
shard_size=shard_size,
|
| 773 |
-
shard_metadata=metadata,
|
| 774 |
-
)
|
| 775 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 776 |
-
|
| 777 |
-
return {
|
| 778 |
-
"project_slug": project_slug,
|
| 779 |
-
"dataset_repo": dataset_repo,
|
| 780 |
-
"total_detections": manifest["index"]["total_detections"],
|
| 781 |
-
"total_audio_files": manifest["index"]["total_audio_files"],
|
| 782 |
-
"total_shards": len(metadata),
|
| 783 |
-
}
|
| 784 |
-
|
| 785 |
-
|
| 786 |
-
def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
|
| 787 |
-
errors: list[str] = []
|
| 788 |
-
|
| 789 |
-
for prefix in REQUIRED_PROJECT_PREFIXES:
|
| 790 |
-
if not any(path.startswith(prefix) for path in repo_files):
|
| 791 |
-
errors.append(f"Missing prefix in dataset repo: {prefix}")
|
| 792 |
-
|
| 793 |
-
if manifest_payload.get("schema_version") != SCHEMA_VERSION:
|
| 794 |
-
errors.append(
|
| 795 |
-
f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
|
| 796 |
-
)
|
| 797 |
-
|
| 798 |
-
if manifest_payload.get("project_slug") != project_slug:
|
| 799 |
-
errors.append(
|
| 800 |
-
f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
|
| 801 |
-
)
|
| 802 |
-
|
| 803 |
-
index_section = manifest_payload.get("index", {})
|
| 804 |
-
shards = index_section.get("shards", [])
|
| 805 |
-
if not isinstance(shards, list):
|
| 806 |
-
errors.append("Manifest index.shards must be a list")
|
| 807 |
-
return errors
|
| 808 |
-
|
| 809 |
-
for shard in shards:
|
| 810 |
-
shard_path = shard.get("path")
|
| 811 |
-
if not shard_path or not isinstance(shard_path, str):
|
| 812 |
-
errors.append("Shard entry missing path")
|
| 813 |
-
continue
|
| 814 |
-
if shard_path not in repo_files:
|
| 815 |
-
errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
|
| 816 |
-
|
| 817 |
-
return errors
|
| 818 |
-
|
| 819 |
-
|
| 820 |
-
def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
|
| 821 |
-
repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 822 |
-
if "manifest.json" not in repo_files:
|
| 823 |
-
return {
|
| 824 |
-
"ok": False,
|
| 825 |
-
"errors": ["manifest.json not found in dataset repository"],
|
| 826 |
-
}
|
| 827 |
-
|
| 828 |
-
manifest_local = hf_hub_download(
|
| 829 |
-
repo_id=dataset_repo,
|
| 830 |
-
repo_type="dataset",
|
| 831 |
-
filename="manifest.json",
|
| 832 |
-
)
|
| 833 |
-
manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
|
| 834 |
-
errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
|
| 835 |
-
|
| 836 |
-
return {
|
| 837 |
-
"ok": len(errors) == 0,
|
| 838 |
-
"errors": errors,
|
| 839 |
-
"total_files": len(repo_files),
|
| 840 |
-
"total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
|
| 841 |
-
}
|
| 842 |
-
|
| 843 |
-
|
| 844 |
-
def build_parser() -> argparse.ArgumentParser:
|
| 845 |
-
parser = argparse.ArgumentParser(description="HF dataset project CLI")
|
| 846 |
-
sub = parser.add_subparsers(dest="command", required=True)
|
| 847 |
-
|
| 848 |
-
create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
|
| 849 |
-
create_project.add_argument("--project-slug", required=True)
|
| 850 |
-
create_project.add_argument("--dataset-repo", required=True)
|
| 851 |
-
create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
|
| 852 |
-
|
| 853 |
-
build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
|
| 854 |
-
build_index.add_argument("--project-slug", required=True)
|
| 855 |
-
build_index.add_argument("--dataset-repo", required=True)
|
| 856 |
-
build_index.add_argument("--detections-file", required=True)
|
| 857 |
-
build_index.add_argument("--shard-size", type=int, default=10000)
|
| 858 |
-
|
| 859 |
-
sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
|
| 860 |
-
sync_audio.add_argument("--project-slug", required=True)
|
| 861 |
-
sync_audio.add_argument("--dataset-repo", required=True)
|
| 862 |
-
sync_audio.add_argument("--local-audio-dir", required=True)
|
| 863 |
-
sync_audio.add_argument("--batch-size", type=int, default=100)
|
| 864 |
-
sync_audio.add_argument("--max-retries", type=int, default=3)
|
| 865 |
-
sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 866 |
-
sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
|
| 867 |
-
|
| 868 |
-
verify_project = sub.add_parser("verify-project", help="Verify project integrity")
|
| 869 |
-
verify_project.add_argument("--project-slug", required=True)
|
| 870 |
-
verify_project.add_argument("--dataset-repo", required=True)
|
| 871 |
-
|
| 872 |
-
ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
|
| 873 |
-
ingest_segments.add_argument("--project-slug", required=True)
|
| 874 |
-
ingest_segments.add_argument("--dataset-repo", required=True)
|
| 875 |
-
ingest_segments.add_argument("--detections-csv", required=True)
|
| 876 |
-
ingest_segments.add_argument("--segments-root", required=True)
|
| 877 |
-
ingest_segments.add_argument("--batch-size", type=int, default=200)
|
| 878 |
-
ingest_segments.add_argument("--shard-size", type=int, default=10000)
|
| 879 |
-
ingest_segments.add_argument("--max-retries", type=int, default=3)
|
| 880 |
-
ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 881 |
-
ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
|
| 882 |
-
ingest_segments.add_argument("--dry-run", action="store_true")
|
| 883 |
-
ingest_segments.add_argument("--report-file", default="")
|
| 884 |
-
|
| 885 |
-
return parser
|
| 886 |
-
|
| 887 |
-
|
| 888 |
-
def main() -> int:
|
| 889 |
-
parser = build_parser()
|
| 890 |
-
args = parser.parse_args()
|
| 891 |
-
api = HfApi()
|
| 892 |
-
|
| 893 |
-
if args.command == "create-project":
|
| 894 |
-
result = ensure_project_dataset_structure(
|
| 895 |
-
api=api,
|
| 896 |
-
project_slug=args.project_slug,
|
| 897 |
-
dataset_repo=args.dataset_repo,
|
| 898 |
-
create_private_repo=bool(args.private),
|
| 899 |
-
)
|
| 900 |
-
print(json.dumps(result, indent=2))
|
| 901 |
-
return 0
|
| 902 |
-
|
| 903 |
-
if args.command == "build-index":
|
| 904 |
-
result = build_and_upload_index(
|
| 905 |
-
api=api,
|
| 906 |
-
project_slug=args.project_slug,
|
| 907 |
-
dataset_repo=args.dataset_repo,
|
| 908 |
-
detections_file=args.detections_file,
|
| 909 |
-
shard_size=args.shard_size,
|
| 910 |
-
)
|
| 911 |
-
print(json.dumps(result, indent=2))
|
| 912 |
-
return 0
|
| 913 |
-
|
| 914 |
-
if args.command == "sync-audio":
|
| 915 |
-
result = sync_audio_batches(
|
| 916 |
-
api=api,
|
| 917 |
-
project_slug=args.project_slug,
|
| 918 |
-
dataset_repo=args.dataset_repo,
|
| 919 |
-
local_audio_dir=args.local_audio_dir,
|
| 920 |
-
batch_size=args.batch_size,
|
| 921 |
-
max_retries=args.max_retries,
|
| 922 |
-
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 923 |
-
resume_state_file=args.resume_state_file,
|
| 924 |
-
)
|
| 925 |
-
print(json.dumps(result, indent=2))
|
| 926 |
-
return 0 if result["failed"] == 0 else 1
|
| 927 |
-
|
| 928 |
-
if args.command == "verify-project":
|
| 929 |
-
result = verify_project(
|
| 930 |
-
api=api,
|
| 931 |
-
project_slug=args.project_slug,
|
| 932 |
-
dataset_repo=args.dataset_repo,
|
| 933 |
-
)
|
| 934 |
-
print(json.dumps(result, indent=2))
|
| 935 |
-
return 0 if result["ok"] else 1
|
| 936 |
-
|
| 937 |
-
if args.command == "ingest-segments":
|
| 938 |
-
if args.dry_run:
|
| 939 |
-
result = run_ingest_segments_dry_run(
|
| 940 |
-
project_slug=args.project_slug,
|
| 941 |
-
detections_csv=args.detections_csv,
|
| 942 |
-
segments_root=args.segments_root,
|
| 943 |
-
)
|
| 944 |
-
else:
|
| 945 |
-
result = ingest_segments_to_hf(
|
| 946 |
-
api=api,
|
| 947 |
-
project_slug=args.project_slug,
|
| 948 |
-
dataset_repo=args.dataset_repo,
|
| 949 |
-
detections_csv=args.detections_csv,
|
| 950 |
-
segments_root=args.segments_root,
|
| 951 |
-
batch_size=args.batch_size,
|
| 952 |
-
shard_size=args.shard_size,
|
| 953 |
-
max_retries=args.max_retries,
|
| 954 |
-
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 955 |
-
resume_state_file=args.resume_state_file,
|
| 956 |
-
)
|
| 957 |
-
|
| 958 |
-
if args.report_file:
|
| 959 |
-
Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
| 960 |
-
|
| 961 |
-
print(json.dumps(result, indent=2))
|
| 962 |
-
return 0 if result.get("failed_uploads", 0) == 0 else 1
|
| 963 |
-
|
| 964 |
-
parser.error("Unknown command")
|
| 965 |
-
return 2
|
| 966 |
-
|
| 967 |
-
|
| 968 |
-
if __name__ == "__main__":
|
| 969 |
-
raise SystemExit(main())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Uploader App/pyproject.toml
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
[project]
|
| 2 |
-
name = "BirdNET-validator-App"
|
| 3 |
-
version = "0.1.0"
|
| 4 |
-
description = "BirdNET detection validator for Hugging Face Spaces"
|
| 5 |
-
readme = "README.md"
|
| 6 |
-
requires-python = ">=3.11"
|
| 7 |
-
|
| 8 |
-
[tool.pytest.ini_options]
|
| 9 |
-
testpaths = ["tests"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Uploader App/requirements.txt
DELETED
|
@@ -1,7 +0,0 @@
|
|
| 1 |
-
gradio>=5.0.0
|
| 2 |
-
huggingface_hub>=0.30.0
|
| 3 |
-
pydantic>=2.8.0
|
| 4 |
-
pandas>=2.2.0
|
| 5 |
-
pyarrow>=17.0.0
|
| 6 |
-
soundfile>=0.12.1
|
| 7 |
-
pytest>=8.3.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Uploader App/src/__init__.py
DELETED
|
File without changes
|
BirdNET Uploader App/src/ui/__init__.py
DELETED
|
File without changes
|
BirdNET Uploader App/src/ui/upload_app_factory.py
DELETED
|
@@ -1,177 +0,0 @@
|
|
| 1 |
-
import json
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
from typing import Any
|
| 4 |
-
|
| 5 |
-
import gradio as gr
|
| 6 |
-
from huggingface_hub import HfApi
|
| 7 |
-
|
| 8 |
-
from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
def _as_pretty_json(payload: dict[str, Any]) -> str:
|
| 12 |
-
return json.dumps(payload, indent=2, ensure_ascii=False)
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
def _build_api(token: str) -> HfApi:
|
| 16 |
-
clean_token = (token or "").strip()
|
| 17 |
-
if clean_token:
|
| 18 |
-
return HfApi(token=clean_token)
|
| 19 |
-
return HfApi()
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
def build_upload_app() -> gr.Blocks:
|
| 23 |
-
with gr.Blocks(title="BirdNET Segments Uploader") as demo:
|
| 24 |
-
gr.Markdown("# BirdNET Segments Uploader")
|
| 25 |
-
gr.Markdown(
|
| 26 |
-
"Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
|
| 27 |
-
"Recomendado: executar dry-run antes do upload real."
|
| 28 |
-
)
|
| 29 |
-
|
| 30 |
-
with gr.Row():
|
| 31 |
-
project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
|
| 32 |
-
dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
|
| 33 |
-
|
| 34 |
-
detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
|
| 35 |
-
segments_root = gr.Textbox(
|
| 36 |
-
label="Pasta raiz de segmentos",
|
| 37 |
-
placeholder=r"ex: C:\dados\BirdNET Segments",
|
| 38 |
-
)
|
| 39 |
-
|
| 40 |
-
with gr.Accordion("Configuracao avancada", open=False):
|
| 41 |
-
hf_token = gr.Textbox(
|
| 42 |
-
label="HF Token (opcional)",
|
| 43 |
-
type="password",
|
| 44 |
-
placeholder="Se vazio, usa sessao ja autenticada no ambiente",
|
| 45 |
-
)
|
| 46 |
-
with gr.Row():
|
| 47 |
-
batch_size = gr.Number(label="Batch size", value=200, precision=0)
|
| 48 |
-
shard_size = gr.Number(label="Shard size", value=10000, precision=0)
|
| 49 |
-
with gr.Row():
|
| 50 |
-
max_retries = gr.Number(label="Max retries", value=3, precision=0)
|
| 51 |
-
retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
|
| 52 |
-
resume_state_file = gr.Textbox(
|
| 53 |
-
label="Resume state file",
|
| 54 |
-
value=".ingest-segments-state.json",
|
| 55 |
-
)
|
| 56 |
-
report_file = gr.Textbox(
|
| 57 |
-
label="Report file local (opcional)",
|
| 58 |
-
placeholder="ex: .ingest-run-report.json",
|
| 59 |
-
)
|
| 60 |
-
|
| 61 |
-
with gr.Row():
|
| 62 |
-
dry_run_button = gr.Button("Dry-run", variant="secondary")
|
| 63 |
-
upload_button = gr.Button("Upload real", variant="primary")
|
| 64 |
-
|
| 65 |
-
status = gr.Markdown(value="Pronto")
|
| 66 |
-
result_json = gr.Code(label="Resultado JSON", language="json")
|
| 67 |
-
|
| 68 |
-
def run_dry_run(
|
| 69 |
-
project: str,
|
| 70 |
-
repo: str,
|
| 71 |
-
csv_path: str | None,
|
| 72 |
-
segments_path: str,
|
| 73 |
-
report_path: str,
|
| 74 |
-
) -> tuple[str, str]:
|
| 75 |
-
project = (project or "").strip()
|
| 76 |
-
if not project:
|
| 77 |
-
return "❌ Informe o project slug", "{}"
|
| 78 |
-
|
| 79 |
-
if not csv_path:
|
| 80 |
-
return "❌ Selecione o CSV de deteccoes", "{}"
|
| 81 |
-
|
| 82 |
-
if not segments_path or not Path(segments_path).exists():
|
| 83 |
-
return "❌ Pasta de segmentos invalida", "{}"
|
| 84 |
-
|
| 85 |
-
try:
|
| 86 |
-
result = run_ingest_segments_dry_run(
|
| 87 |
-
project_slug=project,
|
| 88 |
-
detections_csv=csv_path,
|
| 89 |
-
segments_root=segments_path,
|
| 90 |
-
)
|
| 91 |
-
if report_path.strip():
|
| 92 |
-
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 93 |
-
summary = (
|
| 94 |
-
"✅ Dry-run finalizado | "
|
| 95 |
-
f"Matched: {result['matched_rows']} | "
|
| 96 |
-
f"Unmatched: {result['unmatched_rows']} | "
|
| 97 |
-
f"Ambiguous: {result['ambiguous_rows']}"
|
| 98 |
-
)
|
| 99 |
-
return summary, _as_pretty_json(result)
|
| 100 |
-
except Exception as exc:
|
| 101 |
-
return f"❌ Dry-run falhou: {exc}", "{}"
|
| 102 |
-
|
| 103 |
-
def run_upload(
|
| 104 |
-
project: str,
|
| 105 |
-
repo: str,
|
| 106 |
-
csv_path: str | None,
|
| 107 |
-
segments_path: str,
|
| 108 |
-
token: str,
|
| 109 |
-
batch: float,
|
| 110 |
-
shard: float,
|
| 111 |
-
retries: float,
|
| 112 |
-
backoff: float,
|
| 113 |
-
resume_file: str,
|
| 114 |
-
report_path: str,
|
| 115 |
-
) -> tuple[str, str]:
|
| 116 |
-
project = (project or "").strip()
|
| 117 |
-
repo = (repo or "").strip()
|
| 118 |
-
if not project:
|
| 119 |
-
return "❌ Informe o project slug", "{}"
|
| 120 |
-
if not repo or "/" not in repo:
|
| 121 |
-
return "❌ Informe dataset repo no formato owner/repo", "{}"
|
| 122 |
-
if not csv_path:
|
| 123 |
-
return "❌ Selecione o CSV de deteccoes", "{}"
|
| 124 |
-
if not segments_path or not Path(segments_path).exists():
|
| 125 |
-
return "❌ Pasta de segmentos invalida", "{}"
|
| 126 |
-
|
| 127 |
-
try:
|
| 128 |
-
api = _build_api(token)
|
| 129 |
-
result = ingest_segments_to_hf(
|
| 130 |
-
api=api,
|
| 131 |
-
project_slug=project,
|
| 132 |
-
dataset_repo=repo,
|
| 133 |
-
detections_csv=csv_path,
|
| 134 |
-
segments_root=segments_path,
|
| 135 |
-
batch_size=int(batch),
|
| 136 |
-
shard_size=int(shard),
|
| 137 |
-
max_retries=int(retries),
|
| 138 |
-
retry_backoff_seconds=float(backoff),
|
| 139 |
-
resume_state_file=resume_file,
|
| 140 |
-
)
|
| 141 |
-
if report_path.strip():
|
| 142 |
-
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 143 |
-
summary = (
|
| 144 |
-
"✅ Upload finalizado | "
|
| 145 |
-
f"Uploaded now: {result['uploaded_audio_now']} | "
|
| 146 |
-
f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
|
| 147 |
-
f"Failed uploads: {result['failed_uploads']}"
|
| 148 |
-
)
|
| 149 |
-
return summary, _as_pretty_json(result)
|
| 150 |
-
except Exception as exc:
|
| 151 |
-
return f"❌ Upload falhou: {exc}", "{}"
|
| 152 |
-
|
| 153 |
-
dry_run_button.click(
|
| 154 |
-
fn=run_dry_run,
|
| 155 |
-
inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
|
| 156 |
-
outputs=[status, result_json],
|
| 157 |
-
)
|
| 158 |
-
|
| 159 |
-
upload_button.click(
|
| 160 |
-
fn=run_upload,
|
| 161 |
-
inputs=[
|
| 162 |
-
project_slug,
|
| 163 |
-
dataset_repo,
|
| 164 |
-
detections_csv,
|
| 165 |
-
segments_root,
|
| 166 |
-
hf_token,
|
| 167 |
-
batch_size,
|
| 168 |
-
shard_size,
|
| 169 |
-
max_retries,
|
| 170 |
-
retry_backoff_seconds,
|
| 171 |
-
resume_state_file,
|
| 172 |
-
report_file,
|
| 173 |
-
],
|
| 174 |
-
outputs=[status, result_json],
|
| 175 |
-
)
|
| 176 |
-
|
| 177 |
-
return demo
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/README.md
DELETED
|
@@ -1,83 +0,0 @@
|
|
| 1 |
-
# BirdNET Validator App
|
| 2 |
-
|
| 3 |
-
Aplicacao de validacao de deteccoes para Hugging Face Spaces (Gradio), com foco em:
|
| 4 |
-
- Multi-projeto
|
| 5 |
-
- Audio sob demanda
|
| 6 |
-
- Persistencia de validacoes
|
| 7 |
-
- Escalabilidade para datasets grandes
|
| 8 |
-
|
| 9 |
-
## Desenvolvimento local
|
| 10 |
-
1. Criar ambiente virtual Python 3.11+
|
| 11 |
-
2. Instalar dependencias:
|
| 12 |
-
pip install -r requirements.txt
|
| 13 |
-
3. Rodar app:
|
| 14 |
-
python app.py
|
| 15 |
-
|
| 16 |
-
## Objetivo desta pasta
|
| 17 |
-
Contem os arquivos do app de validacao para evolucao/publicacao independente.
|
| 18 |
-
|
| 19 |
-
## Fluxo recomendado (separado)
|
| 20 |
-
1. Uploader app (porta 7862 por padrao):
|
| 21 |
-
- Rodar dry-run com CSV + pasta de segmentos
|
| 22 |
-
- Rodar upload real para owner/repo do projeto
|
| 23 |
-
2. Validator app (porta 7860):
|
| 24 |
-
- Validar deteccoes do projeto com dados ja publicados
|
| 25 |
-
|
| 26 |
-
Essa separacao evita impacto de performance na validacao durante uploads grandes.
|
| 27 |
-
|
| 28 |
-
## CLI de dataset (Sprint 1)
|
| 29 |
-
Comandos principais:
|
| 30 |
-
|
| 31 |
-
1. Criar estrutura do projeto no dataset Hugging Face:
|
| 32 |
-
python -m cli.hf_dataset_cli create-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 33 |
-
|
| 34 |
-
2. Gerar manifesto e shards iniciais de indice:
|
| 35 |
-
python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
|
| 36 |
-
|
| 37 |
-
3. Fazer upload de audio em lotes com retry e resume:
|
| 38 |
-
python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
|
| 39 |
-
|
| 40 |
-
4. Verificar consistencia basica do projeto:
|
| 41 |
-
python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 42 |
-
|
| 43 |
-
5. Ingestao BirdNET por projeto (dry-run):
|
| 44 |
-
python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --dry-run --report-file .ingest-dry-run-report.json
|
| 45 |
-
|
| 46 |
-
6. Ingestao BirdNET por projeto (upload real):
|
| 47 |
-
python -m cli.hf_dataset_cli ingest-segments --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-csv detections.csv --segments-root "C:\\BirdNET Segments" --batch-size 200 --shard-size 10000 --max-retries 3 --retry-backoff-seconds 1.0 --resume-state-file .ingest-segments-state.json --report-file .ingest-run-report.json
|
| 48 |
-
|
| 49 |
-
Estrutura base criada por projeto/dataset:
|
| 50 |
-
- audio/
|
| 51 |
-
- index/
|
| 52 |
-
- index/shards/
|
| 53 |
-
- validations/
|
| 54 |
-
- audit/
|
| 55 |
-
- manifest.json
|
| 56 |
-
|
| 57 |
-
## Estrutura
|
| 58 |
-
- src/domain: modelos de dominio
|
| 59 |
-
- src/repositories: contratos de persistencia
|
| 60 |
-
- src/services: servicos de aplicacao
|
| 61 |
-
- src/auth: autenticacao e autorizacao
|
| 62 |
-
- src/ui: montagem da interface
|
| 63 |
-
- src/cache: cache efemero local
|
| 64 |
-
- cli: comandos de ingestao/publicacao
|
| 65 |
-
- tests: testes unitarios e integracao
|
| 66 |
-
|
| 67 |
-
## Estado atual de implementacao
|
| 68 |
-
- Sprint 1: CLI create-project, sync-audio, build-index, verify-project
|
| 69 |
-
- Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
|
| 70 |
-
- Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
|
| 71 |
-
- Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
|
| 72 |
-
- Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
|
| 73 |
-
- Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
|
| 74 |
-
- Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
|
| 75 |
-
- Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
|
| 76 |
-
- Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
|
| 77 |
-
- Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
|
| 78 |
-
- Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
|
| 79 |
-
- Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
|
| 80 |
-
- Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
|
| 81 |
-
- Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
|
| 82 |
-
- Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
|
| 83 |
-
- Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/app.py
DELETED
|
@@ -1,6 +0,0 @@
|
|
| 1 |
-
from src.ui.app_factory import create_app
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
if __name__ == "__main__":
|
| 5 |
-
app = create_app()
|
| 6 |
-
app.launch(server_name="0.0.0.0", server_port=7860)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/pyproject.toml
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
[project]
|
| 2 |
-
name = "BirdNET-validator-App"
|
| 3 |
-
version = "0.1.0"
|
| 4 |
-
description = "BirdNET detection validator for Hugging Face Spaces"
|
| 5 |
-
readme = "README.md"
|
| 6 |
-
requires-python = ">=3.11"
|
| 7 |
-
|
| 8 |
-
[tool.pytest.ini_options]
|
| 9 |
-
testpaths = ["tests"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/requirements.txt
DELETED
|
@@ -1,7 +0,0 @@
|
|
| 1 |
-
gradio>=5.0.0
|
| 2 |
-
huggingface_hub>=0.30.0
|
| 3 |
-
pydantic>=2.8.0
|
| 4 |
-
pandas>=2.2.0
|
| 5 |
-
pyarrow>=17.0.0
|
| 6 |
-
soundfile>=0.12.1
|
| 7 |
-
pytest>=8.3.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/__init__.py
DELETED
|
File without changes
|
BirdNET Validator App/src/auth/__init__.py
DELETED
|
File without changes
|
BirdNET Validator App/src/auth/auth_service.py
DELETED
|
@@ -1,200 +0,0 @@
|
|
| 1 |
-
"""AuthService: User authentication, session management, and project ACL."""
|
| 2 |
-
|
| 3 |
-
from dataclasses import dataclass, field
|
| 4 |
-
from datetime import datetime, timedelta
|
| 5 |
-
from typing import Dict, List, Optional
|
| 6 |
-
import uuid
|
| 7 |
-
|
| 8 |
-
from src.domain.models import Role, User
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
@dataclass
|
| 12 |
-
class UserProjectAccess:
|
| 13 |
-
"""Maps a user to projects with specific roles."""
|
| 14 |
-
|
| 15 |
-
username: str
|
| 16 |
-
project_slugs: Dict[str, Role] # Maps project_slug -> role (admin or validator)
|
| 17 |
-
is_active: bool = True
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
@dataclass
|
| 21 |
-
class Session:
|
| 22 |
-
"""Represents an authenticated user session."""
|
| 23 |
-
|
| 24 |
-
session_id: str
|
| 25 |
-
username: str
|
| 26 |
-
role: Role
|
| 27 |
-
authorized_projects: List[str] # Projects this user can access
|
| 28 |
-
created_at: datetime
|
| 29 |
-
last_activity: datetime
|
| 30 |
-
expires_at: datetime
|
| 31 |
-
|
| 32 |
-
def is_expired(self) -> bool:
|
| 33 |
-
"""Check if session has expired."""
|
| 34 |
-
return datetime.utcnow() > self.expires_at
|
| 35 |
-
|
| 36 |
-
def update_activity(self, ttl_minutes: int = 120) -> None:
|
| 37 |
-
"""Update last activity timestamp and extend expiration."""
|
| 38 |
-
self.last_activity = datetime.utcnow()
|
| 39 |
-
self.expires_at = datetime.utcnow() + timedelta(minutes=ttl_minutes)
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
class AuthService:
|
| 43 |
-
"""Manages user authentication, sessions, and project access control."""
|
| 44 |
-
|
| 45 |
-
def __init__(self, session_ttl_minutes: int = 120):
|
| 46 |
-
"""Initialize AuthService with session TTL.
|
| 47 |
-
|
| 48 |
-
Args:
|
| 49 |
-
session_ttl_minutes: Session time-to-live in minutes (default: 2 hours)
|
| 50 |
-
"""
|
| 51 |
-
self.session_ttl_minutes = session_ttl_minutes
|
| 52 |
-
self._sessions: Dict[str, Session] = {} # session_id -> Session
|
| 53 |
-
self._user_access: Dict[str, UserProjectAccess] = {} # username -> UserProjectAccess
|
| 54 |
-
|
| 55 |
-
def register_user_project_access(
|
| 56 |
-
self, username: str, project_access: Dict[str, Role]
|
| 57 |
-
) -> None:
|
| 58 |
-
"""Register or update a user's access to projects.
|
| 59 |
-
|
| 60 |
-
Args:
|
| 61 |
-
username: User name
|
| 62 |
-
project_access: Dict mapping project_slug to Role (admin or validator)
|
| 63 |
-
"""
|
| 64 |
-
self._user_access[username] = UserProjectAccess(
|
| 65 |
-
username=username,
|
| 66 |
-
project_slugs=project_access,
|
| 67 |
-
is_active=True,
|
| 68 |
-
)
|
| 69 |
-
|
| 70 |
-
def login(self, username: str) -> Optional[Session]:
|
| 71 |
-
"""Authenticate a user and create a new session.
|
| 72 |
-
|
| 73 |
-
Args:
|
| 74 |
-
username: User name to authenticate
|
| 75 |
-
|
| 76 |
-
Returns:
|
| 77 |
-
Session if user exists and is active, None otherwise
|
| 78 |
-
"""
|
| 79 |
-
if username not in self._user_access:
|
| 80 |
-
return None
|
| 81 |
-
|
| 82 |
-
access = self._user_access[username]
|
| 83 |
-
if not access.is_active:
|
| 84 |
-
return None
|
| 85 |
-
|
| 86 |
-
# Determine highest role across all projects (admin > validator)
|
| 87 |
-
role = Role.validator
|
| 88 |
-
for proj_role in access.project_slugs.values():
|
| 89 |
-
if proj_role == Role.admin:
|
| 90 |
-
role = Role.admin
|
| 91 |
-
break
|
| 92 |
-
|
| 93 |
-
session_id = str(uuid.uuid4())
|
| 94 |
-
now = datetime.utcnow()
|
| 95 |
-
session = Session(
|
| 96 |
-
session_id=session_id,
|
| 97 |
-
username=username,
|
| 98 |
-
role=role,
|
| 99 |
-
authorized_projects=list(access.project_slugs.keys()),
|
| 100 |
-
created_at=now,
|
| 101 |
-
last_activity=now,
|
| 102 |
-
expires_at=now + timedelta(minutes=self.session_ttl_minutes),
|
| 103 |
-
)
|
| 104 |
-
|
| 105 |
-
self._sessions[session_id] = session
|
| 106 |
-
return session
|
| 107 |
-
|
| 108 |
-
def get_session(self, session_id: str) -> Optional[Session]:
|
| 109 |
-
"""Retrieve an active session by ID.
|
| 110 |
-
|
| 111 |
-
Args:
|
| 112 |
-
session_id: Session ID
|
| 113 |
-
|
| 114 |
-
Returns:
|
| 115 |
-
Session if valid and not expired, None otherwise
|
| 116 |
-
"""
|
| 117 |
-
session = self._sessions.get(session_id)
|
| 118 |
-
if session is None:
|
| 119 |
-
return None
|
| 120 |
-
|
| 121 |
-
if session.is_expired():
|
| 122 |
-
self._sessions.pop(session_id, None)
|
| 123 |
-
return None
|
| 124 |
-
|
| 125 |
-
session.update_activity(self.session_ttl_minutes)
|
| 126 |
-
return session
|
| 127 |
-
|
| 128 |
-
def logout(self, session_id: str) -> None:
|
| 129 |
-
"""Delete a session.
|
| 130 |
-
|
| 131 |
-
Args:
|
| 132 |
-
session_id: Session ID to invalidate
|
| 133 |
-
"""
|
| 134 |
-
self._sessions.pop(session_id, None)
|
| 135 |
-
|
| 136 |
-
def is_user_authorized_for_project(self, username: str, project_slug: str) -> bool:
|
| 137 |
-
"""Check if a user has access to a specific project.
|
| 138 |
-
|
| 139 |
-
Args:
|
| 140 |
-
username: User name
|
| 141 |
-
project_slug: Project slug
|
| 142 |
-
|
| 143 |
-
Returns:
|
| 144 |
-
True if user has access (admin or validator), False otherwise
|
| 145 |
-
"""
|
| 146 |
-
access = self._user_access.get(username)
|
| 147 |
-
if access is None or not access.is_active:
|
| 148 |
-
return False
|
| 149 |
-
|
| 150 |
-
return project_slug in access.project_slugs
|
| 151 |
-
|
| 152 |
-
def get_user_role_for_project(self, username: str, project_slug: str) -> Optional[Role]:
|
| 153 |
-
"""Get the specific role a user has for a project.
|
| 154 |
-
|
| 155 |
-
Args:
|
| 156 |
-
username: User name
|
| 157 |
-
project_slug: Project slug
|
| 158 |
-
|
| 159 |
-
Returns:
|
| 160 |
-
Role if user has access, None otherwise
|
| 161 |
-
"""
|
| 162 |
-
access = self._user_access.get(username)
|
| 163 |
-
if access is None or not access.is_active:
|
| 164 |
-
return None
|
| 165 |
-
|
| 166 |
-
return access.project_slugs.get(project_slug)
|
| 167 |
-
|
| 168 |
-
def list_user_projects(self, username: str) -> List[str]:
|
| 169 |
-
"""List all projects a user has access to.
|
| 170 |
-
|
| 171 |
-
Args:
|
| 172 |
-
username: User name
|
| 173 |
-
|
| 174 |
-
Returns:
|
| 175 |
-
List of project slugs
|
| 176 |
-
"""
|
| 177 |
-
access = self._user_access.get(username)
|
| 178 |
-
if access is None or not access.is_active:
|
| 179 |
-
return []
|
| 180 |
-
|
| 181 |
-
return list(access.project_slugs.keys())
|
| 182 |
-
|
| 183 |
-
def cleanup_expired_sessions(self) -> None:
|
| 184 |
-
"""Remove all expired sessions (maintenance cleanup)."""
|
| 185 |
-
now = datetime.utcnow()
|
| 186 |
-
expired = [
|
| 187 |
-
sid for sid, session in self._sessions.items() if session.is_expired()
|
| 188 |
-
]
|
| 189 |
-
for sid in expired:
|
| 190 |
-
self._sessions.pop(sid)
|
| 191 |
-
|
| 192 |
-
def set_user_active(self, username: str, active: bool) -> None:
|
| 193 |
-
"""Enable or disable a user account.
|
| 194 |
-
|
| 195 |
-
Args:
|
| 196 |
-
username: User name
|
| 197 |
-
active: Whether the user should be active
|
| 198 |
-
"""
|
| 199 |
-
if username in self._user_access:
|
| 200 |
-
self._user_access[username].is_active = active
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/cache/__init__.py
DELETED
|
File without changes
|
BirdNET Validator App/src/cache/ephemeral_cache_manager.py
DELETED
|
@@ -1,82 +0,0 @@
|
|
| 1 |
-
import hashlib
|
| 2 |
-
import tempfile
|
| 3 |
-
import time
|
| 4 |
-
from dataclasses import dataclass
|
| 5 |
-
from pathlib import Path
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
@dataclass(slots=True)
|
| 9 |
-
class CacheEntry:
|
| 10 |
-
key: str
|
| 11 |
-
path: Path
|
| 12 |
-
created_at: float
|
| 13 |
-
expires_at: float
|
| 14 |
-
size_bytes: int
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
class EphemeralCacheManager:
|
| 18 |
-
def __init__(
|
| 19 |
-
self,
|
| 20 |
-
cache_dir: str | None = None,
|
| 21 |
-
ttl_seconds: int = 300,
|
| 22 |
-
max_files: int = 256,
|
| 23 |
-
) -> None:
|
| 24 |
-
self._base_dir = Path(cache_dir) if cache_dir else Path(tempfile.gettempdir()) / "birdnet-validator-cache"
|
| 25 |
-
self._base_dir.mkdir(parents=True, exist_ok=True)
|
| 26 |
-
self._ttl_seconds = ttl_seconds
|
| 27 |
-
self._max_files = max_files
|
| 28 |
-
self._entries: dict[str, CacheEntry] = {}
|
| 29 |
-
|
| 30 |
-
def get(self, key: str) -> Path | None:
|
| 31 |
-
self.cleanup_expired()
|
| 32 |
-
entry = self._entries.get(key)
|
| 33 |
-
if not entry:
|
| 34 |
-
return None
|
| 35 |
-
if not entry.path.exists():
|
| 36 |
-
self._entries.pop(key, None)
|
| 37 |
-
return None
|
| 38 |
-
return entry.path
|
| 39 |
-
|
| 40 |
-
def put_bytes(self, key: str, data: bytes, suffix: str = ".bin") -> Path:
|
| 41 |
-
self.cleanup_expired()
|
| 42 |
-
self._enforce_capacity()
|
| 43 |
-
|
| 44 |
-
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
|
| 45 |
-
file_path = self._base_dir / f"{digest}{suffix}"
|
| 46 |
-
file_path.write_bytes(data)
|
| 47 |
-
|
| 48 |
-
now = time.time()
|
| 49 |
-
self._entries[key] = CacheEntry(
|
| 50 |
-
key=key,
|
| 51 |
-
path=file_path,
|
| 52 |
-
created_at=now,
|
| 53 |
-
expires_at=now + self._ttl_seconds,
|
| 54 |
-
size_bytes=len(data),
|
| 55 |
-
)
|
| 56 |
-
return file_path
|
| 57 |
-
|
| 58 |
-
def cleanup_key(self, key: str) -> None:
|
| 59 |
-
entry = self._entries.pop(key, None)
|
| 60 |
-
if not entry:
|
| 61 |
-
return
|
| 62 |
-
if entry.path.exists():
|
| 63 |
-
entry.path.unlink()
|
| 64 |
-
|
| 65 |
-
def cleanup_expired(self) -> None:
|
| 66 |
-
now = time.time()
|
| 67 |
-
expired_keys = [key for key, entry in self._entries.items() if entry.expires_at <= now or not entry.path.exists()]
|
| 68 |
-
for key in expired_keys:
|
| 69 |
-
self.cleanup_key(key)
|
| 70 |
-
|
| 71 |
-
def clear(self) -> None:
|
| 72 |
-
for key in list(self._entries.keys()):
|
| 73 |
-
self.cleanup_key(key)
|
| 74 |
-
|
| 75 |
-
def _enforce_capacity(self) -> None:
|
| 76 |
-
if len(self._entries) < self._max_files:
|
| 77 |
-
return
|
| 78 |
-
|
| 79 |
-
oldest_keys = sorted(self._entries.keys(), key=lambda key: self._entries[key].created_at)
|
| 80 |
-
overflow = (len(self._entries) - self._max_files) + 1
|
| 81 |
-
for key in oldest_keys[:overflow]:
|
| 82 |
-
self.cleanup_key(key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/domain/__init__.py
DELETED
|
File without changes
|
BirdNET Validator App/src/domain/models.py
DELETED
|
@@ -1,46 +0,0 @@
|
|
| 1 |
-
from enum import Enum
|
| 2 |
-
from typing import Optional
|
| 3 |
-
|
| 4 |
-
from pydantic import BaseModel, Field
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
class Role(str, Enum):
|
| 8 |
-
admin = "admin"
|
| 9 |
-
validator = "validator"
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
class User(BaseModel):
|
| 13 |
-
username: str = Field(min_length=1)
|
| 14 |
-
role: Role
|
| 15 |
-
active: bool = True
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
class Project(BaseModel):
|
| 19 |
-
project_slug: str = Field(min_length=3)
|
| 20 |
-
name: str = Field(min_length=1)
|
| 21 |
-
dataset_repo_id: str = Field(min_length=3)
|
| 22 |
-
active: bool = True
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
class Detection(BaseModel):
|
| 26 |
-
detection_key: str = Field(min_length=16)
|
| 27 |
-
audio_id: str = Field(min_length=1)
|
| 28 |
-
scientific_name: str = Field(min_length=1)
|
| 29 |
-
confidence: float = Field(ge=0.0, le=1.0)
|
| 30 |
-
start_time: float = Field(ge=0.0)
|
| 31 |
-
end_time: float = Field(gt=0.0)
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
class Validation(BaseModel):
|
| 35 |
-
detection_key: str = Field(min_length=16)
|
| 36 |
-
status: str = Field(min_length=1)
|
| 37 |
-
corrected_species: Optional[str] = None
|
| 38 |
-
notes: str = ""
|
| 39 |
-
validator: str = Field(min_length=1)
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
class IndexManifest(BaseModel):
|
| 43 |
-
schema_version: str = "1.0.0"
|
| 44 |
-
project_slug: str
|
| 45 |
-
total_detections: int = Field(ge=0)
|
| 46 |
-
total_audio_files: int = Field(ge=0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/repositories/__init__.py
DELETED
|
File without changes
|
BirdNET Validator App/src/repositories/append_only_validation_repository.py
DELETED
|
@@ -1,89 +0,0 @@
|
|
| 1 |
-
import json
|
| 2 |
-
from datetime import UTC, datetime
|
| 3 |
-
from pathlib import Path
|
| 4 |
-
from uuid import uuid4
|
| 5 |
-
|
| 6 |
-
from src.domain.models import Validation
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class OptimisticLockError(Exception):
|
| 10 |
-
def __init__(self, detection_key: str, expected_version: int, current_version: int) -> None:
|
| 11 |
-
self.detection_key = detection_key
|
| 12 |
-
self.expected_version = expected_version
|
| 13 |
-
self.current_version = current_version
|
| 14 |
-
super().__init__(
|
| 15 |
-
f"Optimistic lock conflict for detection_key={detection_key}: "
|
| 16 |
-
f"expected_version={expected_version}, current_version={current_version}"
|
| 17 |
-
)
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
class AppendOnlyValidationRepository:
|
| 21 |
-
def __init__(self, base_dir: str) -> None:
|
| 22 |
-
self._base_dir = Path(base_dir)
|
| 23 |
-
|
| 24 |
-
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 25 |
-
project_dir = self._base_dir / project_slug / "validations"
|
| 26 |
-
project_dir.mkdir(parents=True, exist_ok=True)
|
| 27 |
-
|
| 28 |
-
current_file = project_dir / "current.json"
|
| 29 |
-
if current_file.exists():
|
| 30 |
-
current_payload = json.loads(current_file.read_text(encoding="utf-8"))
|
| 31 |
-
else:
|
| 32 |
-
current_payload = {}
|
| 33 |
-
|
| 34 |
-
current_item = current_payload.get(item.detection_key, {})
|
| 35 |
-
current_version = int(current_item.get("version", 0))
|
| 36 |
-
expected = expected_version if expected_version is not None else current_version
|
| 37 |
-
if expected != current_version:
|
| 38 |
-
raise OptimisticLockError(
|
| 39 |
-
detection_key=item.detection_key,
|
| 40 |
-
expected_version=expected,
|
| 41 |
-
current_version=current_version,
|
| 42 |
-
)
|
| 43 |
-
|
| 44 |
-
new_version = current_version + 1
|
| 45 |
-
|
| 46 |
-
event_date = datetime.now(UTC).strftime("%Y%m%d")
|
| 47 |
-
events_file = project_dir / f"events-{event_date}.jsonl"
|
| 48 |
-
event = {
|
| 49 |
-
"event_id": str(uuid4()),
|
| 50 |
-
"timestamp": datetime.now(UTC).isoformat(),
|
| 51 |
-
"project_slug": project_slug,
|
| 52 |
-
"expected_version": expected,
|
| 53 |
-
"previous_version": current_version,
|
| 54 |
-
"new_version": new_version,
|
| 55 |
-
**item.model_dump(),
|
| 56 |
-
}
|
| 57 |
-
with events_file.open("a", encoding="utf-8") as handle:
|
| 58 |
-
handle.write(json.dumps(event, ensure_ascii=True) + "\n")
|
| 59 |
-
|
| 60 |
-
current_payload[item.detection_key] = {
|
| 61 |
-
"status": item.status,
|
| 62 |
-
"corrected_species": item.corrected_species,
|
| 63 |
-
"notes": item.notes,
|
| 64 |
-
"validator": item.validator,
|
| 65 |
-
"updated_at": event["timestamp"],
|
| 66 |
-
"version": new_version,
|
| 67 |
-
}
|
| 68 |
-
current_file.write_text(json.dumps(current_payload, indent=2), encoding="utf-8")
|
| 69 |
-
return new_version
|
| 70 |
-
|
| 71 |
-
def list_events(self, project_slug: str) -> list[dict[str, object]]:
|
| 72 |
-
project_dir = self._base_dir / project_slug / "validations"
|
| 73 |
-
if not project_dir.exists():
|
| 74 |
-
return []
|
| 75 |
-
|
| 76 |
-
events: list[dict[str, object]] = []
|
| 77 |
-
for path in sorted(project_dir.glob("events-*.jsonl")):
|
| 78 |
-
with path.open("r", encoding="utf-8") as handle:
|
| 79 |
-
for line in handle:
|
| 80 |
-
payload = line.strip()
|
| 81 |
-
if payload:
|
| 82 |
-
events.append(json.loads(payload))
|
| 83 |
-
return events
|
| 84 |
-
|
| 85 |
-
def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]:
|
| 86 |
-
current_file = self._base_dir / project_slug / "validations" / "current.json"
|
| 87 |
-
if not current_file.exists():
|
| 88 |
-
return {}
|
| 89 |
-
return json.loads(current_file.read_text(encoding="utf-8"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/repositories/contracts.py
DELETED
|
@@ -1,35 +0,0 @@
|
|
| 1 |
-
from typing import Protocol
|
| 2 |
-
|
| 3 |
-
from src.domain.models import Detection, Project, User, Validation
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
class ProjectRepository(Protocol):
|
| 7 |
-
def get_project(self, project_slug: str) -> Project: ...
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
class DetectionRepository(Protocol):
|
| 11 |
-
def list_detections(
|
| 12 |
-
self,
|
| 13 |
-
project_slug: str,
|
| 14 |
-
page: int,
|
| 15 |
-
page_size: int,
|
| 16 |
-
scientific_name: str | None = None,
|
| 17 |
-
min_confidence: float | None = None,
|
| 18 |
-
max_confidence: float | None = None,
|
| 19 |
-
) -> list[Detection]: ...
|
| 20 |
-
|
| 21 |
-
def count_detections(
|
| 22 |
-
self,
|
| 23 |
-
project_slug: str,
|
| 24 |
-
scientific_name: str | None = None,
|
| 25 |
-
min_confidence: float | None = None,
|
| 26 |
-
max_confidence: float | None = None,
|
| 27 |
-
) -> int: ...
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
class ValidationRepository(Protocol):
|
| 31 |
-
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int: ...
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
class AuthRepository(Protocol):
|
| 35 |
-
def authenticate(self, username: str) -> User: ...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/repositories/in_memory_detection_repository.py
DELETED
|
@@ -1,73 +0,0 @@
|
|
| 1 |
-
from collections import defaultdict
|
| 2 |
-
from typing import DefaultDict
|
| 3 |
-
|
| 4 |
-
from src.domain.models import Detection
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
class InMemoryDetectionRepository:
|
| 8 |
-
def __init__(self) -> None:
|
| 9 |
-
self._project_items: DefaultDict[str, list[Detection]] = defaultdict(list)
|
| 10 |
-
|
| 11 |
-
def seed(self, project_slug: str, detections: list[Detection]) -> None:
|
| 12 |
-
self._project_items[project_slug] = list(detections)
|
| 13 |
-
|
| 14 |
-
def list_detections(
|
| 15 |
-
self,
|
| 16 |
-
project_slug: str,
|
| 17 |
-
page: int,
|
| 18 |
-
page_size: int,
|
| 19 |
-
scientific_name: str | None = None,
|
| 20 |
-
min_confidence: float | None = None,
|
| 21 |
-
max_confidence: float | None = None,
|
| 22 |
-
) -> list[Detection]:
|
| 23 |
-
if page < 1:
|
| 24 |
-
raise ValueError("page must be >= 1")
|
| 25 |
-
if page_size < 1:
|
| 26 |
-
raise ValueError("page_size must be >= 1")
|
| 27 |
-
|
| 28 |
-
filtered = self._apply_filters(
|
| 29 |
-
project_slug=project_slug,
|
| 30 |
-
scientific_name=scientific_name,
|
| 31 |
-
min_confidence=min_confidence,
|
| 32 |
-
max_confidence=max_confidence,
|
| 33 |
-
)
|
| 34 |
-
|
| 35 |
-
start = (page - 1) * page_size
|
| 36 |
-
end = start + page_size
|
| 37 |
-
return filtered[start:end]
|
| 38 |
-
|
| 39 |
-
def count_detections(
|
| 40 |
-
self,
|
| 41 |
-
project_slug: str,
|
| 42 |
-
scientific_name: str | None = None,
|
| 43 |
-
min_confidence: float | None = None,
|
| 44 |
-
max_confidence: float | None = None,
|
| 45 |
-
) -> int:
|
| 46 |
-
return len(
|
| 47 |
-
self._apply_filters(
|
| 48 |
-
project_slug=project_slug,
|
| 49 |
-
scientific_name=scientific_name,
|
| 50 |
-
min_confidence=min_confidence,
|
| 51 |
-
max_confidence=max_confidence,
|
| 52 |
-
)
|
| 53 |
-
)
|
| 54 |
-
|
| 55 |
-
def _apply_filters(
|
| 56 |
-
self,
|
| 57 |
-
project_slug: str,
|
| 58 |
-
scientific_name: str | None,
|
| 59 |
-
min_confidence: float | None,
|
| 60 |
-
max_confidence: float | None,
|
| 61 |
-
) -> list[Detection]:
|
| 62 |
-
items = self._project_items.get(project_slug, [])
|
| 63 |
-
|
| 64 |
-
def match(item: Detection) -> bool:
|
| 65 |
-
if scientific_name and item.scientific_name != scientific_name:
|
| 66 |
-
return False
|
| 67 |
-
if min_confidence is not None and item.confidence < min_confidence:
|
| 68 |
-
return False
|
| 69 |
-
if max_confidence is not None and item.confidence > max_confidence:
|
| 70 |
-
return False
|
| 71 |
-
return True
|
| 72 |
-
|
| 73 |
-
return [item for item in items if match(item)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/repositories/in_memory_validation_repository.py
DELETED
|
@@ -1,25 +0,0 @@
|
|
| 1 |
-
from collections import defaultdict
|
| 2 |
-
from typing import DefaultDict
|
| 3 |
-
|
| 4 |
-
from src.domain.models import Validation
|
| 5 |
-
from src.repositories.append_only_validation_repository import OptimisticLockError
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
class InMemoryValidationRepository:
|
| 9 |
-
def __init__(self) -> None:
|
| 10 |
-
self._by_project: DefaultDict[str, list[Validation]] = defaultdict(list)
|
| 11 |
-
self._versions: DefaultDict[str, dict[str, int]] = defaultdict(dict)
|
| 12 |
-
|
| 13 |
-
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 14 |
-
current_version = int(self._versions[project_slug].get(item.detection_key, 0))
|
| 15 |
-
expected = expected_version if expected_version is not None else current_version
|
| 16 |
-
if expected != current_version:
|
| 17 |
-
raise OptimisticLockError(item.detection_key, expected, current_version)
|
| 18 |
-
|
| 19 |
-
self._by_project[project_slug].append(item)
|
| 20 |
-
new_version = current_version + 1
|
| 21 |
-
self._versions[project_slug][item.detection_key] = new_version
|
| 22 |
-
return new_version
|
| 23 |
-
|
| 24 |
-
def list_validations(self, project_slug: str) -> list[Validation]:
|
| 25 |
-
return list(self._by_project.get(project_slug, []))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/services/__init__.py
DELETED
|
File without changes
|
BirdNET Validator App/src/services/audio_fetch_service.py
DELETED
|
@@ -1,54 +0,0 @@
|
|
| 1 |
-
from dataclasses import dataclass
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
|
| 4 |
-
from huggingface_hub import hf_hub_download
|
| 5 |
-
|
| 6 |
-
from src.cache.ephemeral_cache_manager import EphemeralCacheManager
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
SUPPORTED_AUDIO_EXTENSIONS = (".wav", ".mp3", ".flac", ".ogg", ".m4a")
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
@dataclass(slots=True)
|
| 13 |
-
class AudioFetchResult:
|
| 14 |
-
cache_key: str
|
| 15 |
-
local_path: str
|
| 16 |
-
source: str
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
class AudioFetchService:
|
| 20 |
-
def __init__(self, cache_manager: EphemeralCacheManager) -> None:
|
| 21 |
-
self._cache = cache_manager
|
| 22 |
-
|
| 23 |
-
def fetch(self, dataset_repo: str, audio_id: str) -> AudioFetchResult:
|
| 24 |
-
cache_key = f"{dataset_repo}:{audio_id}"
|
| 25 |
-
cached_path = self._cache.get(cache_key)
|
| 26 |
-
if cached_path:
|
| 27 |
-
return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="cache")
|
| 28 |
-
|
| 29 |
-
target_filename, downloaded_path = self._resolve_remote_filename(dataset_repo=dataset_repo, audio_id=audio_id)
|
| 30 |
-
if downloaded_path is None:
|
| 31 |
-
downloaded = hf_hub_download(repo_id=dataset_repo, repo_type="dataset", filename=target_filename)
|
| 32 |
-
downloaded_path = Path(downloaded)
|
| 33 |
-
|
| 34 |
-
suffix = downloaded_path.suffix if downloaded_path.suffix else ".bin"
|
| 35 |
-
local_path = self._cache.put_bytes(cache_key, downloaded_path.read_bytes(), suffix=suffix)
|
| 36 |
-
return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="remote")
|
| 37 |
-
|
| 38 |
-
def cleanup_after_validation(self, cache_key: str) -> None:
|
| 39 |
-
self._cache.cleanup_key(cache_key)
|
| 40 |
-
|
| 41 |
-
def _resolve_remote_filename(self, dataset_repo: str, audio_id: str) -> tuple[str, Path | None]:
|
| 42 |
-
audio_path = Path(audio_id)
|
| 43 |
-
if audio_path.suffix:
|
| 44 |
-
return f"audio/{audio_id}", None
|
| 45 |
-
|
| 46 |
-
for extension in SUPPORTED_AUDIO_EXTENSIONS:
|
| 47 |
-
candidate = f"audio/{audio_id}{extension}"
|
| 48 |
-
try:
|
| 49 |
-
downloaded = hf_hub_download(repo_id=dataset_repo, repo_type="dataset", filename=candidate)
|
| 50 |
-
return candidate, Path(downloaded)
|
| 51 |
-
except Exception:
|
| 52 |
-
continue
|
| 53 |
-
|
| 54 |
-
raise FileNotFoundError(f"Unable to locate audio file for audio_id: {audio_id}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/services/detection_queue_service.py
DELETED
|
@@ -1,64 +0,0 @@
|
|
| 1 |
-
from dataclasses import dataclass
|
| 2 |
-
|
| 3 |
-
from src.domain.models import Detection
|
| 4 |
-
from src.repositories.contracts import DetectionRepository
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
@dataclass(slots=True)
|
| 8 |
-
class DetectionPage:
|
| 9 |
-
project_slug: str
|
| 10 |
-
page: int
|
| 11 |
-
page_size: int
|
| 12 |
-
total_items: int
|
| 13 |
-
total_pages: int
|
| 14 |
-
has_next: bool
|
| 15 |
-
has_previous: bool
|
| 16 |
-
items: list[Detection]
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
class DetectionQueueService:
|
| 20 |
-
def __init__(self, repository: DetectionRepository) -> None:
|
| 21 |
-
self._repository = repository
|
| 22 |
-
|
| 23 |
-
def get_page(
|
| 24 |
-
self,
|
| 25 |
-
project_slug: str,
|
| 26 |
-
page: int,
|
| 27 |
-
page_size: int,
|
| 28 |
-
scientific_name: str | None = None,
|
| 29 |
-
min_confidence: float | None = None,
|
| 30 |
-
max_confidence: float | None = None,
|
| 31 |
-
) -> DetectionPage:
|
| 32 |
-
if page < 1:
|
| 33 |
-
raise ValueError("page must be >= 1")
|
| 34 |
-
if page_size < 1:
|
| 35 |
-
raise ValueError("page_size must be >= 1")
|
| 36 |
-
|
| 37 |
-
total_items = self._repository.count_detections(
|
| 38 |
-
project_slug=project_slug,
|
| 39 |
-
scientific_name=scientific_name,
|
| 40 |
-
min_confidence=min_confidence,
|
| 41 |
-
max_confidence=max_confidence,
|
| 42 |
-
)
|
| 43 |
-
total_pages = max(1, ((total_items - 1) // page_size) + 1) if total_items else 1
|
| 44 |
-
page = min(page, total_pages)
|
| 45 |
-
|
| 46 |
-
items = self._repository.list_detections(
|
| 47 |
-
project_slug=project_slug,
|
| 48 |
-
page=page,
|
| 49 |
-
page_size=page_size,
|
| 50 |
-
scientific_name=scientific_name,
|
| 51 |
-
min_confidence=min_confidence,
|
| 52 |
-
max_confidence=max_confidence,
|
| 53 |
-
)
|
| 54 |
-
|
| 55 |
-
return DetectionPage(
|
| 56 |
-
project_slug=project_slug,
|
| 57 |
-
page=page,
|
| 58 |
-
page_size=page_size,
|
| 59 |
-
total_items=total_items,
|
| 60 |
-
total_pages=total_pages,
|
| 61 |
-
has_next=page < total_pages,
|
| 62 |
-
has_previous=page > 1,
|
| 63 |
-
items=items,
|
| 64 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/services/validation_service.py
DELETED
|
@@ -1,39 +0,0 @@
|
|
| 1 |
-
from dataclasses import dataclass
|
| 2 |
-
|
| 3 |
-
from src.domain.models import Validation
|
| 4 |
-
from src.repositories.contracts import ValidationRepository
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
@dataclass(slots=True)
|
| 8 |
-
class ValidationWriteResult:
|
| 9 |
-
item: Validation
|
| 10 |
-
new_version: int
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
class ValidationService:
|
| 14 |
-
def __init__(self, repository: ValidationRepository) -> None:
|
| 15 |
-
self._repository = repository
|
| 16 |
-
|
| 17 |
-
def validate_detection(
|
| 18 |
-
self,
|
| 19 |
-
project_slug: str,
|
| 20 |
-
detection_key: str,
|
| 21 |
-
status: str,
|
| 22 |
-
validator: str,
|
| 23 |
-
notes: str = "",
|
| 24 |
-
corrected_species: str | None = None,
|
| 25 |
-
expected_version: int | None = None,
|
| 26 |
-
) -> ValidationWriteResult:
|
| 27 |
-
item = Validation(
|
| 28 |
-
detection_key=detection_key,
|
| 29 |
-
status=status,
|
| 30 |
-
corrected_species=corrected_species,
|
| 31 |
-
notes=notes,
|
| 32 |
-
validator=validator,
|
| 33 |
-
)
|
| 34 |
-
new_version = self._repository.save_validation(
|
| 35 |
-
project_slug=project_slug,
|
| 36 |
-
item=item,
|
| 37 |
-
expected_version=expected_version,
|
| 38 |
-
)
|
| 39 |
-
return ValidationWriteResult(item=item, new_version=new_version)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/telemetry/__init__.py
DELETED
|
File without changes
|
BirdNET Validator App/src/ui/__init__.py
DELETED
|
File without changes
|
BirdNET Validator App/src/ui/admin_panel.py
DELETED
|
@@ -1,281 +0,0 @@
|
|
| 1 |
-
"""Gradio admin panel for managing projects and user assignments."""
|
| 2 |
-
|
| 3 |
-
from typing import List, Tuple
|
| 4 |
-
|
| 5 |
-
import gradio as gr
|
| 6 |
-
|
| 7 |
-
from src.auth.auth_service import AuthService, Session
|
| 8 |
-
from src.domain.models import Project, Role
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
class AdminPanelManager:
|
| 12 |
-
"""Management backend for the admin panel."""
|
| 13 |
-
|
| 14 |
-
def __init__(self, auth_service: AuthService):
|
| 15 |
-
"""Initialize admin panel manager.
|
| 16 |
-
|
| 17 |
-
Args:
|
| 18 |
-
auth_service: AuthService instance
|
| 19 |
-
"""
|
| 20 |
-
self.auth_service = auth_service
|
| 21 |
-
self._projects: dict[str, Project] = {} # project_slug -> Project
|
| 22 |
-
self._user_project_assignments: dict[str, dict[str, str]] = {} # username -> {project_slug -> role}
|
| 23 |
-
|
| 24 |
-
def register_project(self, project: Project) -> bool:
|
| 25 |
-
"""Register a new project (idempotent).
|
| 26 |
-
|
| 27 |
-
Args:
|
| 28 |
-
project: Project to register
|
| 29 |
-
|
| 30 |
-
Returns:
|
| 31 |
-
True if new, False if already existed
|
| 32 |
-
"""
|
| 33 |
-
if project.project_slug in self._projects:
|
| 34 |
-
return False
|
| 35 |
-
|
| 36 |
-
self._projects[project.project_slug] = project
|
| 37 |
-
return True
|
| 38 |
-
|
| 39 |
-
def list_projects(self) -> List[dict]:
|
| 40 |
-
"""List all projects as dictionaries for Gradio display.
|
| 41 |
-
|
| 42 |
-
Returns:
|
| 43 |
-
List of project dicts with slug, name, repo_id, active status
|
| 44 |
-
"""
|
| 45 |
-
return [
|
| 46 |
-
{
|
| 47 |
-
"project_slug": p.project_slug,
|
| 48 |
-
"name": p.name,
|
| 49 |
-
"dataset_repo_id": p.dataset_repo_id,
|
| 50 |
-
"active": p.active,
|
| 51 |
-
}
|
| 52 |
-
for p in self._projects.values()
|
| 53 |
-
]
|
| 54 |
-
|
| 55 |
-
def get_project(self, project_slug: str) -> Project | None:
|
| 56 |
-
"""Get project by slug.
|
| 57 |
-
|
| 58 |
-
Args:
|
| 59 |
-
project_slug: Project slug
|
| 60 |
-
|
| 61 |
-
Returns:
|
| 62 |
-
Project if found, None otherwise
|
| 63 |
-
"""
|
| 64 |
-
return self._projects.get(project_slug)
|
| 65 |
-
|
| 66 |
-
def list_users_for_project(self, project_slug: str) -> List[dict]:
|
| 67 |
-
"""List all users assigned to a project.
|
| 68 |
-
|
| 69 |
-
Args:
|
| 70 |
-
project_slug: Project slug
|
| 71 |
-
|
| 72 |
-
Returns:
|
| 73 |
-
List of dicts with username and role
|
| 74 |
-
"""
|
| 75 |
-
result = []
|
| 76 |
-
for username in self.auth_service._user_access.keys():
|
| 77 |
-
role = self.auth_service.get_user_role_for_project(username, project_slug)
|
| 78 |
-
if role is not None:
|
| 79 |
-
result.append({"username": username, "role": role.value})
|
| 80 |
-
|
| 81 |
-
return result
|
| 82 |
-
|
| 83 |
-
def assign_user_to_project(
|
| 84 |
-
self, username: str, project_slug: str, role: str
|
| 85 |
-
) -> Tuple[bool, str]:
|
| 86 |
-
"""Assign a user to a project with a role.
|
| 87 |
-
|
| 88 |
-
Args:
|
| 89 |
-
username: Username
|
| 90 |
-
project_slug: Project slug
|
| 91 |
-
role: "admin" or "validator"
|
| 92 |
-
|
| 93 |
-
Returns:
|
| 94 |
-
Tuple of (success, message)
|
| 95 |
-
"""
|
| 96 |
-
if project_slug not in self._projects:
|
| 97 |
-
return False, f"Project '{project_slug}' not found"
|
| 98 |
-
|
| 99 |
-
if role not in ["admin", "validator"]:
|
| 100 |
-
return False, f"Invalid role: {role}"
|
| 101 |
-
|
| 102 |
-
# Get or create user's project access
|
| 103 |
-
if username not in self.auth_service._user_access:
|
| 104 |
-
self.auth_service.register_user_project_access(username, {})
|
| 105 |
-
|
| 106 |
-
access = self.auth_service._user_access[username]
|
| 107 |
-
access.project_slugs[project_slug] = Role(role)
|
| 108 |
-
|
| 109 |
-
return True, f"✅ Assigned {username} to {project_slug} as {role}"
|
| 110 |
-
|
| 111 |
-
def remove_user_from_project(self, username: str, project_slug: str) -> Tuple[bool, str]:
|
| 112 |
-
"""Remove a user's access to a project.
|
| 113 |
-
|
| 114 |
-
Args:
|
| 115 |
-
username: Username
|
| 116 |
-
project_slug: Project slug
|
| 117 |
-
|
| 118 |
-
Returns:
|
| 119 |
-
Tuple of (success, message)
|
| 120 |
-
"""
|
| 121 |
-
if username not in self.auth_service._user_access:
|
| 122 |
-
return False, f"User '{username}' not found"
|
| 123 |
-
|
| 124 |
-
access = self.auth_service._user_access[username]
|
| 125 |
-
if project_slug not in access.project_slugs:
|
| 126 |
-
return False, f"User '{username}' is not assigned to project '{project_slug}'"
|
| 127 |
-
|
| 128 |
-
del access.project_slugs[project_slug]
|
| 129 |
-
return True, f"✅ Removed {username} from {project_slug}"
|
| 130 |
-
|
| 131 |
-
def toggleproject_active(self, project_slug: str, active: bool) -> Tuple[bool, str]:
|
| 132 |
-
"""Enable or disable a project.
|
| 133 |
-
|
| 134 |
-
Args:
|
| 135 |
-
project_slug: Project slug
|
| 136 |
-
active: Whether to activate or deactivate
|
| 137 |
-
|
| 138 |
-
Returns:
|
| 139 |
-
Tuple of (success, message)
|
| 140 |
-
"""
|
| 141 |
-
if project_slug not in self._projects:
|
| 142 |
-
return False, f"Project '{project_slug}' not found"
|
| 143 |
-
|
| 144 |
-
self._projects[project_slug].active = active
|
| 145 |
-
status = "activated" if active else "deactivated"
|
| 146 |
-
return True, f"✅ Project {project_slug} {status}"
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
def create_admin_panel(admin_manager: AdminPanelManager, current_session: Session) -> gr.Blocks:
|
| 150 |
-
"""Create Gradio admin panel UI.
|
| 151 |
-
|
| 152 |
-
Args:
|
| 153 |
-
admin_manager: AdminPanelManager instance
|
| 154 |
-
current_session: Current user's session
|
| 155 |
-
|
| 156 |
-
Returns:
|
| 157 |
-
Gradio Blocks with admin panel tabs
|
| 158 |
-
"""
|
| 159 |
-
if current_session.role != Role.admin:
|
| 160 |
-
with gr.Blocks() as restricted:
|
| 161 |
-
gr.Markdown("❌ **Access Denied**\n\nOnly administrators can access this panel.")
|
| 162 |
-
return restricted
|
| 163 |
-
|
| 164 |
-
with gr.Blocks(title="BirdNET Admin Panel") as admin_block:
|
| 165 |
-
gr.Markdown("# Admin Panel")
|
| 166 |
-
gr.Markdown(f"Logged in as: **{current_session.username}** (Admin)")
|
| 167 |
-
|
| 168 |
-
with gr.Tabs():
|
| 169 |
-
# Projects Tab
|
| 170 |
-
with gr.Tab("Projects"):
|
| 171 |
-
gr.Markdown("## Manage Projects")
|
| 172 |
-
|
| 173 |
-
with gr.Row():
|
| 174 |
-
with gr.Column(scale=2):
|
| 175 |
-
project_slug_input = gr.Textbox(
|
| 176 |
-
label="Project Slug",
|
| 177 |
-
placeholder="e.g., kenya-2024",
|
| 178 |
-
lines=1,
|
| 179 |
-
)
|
| 180 |
-
project_name_input = gr.Textbox(
|
| 181 |
-
label="Project Name",
|
| 182 |
-
placeholder="e.g., Kenya Survey 2024",
|
| 183 |
-
lines=1,
|
| 184 |
-
)
|
| 185 |
-
|
| 186 |
-
with gr.Column(scale=1):
|
| 187 |
-
repo_id_input = gr.Textbox(
|
| 188 |
-
label="HF Dataset Repo ID",
|
| 189 |
-
placeholder="e.g., org/dataset-name",
|
| 190 |
-
lines=1,
|
| 191 |
-
)
|
| 192 |
-
|
| 193 |
-
project_message = gr.Markdown()
|
| 194 |
-
|
| 195 |
-
# TODO: Implement create project button
|
| 196 |
-
# create_project_button = gr.Button("Create Project", variant="primary")
|
| 197 |
-
|
| 198 |
-
# Projects list
|
| 199 |
-
with gr.Row():
|
| 200 |
-
refresh_projects_button = gr.Button("Refresh Projects List")
|
| 201 |
-
projects_table = gr.Dataframe(
|
| 202 |
-
value=admin_manager.list_projects(),
|
| 203 |
-
headers=["project_slug", "name", "dataset_repo_id", "active"],
|
| 204 |
-
interactive=False,
|
| 205 |
-
)
|
| 206 |
-
|
| 207 |
-
refresh_projects_button.click(
|
| 208 |
-
fn=lambda: admin_manager.list_projects(),
|
| 209 |
-
outputs=[projects_table],
|
| 210 |
-
)
|
| 211 |
-
|
| 212 |
-
# Users Tab
|
| 213 |
-
with gr.Tab("Users"):
|
| 214 |
-
gr.Markdown("## Manage User Access")
|
| 215 |
-
|
| 216 |
-
with gr.Row():
|
| 217 |
-
username_input = gr.Textbox(
|
| 218 |
-
label="Username",
|
| 219 |
-
placeholder="e.g., validator_001",
|
| 220 |
-
lines=1,
|
| 221 |
-
)
|
| 222 |
-
|
| 223 |
-
project_select = gr.Dropdown(
|
| 224 |
-
choices=[p["project_slug"] for p in admin_manager.list_projects()],
|
| 225 |
-
label="Project",
|
| 226 |
-
)
|
| 227 |
-
|
| 228 |
-
role_select = gr.Dropdown(
|
| 229 |
-
choices=["admin", "validator"],
|
| 230 |
-
value="validator",
|
| 231 |
-
label="Role",
|
| 232 |
-
)
|
| 233 |
-
|
| 234 |
-
user_message = gr.Markdown()
|
| 235 |
-
|
| 236 |
-
assign_button = gr.Button("Assign User", variant="primary")
|
| 237 |
-
remove_button = gr.Button("Remove User", variant="stop")
|
| 238 |
-
|
| 239 |
-
def assign_user(username: str, project_slug: str, role: str) -> str:
|
| 240 |
-
success, msg = admin_manager.assign_user_to_project(username, project_slug, role)
|
| 241 |
-
return msg
|
| 242 |
-
|
| 243 |
-
def remove_user(username: str, project_slug: str) -> str:
|
| 244 |
-
success, msg = admin_manager.remove_user_from_project(username, project_slug)
|
| 245 |
-
return msg
|
| 246 |
-
|
| 247 |
-
assign_button.click(
|
| 248 |
-
fn=assign_user,
|
| 249 |
-
inputs=[username_input, project_select, role_select],
|
| 250 |
-
outputs=[user_message],
|
| 251 |
-
)
|
| 252 |
-
|
| 253 |
-
remove_button.click(
|
| 254 |
-
fn=remove_user,
|
| 255 |
-
inputs=[username_input, project_select],
|
| 256 |
-
outputs=[user_message],
|
| 257 |
-
)
|
| 258 |
-
|
| 259 |
-
# Users per project view
|
| 260 |
-
with gr.Row():
|
| 261 |
-
project_filter = gr.Dropdown(
|
| 262 |
-
choices=[p["project_slug"] for p in admin_manager.list_projects()],
|
| 263 |
-
label="View Users for Project",
|
| 264 |
-
)
|
| 265 |
-
|
| 266 |
-
users_table = gr.Dataframe(
|
| 267 |
-
value=[],
|
| 268 |
-
headers=["username", "role"],
|
| 269 |
-
interactive=False,
|
| 270 |
-
)
|
| 271 |
-
|
| 272 |
-
def update_users_table(project_slug: str):
|
| 273 |
-
return admin_manager.list_users_for_project(project_slug)
|
| 274 |
-
|
| 275 |
-
project_filter.change(
|
| 276 |
-
fn=update_users_table,
|
| 277 |
-
inputs=[project_filter],
|
| 278 |
-
outputs=[users_table],
|
| 279 |
-
)
|
| 280 |
-
|
| 281 |
-
return admin_block
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/ui/app_factory.py
DELETED
|
@@ -1,1542 +0,0 @@
|
|
| 1 |
-
import gradio as gr
|
| 2 |
-
import tempfile
|
| 3 |
-
from datetime import date, datetime
|
| 4 |
-
from pathlib import Path
|
| 5 |
-
from typing import Protocol
|
| 6 |
-
|
| 7 |
-
from src.cache.ephemeral_cache_manager import EphemeralCacheManager
|
| 8 |
-
from src.domain.models import Detection, Project, Role
|
| 9 |
-
from src.repositories.append_only_validation_repository import AppendOnlyValidationRepository, OptimisticLockError
|
| 10 |
-
from src.repositories.in_memory_detection_repository import InMemoryDetectionRepository
|
| 11 |
-
from src.services.audio_fetch_service import AudioFetchService
|
| 12 |
-
from src.services.detection_queue_service import DetectionQueueService
|
| 13 |
-
from src.services.validation_service import ValidationService
|
| 14 |
-
from src.auth.auth_service import AuthService
|
| 15 |
-
from src.ui.login_page import create_login_page
|
| 16 |
-
from src.ui.project_selector import create_project_selector
|
| 17 |
-
from src.ui.admin_panel import AdminPanelManager, create_admin_panel
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
class _AudioFetchResultProtocol(Protocol):
|
| 21 |
-
cache_key: str
|
| 22 |
-
local_path: str
|
| 23 |
-
source: str
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
class _AudioServiceProtocol(Protocol):
|
| 27 |
-
def fetch(self, dataset_repo: str, audio_id: str) -> _AudioFetchResultProtocol: ...
|
| 28 |
-
|
| 29 |
-
def cleanup_after_validation(self, cache_key: str) -> None: ...
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
class _ValidationServiceProtocol(Protocol):
|
| 33 |
-
def validate_detection(
|
| 34 |
-
self,
|
| 35 |
-
project_slug: str,
|
| 36 |
-
detection_key: str,
|
| 37 |
-
status: str,
|
| 38 |
-
validator: str,
|
| 39 |
-
notes: str = "",
|
| 40 |
-
corrected_species: str | None = None,
|
| 41 |
-
expected_version: int | None = None,
|
| 42 |
-
) -> object: ...
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
class _ValidationReadRepositoryProtocol(Protocol):
|
| 46 |
-
def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]: ...
|
| 47 |
-
|
| 48 |
-
def list_events(self, project_slug: str) -> list[dict[str, object]]: ...
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
class _QueueServiceProtocol(Protocol):
|
| 52 |
-
def get_page(
|
| 53 |
-
self,
|
| 54 |
-
project_slug: str,
|
| 55 |
-
page: int,
|
| 56 |
-
page_size: int,
|
| 57 |
-
scientific_name: str | None = None,
|
| 58 |
-
min_confidence: float | None = None,
|
| 59 |
-
max_confidence: float | None = None,
|
| 60 |
-
) -> object: ...
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
def _seed_service() -> DetectionQueueService:
|
| 64 |
-
repo = InMemoryDetectionRepository()
|
| 65 |
-
demo_items = [
|
| 66 |
-
Detection(
|
| 67 |
-
detection_key="0000000000001001",
|
| 68 |
-
audio_id="audio_1001",
|
| 69 |
-
scientific_name="Cyanocorax cyanopogon",
|
| 70 |
-
confidence=0.93,
|
| 71 |
-
start_time=1.2,
|
| 72 |
-
end_time=2.5,
|
| 73 |
-
),
|
| 74 |
-
Detection(
|
| 75 |
-
detection_key="0000000000001002",
|
| 76 |
-
audio_id="audio_1002",
|
| 77 |
-
scientific_name="Ramphastos toco",
|
| 78 |
-
confidence=0.88,
|
| 79 |
-
start_time=0.8,
|
| 80 |
-
end_time=2.1,
|
| 81 |
-
),
|
| 82 |
-
Detection(
|
| 83 |
-
detection_key="0000000000001003",
|
| 84 |
-
audio_id="audio_1003",
|
| 85 |
-
scientific_name="Cyanocorax cyanopogon",
|
| 86 |
-
confidence=0.72,
|
| 87 |
-
start_time=3.1,
|
| 88 |
-
end_time=4.0,
|
| 89 |
-
),
|
| 90 |
-
Detection(
|
| 91 |
-
detection_key="0000000000001004",
|
| 92 |
-
audio_id="audio_1004",
|
| 93 |
-
scientific_name="Psarocolius decumanus",
|
| 94 |
-
confidence=0.67,
|
| 95 |
-
start_time=5.0,
|
| 96 |
-
end_time=6.3,
|
| 97 |
-
),
|
| 98 |
-
]
|
| 99 |
-
repo.seed("demo-project", demo_items)
|
| 100 |
-
return DetectionQueueService(repo)
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
def _page_to_table(
|
| 104 |
-
service: _QueueServiceProtocol,
|
| 105 |
-
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 106 |
-
project_slug: str,
|
| 107 |
-
page: int,
|
| 108 |
-
scientific_name: str,
|
| 109 |
-
min_confidence: float,
|
| 110 |
-
validator_filter: str = "",
|
| 111 |
-
status_filter: str = "all",
|
| 112 |
-
updated_after: object = None,
|
| 113 |
-
conflict_detection_key: str = "",
|
| 114 |
-
show_conflicts_only: bool = False,
|
| 115 |
-
):
|
| 116 |
-
filter_name = scientific_name.strip() if scientific_name.strip() else None
|
| 117 |
-
page_obj = service.get_page(
|
| 118 |
-
project_slug="demo-project",
|
| 119 |
-
page=page,
|
| 120 |
-
page_size=2,
|
| 121 |
-
scientific_name=filter_name,
|
| 122 |
-
min_confidence=min_confidence,
|
| 123 |
-
)
|
| 124 |
-
|
| 125 |
-
snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
|
| 126 |
-
|
| 127 |
-
normalized_status_filter = status_filter.strip().lower() if status_filter else "all"
|
| 128 |
-
normalized_validator_filter = validator_filter.strip().lower()
|
| 129 |
-
updated_after_date: date | None = None
|
| 130 |
-
if updated_after is not None:
|
| 131 |
-
if isinstance(updated_after, datetime):
|
| 132 |
-
updated_after_date = updated_after.date()
|
| 133 |
-
elif isinstance(updated_after, date):
|
| 134 |
-
updated_after_date = updated_after
|
| 135 |
-
elif isinstance(updated_after, (int, float)):
|
| 136 |
-
updated_after_date = datetime.fromtimestamp(float(updated_after)).date()
|
| 137 |
-
else:
|
| 138 |
-
updated_after_text = str(updated_after).strip()
|
| 139 |
-
if updated_after_text:
|
| 140 |
-
try:
|
| 141 |
-
updated_after_date = datetime.strptime(updated_after_text, "%Y-%m-%d").date()
|
| 142 |
-
except ValueError:
|
| 143 |
-
try:
|
| 144 |
-
updated_after_date = datetime.fromisoformat(updated_after_text.replace("Z", "+00:00")).date()
|
| 145 |
-
except ValueError:
|
| 146 |
-
updated_after_date = None
|
| 147 |
-
|
| 148 |
-
rows = [
|
| 149 |
-
[
|
| 150 |
-
item.detection_key,
|
| 151 |
-
item.audio_id,
|
| 152 |
-
item.scientific_name,
|
| 153 |
-
round(item.confidence, 3),
|
| 154 |
-
item.start_time,
|
| 155 |
-
item.end_time,
|
| 156 |
-
str(snapshot.get(item.detection_key, {}).get("status", "pending")),
|
| 157 |
-
int(snapshot.get(item.detection_key, {}).get("version", 0)),
|
| 158 |
-
"CONFLICT" if conflict_detection_key and item.detection_key == conflict_detection_key else "",
|
| 159 |
-
"HIGH" if conflict_detection_key and item.detection_key == conflict_detection_key else "",
|
| 160 |
-
]
|
| 161 |
-
for item in page_obj.items
|
| 162 |
-
]
|
| 163 |
-
|
| 164 |
-
if normalized_validator_filter:
|
| 165 |
-
rows = [
|
| 166 |
-
row
|
| 167 |
-
for row in rows
|
| 168 |
-
if normalized_validator_filter in str(snapshot.get(str(row[0]), {}).get("validator", "")).strip().lower()
|
| 169 |
-
]
|
| 170 |
-
|
| 171 |
-
if normalized_status_filter and normalized_status_filter != "all":
|
| 172 |
-
rows = [row for row in rows if str(row[6]).strip().lower() == normalized_status_filter]
|
| 173 |
-
|
| 174 |
-
if updated_after_date:
|
| 175 |
-
filtered_rows: list[list[object]] = []
|
| 176 |
-
for row in rows:
|
| 177 |
-
snapshot_item = snapshot.get(str(row[0]), {})
|
| 178 |
-
updated_at_value = str(snapshot_item.get("updated_at", "")).strip()
|
| 179 |
-
if not updated_at_value:
|
| 180 |
-
continue
|
| 181 |
-
try:
|
| 182 |
-
item_date = datetime.fromisoformat(updated_at_value.replace("Z", "+00:00")).date()
|
| 183 |
-
if item_date >= updated_after_date:
|
| 184 |
-
filtered_rows.append(row)
|
| 185 |
-
except ValueError:
|
| 186 |
-
continue
|
| 187 |
-
rows = filtered_rows
|
| 188 |
-
|
| 189 |
-
if show_conflicts_only:
|
| 190 |
-
rows = [row for row in rows if str(row[8]) == "CONFLICT"]
|
| 191 |
-
|
| 192 |
-
status = f"Pagina {page_obj.page}/{page_obj.total_pages} | Total base: {page_obj.total_items} | Exibidos: {len(rows)}"
|
| 193 |
-
if show_conflicts_only:
|
| 194 |
-
status = f"{status} | Apenas conflitos: {len(rows)} item(ns)"
|
| 195 |
-
return rows, status, page_obj.page
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
def _build_validation_report(snapshot_reader: _ValidationReadRepositoryProtocol, project_slug: str) -> str:
|
| 199 |
-
snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
|
| 200 |
-
events = snapshot_reader.list_events(project_slug=project_slug)
|
| 201 |
-
|
| 202 |
-
counts: dict[str, int] = {}
|
| 203 |
-
for payload in snapshot.values():
|
| 204 |
-
status_value = str(payload.get("status", "unknown"))
|
| 205 |
-
counts[status_value] = counts.get(status_value, 0) + 1
|
| 206 |
-
|
| 207 |
-
parts = [
|
| 208 |
-
f"Projeto: {project_slug}",
|
| 209 |
-
f"Eventos append-only: {len(events)}",
|
| 210 |
-
f"Deteccoes com estado atual: {len(snapshot)}",
|
| 211 |
-
]
|
| 212 |
-
if counts:
|
| 213 |
-
summary = ", ".join(f"{key}={value}" for key, value in sorted(counts.items()))
|
| 214 |
-
parts.append(f"Status atual: {summary}")
|
| 215 |
-
else:
|
| 216 |
-
parts.append("Status atual: sem validacoes")
|
| 217 |
-
return " | ".join(parts)
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
def _extract_audio_id(rows: object, selected_index: int) -> str:
|
| 221 |
-
normalized_rows: list[list[object]]
|
| 222 |
-
|
| 223 |
-
if hasattr(rows, "values"):
|
| 224 |
-
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 225 |
-
else:
|
| 226 |
-
normalized_rows = [list(item) for item in rows] if rows else []
|
| 227 |
-
|
| 228 |
-
if not normalized_rows:
|
| 229 |
-
raise ValueError("Nenhuma deteccao carregada na tabela")
|
| 230 |
-
if selected_index < 0 or selected_index >= len(normalized_rows):
|
| 231 |
-
raise ValueError("Selecione uma deteccao valida na tabela")
|
| 232 |
-
|
| 233 |
-
value = normalized_rows[selected_index][1]
|
| 234 |
-
audio_id = str(value).strip()
|
| 235 |
-
if not audio_id:
|
| 236 |
-
raise ValueError("audio_id invalido na deteccao selecionada")
|
| 237 |
-
return audio_id
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
def _extract_detection_key(rows: object, selected_index: int) -> str:
|
| 241 |
-
normalized_rows: list[list[object]]
|
| 242 |
-
|
| 243 |
-
if hasattr(rows, "values"):
|
| 244 |
-
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 245 |
-
else:
|
| 246 |
-
normalized_rows = [list(item) for item in rows] if rows else []
|
| 247 |
-
|
| 248 |
-
if not normalized_rows:
|
| 249 |
-
raise ValueError("Nenhuma deteccao carregada na tabela")
|
| 250 |
-
if selected_index < 0 or selected_index >= len(normalized_rows):
|
| 251 |
-
raise ValueError("Selecione uma deteccao valida na tabela")
|
| 252 |
-
|
| 253 |
-
value = normalized_rows[selected_index][0]
|
| 254 |
-
detection_key = str(value).strip()
|
| 255 |
-
if not detection_key:
|
| 256 |
-
raise ValueError("detection_key invalido na deteccao selecionada")
|
| 257 |
-
return detection_key
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
def _find_detection_row_index(rows: object, detection_key: str) -> int:
|
| 261 |
-
normalized_rows: list[list[object]]
|
| 262 |
-
|
| 263 |
-
if hasattr(rows, "values"):
|
| 264 |
-
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 265 |
-
else:
|
| 266 |
-
normalized_rows = [list(item) for item in rows] if rows else []
|
| 267 |
-
|
| 268 |
-
for index, row in enumerate(normalized_rows):
|
| 269 |
-
if str(row[0]).strip() == detection_key:
|
| 270 |
-
return index
|
| 271 |
-
return 0
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
def _extract_expected_version(rows: object, selected_index: int) -> int:
|
| 275 |
-
normalized_rows: list[list[object]]
|
| 276 |
-
|
| 277 |
-
if hasattr(rows, "values"):
|
| 278 |
-
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 279 |
-
else:
|
| 280 |
-
normalized_rows = [list(item) for item in rows] if rows else []
|
| 281 |
-
|
| 282 |
-
if not normalized_rows:
|
| 283 |
-
raise ValueError("Nenhuma deteccao carregada na tabela")
|
| 284 |
-
if selected_index < 0 or selected_index >= len(normalized_rows):
|
| 285 |
-
raise ValueError("Selecione uma deteccao valida na tabela")
|
| 286 |
-
|
| 287 |
-
value = normalized_rows[selected_index][7]
|
| 288 |
-
return int(value)
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
def _fetch_selected_audio(
|
| 292 |
-
audio_service: _AudioServiceProtocol,
|
| 293 |
-
dataset_repo: str,
|
| 294 |
-
rows: object,
|
| 295 |
-
selected_index: int,
|
| 296 |
-
previous_cache_key: str,
|
| 297 |
-
) -> tuple[str | None, str, str]:
|
| 298 |
-
repo = dataset_repo.strip()
|
| 299 |
-
if not repo:
|
| 300 |
-
return None, "", "Informe dataset repo no formato owner/repo"
|
| 301 |
-
|
| 302 |
-
try:
|
| 303 |
-
audio_id = _extract_audio_id(rows=rows, selected_index=selected_index)
|
| 304 |
-
result = audio_service.fetch(dataset_repo=repo, audio_id=audio_id)
|
| 305 |
-
status = f"Audio carregado ({result.source}) para audio_id={audio_id}"
|
| 306 |
-
return result.local_path, result.cache_key, status
|
| 307 |
-
except Exception as exc:
|
| 308 |
-
if previous_cache_key:
|
| 309 |
-
return None, previous_cache_key, f"Falha ao carregar audio: {exc}"
|
| 310 |
-
return None, "", f"Falha ao carregar audio: {exc}"
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
def _cleanup_selected_audio(audio_service: _AudioServiceProtocol, cache_key: str) -> tuple[str, str | None]:
|
| 314 |
-
if not cache_key:
|
| 315 |
-
return "Nenhum audio em cache para limpar", None
|
| 316 |
-
|
| 317 |
-
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 318 |
-
return "Cache de audio limpo apos validacao", None
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
def _save_selected_validation(
|
| 322 |
-
validation_service: _ValidationServiceProtocol,
|
| 323 |
-
audio_service: _AudioServiceProtocol,
|
| 324 |
-
project_slug: str,
|
| 325 |
-
rows: object,
|
| 326 |
-
selected_index: int,
|
| 327 |
-
status_value: str,
|
| 328 |
-
validator: str,
|
| 329 |
-
notes: str,
|
| 330 |
-
cache_key: str,
|
| 331 |
-
) -> tuple[str, str, str | None]:
|
| 332 |
-
validator_name = validator.strip()
|
| 333 |
-
if not validator_name:
|
| 334 |
-
return "Informe o nome do validador", cache_key, None
|
| 335 |
-
|
| 336 |
-
try:
|
| 337 |
-
detection_key = _extract_detection_key(rows=rows, selected_index=selected_index)
|
| 338 |
-
expected_version = _extract_expected_version(rows=rows, selected_index=selected_index)
|
| 339 |
-
_ = validation_service.validate_detection(
|
| 340 |
-
project_slug=project_slug,
|
| 341 |
-
detection_key=detection_key,
|
| 342 |
-
status=status_value,
|
| 343 |
-
validator=validator_name,
|
| 344 |
-
notes=notes.strip(),
|
| 345 |
-
expected_version=expected_version,
|
| 346 |
-
)
|
| 347 |
-
if cache_key:
|
| 348 |
-
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 349 |
-
return f"Validacao salva: {detection_key} -> {status_value}", "", None
|
| 350 |
-
except OptimisticLockError as exc:
|
| 351 |
-
return (
|
| 352 |
-
"Conflito de concorrencia: esta deteccao foi atualizada por outro validador "
|
| 353 |
-
f"(detection_key={exc.detection_key}, versao atual={exc.current_version}, esperada={exc.expected_version}). "
|
| 354 |
-
"Recarregue a tabela.",
|
| 355 |
-
cache_key,
|
| 356 |
-
None,
|
| 357 |
-
)
|
| 358 |
-
except Exception as exc:
|
| 359 |
-
return f"Falha ao salvar validacao: {exc}", cache_key, None
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
def _save_selected_validation_with_refresh(
|
| 363 |
-
validation_service: _ValidationServiceProtocol,
|
| 364 |
-
audio_service: _AudioServiceProtocol,
|
| 365 |
-
queue_service: _QueueServiceProtocol,
|
| 366 |
-
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 367 |
-
project_slug: str,
|
| 368 |
-
rows: object,
|
| 369 |
-
selected_index: int,
|
| 370 |
-
status_value: str,
|
| 371 |
-
validator: str,
|
| 372 |
-
notes: str,
|
| 373 |
-
cache_key: str,
|
| 374 |
-
page: int,
|
| 375 |
-
scientific_name: str,
|
| 376 |
-
min_confidence: float,
|
| 377 |
-
validator_filter: str,
|
| 378 |
-
status_filter: str,
|
| 379 |
-
updated_after: object,
|
| 380 |
-
show_conflicts_only: bool,
|
| 381 |
-
) -> tuple[str, str, str | None, list[list[object]], int, int, str, str]:
|
| 382 |
-
selected_key = ""
|
| 383 |
-
try:
|
| 384 |
-
selected_key = _extract_detection_key(rows=rows, selected_index=selected_index)
|
| 385 |
-
except Exception:
|
| 386 |
-
selected_key = ""
|
| 387 |
-
|
| 388 |
-
save_status, updated_cache_key, audio_path = _save_selected_validation(
|
| 389 |
-
validation_service=validation_service,
|
| 390 |
-
audio_service=audio_service,
|
| 391 |
-
project_slug=project_slug,
|
| 392 |
-
rows=rows,
|
| 393 |
-
selected_index=selected_index,
|
| 394 |
-
status_value=status_value,
|
| 395 |
-
validator=validator,
|
| 396 |
-
notes=notes,
|
| 397 |
-
cache_key=cache_key,
|
| 398 |
-
)
|
| 399 |
-
|
| 400 |
-
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 401 |
-
service=queue_service,
|
| 402 |
-
snapshot_reader=snapshot_reader,
|
| 403 |
-
project_slug=project_slug,
|
| 404 |
-
page=page,
|
| 405 |
-
scientific_name=scientific_name,
|
| 406 |
-
min_confidence=min_confidence,
|
| 407 |
-
validator_filter=validator_filter,
|
| 408 |
-
status_filter=status_filter,
|
| 409 |
-
updated_after=updated_after,
|
| 410 |
-
show_conflicts_only=show_conflicts_only,
|
| 411 |
-
)
|
| 412 |
-
|
| 413 |
-
if selected_key:
|
| 414 |
-
refreshed_index = _find_detection_row_index(refreshed_rows, selected_key)
|
| 415 |
-
else:
|
| 416 |
-
refreshed_index = 0
|
| 417 |
-
|
| 418 |
-
if "Conflito de concorrencia" in save_status:
|
| 419 |
-
conflict_key = selected_key
|
| 420 |
-
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 421 |
-
service=queue_service,
|
| 422 |
-
snapshot_reader=snapshot_reader,
|
| 423 |
-
project_slug=project_slug,
|
| 424 |
-
page=refreshed_page,
|
| 425 |
-
scientific_name=scientific_name,
|
| 426 |
-
min_confidence=min_confidence,
|
| 427 |
-
validator_filter=validator_filter,
|
| 428 |
-
status_filter=status_filter,
|
| 429 |
-
updated_after=updated_after,
|
| 430 |
-
conflict_detection_key=conflict_key,
|
| 431 |
-
show_conflicts_only=show_conflicts_only,
|
| 432 |
-
)
|
| 433 |
-
refreshed_index = _find_detection_row_index(refreshed_rows, selected_key) if selected_key else 0
|
| 434 |
-
pending_status_value = status_value
|
| 435 |
-
status = f"{save_status} Tabela recarregada para resolver conflito."
|
| 436 |
-
else:
|
| 437 |
-
conflict_key = ""
|
| 438 |
-
pending_status_value = ""
|
| 439 |
-
status = f"{save_status} | {page_status}"
|
| 440 |
-
|
| 441 |
-
return (
|
| 442 |
-
status,
|
| 443 |
-
updated_cache_key,
|
| 444 |
-
audio_path,
|
| 445 |
-
refreshed_rows,
|
| 446 |
-
refreshed_page,
|
| 447 |
-
refreshed_index,
|
| 448 |
-
pending_status_value,
|
| 449 |
-
conflict_key,
|
| 450 |
-
)
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
def _reapply_last_conflict_validation_with_refresh(
|
| 454 |
-
validation_service: _ValidationServiceProtocol,
|
| 455 |
-
audio_service: _AudioServiceProtocol,
|
| 456 |
-
queue_service: _QueueServiceProtocol,
|
| 457 |
-
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 458 |
-
project_slug: str,
|
| 459 |
-
rows: object,
|
| 460 |
-
selected_index: int,
|
| 461 |
-
pending_status_value: str,
|
| 462 |
-
conflict_detection_key: str,
|
| 463 |
-
validator: str,
|
| 464 |
-
notes: str,
|
| 465 |
-
cache_key: str,
|
| 466 |
-
page: int,
|
| 467 |
-
scientific_name: str,
|
| 468 |
-
min_confidence: float,
|
| 469 |
-
validator_filter: str,
|
| 470 |
-
status_filter: str,
|
| 471 |
-
updated_after: object,
|
| 472 |
-
show_conflicts_only: bool,
|
| 473 |
-
) -> tuple[str, str, str | None, list[list[object]], int, int, str, str]:
|
| 474 |
-
if not pending_status_value:
|
| 475 |
-
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 476 |
-
service=queue_service,
|
| 477 |
-
snapshot_reader=snapshot_reader,
|
| 478 |
-
project_slug=project_slug,
|
| 479 |
-
page=page,
|
| 480 |
-
scientific_name=scientific_name,
|
| 481 |
-
min_confidence=min_confidence,
|
| 482 |
-
validator_filter=validator_filter,
|
| 483 |
-
status_filter=status_filter,
|
| 484 |
-
updated_after=updated_after,
|
| 485 |
-
show_conflicts_only=show_conflicts_only,
|
| 486 |
-
)
|
| 487 |
-
return (
|
| 488 |
-
f"Nenhuma validacao pendente para reaplicar | {page_status}",
|
| 489 |
-
cache_key,
|
| 490 |
-
None,
|
| 491 |
-
refreshed_rows,
|
| 492 |
-
refreshed_page,
|
| 493 |
-
selected_index,
|
| 494 |
-
"",
|
| 495 |
-
"",
|
| 496 |
-
)
|
| 497 |
-
|
| 498 |
-
target_index = _find_detection_row_index(rows, conflict_detection_key) if conflict_detection_key else selected_index
|
| 499 |
-
return _save_selected_validation_with_refresh(
|
| 500 |
-
validation_service=validation_service,
|
| 501 |
-
audio_service=audio_service,
|
| 502 |
-
queue_service=queue_service,
|
| 503 |
-
snapshot_reader=snapshot_reader,
|
| 504 |
-
project_slug=project_slug,
|
| 505 |
-
rows=rows,
|
| 506 |
-
selected_index=target_index,
|
| 507 |
-
status_value=pending_status_value,
|
| 508 |
-
validator=validator,
|
| 509 |
-
notes=notes,
|
| 510 |
-
cache_key=cache_key,
|
| 511 |
-
page=page,
|
| 512 |
-
scientific_name=scientific_name,
|
| 513 |
-
min_confidence=min_confidence,
|
| 514 |
-
validator_filter=validator_filter,
|
| 515 |
-
status_filter=status_filter,
|
| 516 |
-
updated_after=updated_after,
|
| 517 |
-
show_conflicts_only=show_conflicts_only,
|
| 518 |
-
)
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
def _batch_validate_conflicts(
|
| 522 |
-
validation_service: _ValidationServiceProtocol,
|
| 523 |
-
audio_service: _AudioServiceProtocol,
|
| 524 |
-
queue_service: _QueueServiceProtocol,
|
| 525 |
-
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 526 |
-
project_slug: str,
|
| 527 |
-
rows: object,
|
| 528 |
-
status_value: str,
|
| 529 |
-
validator: str,
|
| 530 |
-
notes: str,
|
| 531 |
-
cache_key: str,
|
| 532 |
-
page: int,
|
| 533 |
-
scientific_name: str,
|
| 534 |
-
min_confidence: float,
|
| 535 |
-
validator_filter: str,
|
| 536 |
-
status_filter: str,
|
| 537 |
-
updated_after: object,
|
| 538 |
-
) -> tuple[str, str, str | None, list[list[object]], int]:
|
| 539 |
-
"""Apply the same validation status to all visible conflicts in the table."""
|
| 540 |
-
validator_name = validator.strip()
|
| 541 |
-
if not validator_name:
|
| 542 |
-
return "Informe o nome do validador", "", None, [], page
|
| 543 |
-
|
| 544 |
-
normalized_rows: list[list[object]]
|
| 545 |
-
if hasattr(rows, "values"):
|
| 546 |
-
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 547 |
-
else:
|
| 548 |
-
normalized_rows = [list(item) for item in rows] if rows else []
|
| 549 |
-
|
| 550 |
-
if not normalized_rows:
|
| 551 |
-
return "Nenhuma deteccao com conflito para validar", "", None, [], page
|
| 552 |
-
|
| 553 |
-
conflict_rows = [row for row in normalized_rows if str(row[8]) == "CONFLICT"]
|
| 554 |
-
if not conflict_rows:
|
| 555 |
-
return "Nenhuma deteccao com conflito identificada na tabela", "", None, normalized_rows, page
|
| 556 |
-
|
| 557 |
-
success_count = 0
|
| 558 |
-
failure_count = 0
|
| 559 |
-
conflict_count = 0
|
| 560 |
-
|
| 561 |
-
for row in conflict_rows:
|
| 562 |
-
try:
|
| 563 |
-
detection_key = str(row[0]).strip()
|
| 564 |
-
expected_version = int(row[7])
|
| 565 |
-
|
| 566 |
-
_ = validation_service.validate_detection(
|
| 567 |
-
project_slug=project_slug,
|
| 568 |
-
detection_key=detection_key,
|
| 569 |
-
status=status_value,
|
| 570 |
-
validator=validator_name,
|
| 571 |
-
notes=notes.strip(),
|
| 572 |
-
expected_version=expected_version,
|
| 573 |
-
)
|
| 574 |
-
success_count += 1
|
| 575 |
-
if cache_key:
|
| 576 |
-
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 577 |
-
except OptimisticLockError:
|
| 578 |
-
conflict_count += 1
|
| 579 |
-
except Exception:
|
| 580 |
-
failure_count += 1
|
| 581 |
-
|
| 582 |
-
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 583 |
-
service=queue_service,
|
| 584 |
-
snapshot_reader=snapshot_reader,
|
| 585 |
-
project_slug=project_slug,
|
| 586 |
-
page=page,
|
| 587 |
-
scientific_name=scientific_name,
|
| 588 |
-
min_confidence=min_confidence,
|
| 589 |
-
validator_filter=validator_filter,
|
| 590 |
-
status_filter=status_filter,
|
| 591 |
-
updated_after=updated_after,
|
| 592 |
-
show_conflicts_only=False,
|
| 593 |
-
)
|
| 594 |
-
|
| 595 |
-
summary = f"Processados {len(conflict_rows)} conflitos: {success_count} sucesso, {conflict_count} novos conflitos, {failure_count} falhas"
|
| 596 |
-
status = f"{summary} | {page_status}"
|
| 597 |
-
|
| 598 |
-
return status, "", None, refreshed_rows, refreshed_page
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
def _batch_reapply_all_pending(
|
| 602 |
-
validation_service: _ValidationServiceProtocol,
|
| 603 |
-
audio_service: _AudioServiceProtocol,
|
| 604 |
-
queue_service: _QueueServiceProtocol,
|
| 605 |
-
snapshot_reader: _ValidationReadRepositoryProtocol,
|
| 606 |
-
project_slug: str,
|
| 607 |
-
rows: object,
|
| 608 |
-
pending_statuses: dict[str, str],
|
| 609 |
-
validator: str,
|
| 610 |
-
notes: str,
|
| 611 |
-
cache_key: str,
|
| 612 |
-
page: int,
|
| 613 |
-
scientific_name: str,
|
| 614 |
-
min_confidence: float,
|
| 615 |
-
validator_filter: str,
|
| 616 |
-
status_filter: str,
|
| 617 |
-
updated_after: object,
|
| 618 |
-
) -> tuple[str, str, str | None, list[list[object]], int]:
|
| 619 |
-
"""Reapply all pending validations (stored conflicts) with current version."""
|
| 620 |
-
if not pending_statuses:
|
| 621 |
-
return "Nenhuma validacao pendente para reaplicar", "", None, [], page
|
| 622 |
-
|
| 623 |
-
validator_name = validator.strip()
|
| 624 |
-
if not validator_name:
|
| 625 |
-
return "Informe o nome do validador", "", None, [], page
|
| 626 |
-
|
| 627 |
-
success_count = 0
|
| 628 |
-
conflict_count = 0
|
| 629 |
-
failure_count = 0
|
| 630 |
-
|
| 631 |
-
snapshot = snapshot_reader.load_current_snapshot(project_slug=project_slug)
|
| 632 |
-
|
| 633 |
-
for detection_key, status_value in pending_statuses.items():
|
| 634 |
-
try:
|
| 635 |
-
current_version = int(snapshot.get(detection_key, {}).get("version", 0))
|
| 636 |
-
|
| 637 |
-
_ = validation_service.validate_detection(
|
| 638 |
-
project_slug=project_slug,
|
| 639 |
-
detection_key=detection_key,
|
| 640 |
-
status=status_value,
|
| 641 |
-
validator=validator_name,
|
| 642 |
-
notes=notes.strip(),
|
| 643 |
-
expected_version=current_version,
|
| 644 |
-
)
|
| 645 |
-
success_count += 1
|
| 646 |
-
if cache_key:
|
| 647 |
-
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 648 |
-
except OptimisticLockError:
|
| 649 |
-
conflict_count += 1
|
| 650 |
-
except Exception:
|
| 651 |
-
failure_count += 1
|
| 652 |
-
|
| 653 |
-
refreshed_rows, page_status, refreshed_page = _page_to_table(
|
| 654 |
-
service=queue_service,
|
| 655 |
-
snapshot_reader=snapshot_reader,
|
| 656 |
-
project_slug=project_slug,
|
| 657 |
-
page=page,
|
| 658 |
-
scientific_name=scientific_name,
|
| 659 |
-
min_confidence=min_confidence,
|
| 660 |
-
validator_filter=validator_filter,
|
| 661 |
-
status_filter=status_filter,
|
| 662 |
-
updated_after=updated_after,
|
| 663 |
-
show_conflicts_only=False,
|
| 664 |
-
)
|
| 665 |
-
|
| 666 |
-
summary = f"Reaplicadas {len(pending_statuses)} validacoes: {success_count} sucesso, {conflict_count} novos conflitos, {failure_count} falhas"
|
| 667 |
-
status = f"{summary} | {page_status}"
|
| 668 |
-
|
| 669 |
-
return status, "", None, refreshed_rows, refreshed_page
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
def build_demo_app(project_slug: str = "demo-project") -> gr.Blocks:
|
| 673 |
-
"""Build the demo validation app for a given project.
|
| 674 |
-
|
| 675 |
-
Args:
|
| 676 |
-
project_slug: Project identifier (default: demo-project)
|
| 677 |
-
|
| 678 |
-
Returns:
|
| 679 |
-
Gradio Blocks with validation interface
|
| 680 |
-
"""
|
| 681 |
-
service = _seed_service()
|
| 682 |
-
audio_service = AudioFetchService(EphemeralCacheManager(ttl_seconds=300, max_files=128))
|
| 683 |
-
validation_base_dir = str(Path(tempfile.gettempdir()) / "birdnet-validator-validations")
|
| 684 |
-
validation_repository = AppendOnlyValidationRepository(base_dir=validation_base_dir)
|
| 685 |
-
validation_service = ValidationService(validation_repository)
|
| 686 |
-
|
| 687 |
-
with gr.Blocks(title="BirdNET Validator HF") as demo:
|
| 688 |
-
gr.Markdown("# BirdNET Validator HF")
|
| 689 |
-
gr.Markdown("Sprint 2: fila paginada + audio sob demanda com cache efemero.")
|
| 690 |
-
|
| 691 |
-
dataset_repo = gr.Textbox(label="Dataset repo", value="SEU_USUARIO/birdnet-projeto-dataset")
|
| 692 |
-
|
| 693 |
-
with gr.Row():
|
| 694 |
-
species_filter = gr.Textbox(label="Filtro especie", placeholder="Ex: Cyanocorax cyanopogon")
|
| 695 |
-
min_confidence = gr.Slider(label="Confianca minima", minimum=0.0, maximum=1.0, step=0.01, value=0.0)
|
| 696 |
-
show_conflicts_only = gr.Checkbox(label="Mostrar apenas conflitos", value=False)
|
| 697 |
-
|
| 698 |
-
with gr.Row():
|
| 699 |
-
validator_filter = gr.Textbox(label="Filtro validador", placeholder="Ex: validator-demo")
|
| 700 |
-
validation_status_filter = gr.Dropdown(
|
| 701 |
-
label="Filtro status",
|
| 702 |
-
choices=["all", "pending", "positive", "negative", "uncertain", "skip"],
|
| 703 |
-
value="all",
|
| 704 |
-
)
|
| 705 |
-
updated_after_filter = gr.DateTime(label="Atualizado a partir de", include_time=False, type="string")
|
| 706 |
-
|
| 707 |
-
with gr.Row():
|
| 708 |
-
prev_btn = gr.Button("Pagina anterior")
|
| 709 |
-
next_btn = gr.Button("Proxima pagina")
|
| 710 |
-
refresh_btn = gr.Button("Aplicar filtros")
|
| 711 |
-
|
| 712 |
-
page_state = gr.State(value=1)
|
| 713 |
-
table = gr.Dataframe(
|
| 714 |
-
headers=[
|
| 715 |
-
"detection_key",
|
| 716 |
-
"audio_id",
|
| 717 |
-
"scientific_name",
|
| 718 |
-
"confidence",
|
| 719 |
-
"start_time",
|
| 720 |
-
"end_time",
|
| 721 |
-
"validation_status",
|
| 722 |
-
"version",
|
| 723 |
-
"conflict_flag",
|
| 724 |
-
"conflict_severity",
|
| 725 |
-
],
|
| 726 |
-
label="Deteccoes",
|
| 727 |
-
interactive=False,
|
| 728 |
-
)
|
| 729 |
-
selected_index = gr.Number(label="Linha selecionada", value=0, precision=0)
|
| 730 |
-
|
| 731 |
-
with gr.Row():
|
| 732 |
-
load_audio_btn = gr.Button("Carregar audio selecionado")
|
| 733 |
-
clear_audio_btn = gr.Button("Limpar cache apos validacao")
|
| 734 |
-
|
| 735 |
-
with gr.Row():
|
| 736 |
-
validator_name = gr.Textbox(label="Validador", value="validator-demo")
|
| 737 |
-
validation_notes = gr.Textbox(label="Notas", placeholder="Opcional")
|
| 738 |
-
|
| 739 |
-
with gr.Row():
|
| 740 |
-
approve_btn = gr.Button("Validar positivo")
|
| 741 |
-
reject_btn = gr.Button("Validar negativo")
|
| 742 |
-
uncertain_btn = gr.Button("Indeterminado")
|
| 743 |
-
skip_btn = gr.Button("Pular")
|
| 744 |
-
reapply_btn = gr.Button("Reaplicar validacao apos conflito")
|
| 745 |
-
|
| 746 |
-
with gr.Row():
|
| 747 |
-
batch_approve_conflicts_btn = gr.Button("Aprovar todos os conflitos")
|
| 748 |
-
batch_reject_conflicts_btn = gr.Button("Rejeitar todos os conflitos")
|
| 749 |
-
|
| 750 |
-
report_btn = gr.Button("Gerar relatorio de validacoes")
|
| 751 |
-
|
| 752 |
-
audio_player = gr.Audio(label="Audio sob demanda", type="filepath")
|
| 753 |
-
cache_key_state = gr.State(value="")
|
| 754 |
-
pending_status_state = gr.State(value="")
|
| 755 |
-
conflict_detection_key_state = gr.State(value="")
|
| 756 |
-
status = gr.Textbox(label="Status", interactive=False)
|
| 757 |
-
report_box = gr.Textbox(label="Relatorio", interactive=False)
|
| 758 |
-
|
| 759 |
-
# Keyboard shortcuts: 1=positive, 2=negative, 3=uncertain, 4=skip, R=reapply
|
| 760 |
-
keyboard_shortcuts_info = gr.HTML(
|
| 761 |
-
value="<div style='font-size: 12px; color: #666; padding: 8px; background-color: #f5f5f5; border-radius: 4px; margin-bottom: 10px;'>"
|
| 762 |
-
"<strong>Atalhos de teclado:</strong> 1=Positivo | 2=Negativo | 3=Indeterminado | 4=Pular | R=Reaplicar"
|
| 763 |
-
"</div>"
|
| 764 |
-
"<script>"
|
| 765 |
-
"document.addEventListener('keydown', function(event) {"
|
| 766 |
-
" if (event.target.tagName === 'INPUT' || event.target.tagName === 'TEXTAREA') return;"
|
| 767 |
-
" const key = event.key.toLowerCase();"
|
| 768 |
-
" let buttonText = null;"
|
| 769 |
-
" if (key === '1') buttonText = 'Validar positivo';"
|
| 770 |
-
" else if (key === '2') buttonText = 'Validar negativo';"
|
| 771 |
-
" else if (key === '3') buttonText = 'Indeterminado';"
|
| 772 |
-
" else if (key === '4') buttonText = 'Pular';"
|
| 773 |
-
" else if (key === 'r') buttonText = 'Reaplicar validacao apos conflito';"
|
| 774 |
-
" if (buttonText) {"
|
| 775 |
-
" event.preventDefault();"
|
| 776 |
-
" const buttons = document.querySelectorAll('button');"
|
| 777 |
-
" for (const btn of buttons) {"
|
| 778 |
-
" if (btn.textContent.includes(buttonText)) {"
|
| 779 |
-
" btn.click();"
|
| 780 |
-
" break;"
|
| 781 |
-
" }"
|
| 782 |
-
" }"
|
| 783 |
-
" }"
|
| 784 |
-
"});"
|
| 785 |
-
"</script>"
|
| 786 |
-
)
|
| 787 |
-
|
| 788 |
-
def refresh(
|
| 789 |
-
page: int,
|
| 790 |
-
species: str,
|
| 791 |
-
confidence: float,
|
| 792 |
-
validator_filter_value: str,
|
| 793 |
-
status_filter_value: str,
|
| 794 |
-
updated_after_value: object,
|
| 795 |
-
only_conflicts: bool,
|
| 796 |
-
):
|
| 797 |
-
return _page_to_table(
|
| 798 |
-
service=service,
|
| 799 |
-
snapshot_reader=validation_repository,
|
| 800 |
-
project_slug=project_slug,
|
| 801 |
-
page=page,
|
| 802 |
-
scientific_name=species,
|
| 803 |
-
min_confidence=confidence,
|
| 804 |
-
validator_filter=validator_filter_value,
|
| 805 |
-
status_filter=status_filter_value,
|
| 806 |
-
updated_after=updated_after_value,
|
| 807 |
-
show_conflicts_only=only_conflicts,
|
| 808 |
-
)
|
| 809 |
-
|
| 810 |
-
def go_next(
|
| 811 |
-
page: int,
|
| 812 |
-
species: str,
|
| 813 |
-
confidence: float,
|
| 814 |
-
validator_filter_value: str,
|
| 815 |
-
status_filter_value: str,
|
| 816 |
-
updated_after_value: object,
|
| 817 |
-
only_conflicts: bool,
|
| 818 |
-
):
|
| 819 |
-
return refresh(
|
| 820 |
-
page + 1,
|
| 821 |
-
species,
|
| 822 |
-
confidence,
|
| 823 |
-
validator_filter_value,
|
| 824 |
-
status_filter_value,
|
| 825 |
-
updated_after_value,
|
| 826 |
-
only_conflicts,
|
| 827 |
-
)
|
| 828 |
-
|
| 829 |
-
def go_prev(
|
| 830 |
-
page: int,
|
| 831 |
-
species: str,
|
| 832 |
-
confidence: float,
|
| 833 |
-
validator_filter_value: str,
|
| 834 |
-
status_filter_value: str,
|
| 835 |
-
updated_after_value: object,
|
| 836 |
-
only_conflicts: bool,
|
| 837 |
-
):
|
| 838 |
-
return refresh(
|
| 839 |
-
max(1, page - 1),
|
| 840 |
-
species,
|
| 841 |
-
confidence,
|
| 842 |
-
validator_filter_value,
|
| 843 |
-
status_filter_value,
|
| 844 |
-
updated_after_value,
|
| 845 |
-
only_conflicts,
|
| 846 |
-
)
|
| 847 |
-
|
| 848 |
-
def on_select(evt: gr.SelectData):
|
| 849 |
-
if isinstance(evt.index, tuple):
|
| 850 |
-
return int(evt.index[0])
|
| 851 |
-
if isinstance(evt.index, int):
|
| 852 |
-
return int(evt.index)
|
| 853 |
-
return 0
|
| 854 |
-
|
| 855 |
-
demo.load(
|
| 856 |
-
fn=refresh,
|
| 857 |
-
inputs=[
|
| 858 |
-
page_state,
|
| 859 |
-
species_filter,
|
| 860 |
-
min_confidence,
|
| 861 |
-
validator_filter,
|
| 862 |
-
validation_status_filter,
|
| 863 |
-
updated_after_filter,
|
| 864 |
-
show_conflicts_only,
|
| 865 |
-
],
|
| 866 |
-
outputs=[table, status, page_state],
|
| 867 |
-
)
|
| 868 |
-
refresh_btn.click(
|
| 869 |
-
fn=lambda species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: refresh(
|
| 870 |
-
1,
|
| 871 |
-
species,
|
| 872 |
-
confidence,
|
| 873 |
-
validator_filter_value,
|
| 874 |
-
status_filter_value,
|
| 875 |
-
updated_after_value,
|
| 876 |
-
only_conflicts,
|
| 877 |
-
),
|
| 878 |
-
inputs=[
|
| 879 |
-
species_filter,
|
| 880 |
-
min_confidence,
|
| 881 |
-
validator_filter,
|
| 882 |
-
validation_status_filter,
|
| 883 |
-
updated_after_filter,
|
| 884 |
-
show_conflicts_only,
|
| 885 |
-
],
|
| 886 |
-
outputs=[table, status, page_state],
|
| 887 |
-
)
|
| 888 |
-
next_btn.click(
|
| 889 |
-
fn=go_next,
|
| 890 |
-
inputs=[
|
| 891 |
-
page_state,
|
| 892 |
-
species_filter,
|
| 893 |
-
min_confidence,
|
| 894 |
-
validator_filter,
|
| 895 |
-
validation_status_filter,
|
| 896 |
-
updated_after_filter,
|
| 897 |
-
show_conflicts_only,
|
| 898 |
-
],
|
| 899 |
-
outputs=[table, status, page_state],
|
| 900 |
-
)
|
| 901 |
-
prev_btn.click(
|
| 902 |
-
fn=go_prev,
|
| 903 |
-
inputs=[
|
| 904 |
-
page_state,
|
| 905 |
-
species_filter,
|
| 906 |
-
min_confidence,
|
| 907 |
-
validator_filter,
|
| 908 |
-
validation_status_filter,
|
| 909 |
-
updated_after_filter,
|
| 910 |
-
show_conflicts_only,
|
| 911 |
-
],
|
| 912 |
-
outputs=[table, status, page_state],
|
| 913 |
-
)
|
| 914 |
-
table.select(fn=on_select, inputs=None, outputs=[selected_index])
|
| 915 |
-
load_audio_btn.click(
|
| 916 |
-
fn=lambda repo, rows, idx, cache_key: _fetch_selected_audio(
|
| 917 |
-
audio_service=audio_service,
|
| 918 |
-
dataset_repo=repo,
|
| 919 |
-
rows=rows,
|
| 920 |
-
selected_index=int(idx),
|
| 921 |
-
previous_cache_key=cache_key,
|
| 922 |
-
),
|
| 923 |
-
inputs=[dataset_repo, table, selected_index, cache_key_state],
|
| 924 |
-
outputs=[audio_player, cache_key_state, status],
|
| 925 |
-
)
|
| 926 |
-
clear_audio_btn.click(
|
| 927 |
-
fn=lambda cache_key: _cleanup_selected_audio(audio_service=audio_service, cache_key=cache_key),
|
| 928 |
-
inputs=[cache_key_state],
|
| 929 |
-
outputs=[status, audio_player],
|
| 930 |
-
)
|
| 931 |
-
approve_btn.click(
|
| 932 |
-
fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
|
| 933 |
-
validation_service=validation_service,
|
| 934 |
-
audio_service=audio_service,
|
| 935 |
-
queue_service=service,
|
| 936 |
-
snapshot_reader=validation_repository,
|
| 937 |
-
project_slug=project_slug,
|
| 938 |
-
rows=rows,
|
| 939 |
-
selected_index=int(idx),
|
| 940 |
-
status_value="positive",
|
| 941 |
-
validator=name,
|
| 942 |
-
notes=notes,
|
| 943 |
-
cache_key=cache_key,
|
| 944 |
-
page=int(page),
|
| 945 |
-
scientific_name=species,
|
| 946 |
-
min_confidence=float(confidence),
|
| 947 |
-
validator_filter=validator_filter_value,
|
| 948 |
-
status_filter=status_filter_value,
|
| 949 |
-
updated_after=updated_after_value,
|
| 950 |
-
show_conflicts_only=bool(only_conflicts),
|
| 951 |
-
),
|
| 952 |
-
inputs=[
|
| 953 |
-
table,
|
| 954 |
-
selected_index,
|
| 955 |
-
validator_name,
|
| 956 |
-
validation_notes,
|
| 957 |
-
cache_key_state,
|
| 958 |
-
page_state,
|
| 959 |
-
species_filter,
|
| 960 |
-
min_confidence,
|
| 961 |
-
validator_filter,
|
| 962 |
-
validation_status_filter,
|
| 963 |
-
updated_after_filter,
|
| 964 |
-
show_conflicts_only,
|
| 965 |
-
],
|
| 966 |
-
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 967 |
-
)
|
| 968 |
-
reject_btn.click(
|
| 969 |
-
fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
|
| 970 |
-
validation_service=validation_service,
|
| 971 |
-
audio_service=audio_service,
|
| 972 |
-
queue_service=service,
|
| 973 |
-
snapshot_reader=validation_repository,
|
| 974 |
-
project_slug=project_slug,
|
| 975 |
-
rows=rows,
|
| 976 |
-
selected_index=int(idx),
|
| 977 |
-
status_value="negative",
|
| 978 |
-
validator=name,
|
| 979 |
-
notes=notes,
|
| 980 |
-
cache_key=cache_key,
|
| 981 |
-
page=int(page),
|
| 982 |
-
scientific_name=species,
|
| 983 |
-
min_confidence=float(confidence),
|
| 984 |
-
validator_filter=validator_filter_value,
|
| 985 |
-
status_filter=status_filter_value,
|
| 986 |
-
updated_after=updated_after_value,
|
| 987 |
-
show_conflicts_only=bool(only_conflicts),
|
| 988 |
-
),
|
| 989 |
-
inputs=[
|
| 990 |
-
table,
|
| 991 |
-
selected_index,
|
| 992 |
-
validator_name,
|
| 993 |
-
validation_notes,
|
| 994 |
-
cache_key_state,
|
| 995 |
-
page_state,
|
| 996 |
-
species_filter,
|
| 997 |
-
min_confidence,
|
| 998 |
-
validator_filter,
|
| 999 |
-
validation_status_filter,
|
| 1000 |
-
updated_after_filter,
|
| 1001 |
-
show_conflicts_only,
|
| 1002 |
-
],
|
| 1003 |
-
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 1004 |
-
)
|
| 1005 |
-
uncertain_btn.click(
|
| 1006 |
-
fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
|
| 1007 |
-
validation_service=validation_service,
|
| 1008 |
-
audio_service=audio_service,
|
| 1009 |
-
queue_service=service,
|
| 1010 |
-
snapshot_reader=validation_repository,
|
| 1011 |
-
project_slug=project_slug,
|
| 1012 |
-
rows=rows,
|
| 1013 |
-
selected_index=int(idx),
|
| 1014 |
-
status_value="uncertain",
|
| 1015 |
-
validator=name,
|
| 1016 |
-
notes=notes,
|
| 1017 |
-
cache_key=cache_key,
|
| 1018 |
-
page=int(page),
|
| 1019 |
-
scientific_name=species,
|
| 1020 |
-
min_confidence=float(confidence),
|
| 1021 |
-
validator_filter=validator_filter_value,
|
| 1022 |
-
status_filter=status_filter_value,
|
| 1023 |
-
updated_after=updated_after_value,
|
| 1024 |
-
show_conflicts_only=bool(only_conflicts),
|
| 1025 |
-
),
|
| 1026 |
-
inputs=[
|
| 1027 |
-
table,
|
| 1028 |
-
selected_index,
|
| 1029 |
-
validator_name,
|
| 1030 |
-
validation_notes,
|
| 1031 |
-
cache_key_state,
|
| 1032 |
-
page_state,
|
| 1033 |
-
species_filter,
|
| 1034 |
-
min_confidence,
|
| 1035 |
-
validator_filter,
|
| 1036 |
-
validation_status_filter,
|
| 1037 |
-
updated_after_filter,
|
| 1038 |
-
show_conflicts_only,
|
| 1039 |
-
],
|
| 1040 |
-
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 1041 |
-
)
|
| 1042 |
-
skip_btn.click(
|
| 1043 |
-
fn=lambda rows, idx, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _save_selected_validation_with_refresh(
|
| 1044 |
-
validation_service=validation_service,
|
| 1045 |
-
audio_service=audio_service,
|
| 1046 |
-
queue_service=service,
|
| 1047 |
-
snapshot_reader=validation_repository,
|
| 1048 |
-
project_slug=project_slug,
|
| 1049 |
-
rows=rows,
|
| 1050 |
-
selected_index=int(idx),
|
| 1051 |
-
status_value="skip",
|
| 1052 |
-
validator=name,
|
| 1053 |
-
notes=notes,
|
| 1054 |
-
cache_key=cache_key,
|
| 1055 |
-
page=int(page),
|
| 1056 |
-
scientific_name=species,
|
| 1057 |
-
min_confidence=float(confidence),
|
| 1058 |
-
validator_filter=validator_filter_value,
|
| 1059 |
-
status_filter=status_filter_value,
|
| 1060 |
-
updated_after=updated_after_value,
|
| 1061 |
-
show_conflicts_only=bool(only_conflicts),
|
| 1062 |
-
),
|
| 1063 |
-
inputs=[
|
| 1064 |
-
table,
|
| 1065 |
-
selected_index,
|
| 1066 |
-
validator_name,
|
| 1067 |
-
validation_notes,
|
| 1068 |
-
cache_key_state,
|
| 1069 |
-
page_state,
|
| 1070 |
-
species_filter,
|
| 1071 |
-
min_confidence,
|
| 1072 |
-
validator_filter,
|
| 1073 |
-
validation_status_filter,
|
| 1074 |
-
updated_after_filter,
|
| 1075 |
-
show_conflicts_only,
|
| 1076 |
-
],
|
| 1077 |
-
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 1078 |
-
)
|
| 1079 |
-
reapply_btn.click(
|
| 1080 |
-
fn=lambda rows, idx, pending_status, conflict_key, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value, only_conflicts: _reapply_last_conflict_validation_with_refresh(
|
| 1081 |
-
validation_service=validation_service,
|
| 1082 |
-
audio_service=audio_service,
|
| 1083 |
-
queue_service=service,
|
| 1084 |
-
snapshot_reader=validation_repository,
|
| 1085 |
-
project_slug=project_slug,
|
| 1086 |
-
rows=rows,
|
| 1087 |
-
selected_index=int(idx),
|
| 1088 |
-
pending_status_value=pending_status,
|
| 1089 |
-
conflict_detection_key=conflict_key,
|
| 1090 |
-
validator=name,
|
| 1091 |
-
notes=notes,
|
| 1092 |
-
cache_key=cache_key,
|
| 1093 |
-
page=int(page),
|
| 1094 |
-
scientific_name=species,
|
| 1095 |
-
min_confidence=float(confidence),
|
| 1096 |
-
validator_filter=validator_filter_value,
|
| 1097 |
-
status_filter=status_filter_value,
|
| 1098 |
-
updated_after=updated_after_value,
|
| 1099 |
-
show_conflicts_only=bool(only_conflicts),
|
| 1100 |
-
),
|
| 1101 |
-
inputs=[
|
| 1102 |
-
table,
|
| 1103 |
-
selected_index,
|
| 1104 |
-
pending_status_state,
|
| 1105 |
-
conflict_detection_key_state,
|
| 1106 |
-
validator_name,
|
| 1107 |
-
validation_notes,
|
| 1108 |
-
cache_key_state,
|
| 1109 |
-
page_state,
|
| 1110 |
-
species_filter,
|
| 1111 |
-
min_confidence,
|
| 1112 |
-
validator_filter,
|
| 1113 |
-
validation_status_filter,
|
| 1114 |
-
updated_after_filter,
|
| 1115 |
-
show_conflicts_only,
|
| 1116 |
-
],
|
| 1117 |
-
outputs=[status, cache_key_state, audio_player, table, page_state, selected_index, pending_status_state, conflict_detection_key_state],
|
| 1118 |
-
)
|
| 1119 |
-
batch_approve_conflicts_btn.click(
|
| 1120 |
-
fn=lambda rows, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value: _batch_validate_conflicts(
|
| 1121 |
-
validation_service=validation_service,
|
| 1122 |
-
audio_service=audio_service,
|
| 1123 |
-
queue_service=service,
|
| 1124 |
-
snapshot_reader=validation_repository,
|
| 1125 |
-
project_slug=project_slug,
|
| 1126 |
-
rows=rows,
|
| 1127 |
-
status_value="positive",
|
| 1128 |
-
validator=name,
|
| 1129 |
-
notes=notes,
|
| 1130 |
-
cache_key=cache_key,
|
| 1131 |
-
page=int(page),
|
| 1132 |
-
scientific_name=species,
|
| 1133 |
-
min_confidence=float(confidence),
|
| 1134 |
-
validator_filter=validator_filter_value,
|
| 1135 |
-
status_filter=status_filter_value,
|
| 1136 |
-
updated_after=updated_after_value,
|
| 1137 |
-
),
|
| 1138 |
-
inputs=[
|
| 1139 |
-
table,
|
| 1140 |
-
validator_name,
|
| 1141 |
-
validation_notes,
|
| 1142 |
-
cache_key_state,
|
| 1143 |
-
page_state,
|
| 1144 |
-
species_filter,
|
| 1145 |
-
min_confidence,
|
| 1146 |
-
validator_filter,
|
| 1147 |
-
validation_status_filter,
|
| 1148 |
-
updated_after_filter,
|
| 1149 |
-
],
|
| 1150 |
-
outputs=[status, cache_key_state, audio_player, table, page_state],
|
| 1151 |
-
)
|
| 1152 |
-
batch_reject_conflicts_btn.click(
|
| 1153 |
-
fn=lambda rows, name, notes, cache_key, page, species, confidence, validator_filter_value, status_filter_value, updated_after_value: _batch_validate_conflicts(
|
| 1154 |
-
validation_service=validation_service,
|
| 1155 |
-
audio_service=audio_service,
|
| 1156 |
-
queue_service=service,
|
| 1157 |
-
snapshot_reader=validation_repository,
|
| 1158 |
-
project_slug=project_slug,
|
| 1159 |
-
rows=rows,
|
| 1160 |
-
status_value="negative",
|
| 1161 |
-
validator=name,
|
| 1162 |
-
notes=notes,
|
| 1163 |
-
cache_key=cache_key,
|
| 1164 |
-
page=int(page),
|
| 1165 |
-
scientific_name=species,
|
| 1166 |
-
min_confidence=float(confidence),
|
| 1167 |
-
validator_filter=validator_filter_value,
|
| 1168 |
-
status_filter=status_filter_value,
|
| 1169 |
-
updated_after=updated_after_value,
|
| 1170 |
-
),
|
| 1171 |
-
inputs=[
|
| 1172 |
-
table,
|
| 1173 |
-
validator_name,
|
| 1174 |
-
validation_notes,
|
| 1175 |
-
cache_key_state,
|
| 1176 |
-
page_state,
|
| 1177 |
-
species_filter,
|
| 1178 |
-
min_confidence,
|
| 1179 |
-
validator_filter,
|
| 1180 |
-
validation_status_filter,
|
| 1181 |
-
updated_after_filter,
|
| 1182 |
-
],
|
| 1183 |
-
outputs=[status, cache_key_state, audio_player, table, page_state],
|
| 1184 |
-
)
|
| 1185 |
-
report_btn.click(
|
| 1186 |
-
fn=lambda: _build_validation_report(validation_repository, project_slug),
|
| 1187 |
-
inputs=None,
|
| 1188 |
-
outputs=[report_box],
|
| 1189 |
-
)
|
| 1190 |
-
|
| 1191 |
-
return demo
|
| 1192 |
-
|
| 1193 |
-
|
| 1194 |
-
def create_app() -> gr.Blocks:
|
| 1195 |
-
"""Build the BirdNET Validator app with multi-project auth integration.
|
| 1196 |
-
|
| 1197 |
-
Returns multi-tab interface with:
|
| 1198 |
-
- Login tab for user authentication
|
| 1199 |
-
- Project selection for authorized projects
|
| 1200 |
-
- Admin panel for project/user management (admin only)
|
| 1201 |
-
- Validation interface for selected project
|
| 1202 |
-
|
| 1203 |
-
Returns:
|
| 1204 |
-
Gradio Blocks with full auth-integrated app
|
| 1205 |
-
"""
|
| 1206 |
-
# Initialize auth service
|
| 1207 |
-
auth_service = AuthService(session_ttl_minutes=120)
|
| 1208 |
-
|
| 1209 |
-
# Setup demo users (in production: database/LDAP/OAuth)
|
| 1210 |
-
auth_service.register_user_project_access(
|
| 1211 |
-
"admin_user",
|
| 1212 |
-
{"kenya-2024": Role.admin, "nairobi-2023": Role.admin},
|
| 1213 |
-
)
|
| 1214 |
-
auth_service.register_user_project_access(
|
| 1215 |
-
"validator_demo",
|
| 1216 |
-
{"demo-project": Role.validator, "kenya-2024": Role.validator},
|
| 1217 |
-
)
|
| 1218 |
-
auth_service.register_user_project_access(
|
| 1219 |
-
"validator_other",
|
| 1220 |
-
{"nairobi-2023": Role.validator},
|
| 1221 |
-
)
|
| 1222 |
-
|
| 1223 |
-
# Initialize admin panel manager
|
| 1224 |
-
admin_manager = AdminPanelManager(auth_service)
|
| 1225 |
-
|
| 1226 |
-
# Register projects
|
| 1227 |
-
admin_manager.register_project(
|
| 1228 |
-
Project(
|
| 1229 |
-
project_slug="kenya-2024",
|
| 1230 |
-
name="Kenya Survey 2024",
|
| 1231 |
-
dataset_repo_id="birdnet/kenya-2024-dataset",
|
| 1232 |
-
active=True,
|
| 1233 |
-
)
|
| 1234 |
-
)
|
| 1235 |
-
admin_manager.register_project(
|
| 1236 |
-
Project(
|
| 1237 |
-
project_slug="nairobi-2023",
|
| 1238 |
-
name="Nairobi Survey 2023",
|
| 1239 |
-
dataset_repo_id="birdnet/nairobi-2023-dataset",
|
| 1240 |
-
active=True,
|
| 1241 |
-
)
|
| 1242 |
-
)
|
| 1243 |
-
admin_manager.register_project(
|
| 1244 |
-
Project(
|
| 1245 |
-
project_slug="demo-project",
|
| 1246 |
-
name="Demo Project",
|
| 1247 |
-
dataset_repo_id="birdnet/demo-dataset",
|
| 1248 |
-
active=True,
|
| 1249 |
-
)
|
| 1250 |
-
)
|
| 1251 |
-
|
| 1252 |
-
with gr.Blocks(title="BirdNET Validator HF - Multi-Project") as wrapper:
|
| 1253 |
-
gr.Markdown("# BirdNET Validator HF - Sprint 4: Segurança Multi-Projeto")
|
| 1254 |
-
gr.Markdown("**Versão com autenticação, autorização por projeto, e painel admin**")
|
| 1255 |
-
|
| 1256 |
-
# Session state
|
| 1257 |
-
session_state = gr.State(value=None)
|
| 1258 |
-
selected_project_state = gr.State(value=None)
|
| 1259 |
-
|
| 1260 |
-
def _project_rows() -> list[list[object]]:
|
| 1261 |
-
projects = admin_manager.list_projects()
|
| 1262 |
-
return [
|
| 1263 |
-
[
|
| 1264 |
-
p["project_slug"],
|
| 1265 |
-
p["name"],
|
| 1266 |
-
p["dataset_repo_id"],
|
| 1267 |
-
"yes" if bool(p["active"]) else "no",
|
| 1268 |
-
]
|
| 1269 |
-
for p in projects
|
| 1270 |
-
]
|
| 1271 |
-
|
| 1272 |
-
def _project_slugs() -> list[str]:
|
| 1273 |
-
return [p["project_slug"] for p in admin_manager.list_projects()]
|
| 1274 |
-
|
| 1275 |
-
with gr.Tabs():
|
| 1276 |
-
# ===== TAB 1: Login =====
|
| 1277 |
-
with gr.Tab("🔐 Login", id="login_tab"):
|
| 1278 |
-
login_block, username_input, session_output, login_button, error_message = (
|
| 1279 |
-
create_login_page(auth_service)
|
| 1280 |
-
)
|
| 1281 |
-
|
| 1282 |
-
# Store session ID when login succeeds
|
| 1283 |
-
def handle_login_success(session_id: str):
|
| 1284 |
-
"""Process successful login and store session."""
|
| 1285 |
-
if session_id:
|
| 1286 |
-
return auth_service.get_session(session_id)
|
| 1287 |
-
return None
|
| 1288 |
-
|
| 1289 |
-
session_output.change(
|
| 1290 |
-
fn=handle_login_success,
|
| 1291 |
-
inputs=[session_output],
|
| 1292 |
-
outputs=[session_state],
|
| 1293 |
-
)
|
| 1294 |
-
|
| 1295 |
-
# ===== TAB 2: Admin Panel =====
|
| 1296 |
-
with gr.Tab("⚙️ Admin", id="admin_tab"):
|
| 1297 |
-
admin_info = gr.Markdown(value="⚠️ Faça login primeiro")
|
| 1298 |
-
|
| 1299 |
-
def create_admin_display(session):
|
| 1300 |
-
"""Show admin panel or access denied message."""
|
| 1301 |
-
if session is None:
|
| 1302 |
-
return (
|
| 1303 |
-
"❌ **Not Logged In** — Please login first in the Login tab.",
|
| 1304 |
-
gr.update(visible=False),
|
| 1305 |
-
)
|
| 1306 |
-
if session.role.value != "admin":
|
| 1307 |
-
return (
|
| 1308 |
-
"❌ **Access Denied** — Only administrators can access this panel.",
|
| 1309 |
-
gr.update(visible=False),
|
| 1310 |
-
)
|
| 1311 |
-
return (
|
| 1312 |
-
f"✅ **Admin Panel** — Welcome {session.username}! Click below to manage projects and users.",
|
| 1313 |
-
gr.update(visible=True),
|
| 1314 |
-
)
|
| 1315 |
-
|
| 1316 |
-
with gr.Group(visible=False) as admin_controls:
|
| 1317 |
-
gr.Markdown("#### Projetos Cadastrados")
|
| 1318 |
-
|
| 1319 |
-
with gr.Row():
|
| 1320 |
-
create_project_slug = gr.Textbox(
|
| 1321 |
-
label="Novo Project Slug",
|
| 1322 |
-
placeholder="ex: amazonas-2026",
|
| 1323 |
-
)
|
| 1324 |
-
create_project_name = gr.Textbox(
|
| 1325 |
-
label="Nome do Projeto",
|
| 1326 |
-
placeholder="ex: Amazonas Survey 2026",
|
| 1327 |
-
)
|
| 1328 |
-
create_project_repo = gr.Textbox(
|
| 1329 |
-
label="HF Dataset Repo ID",
|
| 1330 |
-
placeholder="ex: birdnet/amazonas-2026-dataset",
|
| 1331 |
-
)
|
| 1332 |
-
|
| 1333 |
-
create_project_message = gr.Markdown()
|
| 1334 |
-
|
| 1335 |
-
def create_project(session, slug: str, name: str, repo_id: str):
|
| 1336 |
-
if session is None or session.role.value != "admin":
|
| 1337 |
-
return "❌ Access denied. Apenas admin pode criar projetos.", gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
|
| 1338 |
-
|
| 1339 |
-
slug = (slug or "").strip()
|
| 1340 |
-
name = (name or "").strip()
|
| 1341 |
-
repo_id = (repo_id or "").strip()
|
| 1342 |
-
if not slug or not name or not repo_id:
|
| 1343 |
-
return "⚠️ Preencha slug, nome e repo id.", gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
|
| 1344 |
-
|
| 1345 |
-
created = admin_manager.register_project(
|
| 1346 |
-
Project(
|
| 1347 |
-
project_slug=slug,
|
| 1348 |
-
name=name,
|
| 1349 |
-
dataset_repo_id=repo_id,
|
| 1350 |
-
active=True,
|
| 1351 |
-
)
|
| 1352 |
-
)
|
| 1353 |
-
if not created:
|
| 1354 |
-
return (
|
| 1355 |
-
f"⚠️ Projeto '{slug}' já existe.",
|
| 1356 |
-
_project_rows(),
|
| 1357 |
-
gr.update(choices=_project_slugs()),
|
| 1358 |
-
gr.update(),
|
| 1359 |
-
gr.update(),
|
| 1360 |
-
gr.update(),
|
| 1361 |
-
)
|
| 1362 |
-
|
| 1363 |
-
return (
|
| 1364 |
-
f"✅ Projeto '{slug}' criado com sucesso.",
|
| 1365 |
-
_project_rows(),
|
| 1366 |
-
gr.update(choices=_project_slugs(), value=slug),
|
| 1367 |
-
gr.update(value=""),
|
| 1368 |
-
gr.update(value=""),
|
| 1369 |
-
gr.update(value=""),
|
| 1370 |
-
)
|
| 1371 |
-
|
| 1372 |
-
create_project_btn = gr.Button("➕ Criar Projeto", variant="primary")
|
| 1373 |
-
projects_table = gr.Dataframe(
|
| 1374 |
-
value=_project_rows(),
|
| 1375 |
-
headers=["project_slug", "name", "dataset_repo_id", "active"],
|
| 1376 |
-
interactive=False,
|
| 1377 |
-
)
|
| 1378 |
-
refresh_projects_btn = gr.Button("🔄 Atualizar Lista")
|
| 1379 |
-
|
| 1380 |
-
def refresh_projects(session):
|
| 1381 |
-
if session is None or session.role.value != "admin":
|
| 1382 |
-
return []
|
| 1383 |
-
return _project_rows()
|
| 1384 |
-
|
| 1385 |
-
refresh_projects_btn.click(
|
| 1386 |
-
fn=refresh_projects,
|
| 1387 |
-
inputs=[session_state],
|
| 1388 |
-
outputs=[projects_table],
|
| 1389 |
-
)
|
| 1390 |
-
|
| 1391 |
-
with gr.Group(visible=False) as admin_users_controls:
|
| 1392 |
-
gr.Markdown("#### Assignar Usuário a Projeto")
|
| 1393 |
-
with gr.Row():
|
| 1394 |
-
admin_username = gr.Textbox(
|
| 1395 |
-
label="Usuário", placeholder="validator_001"
|
| 1396 |
-
)
|
| 1397 |
-
admin_project = gr.Dropdown(
|
| 1398 |
-
choices=_project_slugs(),
|
| 1399 |
-
label="Projeto",
|
| 1400 |
-
)
|
| 1401 |
-
admin_role = gr.Dropdown(
|
| 1402 |
-
choices=["admin", "validator"],
|
| 1403 |
-
value="validator",
|
| 1404 |
-
label="Role",
|
| 1405 |
-
)
|
| 1406 |
-
|
| 1407 |
-
admin_message = gr.Markdown()
|
| 1408 |
-
|
| 1409 |
-
def assign_user(session, username: str, project: str, role: str):
|
| 1410 |
-
if session is None or session.role.value != "admin":
|
| 1411 |
-
return "❌ Access denied. Apenas admin pode atribuir usuários.", gr.update(), gr.update(), gr.update()
|
| 1412 |
-
success, msg = admin_manager.assign_user_to_project(
|
| 1413 |
-
username, project, role
|
| 1414 |
-
)
|
| 1415 |
-
if success:
|
| 1416 |
-
return msg, gr.update(value=""), gr.update(value=None), gr.update(value="validator")
|
| 1417 |
-
return msg, gr.update(), gr.update(), gr.update()
|
| 1418 |
-
|
| 1419 |
-
assign_btn = gr.Button("✅ Assignar", variant="primary")
|
| 1420 |
-
assign_btn.click(
|
| 1421 |
-
fn=assign_user,
|
| 1422 |
-
inputs=[session_state, admin_username, admin_project, admin_role],
|
| 1423 |
-
outputs=[admin_message, admin_username, admin_project, admin_role],
|
| 1424 |
-
)
|
| 1425 |
-
|
| 1426 |
-
create_project_btn.click(
|
| 1427 |
-
fn=create_project,
|
| 1428 |
-
inputs=[session_state, create_project_slug, create_project_name, create_project_repo],
|
| 1429 |
-
outputs=[
|
| 1430 |
-
create_project_message,
|
| 1431 |
-
projects_table,
|
| 1432 |
-
admin_project,
|
| 1433 |
-
create_project_slug,
|
| 1434 |
-
create_project_name,
|
| 1435 |
-
create_project_repo,
|
| 1436 |
-
],
|
| 1437 |
-
)
|
| 1438 |
-
|
| 1439 |
-
session_state.change(
|
| 1440 |
-
fn=create_admin_display,
|
| 1441 |
-
inputs=[session_state],
|
| 1442 |
-
outputs=[admin_info, admin_controls],
|
| 1443 |
-
)
|
| 1444 |
-
|
| 1445 |
-
session_state.change(
|
| 1446 |
-
fn=lambda s: gr.update(visible=bool(s is not None and s.role.value == "admin")),
|
| 1447 |
-
inputs=[session_state],
|
| 1448 |
-
outputs=[admin_users_controls],
|
| 1449 |
-
)
|
| 1450 |
-
|
| 1451 |
-
# ===== TAB 3: Project Selection =====
|
| 1452 |
-
with gr.Tab("📁 Selecionar Projeto", id="project_tab"):
|
| 1453 |
-
project_info_display = gr.Markdown(
|
| 1454 |
-
value="⚠️ Faça login primeiro na aba **Login**"
|
| 1455 |
-
)
|
| 1456 |
-
project_selector = gr.Dropdown(
|
| 1457 |
-
choices=[],
|
| 1458 |
-
label="Projeto Autorizado",
|
| 1459 |
-
interactive=False,
|
| 1460 |
-
)
|
| 1461 |
-
|
| 1462 |
-
def update_project_selector(session):
|
| 1463 |
-
"""Update project dropdown when user logs in."""
|
| 1464 |
-
if session is None:
|
| 1465 |
-
return (
|
| 1466 |
-
gr.Dropdown(choices=[], value=None, interactive=False),
|
| 1467 |
-
"❌ Not logged in. Please login first.",
|
| 1468 |
-
None,
|
| 1469 |
-
)
|
| 1470 |
-
|
| 1471 |
-
projects = session.authorized_projects
|
| 1472 |
-
if not projects:
|
| 1473 |
-
return (
|
| 1474 |
-
gr.Dropdown(choices=[], value=None, interactive=False),
|
| 1475 |
-
"⚠️ **No Projects Assigned**\n\nYou don't have access to any projects yet. Contact an administrator.",
|
| 1476 |
-
None,
|
| 1477 |
-
)
|
| 1478 |
-
|
| 1479 |
-
selected = projects[0]
|
| 1480 |
-
role = auth_service.get_user_role_for_project(session.username, selected)
|
| 1481 |
-
role_label = role.value.upper() if role else "UNKNOWN"
|
| 1482 |
-
return (
|
| 1483 |
-
gr.Dropdown(choices=projects, value=selected, interactive=True),
|
| 1484 |
-
f"📁 **Project:** {selected} | **Your Role:** {role_label}",
|
| 1485 |
-
selected,
|
| 1486 |
-
)
|
| 1487 |
-
|
| 1488 |
-
session_state.change(
|
| 1489 |
-
fn=update_project_selector,
|
| 1490 |
-
inputs=[session_state],
|
| 1491 |
-
outputs=[project_selector, project_info_display, selected_project_state],
|
| 1492 |
-
)
|
| 1493 |
-
|
| 1494 |
-
def update_selected_project(selected: str, session):
|
| 1495 |
-
"""Update state when project is selected."""
|
| 1496 |
-
if session and selected:
|
| 1497 |
-
return selected
|
| 1498 |
-
return None
|
| 1499 |
-
|
| 1500 |
-
project_selector.change(
|
| 1501 |
-
fn=update_selected_project,
|
| 1502 |
-
inputs=[project_selector, session_state],
|
| 1503 |
-
outputs=[selected_project_state],
|
| 1504 |
-
)
|
| 1505 |
-
|
| 1506 |
-
# ===== TAB 4: Validation =====
|
| 1507 |
-
with gr.Tab("✓ Validação", id="validation_tab"):
|
| 1508 |
-
validation_status = gr.Markdown(
|
| 1509 |
-
value="ℹ️ Faça login e selecione um projeto para começar"
|
| 1510 |
-
)
|
| 1511 |
-
|
| 1512 |
-
def get_validation_status(session, selected_project):
|
| 1513 |
-
"""Show status message based on login/project state."""
|
| 1514 |
-
if session is None:
|
| 1515 |
-
return "❌ **Não autenticado** — Faça login na aba **Login** primeiro"
|
| 1516 |
-
if selected_project is None:
|
| 1517 |
-
return f"⚠️ **Projeto não selecionado** — Selecione um projeto na aba **Selecionar Projeto**"
|
| 1518 |
-
return f"✅ **Pronto para validar** — Projeto: **{selected_project}** | Usuário: **{session.username}**\n\n(Validação será carregada abaixo)"
|
| 1519 |
-
|
| 1520 |
-
session_state.change(
|
| 1521 |
-
fn=lambda s, p: get_validation_status(s, p),
|
| 1522 |
-
inputs=[session_state, selected_project_state],
|
| 1523 |
-
outputs=[validation_status],
|
| 1524 |
-
)
|
| 1525 |
-
|
| 1526 |
-
selected_project_state.change(
|
| 1527 |
-
fn=lambda s, p: get_validation_status(s, p),
|
| 1528 |
-
inputs=[session_state, selected_project_state],
|
| 1529 |
-
outputs=[validation_status],
|
| 1530 |
-
)
|
| 1531 |
-
|
| 1532 |
-
# Note: In a real implementation, we would load build_demo_app(selected_project) here
|
| 1533 |
-
# For now, we show a placeholder since Gradio tabs don't support dynamic nested Blocks
|
| 1534 |
-
gr.Markdown("---")
|
| 1535 |
-
validation_placeholder = gr.Textbox(
|
| 1536 |
-
label="Validação",
|
| 1537 |
-
value="📋 Interface de validação será carregada aqui para o projeto selecionado",
|
| 1538 |
-
interactive=False,
|
| 1539 |
-
lines=10,
|
| 1540 |
-
)
|
| 1541 |
-
|
| 1542 |
-
return wrapper
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/ui/login_page.py
DELETED
|
@@ -1,66 +0,0 @@
|
|
| 1 |
-
"""Gradio login page component for multi-project authorization."""
|
| 2 |
-
|
| 3 |
-
from typing import Optional, Tuple
|
| 4 |
-
|
| 5 |
-
import gradio as gr
|
| 6 |
-
|
| 7 |
-
from src.auth.auth_service import AuthService, Session
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def create_login_page(auth_service: AuthService) -> Tuple[gr.Blocks, gr.Textbox, gr.Textbox, gr.Button, gr.Markdown]:
|
| 11 |
-
"""Create a Gradio login page with username input and session tracking.
|
| 12 |
-
|
| 13 |
-
Args:
|
| 14 |
-
auth_service: AuthService instance for login validation
|
| 15 |
-
|
| 16 |
-
Returns:
|
| 17 |
-
Tuple of (login_block, username_input, session_output, login_button, error_message)
|
| 18 |
-
"""
|
| 19 |
-
|
| 20 |
-
with gr.Blocks(title="BirdNET Validator - Login") as login_block:
|
| 21 |
-
gr.Markdown("# BirdNET Validation Platform")
|
| 22 |
-
gr.Markdown("Login to access multi-project validation workflows")
|
| 23 |
-
|
| 24 |
-
username_input = gr.Textbox(
|
| 25 |
-
label="Username",
|
| 26 |
-
placeholder="Enter your username",
|
| 27 |
-
lines=1,
|
| 28 |
-
)
|
| 29 |
-
|
| 30 |
-
error_message = gr.Markdown()
|
| 31 |
-
|
| 32 |
-
login_button = gr.Button("Login", variant="primary", scale=1)
|
| 33 |
-
|
| 34 |
-
session_output = gr.Textbox(
|
| 35 |
-
label="Session ID",
|
| 36 |
-
interactive=False,
|
| 37 |
-
visible=False,
|
| 38 |
-
)
|
| 39 |
-
|
| 40 |
-
def perform_login(username: str) -> Tuple[str, str]:
|
| 41 |
-
"""Attempt login and return session ID or error message.
|
| 42 |
-
|
| 43 |
-
Args:
|
| 44 |
-
username: Username to authenticate
|
| 45 |
-
|
| 46 |
-
Returns:
|
| 47 |
-
Tuple of (session_id, error_message)
|
| 48 |
-
"""
|
| 49 |
-
if not username or not username.strip():
|
| 50 |
-
return "", "❌ Please enter a username"
|
| 51 |
-
|
| 52 |
-
username = username.strip()
|
| 53 |
-
session = auth_service.login(username)
|
| 54 |
-
|
| 55 |
-
if session is None:
|
| 56 |
-
return "", f"❌ User '{username}' not found or inactive"
|
| 57 |
-
|
| 58 |
-
return session.session_id, f"✅ Welcome, {username}! (Admin)" if session.role.value == "admin" else f"✅ Welcome, {username}! (Validator)"
|
| 59 |
-
|
| 60 |
-
login_button.click(
|
| 61 |
-
fn=perform_login,
|
| 62 |
-
inputs=[username_input],
|
| 63 |
-
outputs=[session_output, error_message],
|
| 64 |
-
)
|
| 65 |
-
|
| 66 |
-
return login_block, username_input, session_output, login_button, error_message
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/ui/project_selector.py
DELETED
|
@@ -1,77 +0,0 @@
|
|
| 1 |
-
"""Project selector component for switching between authorized projects."""
|
| 2 |
-
|
| 3 |
-
from typing import List, Tuple
|
| 4 |
-
|
| 5 |
-
import gradio as gr
|
| 6 |
-
|
| 7 |
-
from src.auth.auth_service import AuthService, Session
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def create_project_selector(
|
| 11 |
-
auth_service: AuthService, current_session: Session
|
| 12 |
-
) -> Tuple[gr.Dropdown, gr.Markdown]:
|
| 13 |
-
"""Create a project selector dropdown for the current user.
|
| 14 |
-
|
| 15 |
-
Args:
|
| 16 |
-
auth_service: AuthService instance
|
| 17 |
-
current_session: Current user's session
|
| 18 |
-
|
| 19 |
-
Returns:
|
| 20 |
-
Tuple of (project_dropdown, project_info_display)
|
| 21 |
-
"""
|
| 22 |
-
authorized_projects = current_session.authorized_projects
|
| 23 |
-
|
| 24 |
-
if not authorized_projects:
|
| 25 |
-
with gr.Group():
|
| 26 |
-
gr.Markdown("⚠️ **No Projects Assigned**\n\nYou don't have access to any projects yet. Contact an administrator.")
|
| 27 |
-
|
| 28 |
-
return (
|
| 29 |
-
gr.Dropdown(choices=[], label="Project", interactive=False),
|
| 30 |
-
gr.Markdown(""),
|
| 31 |
-
)
|
| 32 |
-
|
| 33 |
-
with gr.Group():
|
| 34 |
-
project_dropdown = gr.Dropdown(
|
| 35 |
-
choices=authorized_projects,
|
| 36 |
-
value=authorized_projects[0],
|
| 37 |
-
label="Select Project",
|
| 38 |
-
interactive=True,
|
| 39 |
-
)
|
| 40 |
-
|
| 41 |
-
project_info = gr.Markdown()
|
| 42 |
-
|
| 43 |
-
def update_project_info(selected_project: str) -> str:
|
| 44 |
-
"""Display info about the selected project.
|
| 45 |
-
|
| 46 |
-
Args:
|
| 47 |
-
selected_project: Project slug
|
| 48 |
-
|
| 49 |
-
Returns:
|
| 50 |
-
Markdown formatted project info
|
| 51 |
-
"""
|
| 52 |
-
role = auth_service.get_user_role_for_project(
|
| 53 |
-
current_session.username, selected_project
|
| 54 |
-
)
|
| 55 |
-
if role:
|
| 56 |
-
return f"📁 **Project:** {selected_project} | **Your Role:** {role.value.upper()}"
|
| 57 |
-
return f"📁 **Project:** {selected_project}"
|
| 58 |
-
|
| 59 |
-
# Initial display
|
| 60 |
-
initial_project = authorized_projects[0]
|
| 61 |
-
role = auth_service.get_user_role_for_project(
|
| 62 |
-
current_session.username, initial_project
|
| 63 |
-
)
|
| 64 |
-
initial_info = (
|
| 65 |
-
f"📁 **Project:** {initial_project} | **Your Role:** {role.value.upper()}"
|
| 66 |
-
if role
|
| 67 |
-
else f"📁 **Project:** {initial_project}"
|
| 68 |
-
)
|
| 69 |
-
project_info.value = initial_info
|
| 70 |
-
|
| 71 |
-
project_dropdown.change(
|
| 72 |
-
fn=update_project_info,
|
| 73 |
-
inputs=[project_dropdown],
|
| 74 |
-
outputs=[project_info],
|
| 75 |
-
)
|
| 76 |
-
|
| 77 |
-
return project_dropdown, project_info
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BirdNET Validator App/src/ui/upload_app_factory.py
DELETED
|
@@ -1,177 +0,0 @@
|
|
| 1 |
-
import json
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
from typing import Any
|
| 4 |
-
|
| 5 |
-
import gradio as gr
|
| 6 |
-
from huggingface_hub import HfApi
|
| 7 |
-
|
| 8 |
-
from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
def _as_pretty_json(payload: dict[str, Any]) -> str:
|
| 12 |
-
return json.dumps(payload, indent=2, ensure_ascii=False)
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
def _build_api(token: str) -> HfApi:
|
| 16 |
-
clean_token = (token or "").strip()
|
| 17 |
-
if clean_token:
|
| 18 |
-
return HfApi(token=clean_token)
|
| 19 |
-
return HfApi()
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
def build_upload_app() -> gr.Blocks:
|
| 23 |
-
with gr.Blocks(title="BirdNET Segments Uploader") as demo:
|
| 24 |
-
gr.Markdown("# BirdNET Segments Uploader")
|
| 25 |
-
gr.Markdown(
|
| 26 |
-
"Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
|
| 27 |
-
"Recomendado: executar dry-run antes do upload real."
|
| 28 |
-
)
|
| 29 |
-
|
| 30 |
-
with gr.Row():
|
| 31 |
-
project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
|
| 32 |
-
dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
|
| 33 |
-
|
| 34 |
-
detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
|
| 35 |
-
segments_root = gr.Textbox(
|
| 36 |
-
label="Pasta raiz de segmentos",
|
| 37 |
-
placeholder=r"ex: C:\dados\BirdNET Segments",
|
| 38 |
-
)
|
| 39 |
-
|
| 40 |
-
with gr.Accordion("Configuracao avancada", open=False):
|
| 41 |
-
hf_token = gr.Textbox(
|
| 42 |
-
label="HF Token (opcional)",
|
| 43 |
-
type="password",
|
| 44 |
-
placeholder="Se vazio, usa sessao ja autenticada no ambiente",
|
| 45 |
-
)
|
| 46 |
-
with gr.Row():
|
| 47 |
-
batch_size = gr.Number(label="Batch size", value=200, precision=0)
|
| 48 |
-
shard_size = gr.Number(label="Shard size", value=10000, precision=0)
|
| 49 |
-
with gr.Row():
|
| 50 |
-
max_retries = gr.Number(label="Max retries", value=3, precision=0)
|
| 51 |
-
retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
|
| 52 |
-
resume_state_file = gr.Textbox(
|
| 53 |
-
label="Resume state file",
|
| 54 |
-
value=".ingest-segments-state.json",
|
| 55 |
-
)
|
| 56 |
-
report_file = gr.Textbox(
|
| 57 |
-
label="Report file local (opcional)",
|
| 58 |
-
placeholder="ex: .ingest-run-report.json",
|
| 59 |
-
)
|
| 60 |
-
|
| 61 |
-
with gr.Row():
|
| 62 |
-
dry_run_button = gr.Button("Dry-run", variant="secondary")
|
| 63 |
-
upload_button = gr.Button("Upload real", variant="primary")
|
| 64 |
-
|
| 65 |
-
status = gr.Markdown(value="Pronto")
|
| 66 |
-
result_json = gr.Code(label="Resultado JSON", language="json")
|
| 67 |
-
|
| 68 |
-
def run_dry_run(
|
| 69 |
-
project: str,
|
| 70 |
-
repo: str,
|
| 71 |
-
csv_path: str | None,
|
| 72 |
-
segments_path: str,
|
| 73 |
-
report_path: str,
|
| 74 |
-
) -> tuple[str, str]:
|
| 75 |
-
project = (project or "").strip()
|
| 76 |
-
if not project:
|
| 77 |
-
return "❌ Informe o project slug", "{}"
|
| 78 |
-
|
| 79 |
-
if not csv_path:
|
| 80 |
-
return "❌ Selecione o CSV de deteccoes", "{}"
|
| 81 |
-
|
| 82 |
-
if not segments_path or not Path(segments_path).exists():
|
| 83 |
-
return "❌ Pasta de segmentos invalida", "{}"
|
| 84 |
-
|
| 85 |
-
try:
|
| 86 |
-
result = run_ingest_segments_dry_run(
|
| 87 |
-
project_slug=project,
|
| 88 |
-
detections_csv=csv_path,
|
| 89 |
-
segments_root=segments_path,
|
| 90 |
-
)
|
| 91 |
-
if report_path.strip():
|
| 92 |
-
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 93 |
-
summary = (
|
| 94 |
-
"✅ Dry-run finalizado | "
|
| 95 |
-
f"Matched: {result['matched_rows']} | "
|
| 96 |
-
f"Unmatched: {result['unmatched_rows']} | "
|
| 97 |
-
f"Ambiguous: {result['ambiguous_rows']}"
|
| 98 |
-
)
|
| 99 |
-
return summary, _as_pretty_json(result)
|
| 100 |
-
except Exception as exc:
|
| 101 |
-
return f"❌ Dry-run falhou: {exc}", "{}"
|
| 102 |
-
|
| 103 |
-
def run_upload(
|
| 104 |
-
project: str,
|
| 105 |
-
repo: str,
|
| 106 |
-
csv_path: str | None,
|
| 107 |
-
segments_path: str,
|
| 108 |
-
token: str,
|
| 109 |
-
batch: float,
|
| 110 |
-
shard: float,
|
| 111 |
-
retries: float,
|
| 112 |
-
backoff: float,
|
| 113 |
-
resume_file: str,
|
| 114 |
-
report_path: str,
|
| 115 |
-
) -> tuple[str, str]:
|
| 116 |
-
project = (project or "").strip()
|
| 117 |
-
repo = (repo or "").strip()
|
| 118 |
-
if not project:
|
| 119 |
-
return "❌ Informe o project slug", "{}"
|
| 120 |
-
if not repo or "/" not in repo:
|
| 121 |
-
return "❌ Informe dataset repo no formato owner/repo", "{}"
|
| 122 |
-
if not csv_path:
|
| 123 |
-
return "❌ Selecione o CSV de deteccoes", "{}"
|
| 124 |
-
if not segments_path or not Path(segments_path).exists():
|
| 125 |
-
return "❌ Pasta de segmentos invalida", "{}"
|
| 126 |
-
|
| 127 |
-
try:
|
| 128 |
-
api = _build_api(token)
|
| 129 |
-
result = ingest_segments_to_hf(
|
| 130 |
-
api=api,
|
| 131 |
-
project_slug=project,
|
| 132 |
-
dataset_repo=repo,
|
| 133 |
-
detections_csv=csv_path,
|
| 134 |
-
segments_root=segments_path,
|
| 135 |
-
batch_size=int(batch),
|
| 136 |
-
shard_size=int(shard),
|
| 137 |
-
max_retries=int(retries),
|
| 138 |
-
retry_backoff_seconds=float(backoff),
|
| 139 |
-
resume_state_file=resume_file,
|
| 140 |
-
)
|
| 141 |
-
if report_path.strip():
|
| 142 |
-
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 143 |
-
summary = (
|
| 144 |
-
"✅ Upload finalizado | "
|
| 145 |
-
f"Uploaded now: {result['uploaded_audio_now']} | "
|
| 146 |
-
f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
|
| 147 |
-
f"Failed uploads: {result['failed_uploads']}"
|
| 148 |
-
)
|
| 149 |
-
return summary, _as_pretty_json(result)
|
| 150 |
-
except Exception as exc:
|
| 151 |
-
return f"❌ Upload falhou: {exc}", "{}"
|
| 152 |
-
|
| 153 |
-
dry_run_button.click(
|
| 154 |
-
fn=run_dry_run,
|
| 155 |
-
inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
|
| 156 |
-
outputs=[status, result_json],
|
| 157 |
-
)
|
| 158 |
-
|
| 159 |
-
upload_button.click(
|
| 160 |
-
fn=run_upload,
|
| 161 |
-
inputs=[
|
| 162 |
-
project_slug,
|
| 163 |
-
dataset_repo,
|
| 164 |
-
detections_csv,
|
| 165 |
-
segments_root,
|
| 166 |
-
hf_token,
|
| 167 |
-
batch_size,
|
| 168 |
-
shard_size,
|
| 169 |
-
max_retries,
|
| 170 |
-
retry_backoff_seconds,
|
| 171 |
-
resume_state_file,
|
| 172 |
-
report_file,
|
| 173 |
-
],
|
| 174 |
-
outputs=[status, result_json],
|
| 175 |
-
)
|
| 176 |
-
|
| 177 |
-
return demo
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
CHANGED
|
@@ -1,82 +1,64 @@
|
|
| 1 |
-
# BirdNET
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
- src/
|
| 58 |
-
- src/
|
| 59 |
-
- src/
|
| 60 |
-
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
## Estado atual de implementacao
|
| 67 |
-
- Sprint 1: CLI create-project, sync-audio, build-index, verify-project
|
| 68 |
-
- Sprint 2 (inicial): fila paginada de deteccoes com filtros basicos em demo local Gradio
|
| 69 |
-
- Sprint 2 (backend): AudioFetchService com download sob demanda por audio_id + cache efemero com cleanup pos-validacao
|
| 70 |
-
- Sprint 2 (UI): carregar audio da deteccao selecionada e limpar cache apos validacao
|
| 71 |
-
- Sprint 2 (UI fluxo): botoes de validacao rapida com salvamento em memoria e limpeza automatica do cache de audio
|
| 72 |
-
- Sprint 3 (inicial): persistencia append-only de validacoes em eventos JSONL + snapshot current por detection_key
|
| 73 |
-
- Sprint 3 (UI): tabela exibe status atual da validacao por detection_key e botao de relatorio por projeto
|
| 74 |
-
- Sprint 3 (concorrencia): controle otimista por versao esperada com deteccao de conflito e bloqueio de sobrescrita silenciosa
|
| 75 |
-
- Sprint 3 (UX conflito): refresh automatico da tabela em conflito e foco na deteccao impactada para resolucao rapida
|
| 76 |
-
- Sprint 3 (UX conflito+): flag visual de conflito na linha e botao para reaplicar validacao na versao atual
|
| 77 |
-
- Sprint 3 (UX conflito++): sinalizacao explicita com conflict_flag=CONFLICT e conflict_severity=HIGH na deteccao afetada
|
| 78 |
-
- Sprint 3 (UX conflito+++): filtro rapido "mostrar apenas conflitos" integrado em refresh, paginacao e acoes de validacao
|
| 79 |
-
- Sprint 3 (UX conflito++++): atalhos teclado para validacao rapida (1=positivo 2=negativo 3=indeterminado 4=pular R=reaplicar)
|
| 80 |
-
- Sprint 3 (UX conflito+++++): operacoes em lote para aprovar/rejeitar todos os conflitos em uma unica acao
|
| 81 |
-
- Sprint 3 (UX conflito++++++): filtros avancados por validador, status e data de atualizacao em toda a navegacao
|
| 82 |
-
- Sprint 3 (UX conflito+++++++): filtro de data com seletor nativo (DateTime sem hora) para reduzir erros de digitacao
|
|
|
|
| 1 |
+
# BirdNET-Validator-App
|
| 2 |
+
|
| 3 |
+
BirdNET-Validator-App is a Gradio application for human validation of BirdNET detections in multi-project workflows.
|
| 4 |
+
|
| 5 |
+
It provides a practical review interface with authentication, project-level authorization, paginated queues, on-demand audio, and append-only validation history.
|
| 6 |
+
|
| 7 |
+
## Objective
|
| 8 |
+
|
| 9 |
+
This project focuses on high-confidence validation operations by combining:
|
| 10 |
+
|
| 11 |
+
- controlled access by user and project
|
| 12 |
+
- rapid decision flow for detections
|
| 13 |
+
- auditability through append-only events
|
| 14 |
+
- conflict-aware updates for concurrent validators
|
| 15 |
+
|
| 16 |
+
## Key Usability Features
|
| 17 |
+
|
| 18 |
+
1. Multi-project login with role-based access
|
| 19 |
+
2. Project selection for authorized datasets
|
| 20 |
+
3. Detection queue with filters and pagination
|
| 21 |
+
4. Audio loading on demand with ephemeral cache
|
| 22 |
+
5. Fast validation actions (positive, negative, uncertain, skip)
|
| 23 |
+
6. Concurrency-safe writes with optimistic lock feedback
|
| 24 |
+
7. Conflict resolution support and validation reporting
|
| 25 |
+
|
| 26 |
+
## Quick Start (Local)
|
| 27 |
+
|
| 28 |
+
1. Create and activate a Python 3.11+ virtual environment.
|
| 29 |
+
2. Install dependencies:
|
| 30 |
+
|
| 31 |
+
```bash
|
| 32 |
+
pip install -r requirements.txt
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
3. Run the app:
|
| 36 |
+
|
| 37 |
+
```bash
|
| 38 |
+
python app.py
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
Default port is 7860.
|
| 42 |
+
|
| 43 |
+
## Typical Validation Flow
|
| 44 |
+
|
| 45 |
+
1. Login with a valid user.
|
| 46 |
+
2. Select an authorized project.
|
| 47 |
+
3. Load queue items and apply filters when needed.
|
| 48 |
+
4. Listen to selected audio and submit validation status.
|
| 49 |
+
5. Resolve conflicts when concurrent updates occur.
|
| 50 |
+
6. Export or inspect project validation report.
|
| 51 |
+
|
| 52 |
+
## Repository Structure
|
| 53 |
+
|
| 54 |
+
- `app.py`: validator app entrypoint
|
| 55 |
+
- `src/auth`: authentication/session and ACL logic
|
| 56 |
+
- `src/domain`: domain models
|
| 57 |
+
- `src/repositories`: in-memory and append-only persistence
|
| 58 |
+
- `src/services`: queue, validation, and audio fetch services
|
| 59 |
+
- `src/ui`: Gradio interface composition and callbacks
|
| 60 |
+
- `tests`: unit and integration test suites
|
| 61 |
+
|
| 62 |
+
## Project Status
|
| 63 |
+
|
| 64 |
+
Validator workflow is active and under continuous development, with emphasis on reliability, auditability, and operator productivity.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app_uploader.py
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
|
| 3 |
-
from src.ui.upload_app_factory import build_upload_app
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
if __name__ == "__main__":
|
| 7 |
-
app = build_upload_app()
|
| 8 |
-
port = int(os.getenv("BIRDNET_UPLOADER_PORT", "7862"))
|
| 9 |
-
app.launch(server_name="0.0.0.0", server_port=port)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cli/hf_dataset_cli.py
DELETED
|
@@ -1,969 +0,0 @@
|
|
| 1 |
-
import argparse
|
| 2 |
-
import hashlib
|
| 3 |
-
import json
|
| 4 |
-
import math
|
| 5 |
-
import re
|
| 6 |
-
import tempfile
|
| 7 |
-
import time
|
| 8 |
-
from dataclasses import asdict, dataclass
|
| 9 |
-
from datetime import UTC, datetime
|
| 10 |
-
from io import BytesIO
|
| 11 |
-
from pathlib import Path
|
| 12 |
-
from typing import Any
|
| 13 |
-
|
| 14 |
-
import pandas as pd
|
| 15 |
-
from huggingface_hub import HfApi, hf_hub_download
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
SCHEMA_VERSION = "1.0.0"
|
| 19 |
-
REQUIRED_PROJECT_PREFIXES = (
|
| 20 |
-
"audio/",
|
| 21 |
-
"index/",
|
| 22 |
-
"index/shards/",
|
| 23 |
-
"validations/",
|
| 24 |
-
"audit/",
|
| 25 |
-
)
|
| 26 |
-
REQUIRED_PLACEHOLDER_FILES = (
|
| 27 |
-
"audio/.gitkeep",
|
| 28 |
-
"index/.gitkeep",
|
| 29 |
-
"index/shards/.gitkeep",
|
| 30 |
-
"validations/.gitkeep",
|
| 31 |
-
"audit/.gitkeep",
|
| 32 |
-
)
|
| 33 |
-
REQUIRED_DETECTIONS_COLUMNS = (
|
| 34 |
-
"detection_key",
|
| 35 |
-
"audio_id",
|
| 36 |
-
"scientific_name",
|
| 37 |
-
"confidence",
|
| 38 |
-
"start_time",
|
| 39 |
-
"end_time",
|
| 40 |
-
)
|
| 41 |
-
SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
|
| 42 |
-
REQUIRED_BIRDNET_COLUMNS = (
|
| 43 |
-
"source_file",
|
| 44 |
-
"scientific_name",
|
| 45 |
-
"confidence",
|
| 46 |
-
"start_time",
|
| 47 |
-
"end_time",
|
| 48 |
-
)
|
| 49 |
-
SEGMENT_FILENAME_PATTERN = re.compile(
|
| 50 |
-
r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
|
| 51 |
-
)
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
@dataclass(slots=True)
|
| 55 |
-
class ShardMetadata:
|
| 56 |
-
path: str
|
| 57 |
-
rows: int
|
| 58 |
-
sha256: str
|
| 59 |
-
size_bytes: int
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
@dataclass(slots=True)
|
| 63 |
-
class SegmentFileRecord:
|
| 64 |
-
scientific_name: str
|
| 65 |
-
source_stem: str
|
| 66 |
-
start_time: float
|
| 67 |
-
end_time: float
|
| 68 |
-
confidence_pct: int
|
| 69 |
-
absolute_path: Path
|
| 70 |
-
relative_path: str
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
def _utcnow_iso() -> str:
|
| 74 |
-
return datetime.now(UTC).replace(microsecond=0).isoformat()
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
|
| 78 |
-
api.upload_file(
|
| 79 |
-
path_or_fileobj=BytesIO(content.encode("utf-8")),
|
| 80 |
-
path_in_repo=path_in_repo,
|
| 81 |
-
repo_id=dataset_repo,
|
| 82 |
-
repo_type="dataset",
|
| 83 |
-
)
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
|
| 87 |
-
api.upload_file(
|
| 88 |
-
path_or_fileobj=BytesIO(b""),
|
| 89 |
-
path_in_repo=path_in_repo,
|
| 90 |
-
repo_id=dataset_repo,
|
| 91 |
-
repo_type="dataset",
|
| 92 |
-
)
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
def ensure_project_dataset_structure(
|
| 96 |
-
api: HfApi,
|
| 97 |
-
project_slug: str,
|
| 98 |
-
dataset_repo: str,
|
| 99 |
-
create_private_repo: bool,
|
| 100 |
-
) -> dict[str, Any]:
|
| 101 |
-
api.create_repo(
|
| 102 |
-
repo_id=dataset_repo,
|
| 103 |
-
repo_type="dataset",
|
| 104 |
-
private=create_private_repo,
|
| 105 |
-
exist_ok=True,
|
| 106 |
-
)
|
| 107 |
-
|
| 108 |
-
files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 109 |
-
created_paths: list[str] = []
|
| 110 |
-
|
| 111 |
-
for placeholder in REQUIRED_PLACEHOLDER_FILES:
|
| 112 |
-
if placeholder not in files:
|
| 113 |
-
_upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
|
| 114 |
-
created_paths.append(placeholder)
|
| 115 |
-
|
| 116 |
-
manifest_path = "manifest.json"
|
| 117 |
-
if manifest_path not in files:
|
| 118 |
-
manifest = {
|
| 119 |
-
"schema_version": SCHEMA_VERSION,
|
| 120 |
-
"project_slug": project_slug,
|
| 121 |
-
"dataset_repo_id": dataset_repo,
|
| 122 |
-
"created_at": _utcnow_iso(),
|
| 123 |
-
"updated_at": _utcnow_iso(),
|
| 124 |
-
"index": {
|
| 125 |
-
"total_detections": 0,
|
| 126 |
-
"total_audio_files": 0,
|
| 127 |
-
"shard_size": 0,
|
| 128 |
-
"shards": [],
|
| 129 |
-
},
|
| 130 |
-
}
|
| 131 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
|
| 132 |
-
created_paths.append(manifest_path)
|
| 133 |
-
|
| 134 |
-
return {
|
| 135 |
-
"dataset_repo": dataset_repo,
|
| 136 |
-
"project_slug": project_slug,
|
| 137 |
-
"created_paths": created_paths,
|
| 138 |
-
}
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
def load_detections_table(detections_file: str) -> pd.DataFrame:
|
| 142 |
-
input_path = Path(detections_file)
|
| 143 |
-
if not input_path.exists():
|
| 144 |
-
raise FileNotFoundError(f"Detections file not found: {detections_file}")
|
| 145 |
-
|
| 146 |
-
suffix = input_path.suffix.lower()
|
| 147 |
-
if suffix == ".csv":
|
| 148 |
-
frame = pd.read_csv(input_path)
|
| 149 |
-
elif suffix in {".jsonl", ".ndjson"}:
|
| 150 |
-
frame = pd.read_json(input_path, lines=True)
|
| 151 |
-
elif suffix == ".parquet":
|
| 152 |
-
frame = pd.read_parquet(input_path)
|
| 153 |
-
else:
|
| 154 |
-
raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
|
| 155 |
-
|
| 156 |
-
missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
|
| 157 |
-
if missing_columns:
|
| 158 |
-
raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
|
| 159 |
-
|
| 160 |
-
if frame.empty:
|
| 161 |
-
return frame
|
| 162 |
-
|
| 163 |
-
deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
|
| 164 |
-
return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
|
| 168 |
-
input_path = Path(detections_csv)
|
| 169 |
-
if not input_path.exists():
|
| 170 |
-
raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
|
| 171 |
-
|
| 172 |
-
frame = pd.read_csv(input_path)
|
| 173 |
-
missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
|
| 174 |
-
if missing_columns:
|
| 175 |
-
raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
|
| 176 |
-
|
| 177 |
-
if frame.empty:
|
| 178 |
-
return frame
|
| 179 |
-
|
| 180 |
-
normalized = frame.copy()
|
| 181 |
-
normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
|
| 182 |
-
normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
|
| 183 |
-
normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
|
| 184 |
-
normalized["species_key"] = normalized["scientific_name"].str.casefold()
|
| 185 |
-
normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
|
| 186 |
-
normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
|
| 187 |
-
normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
|
| 188 |
-
|
| 189 |
-
normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
|
| 190 |
-
return normalized
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
|
| 194 |
-
payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
|
| 195 |
-
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
|
| 199 |
-
stem = Path(file_name).stem
|
| 200 |
-
match = SEGMENT_FILENAME_PATTERN.match(stem)
|
| 201 |
-
if not match:
|
| 202 |
-
return None
|
| 203 |
-
|
| 204 |
-
source_stem = match.group("source_stem")
|
| 205 |
-
start_time = float(match.group("start"))
|
| 206 |
-
end_time = float(match.group("end"))
|
| 207 |
-
confidence_pct = int(match.group("confidence_pct"))
|
| 208 |
-
return source_stem, start_time, end_time, confidence_pct
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
|
| 212 |
-
root = Path(segments_root)
|
| 213 |
-
if not root.exists() or not root.is_dir():
|
| 214 |
-
raise FileNotFoundError(f"Segments root not found: {segments_root}")
|
| 215 |
-
|
| 216 |
-
records: list[SegmentFileRecord] = []
|
| 217 |
-
for absolute_path in root.rglob("*"):
|
| 218 |
-
if not absolute_path.is_file():
|
| 219 |
-
continue
|
| 220 |
-
if absolute_path.suffix.lower() != ".wav":
|
| 221 |
-
continue
|
| 222 |
-
|
| 223 |
-
rel = absolute_path.relative_to(root)
|
| 224 |
-
scientific_name = rel.parts[0] if rel.parts else ""
|
| 225 |
-
parsed = parse_segment_filename(absolute_path.name)
|
| 226 |
-
if not parsed:
|
| 227 |
-
continue
|
| 228 |
-
source_stem, start_time, end_time, confidence_pct = parsed
|
| 229 |
-
records.append(
|
| 230 |
-
SegmentFileRecord(
|
| 231 |
-
scientific_name=scientific_name,
|
| 232 |
-
source_stem=source_stem,
|
| 233 |
-
start_time=start_time,
|
| 234 |
-
end_time=end_time,
|
| 235 |
-
confidence_pct=confidence_pct,
|
| 236 |
-
absolute_path=absolute_path,
|
| 237 |
-
relative_path=rel.as_posix(),
|
| 238 |
-
)
|
| 239 |
-
)
|
| 240 |
-
|
| 241 |
-
return records
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 245 |
-
frame = load_birdnet_detections_csv(detections_csv=detections_csv)
|
| 246 |
-
segment_records = discover_segment_records(segments_root=segments_root)
|
| 247 |
-
|
| 248 |
-
by_species: dict[str, list[SegmentFileRecord]] = {}
|
| 249 |
-
for record in segment_records:
|
| 250 |
-
by_species.setdefault(record.scientific_name.casefold(), []).append(record)
|
| 251 |
-
|
| 252 |
-
matched_rows: list[dict[str, Any]] = []
|
| 253 |
-
unmatched_rows: list[dict[str, Any]] = []
|
| 254 |
-
ambiguous_rows: list[dict[str, Any]] = []
|
| 255 |
-
|
| 256 |
-
for row in frame.to_dict(orient="records"):
|
| 257 |
-
species_key = str(row.get("species_key", "")).casefold()
|
| 258 |
-
source_stem = str(row.get("source_stem", "")).strip()
|
| 259 |
-
start_time = float(row["start_time"])
|
| 260 |
-
end_time = float(row["end_time"])
|
| 261 |
-
confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
|
| 262 |
-
|
| 263 |
-
species_candidates = by_species.get(species_key, [])
|
| 264 |
-
exact_matches = [
|
| 265 |
-
record
|
| 266 |
-
for record in species_candidates
|
| 267 |
-
if record.source_stem == source_stem
|
| 268 |
-
and abs(record.start_time - start_time) < 1e-6
|
| 269 |
-
and abs(record.end_time - end_time) < 1e-6
|
| 270 |
-
]
|
| 271 |
-
|
| 272 |
-
if not exact_matches:
|
| 273 |
-
unmatched_rows.append(
|
| 274 |
-
{
|
| 275 |
-
"source_file": row.get("source_file", ""),
|
| 276 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 277 |
-
"start_time": start_time,
|
| 278 |
-
"end_time": end_time,
|
| 279 |
-
}
|
| 280 |
-
)
|
| 281 |
-
continue
|
| 282 |
-
|
| 283 |
-
selected = exact_matches[0]
|
| 284 |
-
if len(exact_matches) > 1:
|
| 285 |
-
expected_pct = int(round(confidence * 100))
|
| 286 |
-
exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
|
| 287 |
-
selected = exact_matches[0]
|
| 288 |
-
ambiguous_rows.append(
|
| 289 |
-
{
|
| 290 |
-
"source_file": row.get("source_file", ""),
|
| 291 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 292 |
-
"start_time": start_time,
|
| 293 |
-
"end_time": end_time,
|
| 294 |
-
"candidates": len(exact_matches),
|
| 295 |
-
}
|
| 296 |
-
)
|
| 297 |
-
|
| 298 |
-
matched_rows.append(
|
| 299 |
-
{
|
| 300 |
-
"project_slug": project_slug,
|
| 301 |
-
"detection_key": _build_detection_key(
|
| 302 |
-
project_slug=project_slug,
|
| 303 |
-
source_file=str(row.get("source_file", "")),
|
| 304 |
-
scientific_name=str(row.get("scientific_name", "")),
|
| 305 |
-
start_time=start_time,
|
| 306 |
-
end_time=end_time,
|
| 307 |
-
),
|
| 308 |
-
"audio_id": source_stem,
|
| 309 |
-
"segment_filename": selected.absolute_path.name,
|
| 310 |
-
"segment_relpath": selected.relative_path,
|
| 311 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 312 |
-
"common_name": row.get("common_name", ""),
|
| 313 |
-
"confidence": confidence,
|
| 314 |
-
"start_time": start_time,
|
| 315 |
-
"end_time": end_time,
|
| 316 |
-
"exact_start": row.get("exact_start", start_time),
|
| 317 |
-
"exact_end": row.get("exact_end", end_time),
|
| 318 |
-
"locality": row.get("locality", ""),
|
| 319 |
-
"point": row.get("point", ""),
|
| 320 |
-
"date_folder": row.get("date_folder", ""),
|
| 321 |
-
"min_freq": row.get("min_freq", None),
|
| 322 |
-
"max_freq": row.get("max_freq", None),
|
| 323 |
-
"box_source": row.get("box_source", ""),
|
| 324 |
-
"label": row.get("label", ""),
|
| 325 |
-
"source_file": row.get("source_file", ""),
|
| 326 |
-
"schema_version": SCHEMA_VERSION,
|
| 327 |
-
"ingested_at": _utcnow_iso(),
|
| 328 |
-
}
|
| 329 |
-
)
|
| 330 |
-
|
| 331 |
-
return {
|
| 332 |
-
"frame": frame,
|
| 333 |
-
"segment_records_total": len(segment_records),
|
| 334 |
-
"matched_rows": matched_rows,
|
| 335 |
-
"unmatched_rows": unmatched_rows,
|
| 336 |
-
"ambiguous_rows": ambiguous_rows,
|
| 337 |
-
}
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 341 |
-
match_result = _match_birdnet_rows(
|
| 342 |
-
project_slug=project_slug,
|
| 343 |
-
detections_csv=detections_csv,
|
| 344 |
-
segments_root=segments_root,
|
| 345 |
-
)
|
| 346 |
-
frame = match_result["frame"]
|
| 347 |
-
matched_rows = match_result["matched_rows"]
|
| 348 |
-
unmatched_rows = match_result["unmatched_rows"]
|
| 349 |
-
ambiguous_rows = match_result["ambiguous_rows"]
|
| 350 |
-
|
| 351 |
-
unique_audio = {row["audio_id"] for row in matched_rows}
|
| 352 |
-
result = {
|
| 353 |
-
"mode": "dry-run",
|
| 354 |
-
"project_slug": project_slug,
|
| 355 |
-
"detections_csv": detections_csv,
|
| 356 |
-
"segments_root": segments_root,
|
| 357 |
-
"csv_rows_total": int(len(frame)),
|
| 358 |
-
"segments_found_total": int(match_result["segment_records_total"]),
|
| 359 |
-
"matched_rows": len(matched_rows),
|
| 360 |
-
"unmatched_rows": len(unmatched_rows),
|
| 361 |
-
"ambiguous_rows": len(ambiguous_rows),
|
| 362 |
-
"unique_audio_ids": len(unique_audio),
|
| 363 |
-
"sample_unmatched": unmatched_rows[:20],
|
| 364 |
-
"sample_ambiguous": ambiguous_rows[:20],
|
| 365 |
-
}
|
| 366 |
-
return result
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
def _file_sha256(file_path: Path) -> str:
|
| 370 |
-
hasher = hashlib.sha256()
|
| 371 |
-
with file_path.open("rb") as handle:
|
| 372 |
-
for chunk in iter(lambda: handle.read(65536), b""):
|
| 373 |
-
hasher.update(chunk)
|
| 374 |
-
return hasher.hexdigest()
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
|
| 378 |
-
if shard_size <= 0:
|
| 379 |
-
raise ValueError("shard_size must be greater than zero")
|
| 380 |
-
|
| 381 |
-
output_dir.mkdir(parents=True, exist_ok=True)
|
| 382 |
-
shards: list[ShardMetadata] = []
|
| 383 |
-
|
| 384 |
-
if frame.empty:
|
| 385 |
-
return shards
|
| 386 |
-
|
| 387 |
-
total_rows = len(frame)
|
| 388 |
-
total_shards = math.ceil(total_rows / shard_size)
|
| 389 |
-
|
| 390 |
-
for index in range(total_shards):
|
| 391 |
-
start = index * shard_size
|
| 392 |
-
end = min(start + shard_size, total_rows)
|
| 393 |
-
shard_frame = frame.iloc[start:end]
|
| 394 |
-
shard_filename = f"shard-{index:05d}.parquet"
|
| 395 |
-
local_path = output_dir / shard_filename
|
| 396 |
-
shard_frame.to_parquet(local_path, index=False)
|
| 397 |
-
|
| 398 |
-
shards.append(
|
| 399 |
-
ShardMetadata(
|
| 400 |
-
path=f"index/shards/{shard_filename}",
|
| 401 |
-
rows=len(shard_frame),
|
| 402 |
-
sha256=_file_sha256(local_path),
|
| 403 |
-
size_bytes=local_path.stat().st_size,
|
| 404 |
-
)
|
| 405 |
-
)
|
| 406 |
-
|
| 407 |
-
return shards
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
def build_manifest_payload(
|
| 411 |
-
project_slug: str,
|
| 412 |
-
dataset_repo: str,
|
| 413 |
-
frame: pd.DataFrame,
|
| 414 |
-
shard_size: int,
|
| 415 |
-
shard_metadata: list[ShardMetadata],
|
| 416 |
-
) -> dict[str, Any]:
|
| 417 |
-
now = _utcnow_iso()
|
| 418 |
-
return {
|
| 419 |
-
"schema_version": SCHEMA_VERSION,
|
| 420 |
-
"project_slug": project_slug,
|
| 421 |
-
"dataset_repo_id": dataset_repo,
|
| 422 |
-
"updated_at": now,
|
| 423 |
-
"index": {
|
| 424 |
-
"total_detections": len(frame),
|
| 425 |
-
"total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
|
| 426 |
-
"shard_size": shard_size,
|
| 427 |
-
"shards": [asdict(item) for item in shard_metadata],
|
| 428 |
-
},
|
| 429 |
-
}
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
def discover_audio_files(local_audio_dir: str) -> list[Path]:
|
| 433 |
-
audio_root = Path(local_audio_dir)
|
| 434 |
-
if not audio_root.exists() or not audio_root.is_dir():
|
| 435 |
-
raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
|
| 436 |
-
|
| 437 |
-
files = [
|
| 438 |
-
path
|
| 439 |
-
for path in audio_root.rglob("*")
|
| 440 |
-
if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
|
| 441 |
-
]
|
| 442 |
-
return sorted(files)
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
|
| 446 |
-
if chunk_size <= 0:
|
| 447 |
-
raise ValueError("chunk_size must be greater than zero")
|
| 448 |
-
return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
def _load_resume_state(state_file: Path) -> dict[str, Any]:
|
| 452 |
-
if not state_file.exists():
|
| 453 |
-
return {"uploaded": [], "failed": []}
|
| 454 |
-
|
| 455 |
-
payload = json.loads(state_file.read_text(encoding="utf-8"))
|
| 456 |
-
uploaded = payload.get("uploaded", [])
|
| 457 |
-
failed = payload.get("failed", [])
|
| 458 |
-
return {
|
| 459 |
-
"uploaded": uploaded if isinstance(uploaded, list) else [],
|
| 460 |
-
"failed": failed if isinstance(failed, list) else [],
|
| 461 |
-
}
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
|
| 465 |
-
state_file.parent.mkdir(parents=True, exist_ok=True)
|
| 466 |
-
payload = {
|
| 467 |
-
"updated_at": _utcnow_iso(),
|
| 468 |
-
"uploaded": sorted(uploaded),
|
| 469 |
-
"failed": sorted(failed),
|
| 470 |
-
}
|
| 471 |
-
state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
def _upload_audio_with_retry(
|
| 475 |
-
api: HfApi,
|
| 476 |
-
dataset_repo: str,
|
| 477 |
-
local_path: Path,
|
| 478 |
-
path_in_repo: str,
|
| 479 |
-
max_retries: int,
|
| 480 |
-
retry_backoff_seconds: float,
|
| 481 |
-
) -> None:
|
| 482 |
-
attempts = 0
|
| 483 |
-
while True:
|
| 484 |
-
attempts += 1
|
| 485 |
-
try:
|
| 486 |
-
api.upload_file(
|
| 487 |
-
path_or_fileobj=str(local_path),
|
| 488 |
-
path_in_repo=path_in_repo,
|
| 489 |
-
repo_id=dataset_repo,
|
| 490 |
-
repo_type="dataset",
|
| 491 |
-
)
|
| 492 |
-
return
|
| 493 |
-
except Exception:
|
| 494 |
-
if attempts > max_retries:
|
| 495 |
-
raise
|
| 496 |
-
time.sleep(retry_backoff_seconds * attempts)
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
def ingest_segments_to_hf(
|
| 500 |
-
api: HfApi,
|
| 501 |
-
project_slug: str,
|
| 502 |
-
dataset_repo: str,
|
| 503 |
-
detections_csv: str,
|
| 504 |
-
segments_root: str,
|
| 505 |
-
batch_size: int,
|
| 506 |
-
shard_size: int,
|
| 507 |
-
max_retries: int,
|
| 508 |
-
retry_backoff_seconds: float,
|
| 509 |
-
resume_state_file: str,
|
| 510 |
-
) -> dict[str, Any]:
|
| 511 |
-
ensure_project_dataset_structure(
|
| 512 |
-
api=api,
|
| 513 |
-
project_slug=project_slug,
|
| 514 |
-
dataset_repo=dataset_repo,
|
| 515 |
-
create_private_repo=False,
|
| 516 |
-
)
|
| 517 |
-
|
| 518 |
-
match_result = _match_birdnet_rows(
|
| 519 |
-
project_slug=project_slug,
|
| 520 |
-
detections_csv=detections_csv,
|
| 521 |
-
segments_root=segments_root,
|
| 522 |
-
)
|
| 523 |
-
frame = match_result["frame"]
|
| 524 |
-
matched_rows = match_result["matched_rows"]
|
| 525 |
-
unmatched_rows = match_result["unmatched_rows"]
|
| 526 |
-
ambiguous_rows = match_result["ambiguous_rows"]
|
| 527 |
-
|
| 528 |
-
# Deduplicate by remote path so retries/resume work on a stable key.
|
| 529 |
-
path_to_local: dict[str, Path] = {}
|
| 530 |
-
for row in matched_rows:
|
| 531 |
-
repo_path = f"audio/segments/{row['segment_relpath']}"
|
| 532 |
-
if repo_path in path_to_local:
|
| 533 |
-
continue
|
| 534 |
-
path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
|
| 535 |
-
|
| 536 |
-
state_path = Path(resume_state_file)
|
| 537 |
-
state_payload = _load_resume_state(state_file=state_path)
|
| 538 |
-
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 539 |
-
failed_state = set(str(item) for item in state_payload["failed"])
|
| 540 |
-
|
| 541 |
-
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 542 |
-
|
| 543 |
-
pending: list[tuple[Path, str]] = []
|
| 544 |
-
skipped_existing = 0
|
| 545 |
-
for repo_path, local_path in path_to_local.items():
|
| 546 |
-
if repo_path in remote_files:
|
| 547 |
-
uploaded_state.add(repo_path)
|
| 548 |
-
skipped_existing += 1
|
| 549 |
-
continue
|
| 550 |
-
if repo_path in uploaded_state:
|
| 551 |
-
continue
|
| 552 |
-
pending.append((local_path, repo_path))
|
| 553 |
-
|
| 554 |
-
uploaded_now: set[str] = set()
|
| 555 |
-
failed_now: set[str] = set()
|
| 556 |
-
|
| 557 |
-
for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
|
| 558 |
-
batch_uploaded = 0
|
| 559 |
-
batch_failed = 0
|
| 560 |
-
for local_path, repo_path in batch:
|
| 561 |
-
try:
|
| 562 |
-
_upload_audio_with_retry(
|
| 563 |
-
api=api,
|
| 564 |
-
dataset_repo=dataset_repo,
|
| 565 |
-
local_path=local_path,
|
| 566 |
-
path_in_repo=repo_path,
|
| 567 |
-
max_retries=max_retries,
|
| 568 |
-
retry_backoff_seconds=retry_backoff_seconds,
|
| 569 |
-
)
|
| 570 |
-
uploaded_state.add(repo_path)
|
| 571 |
-
failed_state.discard(repo_path)
|
| 572 |
-
uploaded_now.add(repo_path)
|
| 573 |
-
batch_uploaded += 1
|
| 574 |
-
except Exception:
|
| 575 |
-
failed_state.add(repo_path)
|
| 576 |
-
failed_now.add(repo_path)
|
| 577 |
-
batch_failed += 1
|
| 578 |
-
|
| 579 |
-
print(
|
| 580 |
-
json.dumps(
|
| 581 |
-
{
|
| 582 |
-
"event": "ingest-segments-audio-batch",
|
| 583 |
-
"batch_index": batch_index,
|
| 584 |
-
"batch_size": len(batch),
|
| 585 |
-
"uploaded": batch_uploaded,
|
| 586 |
-
"failed": batch_failed,
|
| 587 |
-
}
|
| 588 |
-
)
|
| 589 |
-
)
|
| 590 |
-
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
|
| 591 |
-
|
| 592 |
-
index_frame = pd.DataFrame(matched_rows)
|
| 593 |
-
if not index_frame.empty:
|
| 594 |
-
index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
|
| 595 |
-
|
| 596 |
-
with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
|
| 597 |
-
temp_dir = Path(temp_dir_str)
|
| 598 |
-
shard_dir = temp_dir / "index" / "shards"
|
| 599 |
-
shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
|
| 600 |
-
|
| 601 |
-
for item in shard_metadata:
|
| 602 |
-
local_path = temp_dir / item.path
|
| 603 |
-
api.upload_file(
|
| 604 |
-
path_or_fileobj=str(local_path),
|
| 605 |
-
path_in_repo=item.path,
|
| 606 |
-
repo_id=dataset_repo,
|
| 607 |
-
repo_type="dataset",
|
| 608 |
-
)
|
| 609 |
-
|
| 610 |
-
manifest = build_manifest_payload(
|
| 611 |
-
project_slug=project_slug,
|
| 612 |
-
dataset_repo=dataset_repo,
|
| 613 |
-
frame=index_frame,
|
| 614 |
-
shard_size=shard_size,
|
| 615 |
-
shard_metadata=shard_metadata,
|
| 616 |
-
)
|
| 617 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 618 |
-
|
| 619 |
-
run_report = {
|
| 620 |
-
"mode": "execute",
|
| 621 |
-
"project_slug": project_slug,
|
| 622 |
-
"dataset_repo": dataset_repo,
|
| 623 |
-
"detections_csv": detections_csv,
|
| 624 |
-
"segments_root": segments_root,
|
| 625 |
-
"csv_rows_total": int(len(frame)),
|
| 626 |
-
"segments_found_total": int(match_result["segment_records_total"]),
|
| 627 |
-
"matched_rows": len(matched_rows),
|
| 628 |
-
"unmatched_rows": len(unmatched_rows),
|
| 629 |
-
"ambiguous_rows": len(ambiguous_rows),
|
| 630 |
-
"unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
|
| 631 |
-
"pending_audio_uploads": len(pending),
|
| 632 |
-
"uploaded_audio_now": len(uploaded_now),
|
| 633 |
-
"uploaded_audio_skipped_existing": skipped_existing,
|
| 634 |
-
"failed_uploads": len(failed_now),
|
| 635 |
-
"resume_state_file": str(state_path),
|
| 636 |
-
"index_rows_written": len(index_frame),
|
| 637 |
-
"shards_written": len(shard_metadata),
|
| 638 |
-
"sample_unmatched": unmatched_rows[:20],
|
| 639 |
-
"sample_ambiguous": ambiguous_rows[:20],
|
| 640 |
-
}
|
| 641 |
-
|
| 642 |
-
audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
|
| 643 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
|
| 644 |
-
return run_report
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
def sync_audio_batches(
|
| 648 |
-
api: HfApi,
|
| 649 |
-
project_slug: str,
|
| 650 |
-
dataset_repo: str,
|
| 651 |
-
local_audio_dir: str,
|
| 652 |
-
batch_size: int,
|
| 653 |
-
max_retries: int,
|
| 654 |
-
retry_backoff_seconds: float,
|
| 655 |
-
resume_state_file: str,
|
| 656 |
-
) -> dict[str, Any]:
|
| 657 |
-
ensure_project_dataset_structure(
|
| 658 |
-
api=api,
|
| 659 |
-
project_slug=project_slug,
|
| 660 |
-
dataset_repo=dataset_repo,
|
| 661 |
-
create_private_repo=False,
|
| 662 |
-
)
|
| 663 |
-
|
| 664 |
-
all_files = discover_audio_files(local_audio_dir=local_audio_dir)
|
| 665 |
-
state_path = Path(resume_state_file)
|
| 666 |
-
state_payload = _load_resume_state(state_file=state_path)
|
| 667 |
-
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 668 |
-
|
| 669 |
-
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 670 |
-
|
| 671 |
-
pending: list[tuple[Path, str]] = []
|
| 672 |
-
for path in all_files:
|
| 673 |
-
rel = path.relative_to(Path(local_audio_dir)).as_posix()
|
| 674 |
-
repo_path = f"audio/{rel}"
|
| 675 |
-
|
| 676 |
-
if repo_path in remote_files:
|
| 677 |
-
uploaded_state.add(repo_path)
|
| 678 |
-
continue
|
| 679 |
-
|
| 680 |
-
if repo_path in uploaded_state:
|
| 681 |
-
continue
|
| 682 |
-
|
| 683 |
-
pending.append((path, repo_path))
|
| 684 |
-
|
| 685 |
-
pending_paths = [item[0] for item in pending]
|
| 686 |
-
by_local_to_repo = {local: repo for local, repo in pending}
|
| 687 |
-
batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
|
| 688 |
-
|
| 689 |
-
uploaded_now: set[str] = set()
|
| 690 |
-
failed: set[str] = set()
|
| 691 |
-
|
| 692 |
-
for batch_index, batch in enumerate(batches, start=1):
|
| 693 |
-
batch_uploaded = 0
|
| 694 |
-
batch_failed = 0
|
| 695 |
-
|
| 696 |
-
for local_path in batch:
|
| 697 |
-
repo_path = by_local_to_repo[local_path]
|
| 698 |
-
try:
|
| 699 |
-
_upload_audio_with_retry(
|
| 700 |
-
api=api,
|
| 701 |
-
dataset_repo=dataset_repo,
|
| 702 |
-
local_path=local_path,
|
| 703 |
-
path_in_repo=repo_path,
|
| 704 |
-
max_retries=max_retries,
|
| 705 |
-
retry_backoff_seconds=retry_backoff_seconds,
|
| 706 |
-
)
|
| 707 |
-
uploaded_now.add(repo_path)
|
| 708 |
-
uploaded_state.add(repo_path)
|
| 709 |
-
batch_uploaded += 1
|
| 710 |
-
except Exception:
|
| 711 |
-
failed.add(repo_path)
|
| 712 |
-
batch_failed += 1
|
| 713 |
-
|
| 714 |
-
print(
|
| 715 |
-
json.dumps(
|
| 716 |
-
{
|
| 717 |
-
"event": "sync-audio-batch",
|
| 718 |
-
"batch_index": batch_index,
|
| 719 |
-
"batch_size": len(batch),
|
| 720 |
-
"uploaded": batch_uploaded,
|
| 721 |
-
"failed": batch_failed,
|
| 722 |
-
}
|
| 723 |
-
)
|
| 724 |
-
)
|
| 725 |
-
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
|
| 726 |
-
|
| 727 |
-
return {
|
| 728 |
-
"project_slug": project_slug,
|
| 729 |
-
"dataset_repo": dataset_repo,
|
| 730 |
-
"total_local_audio_files": len(all_files),
|
| 731 |
-
"pending_uploads": len(pending),
|
| 732 |
-
"uploaded_now": len(uploaded_now),
|
| 733 |
-
"failed": len(failed),
|
| 734 |
-
"resume_state_file": str(state_path),
|
| 735 |
-
}
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
def build_and_upload_index(
|
| 739 |
-
api: HfApi,
|
| 740 |
-
project_slug: str,
|
| 741 |
-
dataset_repo: str,
|
| 742 |
-
detections_file: str,
|
| 743 |
-
shard_size: int,
|
| 744 |
-
) -> dict[str, Any]:
|
| 745 |
-
ensure_project_dataset_structure(
|
| 746 |
-
api=api,
|
| 747 |
-
project_slug=project_slug,
|
| 748 |
-
dataset_repo=dataset_repo,
|
| 749 |
-
create_private_repo=False,
|
| 750 |
-
)
|
| 751 |
-
|
| 752 |
-
frame = load_detections_table(detections_file=detections_file)
|
| 753 |
-
|
| 754 |
-
with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
|
| 755 |
-
temp_dir = Path(temp_dir_str)
|
| 756 |
-
shard_dir = temp_dir / "index" / "shards"
|
| 757 |
-
metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
|
| 758 |
-
|
| 759 |
-
for item in metadata:
|
| 760 |
-
local_path = temp_dir / item.path
|
| 761 |
-
api.upload_file(
|
| 762 |
-
path_or_fileobj=str(local_path),
|
| 763 |
-
path_in_repo=item.path,
|
| 764 |
-
repo_id=dataset_repo,
|
| 765 |
-
repo_type="dataset",
|
| 766 |
-
)
|
| 767 |
-
|
| 768 |
-
manifest = build_manifest_payload(
|
| 769 |
-
project_slug=project_slug,
|
| 770 |
-
dataset_repo=dataset_repo,
|
| 771 |
-
frame=frame,
|
| 772 |
-
shard_size=shard_size,
|
| 773 |
-
shard_metadata=metadata,
|
| 774 |
-
)
|
| 775 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 776 |
-
|
| 777 |
-
return {
|
| 778 |
-
"project_slug": project_slug,
|
| 779 |
-
"dataset_repo": dataset_repo,
|
| 780 |
-
"total_detections": manifest["index"]["total_detections"],
|
| 781 |
-
"total_audio_files": manifest["index"]["total_audio_files"],
|
| 782 |
-
"total_shards": len(metadata),
|
| 783 |
-
}
|
| 784 |
-
|
| 785 |
-
|
| 786 |
-
def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
|
| 787 |
-
errors: list[str] = []
|
| 788 |
-
|
| 789 |
-
for prefix in REQUIRED_PROJECT_PREFIXES:
|
| 790 |
-
if not any(path.startswith(prefix) for path in repo_files):
|
| 791 |
-
errors.append(f"Missing prefix in dataset repo: {prefix}")
|
| 792 |
-
|
| 793 |
-
if manifest_payload.get("schema_version") != SCHEMA_VERSION:
|
| 794 |
-
errors.append(
|
| 795 |
-
f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
|
| 796 |
-
)
|
| 797 |
-
|
| 798 |
-
if manifest_payload.get("project_slug") != project_slug:
|
| 799 |
-
errors.append(
|
| 800 |
-
f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
|
| 801 |
-
)
|
| 802 |
-
|
| 803 |
-
index_section = manifest_payload.get("index", {})
|
| 804 |
-
shards = index_section.get("shards", [])
|
| 805 |
-
if not isinstance(shards, list):
|
| 806 |
-
errors.append("Manifest index.shards must be a list")
|
| 807 |
-
return errors
|
| 808 |
-
|
| 809 |
-
for shard in shards:
|
| 810 |
-
shard_path = shard.get("path")
|
| 811 |
-
if not shard_path or not isinstance(shard_path, str):
|
| 812 |
-
errors.append("Shard entry missing path")
|
| 813 |
-
continue
|
| 814 |
-
if shard_path not in repo_files:
|
| 815 |
-
errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
|
| 816 |
-
|
| 817 |
-
return errors
|
| 818 |
-
|
| 819 |
-
|
| 820 |
-
def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
|
| 821 |
-
repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 822 |
-
if "manifest.json" not in repo_files:
|
| 823 |
-
return {
|
| 824 |
-
"ok": False,
|
| 825 |
-
"errors": ["manifest.json not found in dataset repository"],
|
| 826 |
-
}
|
| 827 |
-
|
| 828 |
-
manifest_local = hf_hub_download(
|
| 829 |
-
repo_id=dataset_repo,
|
| 830 |
-
repo_type="dataset",
|
| 831 |
-
filename="manifest.json",
|
| 832 |
-
)
|
| 833 |
-
manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
|
| 834 |
-
errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
|
| 835 |
-
|
| 836 |
-
return {
|
| 837 |
-
"ok": len(errors) == 0,
|
| 838 |
-
"errors": errors,
|
| 839 |
-
"total_files": len(repo_files),
|
| 840 |
-
"total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
|
| 841 |
-
}
|
| 842 |
-
|
| 843 |
-
|
| 844 |
-
def build_parser() -> argparse.ArgumentParser:
|
| 845 |
-
parser = argparse.ArgumentParser(description="HF dataset project CLI")
|
| 846 |
-
sub = parser.add_subparsers(dest="command", required=True)
|
| 847 |
-
|
| 848 |
-
create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
|
| 849 |
-
create_project.add_argument("--project-slug", required=True)
|
| 850 |
-
create_project.add_argument("--dataset-repo", required=True)
|
| 851 |
-
create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
|
| 852 |
-
|
| 853 |
-
build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
|
| 854 |
-
build_index.add_argument("--project-slug", required=True)
|
| 855 |
-
build_index.add_argument("--dataset-repo", required=True)
|
| 856 |
-
build_index.add_argument("--detections-file", required=True)
|
| 857 |
-
build_index.add_argument("--shard-size", type=int, default=10000)
|
| 858 |
-
|
| 859 |
-
sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
|
| 860 |
-
sync_audio.add_argument("--project-slug", required=True)
|
| 861 |
-
sync_audio.add_argument("--dataset-repo", required=True)
|
| 862 |
-
sync_audio.add_argument("--local-audio-dir", required=True)
|
| 863 |
-
sync_audio.add_argument("--batch-size", type=int, default=100)
|
| 864 |
-
sync_audio.add_argument("--max-retries", type=int, default=3)
|
| 865 |
-
sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 866 |
-
sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
|
| 867 |
-
|
| 868 |
-
verify_project = sub.add_parser("verify-project", help="Verify project integrity")
|
| 869 |
-
verify_project.add_argument("--project-slug", required=True)
|
| 870 |
-
verify_project.add_argument("--dataset-repo", required=True)
|
| 871 |
-
|
| 872 |
-
ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
|
| 873 |
-
ingest_segments.add_argument("--project-slug", required=True)
|
| 874 |
-
ingest_segments.add_argument("--dataset-repo", required=True)
|
| 875 |
-
ingest_segments.add_argument("--detections-csv", required=True)
|
| 876 |
-
ingest_segments.add_argument("--segments-root", required=True)
|
| 877 |
-
ingest_segments.add_argument("--batch-size", type=int, default=200)
|
| 878 |
-
ingest_segments.add_argument("--shard-size", type=int, default=10000)
|
| 879 |
-
ingest_segments.add_argument("--max-retries", type=int, default=3)
|
| 880 |
-
ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 881 |
-
ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
|
| 882 |
-
ingest_segments.add_argument("--dry-run", action="store_true")
|
| 883 |
-
ingest_segments.add_argument("--report-file", default="")
|
| 884 |
-
|
| 885 |
-
return parser
|
| 886 |
-
|
| 887 |
-
|
| 888 |
-
def main() -> int:
|
| 889 |
-
parser = build_parser()
|
| 890 |
-
args = parser.parse_args()
|
| 891 |
-
api = HfApi()
|
| 892 |
-
|
| 893 |
-
if args.command == "create-project":
|
| 894 |
-
result = ensure_project_dataset_structure(
|
| 895 |
-
api=api,
|
| 896 |
-
project_slug=args.project_slug,
|
| 897 |
-
dataset_repo=args.dataset_repo,
|
| 898 |
-
create_private_repo=bool(args.private),
|
| 899 |
-
)
|
| 900 |
-
print(json.dumps(result, indent=2))
|
| 901 |
-
return 0
|
| 902 |
-
|
| 903 |
-
if args.command == "build-index":
|
| 904 |
-
result = build_and_upload_index(
|
| 905 |
-
api=api,
|
| 906 |
-
project_slug=args.project_slug,
|
| 907 |
-
dataset_repo=args.dataset_repo,
|
| 908 |
-
detections_file=args.detections_file,
|
| 909 |
-
shard_size=args.shard_size,
|
| 910 |
-
)
|
| 911 |
-
print(json.dumps(result, indent=2))
|
| 912 |
-
return 0
|
| 913 |
-
|
| 914 |
-
if args.command == "sync-audio":
|
| 915 |
-
result = sync_audio_batches(
|
| 916 |
-
api=api,
|
| 917 |
-
project_slug=args.project_slug,
|
| 918 |
-
dataset_repo=args.dataset_repo,
|
| 919 |
-
local_audio_dir=args.local_audio_dir,
|
| 920 |
-
batch_size=args.batch_size,
|
| 921 |
-
max_retries=args.max_retries,
|
| 922 |
-
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 923 |
-
resume_state_file=args.resume_state_file,
|
| 924 |
-
)
|
| 925 |
-
print(json.dumps(result, indent=2))
|
| 926 |
-
return 0 if result["failed"] == 0 else 1
|
| 927 |
-
|
| 928 |
-
if args.command == "verify-project":
|
| 929 |
-
result = verify_project(
|
| 930 |
-
api=api,
|
| 931 |
-
project_slug=args.project_slug,
|
| 932 |
-
dataset_repo=args.dataset_repo,
|
| 933 |
-
)
|
| 934 |
-
print(json.dumps(result, indent=2))
|
| 935 |
-
return 0 if result["ok"] else 1
|
| 936 |
-
|
| 937 |
-
if args.command == "ingest-segments":
|
| 938 |
-
if args.dry_run:
|
| 939 |
-
result = run_ingest_segments_dry_run(
|
| 940 |
-
project_slug=args.project_slug,
|
| 941 |
-
detections_csv=args.detections_csv,
|
| 942 |
-
segments_root=args.segments_root,
|
| 943 |
-
)
|
| 944 |
-
else:
|
| 945 |
-
result = ingest_segments_to_hf(
|
| 946 |
-
api=api,
|
| 947 |
-
project_slug=args.project_slug,
|
| 948 |
-
dataset_repo=args.dataset_repo,
|
| 949 |
-
detections_csv=args.detections_csv,
|
| 950 |
-
segments_root=args.segments_root,
|
| 951 |
-
batch_size=args.batch_size,
|
| 952 |
-
shard_size=args.shard_size,
|
| 953 |
-
max_retries=args.max_retries,
|
| 954 |
-
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 955 |
-
resume_state_file=args.resume_state_file,
|
| 956 |
-
)
|
| 957 |
-
|
| 958 |
-
if args.report_file:
|
| 959 |
-
Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
| 960 |
-
|
| 961 |
-
print(json.dumps(result, indent=2))
|
| 962 |
-
return 0 if result.get("failed_uploads", 0) == 0 else 1
|
| 963 |
-
|
| 964 |
-
parser.error("Unknown command")
|
| 965 |
-
return 2
|
| 966 |
-
|
| 967 |
-
|
| 968 |
-
if __name__ == "__main__":
|
| 969 |
-
raise SystemExit(main())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
docs/HF_SEGMENTS_INGESTION_PLAN.md
DELETED
|
@@ -1,194 +0,0 @@
|
|
| 1 |
-
# HF Segments Ingestion Plan
|
| 2 |
-
|
| 3 |
-
## Objective
|
| 4 |
-
Build a dedicated ingestion tool (separate from validation UI) that accepts:
|
| 5 |
-
1. A BirdNET detections CSV
|
| 6 |
-
2. A root segments folder (including subfolders)
|
| 7 |
-
|
| 8 |
-
Then uploads both audio segments and a normalized metadata index to a Hugging Face dataset repo.
|
| 9 |
-
|
| 10 |
-
## Why This Comes Next
|
| 11 |
-
- Validation UI is already stable for auth/admin/ACL.
|
| 12 |
-
- Real operations depend on real data availability.
|
| 13 |
-
- Ingestion first avoids UI rework over mock assumptions.
|
| 14 |
-
|
| 15 |
-
## Real Dataset Observations (from local reference)
|
| 16 |
-
- Segments root folders (species folders): 293
|
| 17 |
-
- Segment files: 146562
|
| 18 |
-
- Extension profile: `.wav` only (plus one `_progress_log.txt`)
|
| 19 |
-
- Filename pattern sample:
|
| 20 |
-
- `Accipiter striatus/Catim_LS-1500_U10_20250221_060600_0.0-3.0s_85%.wav`
|
| 21 |
-
- `Cyanocorax cyanopogon/Aiuab_LN-0500_U16_20260121_153400_42.0-45.0s_98%.wav`
|
| 22 |
-
|
| 23 |
-
## Input CSV (reference provided)
|
| 24 |
-
Observed columns include:
|
| 25 |
-
- `locality`
|
| 26 |
-
- `point`
|
| 27 |
-
- `date_folder`
|
| 28 |
-
- `source_file`
|
| 29 |
-
- `scientific_name`
|
| 30 |
-
- `common_name`
|
| 31 |
-
- `confidence`
|
| 32 |
-
- `start_time`
|
| 33 |
-
- `end_time`
|
| 34 |
-
- `exact_start`
|
| 35 |
-
- `exact_end`
|
| 36 |
-
- `min_freq`
|
| 37 |
-
- `max_freq`
|
| 38 |
-
- `box_source`
|
| 39 |
-
- `label`
|
| 40 |
-
|
| 41 |
-
## Proposed HF Dataset Layout
|
| 42 |
-
- `audio/segments/<scientific_name>/<segment_filename>.wav`
|
| 43 |
-
- `index/shards/*.parquet` (normalized row index for fast reads)
|
| 44 |
-
- `manifest.json` (schema/version/counters)
|
| 45 |
-
- `audit/ingestion-runs/<run_id>.json` (run report)
|
| 46 |
-
- `audit/unmatched/<run_id>.parquet` (rows not linked to files)
|
| 47 |
-
|
| 48 |
-
## Canonical Metadata Row (Normalized)
|
| 49 |
-
Required fields for each detection segment:
|
| 50 |
-
- `project_slug`
|
| 51 |
-
- `detection_key` (stable deterministic key)
|
| 52 |
-
- `audio_id` (source audio id, from `source_file` stem)
|
| 53 |
-
- `segment_filename`
|
| 54 |
-
- `segment_relpath`
|
| 55 |
-
- `scientific_name`
|
| 56 |
-
- `common_name`
|
| 57 |
-
- `confidence`
|
| 58 |
-
- `start_time`
|
| 59 |
-
- `end_time`
|
| 60 |
-
- `exact_start`
|
| 61 |
-
- `exact_end`
|
| 62 |
-
- `locality`
|
| 63 |
-
- `point`
|
| 64 |
-
- `date_folder`
|
| 65 |
-
- `min_freq`
|
| 66 |
-
- `max_freq`
|
| 67 |
-
- `box_source`
|
| 68 |
-
- `label`
|
| 69 |
-
- `source_file`
|
| 70 |
-
- `ingested_at`
|
| 71 |
-
- `schema_version`
|
| 72 |
-
|
| 73 |
-
Optional convenience fields:
|
| 74 |
-
- `segment_duration`
|
| 75 |
-
- `confidence_pct_rounded`
|
| 76 |
-
- `match_strategy`
|
| 77 |
-
- `match_score`
|
| 78 |
-
|
| 79 |
-
## Matching Strategy (CSV <-> Segment Files)
|
| 80 |
-
Primary join (deterministic):
|
| 81 |
-
1. species folder = `scientific_name`
|
| 82 |
-
2. segment filename starts with `source_file` stem
|
| 83 |
-
3. filename contains `<start_time>-<end_time>s`
|
| 84 |
-
|
| 85 |
-
Tolerance rules:
|
| 86 |
-
- Use decimal normalization (`3` vs `3.0`) for time token search.
|
| 87 |
-
- Confidence in filename (`85%`) is optional for join, used as tie-breaker only.
|
| 88 |
-
|
| 89 |
-
Tie-break policy:
|
| 90 |
-
- If more than one segment matches, pick exact time token first.
|
| 91 |
-
- If still ambiguous, pick closest confidence percentage.
|
| 92 |
-
- Persist ambiguity in audit file.
|
| 93 |
-
|
| 94 |
-
## Deterministic detection_key
|
| 95 |
-
Generate with stable hash to satisfy model constraints and dedupe safely:
|
| 96 |
-
- input: `project_slug|source_file|scientific_name|start_time|end_time`
|
| 97 |
-
- `detection_key = sha256(input).hexdigest()[:24]`
|
| 98 |
-
|
| 99 |
-
Benefits:
|
| 100 |
-
- deterministic across reruns
|
| 101 |
-
- longer than minimum length constraints
|
| 102 |
-
- independent from local filesystem ordering
|
| 103 |
-
|
| 104 |
-
## CLI Proposal (new command)
|
| 105 |
-
Add command to `cli/hf_dataset_cli.py`:
|
| 106 |
-
- `ingest-segments`
|
| 107 |
-
|
| 108 |
-
Arguments:
|
| 109 |
-
- `--project-slug`
|
| 110 |
-
- `--dataset-repo`
|
| 111 |
-
- `--detections-csv`
|
| 112 |
-
- `--segments-root`
|
| 113 |
-
- `--batch-size` (default 200)
|
| 114 |
-
- `--shard-size` (default 10000)
|
| 115 |
-
- `--dry-run`
|
| 116 |
-
- `--resume-state-file` (default `.ingest-segments-state.json`)
|
| 117 |
-
- `--max-retries` (default 3)
|
| 118 |
-
- `--retry-backoff-seconds` (default 1.0)
|
| 119 |
-
|
| 120 |
-
## Command Behavior
|
| 121 |
-
Dry-run mode:
|
| 122 |
-
- validates input schema
|
| 123 |
-
- computes matching stats
|
| 124 |
-
- reports would-upload counts
|
| 125 |
-
- writes no remote files
|
| 126 |
-
|
| 127 |
-
Execute mode:
|
| 128 |
-
1. ensure dataset scaffold
|
| 129 |
-
2. parse CSV and normalize rows
|
| 130 |
-
3. discover segments recursively
|
| 131 |
-
4. match rows to segment files
|
| 132 |
-
5. upload segment audio in batches with resume/retry
|
| 133 |
-
6. build/upload index shards from matched rows
|
| 134 |
-
7. update `manifest.json`
|
| 135 |
-
8. write audit artifacts
|
| 136 |
-
|
| 137 |
-
## Operational Report (stdout JSON)
|
| 138 |
-
Return key counters:
|
| 139 |
-
- `csv_rows_total`
|
| 140 |
-
- `segments_found_total`
|
| 141 |
-
- `matched_rows`
|
| 142 |
-
- `unmatched_rows`
|
| 143 |
-
- `ambiguous_rows`
|
| 144 |
-
- `uploaded_audio_now`
|
| 145 |
-
- `uploaded_audio_skipped_existing`
|
| 146 |
-
- `index_rows_written`
|
| 147 |
-
- `shards_written`
|
| 148 |
-
- `failed_uploads`
|
| 149 |
-
- `duration_seconds`
|
| 150 |
-
|
| 151 |
-
## Failure Policy
|
| 152 |
-
Hard-fail:
|
| 153 |
-
- missing required CSV columns
|
| 154 |
-
- unreadable segments root
|
| 155 |
-
- HF auth/repo unreachable
|
| 156 |
-
|
| 157 |
-
Soft-fail with audit:
|
| 158 |
-
- unmatched rows
|
| 159 |
-
- ambiguous matches
|
| 160 |
-
- partial audio upload failures
|
| 161 |
-
|
| 162 |
-
Exit code:
|
| 163 |
-
- `0` when no upload failures (unmatched allowed with warning)
|
| 164 |
-
- `1` when upload/index write failures occur
|
| 165 |
-
|
| 166 |
-
## Performance Notes
|
| 167 |
-
Given ~146k WAV files:
|
| 168 |
-
- keep file discovery cached in-memory map by species + source_file stem
|
| 169 |
-
- avoid O(N*M) matching scans
|
| 170 |
-
- process CSV in chunks where possible
|
| 171 |
-
- upload in batches and persist resume state after each batch
|
| 172 |
-
|
| 173 |
-
## Security and Separation
|
| 174 |
-
- Ingestion tool remains separate from validation UI.
|
| 175 |
-
- Validation app reads index/audio already published to HF.
|
| 176 |
-
- ACL remains project-scoped at validation stage.
|
| 177 |
-
|
| 178 |
-
## Implementation Phases
|
| 179 |
-
Phase 1 (MVP ingestion):
|
| 180 |
-
- `ingest-segments --dry-run`
|
| 181 |
-
- deterministic key generation
|
| 182 |
-
- CSV normalization + matching report
|
| 183 |
-
|
| 184 |
-
Phase 2 (publish):
|
| 185 |
-
- batched audio upload + resume/retry
|
| 186 |
-
- index shard generation and manifest update
|
| 187 |
-
|
| 188 |
-
Phase 3 (hardening):
|
| 189 |
-
- audit files for unmatched/ambiguous
|
| 190 |
-
- duplicate handling policy and idempotency checks
|
| 191 |
-
- integration tests with fixture CSV + synthetic segment tree
|
| 192 |
-
|
| 193 |
-
## Suggested Next Action
|
| 194 |
-
Implement Phase 1 first (`dry-run`) so we can validate matching quality on your real CSV and folder before any remote upload.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pyproject.toml
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
[project]
|
| 2 |
-
name = "BirdNET-
|
| 3 |
version = "0.1.0"
|
| 4 |
description = "BirdNET detection validator for Hugging Face Spaces"
|
| 5 |
readme = "README.md"
|
|
|
|
| 1 |
[project]
|
| 2 |
+
name = "BirdNET-Validator-App"
|
| 3 |
version = "0.1.0"
|
| 4 |
description = "BirdNET detection validator for Hugging Face Spaces"
|
| 5 |
readme = "README.md"
|
requirements.txt
CHANGED
|
@@ -1,7 +1,4 @@
|
|
| 1 |
gradio>=5.0.0
|
| 2 |
huggingface_hub>=0.30.0
|
| 3 |
pydantic>=2.8.0
|
| 4 |
-
pandas>=2.2.0
|
| 5 |
-
pyarrow>=17.0.0
|
| 6 |
-
soundfile>=0.12.1
|
| 7 |
pytest>=8.3.0
|
|
|
|
| 1 |
gradio>=5.0.0
|
| 2 |
huggingface_hub>=0.30.0
|
| 3 |
pydantic>=2.8.0
|
|
|
|
|
|
|
|
|
|
| 4 |
pytest>=8.3.0
|
src/ui/upload_app_factory.py
DELETED
|
@@ -1,177 +0,0 @@
|
|
| 1 |
-
import json
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
from typing import Any
|
| 4 |
-
|
| 5 |
-
import gradio as gr
|
| 6 |
-
from huggingface_hub import HfApi
|
| 7 |
-
|
| 8 |
-
from cli.hf_dataset_cli import ingest_segments_to_hf, run_ingest_segments_dry_run
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
def _as_pretty_json(payload: dict[str, Any]) -> str:
|
| 12 |
-
return json.dumps(payload, indent=2, ensure_ascii=False)
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
def _build_api(token: str) -> HfApi:
|
| 16 |
-
clean_token = (token or "").strip()
|
| 17 |
-
if clean_token:
|
| 18 |
-
return HfApi(token=clean_token)
|
| 19 |
-
return HfApi()
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
def build_upload_app() -> gr.Blocks:
|
| 23 |
-
with gr.Blocks(title="BirdNET Segments Uploader") as demo:
|
| 24 |
-
gr.Markdown("# BirdNET Segments Uploader")
|
| 25 |
-
gr.Markdown(
|
| 26 |
-
"Use esta ferramenta para importar segmentos + CSV de deteccoes por projeto. "
|
| 27 |
-
"Recomendado: executar dry-run antes do upload real."
|
| 28 |
-
)
|
| 29 |
-
|
| 30 |
-
with gr.Row():
|
| 31 |
-
project_slug = gr.Textbox(label="Project Slug", placeholder="ex: ppbio-aiuaba")
|
| 32 |
-
dataset_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo")
|
| 33 |
-
|
| 34 |
-
detections_csv = gr.File(label="CSV de deteccoes", file_types=[".csv"], type="filepath")
|
| 35 |
-
segments_root = gr.Textbox(
|
| 36 |
-
label="Pasta raiz de segmentos",
|
| 37 |
-
placeholder=r"ex: C:\dados\BirdNET Segments",
|
| 38 |
-
)
|
| 39 |
-
|
| 40 |
-
with gr.Accordion("Configuracao avancada", open=False):
|
| 41 |
-
hf_token = gr.Textbox(
|
| 42 |
-
label="HF Token (opcional)",
|
| 43 |
-
type="password",
|
| 44 |
-
placeholder="Se vazio, usa sessao ja autenticada no ambiente",
|
| 45 |
-
)
|
| 46 |
-
with gr.Row():
|
| 47 |
-
batch_size = gr.Number(label="Batch size", value=200, precision=0)
|
| 48 |
-
shard_size = gr.Number(label="Shard size", value=10000, precision=0)
|
| 49 |
-
with gr.Row():
|
| 50 |
-
max_retries = gr.Number(label="Max retries", value=3, precision=0)
|
| 51 |
-
retry_backoff_seconds = gr.Number(label="Retry backoff (s)", value=1.0)
|
| 52 |
-
resume_state_file = gr.Textbox(
|
| 53 |
-
label="Resume state file",
|
| 54 |
-
value=".ingest-segments-state.json",
|
| 55 |
-
)
|
| 56 |
-
report_file = gr.Textbox(
|
| 57 |
-
label="Report file local (opcional)",
|
| 58 |
-
placeholder="ex: .ingest-run-report.json",
|
| 59 |
-
)
|
| 60 |
-
|
| 61 |
-
with gr.Row():
|
| 62 |
-
dry_run_button = gr.Button("Dry-run", variant="secondary")
|
| 63 |
-
upload_button = gr.Button("Upload real", variant="primary")
|
| 64 |
-
|
| 65 |
-
status = gr.Markdown(value="Pronto")
|
| 66 |
-
result_json = gr.Code(label="Resultado JSON", language="json")
|
| 67 |
-
|
| 68 |
-
def run_dry_run(
|
| 69 |
-
project: str,
|
| 70 |
-
repo: str,
|
| 71 |
-
csv_path: str | None,
|
| 72 |
-
segments_path: str,
|
| 73 |
-
report_path: str,
|
| 74 |
-
) -> tuple[str, str]:
|
| 75 |
-
project = (project or "").strip()
|
| 76 |
-
if not project:
|
| 77 |
-
return "❌ Informe o project slug", "{}"
|
| 78 |
-
|
| 79 |
-
if not csv_path:
|
| 80 |
-
return "❌ Selecione o CSV de deteccoes", "{}"
|
| 81 |
-
|
| 82 |
-
if not segments_path or not Path(segments_path).exists():
|
| 83 |
-
return "❌ Pasta de segmentos invalida", "{}"
|
| 84 |
-
|
| 85 |
-
try:
|
| 86 |
-
result = run_ingest_segments_dry_run(
|
| 87 |
-
project_slug=project,
|
| 88 |
-
detections_csv=csv_path,
|
| 89 |
-
segments_root=segments_path,
|
| 90 |
-
)
|
| 91 |
-
if report_path.strip():
|
| 92 |
-
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 93 |
-
summary = (
|
| 94 |
-
"✅ Dry-run finalizado | "
|
| 95 |
-
f"Matched: {result['matched_rows']} | "
|
| 96 |
-
f"Unmatched: {result['unmatched_rows']} | "
|
| 97 |
-
f"Ambiguous: {result['ambiguous_rows']}"
|
| 98 |
-
)
|
| 99 |
-
return summary, _as_pretty_json(result)
|
| 100 |
-
except Exception as exc:
|
| 101 |
-
return f"❌ Dry-run falhou: {exc}", "{}"
|
| 102 |
-
|
| 103 |
-
def run_upload(
|
| 104 |
-
project: str,
|
| 105 |
-
repo: str,
|
| 106 |
-
csv_path: str | None,
|
| 107 |
-
segments_path: str,
|
| 108 |
-
token: str,
|
| 109 |
-
batch: float,
|
| 110 |
-
shard: float,
|
| 111 |
-
retries: float,
|
| 112 |
-
backoff: float,
|
| 113 |
-
resume_file: str,
|
| 114 |
-
report_path: str,
|
| 115 |
-
) -> tuple[str, str]:
|
| 116 |
-
project = (project or "").strip()
|
| 117 |
-
repo = (repo or "").strip()
|
| 118 |
-
if not project:
|
| 119 |
-
return "❌ Informe o project slug", "{}"
|
| 120 |
-
if not repo or "/" not in repo:
|
| 121 |
-
return "❌ Informe dataset repo no formato owner/repo", "{}"
|
| 122 |
-
if not csv_path:
|
| 123 |
-
return "❌ Selecione o CSV de deteccoes", "{}"
|
| 124 |
-
if not segments_path or not Path(segments_path).exists():
|
| 125 |
-
return "❌ Pasta de segmentos invalida", "{}"
|
| 126 |
-
|
| 127 |
-
try:
|
| 128 |
-
api = _build_api(token)
|
| 129 |
-
result = ingest_segments_to_hf(
|
| 130 |
-
api=api,
|
| 131 |
-
project_slug=project,
|
| 132 |
-
dataset_repo=repo,
|
| 133 |
-
detections_csv=csv_path,
|
| 134 |
-
segments_root=segments_path,
|
| 135 |
-
batch_size=int(batch),
|
| 136 |
-
shard_size=int(shard),
|
| 137 |
-
max_retries=int(retries),
|
| 138 |
-
retry_backoff_seconds=float(backoff),
|
| 139 |
-
resume_state_file=resume_file,
|
| 140 |
-
)
|
| 141 |
-
if report_path.strip():
|
| 142 |
-
Path(report_path).write_text(_as_pretty_json(result), encoding="utf-8")
|
| 143 |
-
summary = (
|
| 144 |
-
"✅ Upload finalizado | "
|
| 145 |
-
f"Uploaded now: {result['uploaded_audio_now']} | "
|
| 146 |
-
f"Skipped existing: {result['uploaded_audio_skipped_existing']} | "
|
| 147 |
-
f"Failed uploads: {result['failed_uploads']}"
|
| 148 |
-
)
|
| 149 |
-
return summary, _as_pretty_json(result)
|
| 150 |
-
except Exception as exc:
|
| 151 |
-
return f"❌ Upload falhou: {exc}", "{}"
|
| 152 |
-
|
| 153 |
-
dry_run_button.click(
|
| 154 |
-
fn=run_dry_run,
|
| 155 |
-
inputs=[project_slug, dataset_repo, detections_csv, segments_root, report_file],
|
| 156 |
-
outputs=[status, result_json],
|
| 157 |
-
)
|
| 158 |
-
|
| 159 |
-
upload_button.click(
|
| 160 |
-
fn=run_upload,
|
| 161 |
-
inputs=[
|
| 162 |
-
project_slug,
|
| 163 |
-
dataset_repo,
|
| 164 |
-
detections_csv,
|
| 165 |
-
segments_root,
|
| 166 |
-
hf_token,
|
| 167 |
-
batch_size,
|
| 168 |
-
shard_size,
|
| 169 |
-
max_retries,
|
| 170 |
-
retry_backoff_seconds,
|
| 171 |
-
resume_state_file,
|
| 172 |
-
report_file,
|
| 173 |
-
],
|
| 174 |
-
outputs=[status, result_json],
|
| 175 |
-
)
|
| 176 |
-
|
| 177 |
-
return demo
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/unit/test_hf_dataset_cli.py
DELETED
|
@@ -1,231 +0,0 @@
|
|
| 1 |
-
import json
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
|
| 4 |
-
import pandas as pd
|
| 5 |
-
import pytest
|
| 6 |
-
|
| 7 |
-
from cli.hf_dataset_cli import (
|
| 8 |
-
SCHEMA_VERSION,
|
| 9 |
-
_chunk_items,
|
| 10 |
-
_load_resume_state,
|
| 11 |
-
_save_resume_state,
|
| 12 |
-
build_manifest_payload,
|
| 13 |
-
build_shards_in_directory,
|
| 14 |
-
collect_verify_errors,
|
| 15 |
-
discover_audio_files,
|
| 16 |
-
load_detections_table,
|
| 17 |
-
sync_audio_batches,
|
| 18 |
-
)
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
def _sample_rows() -> list[dict[str, object]]:
|
| 22 |
-
return [
|
| 23 |
-
{
|
| 24 |
-
"detection_key": "aaaaaaaaaaaaaaaa",
|
| 25 |
-
"audio_id": "audio_1",
|
| 26 |
-
"scientific_name": "Cyanocorax cyanopogon",
|
| 27 |
-
"confidence": 0.91,
|
| 28 |
-
"start_time": 0.0,
|
| 29 |
-
"end_time": 1.2,
|
| 30 |
-
},
|
| 31 |
-
{
|
| 32 |
-
"detection_key": "bbbbbbbbbbbbbbbb",
|
| 33 |
-
"audio_id": "audio_2",
|
| 34 |
-
"scientific_name": "Ramphastos toco",
|
| 35 |
-
"confidence": 0.88,
|
| 36 |
-
"start_time": 0.5,
|
| 37 |
-
"end_time": 2.2,
|
| 38 |
-
},
|
| 39 |
-
{
|
| 40 |
-
"detection_key": "cccccccccccccccc",
|
| 41 |
-
"audio_id": "audio_2",
|
| 42 |
-
"scientific_name": "Psarocolius decumanus",
|
| 43 |
-
"confidence": 0.77,
|
| 44 |
-
"start_time": 1.1,
|
| 45 |
-
"end_time": 2.8,
|
| 46 |
-
},
|
| 47 |
-
]
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
def test_load_detections_table_csv(tmp_path: Path) -> None:
|
| 51 |
-
csv_path = tmp_path / "detections.csv"
|
| 52 |
-
pd.DataFrame(_sample_rows()).to_csv(csv_path, index=False)
|
| 53 |
-
|
| 54 |
-
loaded = load_detections_table(str(csv_path))
|
| 55 |
-
|
| 56 |
-
assert len(loaded) == 3
|
| 57 |
-
assert list(loaded.columns).count("detection_key") == 1
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
def test_load_detections_missing_columns(tmp_path: Path) -> None:
|
| 61 |
-
csv_path = tmp_path / "bad.csv"
|
| 62 |
-
pd.DataFrame([{"detection_key": "aaaaaaaaaaaaaaaa", "audio_id": "audio_1"}]).to_csv(csv_path, index=False)
|
| 63 |
-
|
| 64 |
-
with pytest.raises(ValueError):
|
| 65 |
-
_ = load_detections_table(str(csv_path))
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
def test_build_shards_in_directory(tmp_path: Path) -> None:
|
| 69 |
-
frame = pd.DataFrame(_sample_rows())
|
| 70 |
-
output_dir = tmp_path / "index" / "shards"
|
| 71 |
-
|
| 72 |
-
shards = build_shards_in_directory(frame=frame, output_dir=output_dir, shard_size=2)
|
| 73 |
-
|
| 74 |
-
assert len(shards) == 2
|
| 75 |
-
assert (output_dir / "shard-00000.parquet").exists()
|
| 76 |
-
assert (output_dir / "shard-00001.parquet").exists()
|
| 77 |
-
assert shards[0].rows == 2
|
| 78 |
-
assert shards[1].rows == 1
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def test_manifest_payload_has_expected_stats() -> None:
|
| 82 |
-
frame = pd.DataFrame(_sample_rows())
|
| 83 |
-
payload = build_manifest_payload(
|
| 84 |
-
project_slug="ppbio-rabeca",
|
| 85 |
-
dataset_repo="org/birdnet-ppbio-rabeca-dataset",
|
| 86 |
-
frame=frame,
|
| 87 |
-
shard_size=1000,
|
| 88 |
-
shard_metadata=[],
|
| 89 |
-
)
|
| 90 |
-
|
| 91 |
-
assert payload["schema_version"] == SCHEMA_VERSION
|
| 92 |
-
assert payload["index"]["total_detections"] == 3
|
| 93 |
-
assert payload["index"]["total_audio_files"] == 2
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
def test_collect_verify_errors_for_missing_prefix_and_shard() -> None:
|
| 97 |
-
manifest_payload = {
|
| 98 |
-
"schema_version": SCHEMA_VERSION,
|
| 99 |
-
"project_slug": "ppbio-rabeca",
|
| 100 |
-
"index": {
|
| 101 |
-
"shards": [
|
| 102 |
-
{
|
| 103 |
-
"path": "index/shards/shard-00000.parquet",
|
| 104 |
-
"rows": 2,
|
| 105 |
-
"sha256": "abc",
|
| 106 |
-
"size_bytes": 10,
|
| 107 |
-
}
|
| 108 |
-
]
|
| 109 |
-
},
|
| 110 |
-
}
|
| 111 |
-
|
| 112 |
-
repo_files = {
|
| 113 |
-
"manifest.json",
|
| 114 |
-
"audio/.gitkeep",
|
| 115 |
-
"index/.gitkeep",
|
| 116 |
-
"validations/.gitkeep",
|
| 117 |
-
"audit/.gitkeep",
|
| 118 |
-
}
|
| 119 |
-
|
| 120 |
-
errors = collect_verify_errors(
|
| 121 |
-
repo_files=repo_files,
|
| 122 |
-
manifest_payload=manifest_payload,
|
| 123 |
-
project_slug="ppbio-rabeca",
|
| 124 |
-
)
|
| 125 |
-
|
| 126 |
-
assert any("index/shards/" in message for message in errors)
|
| 127 |
-
assert any("Shard referenced in manifest" in message for message in errors)
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
def test_manifest_json_roundtrip(tmp_path: Path) -> None:
|
| 131 |
-
payload = {
|
| 132 |
-
"schema_version": SCHEMA_VERSION,
|
| 133 |
-
"project_slug": "ppbio-rabeca",
|
| 134 |
-
"index": {"shards": []},
|
| 135 |
-
}
|
| 136 |
-
manifest_path = tmp_path / "manifest.json"
|
| 137 |
-
manifest_path.write_text(json.dumps(payload), encoding="utf-8")
|
| 138 |
-
|
| 139 |
-
loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
|
| 140 |
-
assert loaded["project_slug"] == "ppbio-rabeca"
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
def test_discover_audio_files_filters_supported_extensions(tmp_path: Path) -> None:
|
| 144 |
-
audio_dir = tmp_path / "audio"
|
| 145 |
-
audio_dir.mkdir(parents=True)
|
| 146 |
-
(audio_dir / "a.wav").write_bytes(b"wav")
|
| 147 |
-
(audio_dir / "b.mp3").write_bytes(b"mp3")
|
| 148 |
-
(audio_dir / "c.txt").write_text("ignore", encoding="utf-8")
|
| 149 |
-
|
| 150 |
-
files = discover_audio_files(str(audio_dir))
|
| 151 |
-
names = [item.name for item in files]
|
| 152 |
-
|
| 153 |
-
assert names == ["a.wav", "b.mp3"]
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
def test_chunk_items() -> None:
|
| 157 |
-
files = [Path(f"f{i}.wav") for i in range(5)]
|
| 158 |
-
chunks = _chunk_items(files, 2)
|
| 159 |
-
|
| 160 |
-
assert len(chunks) == 3
|
| 161 |
-
assert len(chunks[0]) == 2
|
| 162 |
-
assert len(chunks[1]) == 2
|
| 163 |
-
assert len(chunks[2]) == 1
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
def test_resume_state_roundtrip(tmp_path: Path) -> None:
|
| 167 |
-
state_file = tmp_path / "state.json"
|
| 168 |
-
_save_resume_state(state_file=state_file, uploaded={"audio/a.wav"}, failed={"audio/b.wav"})
|
| 169 |
-
|
| 170 |
-
loaded = _load_resume_state(state_file=state_file)
|
| 171 |
-
|
| 172 |
-
assert loaded["uploaded"] == ["audio/a.wav"]
|
| 173 |
-
assert loaded["failed"] == ["audio/b.wav"]
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
def test_sync_audio_batches_skips_remote_and_checkpointed(tmp_path: Path) -> None:
|
| 177 |
-
class FakeApi:
|
| 178 |
-
def __init__(self) -> None:
|
| 179 |
-
self.files = {
|
| 180 |
-
"audio/existing.wav",
|
| 181 |
-
"manifest.json",
|
| 182 |
-
"audio/.gitkeep",
|
| 183 |
-
"index/.gitkeep",
|
| 184 |
-
"index/shards/.gitkeep",
|
| 185 |
-
"validations/.gitkeep",
|
| 186 |
-
"audit/.gitkeep",
|
| 187 |
-
}
|
| 188 |
-
self.uploaded: list[str] = []
|
| 189 |
-
|
| 190 |
-
def create_repo(self, **kwargs: object) -> None:
|
| 191 |
-
_ = kwargs
|
| 192 |
-
|
| 193 |
-
def list_repo_files(self, **kwargs: object) -> list[str]:
|
| 194 |
-
_ = kwargs
|
| 195 |
-
return sorted(self.files)
|
| 196 |
-
|
| 197 |
-
def upload_file(self, *, path_or_fileobj: object, path_in_repo: str, **kwargs: object) -> None:
|
| 198 |
-
_ = path_or_fileobj
|
| 199 |
-
_ = kwargs
|
| 200 |
-
self.uploaded.append(path_in_repo)
|
| 201 |
-
self.files.add(path_in_repo)
|
| 202 |
-
|
| 203 |
-
audio_dir = tmp_path / "audio"
|
| 204 |
-
audio_dir.mkdir(parents=True)
|
| 205 |
-
(audio_dir / "existing.wav").write_bytes(b"x")
|
| 206 |
-
(audio_dir / "new.wav").write_bytes(b"x")
|
| 207 |
-
(audio_dir / "checkpointed.wav").write_bytes(b"x")
|
| 208 |
-
|
| 209 |
-
state_file = tmp_path / "resume.json"
|
| 210 |
-
state_file.write_text(
|
| 211 |
-
json.dumps({"uploaded": ["audio/checkpointed.wav"], "failed": []}),
|
| 212 |
-
encoding="utf-8",
|
| 213 |
-
)
|
| 214 |
-
|
| 215 |
-
api = FakeApi()
|
| 216 |
-
result = sync_audio_batches(
|
| 217 |
-
api=api,
|
| 218 |
-
project_slug="ppbio-rabeca",
|
| 219 |
-
dataset_repo="org/birdnet-ppbio-rabeca-dataset",
|
| 220 |
-
local_audio_dir=str(audio_dir),
|
| 221 |
-
batch_size=2,
|
| 222 |
-
max_retries=1,
|
| 223 |
-
retry_backoff_seconds=0,
|
| 224 |
-
resume_state_file=str(state_file),
|
| 225 |
-
)
|
| 226 |
-
|
| 227 |
-
assert result["total_local_audio_files"] == 3
|
| 228 |
-
assert result["pending_uploads"] == 1
|
| 229 |
-
assert result["uploaded_now"] == 1
|
| 230 |
-
assert result["failed"] == 0
|
| 231 |
-
assert api.uploaded == ["audio/new.wav"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/unit/test_hf_ingest_segments_cli.py
DELETED
|
@@ -1,127 +0,0 @@
|
|
| 1 |
-
from pathlib import Path
|
| 2 |
-
|
| 3 |
-
import pandas as pd
|
| 4 |
-
|
| 5 |
-
from cli.hf_dataset_cli import (
|
| 6 |
-
_build_detection_key,
|
| 7 |
-
build_parser,
|
| 8 |
-
parse_segment_filename,
|
| 9 |
-
run_ingest_segments_dry_run,
|
| 10 |
-
)
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
def test_parse_segment_filename_extracts_components():
|
| 14 |
-
parsed = parse_segment_filename("Aiuab_LN-0500_U16_20260121_153400_42.0-45.0s_98%.wav")
|
| 15 |
-
assert parsed is not None
|
| 16 |
-
source_stem, start_time, end_time, confidence_pct = parsed
|
| 17 |
-
assert source_stem == "Aiuab_LN-0500_U16_20260121_153400"
|
| 18 |
-
assert start_time == 42.0
|
| 19 |
-
assert end_time == 45.0
|
| 20 |
-
assert confidence_pct == 98
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
def test_detection_key_is_stable_and_long_enough():
|
| 24 |
-
key1 = _build_detection_key(
|
| 25 |
-
project_slug="ppbio-aiuaba",
|
| 26 |
-
source_file="Aiuab_LN-0500_U16_20260121_153400.wav",
|
| 27 |
-
scientific_name="Cyanocorax cyanopogon",
|
| 28 |
-
start_time=42.0,
|
| 29 |
-
end_time=45.0,
|
| 30 |
-
)
|
| 31 |
-
key2 = _build_detection_key(
|
| 32 |
-
project_slug="ppbio-aiuaba",
|
| 33 |
-
source_file="Aiuab_LN-0500_U16_20260121_153400.wav",
|
| 34 |
-
scientific_name="Cyanocorax cyanopogon",
|
| 35 |
-
start_time=42.0,
|
| 36 |
-
end_time=45.0,
|
| 37 |
-
)
|
| 38 |
-
assert key1 == key2
|
| 39 |
-
assert len(key1) >= 16
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
def test_run_ingest_segments_dry_run_matches_rows(tmp_path: Path):
|
| 43 |
-
segments_root = tmp_path / "segments"
|
| 44 |
-
species_dir = segments_root / "Cyanocorax cyanopogon"
|
| 45 |
-
species_dir.mkdir(parents=True, exist_ok=True)
|
| 46 |
-
|
| 47 |
-
(species_dir / "Aiuab_LN-0500_U16_20260121_153400_42.0-45.0s_98%.wav").write_bytes(b"dummy")
|
| 48 |
-
(species_dir / "Aiuab_LN-0500_U16_20260121_153400_45.0-48.0s_88%.wav").write_bytes(b"dummy")
|
| 49 |
-
|
| 50 |
-
frame = pd.DataFrame(
|
| 51 |
-
[
|
| 52 |
-
{
|
| 53 |
-
"locality": "PPBIO Aiuaba",
|
| 54 |
-
"point": "Aiuab_LN-0500_U16",
|
| 55 |
-
"date_folder": "20260121",
|
| 56 |
-
"source_file": "Aiuab_LN-0500_U16_20260121_153400.wav",
|
| 57 |
-
"scientific_name": "Cyanocorax cyanopogon",
|
| 58 |
-
"common_name": "White-naped Jay",
|
| 59 |
-
"confidence": 0.98,
|
| 60 |
-
"start_time": 42.0,
|
| 61 |
-
"end_time": 45.0,
|
| 62 |
-
"exact_start": 42.0,
|
| 63 |
-
"exact_end": 45.0,
|
| 64 |
-
"min_freq": 0,
|
| 65 |
-
"max_freq": 15000,
|
| 66 |
-
"box_source": "BirdNET_Original",
|
| 67 |
-
"label": "Cyanocorax cyanopogon_White-naped Jay",
|
| 68 |
-
},
|
| 69 |
-
{
|
| 70 |
-
"locality": "PPBIO Aiuaba",
|
| 71 |
-
"point": "Aiuab_LN-0500_U16",
|
| 72 |
-
"date_folder": "20260121",
|
| 73 |
-
"source_file": "Aiuab_LN-0500_U16_20260121_153400.wav",
|
| 74 |
-
"scientific_name": "Cyanocorax cyanopogon",
|
| 75 |
-
"common_name": "White-naped Jay",
|
| 76 |
-
"confidence": 0.50,
|
| 77 |
-
"start_time": 0.0,
|
| 78 |
-
"end_time": 3.0,
|
| 79 |
-
"exact_start": 0.0,
|
| 80 |
-
"exact_end": 3.0,
|
| 81 |
-
"min_freq": 0,
|
| 82 |
-
"max_freq": 15000,
|
| 83 |
-
"box_source": "BirdNET_Original",
|
| 84 |
-
"label": "Cyanocorax cyanopogon_White-naped Jay",
|
| 85 |
-
},
|
| 86 |
-
]
|
| 87 |
-
)
|
| 88 |
-
|
| 89 |
-
csv_path = tmp_path / "detections.csv"
|
| 90 |
-
frame.to_csv(csv_path, index=False)
|
| 91 |
-
|
| 92 |
-
result = run_ingest_segments_dry_run(
|
| 93 |
-
project_slug="ppbio-aiuaba",
|
| 94 |
-
detections_csv=str(csv_path),
|
| 95 |
-
segments_root=str(segments_root),
|
| 96 |
-
)
|
| 97 |
-
|
| 98 |
-
assert result["mode"] == "dry-run"
|
| 99 |
-
assert result["csv_rows_total"] == 2
|
| 100 |
-
assert result["segments_found_total"] == 2
|
| 101 |
-
assert result["matched_rows"] == 1
|
| 102 |
-
assert result["unmatched_rows"] == 1
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
def test_build_parser_ingest_segments_defaults():
|
| 106 |
-
parser = build_parser()
|
| 107 |
-
args = parser.parse_args(
|
| 108 |
-
[
|
| 109 |
-
"ingest-segments",
|
| 110 |
-
"--project-slug",
|
| 111 |
-
"ppbio-aiuaba",
|
| 112 |
-
"--dataset-repo",
|
| 113 |
-
"owner/repo",
|
| 114 |
-
"--detections-csv",
|
| 115 |
-
"detections.csv",
|
| 116 |
-
"--segments-root",
|
| 117 |
-
"segments",
|
| 118 |
-
]
|
| 119 |
-
)
|
| 120 |
-
|
| 121 |
-
assert args.command == "ingest-segments"
|
| 122 |
-
assert args.batch_size == 200
|
| 123 |
-
assert args.shard_size == 10000
|
| 124 |
-
assert args.max_retries == 3
|
| 125 |
-
assert args.retry_backoff_seconds == 1.0
|
| 126 |
-
assert args.resume_state_file == ".ingest-segments-state.json"
|
| 127 |
-
assert args.dry_run is False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|