Spaces:
Sleeping
Sleeping
Commit ·
baf0e08
1
Parent(s): b728b04
Implement BirdNET Uploader CLI with deduplication, error handling, and session management
Browse files- Add Deduplicator class for managing remote file checks and caching
- Introduce error handling with specific exceptions and hints
- Create session management for upload checkpoints
- Implement repository service for dataset creation and validation
- Develop local scanner for audio file detection and species inference
- Add hash utilities for file integrity checks
- Build command-line interface using Click for user interactions
- Create a minimal TUI for user experience during uploads
- Write comprehensive unit tests for all components
- BIRDNET_UPLOADER_REDESIGN.md +665 -0
- README.md +360 -101
- SPRINT1_KICKOFF.md +769 -0
- SPRINT_ROADMAP.md +649 -0
- pyproject.toml +3 -0
- requirements.txt +4 -0
- src/uploader_cli/__init__.py +7 -0
- src/uploader_cli/auth_service.py +60 -0
- src/uploader_cli/batch_uploader.py +102 -0
- src/uploader_cli/config.py +61 -0
- src/uploader_cli/deduplicator.py +115 -0
- src/uploader_cli/error_handler.py +29 -0
- src/uploader_cli/exceptions.py +26 -0
- src/uploader_cli/hash_utils.py +21 -0
- src/uploader_cli/main.py +107 -0
- src/uploader_cli/repo_service.py +110 -0
- src/uploader_cli/scanner.py +62 -0
- src/uploader_cli/session_manager.py +98 -0
- src/uploader_cli/tui.py +22 -0
- tests/unit/test_batch_uploader.py +58 -0
- tests/unit/test_uploader_cli_auth_service.py +80 -0
- tests/unit/test_uploader_cli_config.py +25 -0
- tests/unit/test_uploader_cli_deduplicator.py +55 -0
- tests/unit/test_uploader_cli_error_handler.py +25 -0
- tests/unit/test_uploader_cli_hash_utils.py +25 -0
- tests/unit/test_uploader_cli_repo_service.py +124 -0
- tests/unit/test_uploader_cli_scanner.py +34 -0
- tests/unit/test_uploader_cli_session.py +15 -0
- tests/unit/test_uploader_cli_session_manager.py +66 -0
BIRDNET_UPLOADER_REDESIGN.md
ADDED
|
@@ -0,0 +1,665 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BirdNET Uploader: Redesign CLI Moderno
|
| 2 |
+
|
| 3 |
+
## Visão Geral
|
| 4 |
+
|
| 5 |
+
Executável local moderno com **um único fluxo integrado**: login → criar repositório → apontar pasta → fazer upload.
|
| 6 |
+
|
| 7 |
+
Interface limpa, baseada em TUI (Textual) ou rich terminal output, compatível com a estrutura esperada pelo BirdNET Validator.
|
| 8 |
+
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
## 1. FLUXO DE UX ÚNICO INTEGRADO
|
| 12 |
+
|
| 13 |
+
### Tela 1: Autenticação
|
| 14 |
+
```
|
| 15 |
+
┌─────────────────────────────────────────────────┐
|
| 16 |
+
│ BirdNET Uploader │
|
| 17 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 18 |
+
│ │
|
| 19 |
+
│ 🐦 Autentique-se no Hugging Face │
|
| 20 |
+
│ │
|
| 21 |
+
│ [i] Cole seu token HF (Settings → Access │
|
| 22 |
+
│ Tokens em huggingface.co) │
|
| 23 |
+
│ │
|
| 24 |
+
│ Token: [●●●●●●●●●●●●●●●●●●●●●●] │
|
| 25 |
+
│ │
|
| 26 |
+
│ [ Continuar ] [ Sair ] │
|
| 27 |
+
│ │
|
| 28 |
+
└─────────────────────────────────────────────────┘
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
### Tela 2: Configuração (Um formulário integrado)
|
| 32 |
+
```
|
| 33 |
+
┌─────────────────────────────────────────────────┐
|
| 34 |
+
│ BirdNET Uploader - Configuração │
|
| 35 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 36 |
+
│ │
|
| 37 |
+
│ Conectado como: alice@example.com │
|
| 38 |
+
│ │
|
| 39 |
+
│ REPOSITÓRIO │
|
| 40 |
+
│ Nome do repo: [my-birds-2026 ] │
|
| 41 |
+
│ Visibilidade: ( ) Público (●) Privado │
|
| 42 |
+
│ │
|
| 43 |
+
│ ARQUIVOS │
|
| 44 |
+
│ Pasta segmentos: [C:\audio\segments] │
|
| 45 |
+
│ Arquivo CSV: [C:\audio\metadata.csv] (opt)
|
| 46 |
+
│ [ Verificar ] │
|
| 47 |
+
│ ✓ 1.247 arquivos encontrados │
|
| 48 |
+
│ ✓ Tamanho total: 53 GB │
|
| 49 |
+
│ ✓ CSV compatível: 1.200 linhas │
|
| 50 |
+
│ │
|
| 51 |
+
│ [ Criar Repo e Fazer Upload ] │
|
| 52 |
+
│ [ Voltar ] │
|
| 53 |
+
│ │
|
| 54 |
+
└─────────────────────────────────────────────────┘
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
### Tela 3: Resumo e Confirmação
|
| 58 |
+
```
|
| 59 |
+
┌─────────────────────────────────────────────────┐
|
| 60 |
+
│ Pronto para Começar │
|
| 61 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 62 |
+
│ │
|
| 63 |
+
│ Repositório: alice/my-birds-2026 │
|
| 64 |
+
│ Visibilidade: Privado │
|
| 65 |
+
│ URL: https://huggingface.co/datasets/alice/.. │
|
| 66 |
+
│ │
|
| 67 |
+
│ Origem: │
|
| 68 |
+
│ Pasta: C:\audio\segments │
|
| 69 |
+
│ Arquivos: 1.247 │
|
| 70 |
+
│ Tamanho: 53 GB │
|
| 71 |
+
│ CSV: metadata.csv (1.200 linhas) │
|
| 72 |
+
│ │
|
| 73 |
+
│ Duração estimada: ~4 horas (com conexão 10MB/s)
|
| 74 |
+
│ │
|
| 75 |
+
│ [ Iniciar Upload ] [ Cancelar ] │
|
| 76 |
+
│ │
|
| 77 |
+
└─────────────────────────────────────────────────┘
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
### Tela 4: Upload em Andamento
|
| 81 |
+
```
|
| 82 |
+
┌─────────────────────────────────────────────────┐
|
| 83 |
+
│ Upload em Andamento │
|
| 84 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 85 |
+
│ │
|
| 86 |
+
│ Progresso Global │
|
| 87 |
+
│ ████████████████░░░░░░░░░░░░░░ 62.3% │
|
| 88 |
+
│ │
|
| 89 |
+
│ Arquivos: 775/1.247 (skipped: 18, erro: 2) │
|
| 90 |
+
│ Dados: 33 GB / 53 GB │
|
| 91 |
+
│ Velocidade: 9.6 MB/s | ETA: 01h 15m │
|
| 92 |
+
│ │
|
| 93 |
+
│ Arquivo Atual: Parrot_Ara_001.wav │
|
| 94 |
+
│ ████████████░░░░░░░░░░░░░░░░░░░ 45.2% │
|
| 95 |
+
│ 3.2 / 7.1 MB │
|
| 96 |
+
│ │
|
| 97 |
+
│ Último evento: ✓ Thrush_042.wav enviado (2s) │
|
| 98 |
+
│ │
|
| 99 |
+
│ [ Pausar ] [ Cancelar ] [ Detalhes ] │
|
| 100 |
+
│ │
|
| 101 |
+
└─────────────────────────────────────────────────┘
|
| 102 |
+
```
|
| 103 |
+
|
| 104 |
+
### Tela 5: Resumo Final
|
| 105 |
+
```
|
| 106 |
+
┌─────────────────────────────────────────────────┐
|
| 107 |
+
│ ✓ Upload Concluído! │
|
| 108 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 109 |
+
│ │
|
| 110 |
+
│ Repositório: alice/my-birds-2026 │
|
| 111 |
+
│ URL: https://huggingface.co/datasets/alice/.. │
|
| 112 |
+
│ │
|
| 113 |
+
│ Resumo: │
|
| 114 |
+
│ ✓ Enviados: 1.227 arquivos │
|
| 115 |
+
│ → Pulados: 18 (já existiam) │
|
| 116 |
+
│ ✗ Erros: 2 (revisar detalhes) │
|
| 117 |
+
│ ⏱ Tempo total: 4h 23min │
|
| 118 |
+
│ 📊 Velocidade média: 9.8 MB/s │
|
| 119 |
+
│ │
|
| 120 |
+
│ Próximos passos: │
|
| 121 |
+
│ 1. Validar dados com BirdNET Validator │
|
| 122 |
+
│ 2. Verificar relatório de ingestão │
|
| 123 |
+
│ │
|
| 124 |
+
│ Relatório salvo em: ~/.birdnet-uploader/ │
|
| 125 |
+
│ sessions/upload-xxx/ │
|
| 126 |
+
│ report.json │
|
| 127 |
+
│ │
|
| 128 |
+
│ [ Ver Relatório Completo ] [ Sair ] │
|
| 129 |
+
│ │
|
| 130 |
+
└─────────────────────────────────────────────────┘
|
| 131 |
+
```
|
| 132 |
+
|
| 133 |
+
---
|
| 134 |
+
|
| 135 |
+
## 2. ARQUITETURA LIMPA E MODULAR
|
| 136 |
+
|
| 137 |
+
### Estrutura de Diretórios do Projeto
|
| 138 |
+
|
| 139 |
+
```
|
| 140 |
+
birdnet-uploader/
|
| 141 |
+
├── src/
|
| 142 |
+
│ ├── __init__.py
|
| 143 |
+
│ ├── main.py # Entrypoint principal
|
| 144 |
+
│ ├── config.py # Config app e constantes
|
| 145 |
+
│ ├── auth/
|
| 146 |
+
│ │ ├── __init__.py
|
| 147 |
+
│ │ ├── auth_service.py # Login, token, keyring
|
| 148 |
+
│ │ └── token_manager.py # Gerenciamento de token
|
| 149 |
+
│ ├── repository/
|
| 150 |
+
│ │ ├── __init__.py
|
| 151 |
+
│ │ ├── repo_service.py # Criar/validar dataset HF
|
| 152 |
+
│ │ └── structure_initializer.py # Init estrutura padrão
|
| 153 |
+
│ ├── upload/
|
| 154 |
+
│ │ ├── __init__.py
|
| 155 |
+
│ │ ├── scanner.py # Varrer pasta local
|
| 156 |
+
│ │ ├── uploader.py # Motor de upload por lotes
|
| 157 |
+
│ │ ├── deduplicator.py # Checagem de duplicação
|
| 158 |
+
│ │ └── batch_processor.py # Processamento de lotes
|
| 159 |
+
│ ├── csv/
|
| 160 |
+
│ │ ├── __init__.py
|
| 161 |
+
│ │ ├── csv_matcher.py # Matching CSV + segmentos
|
| 162 |
+
│ │ └── indexer.py # Geração de índice/shards
|
| 163 |
+
│ ├── session/
|
| 164 |
+
│ │ ├── __init__.py
|
| 165 |
+
│ │ ├── session_manager.py # Gerenciamento de sessão
|
| 166 |
+
│ │ ├── checkpoint.py # Persistência de checkpoint
|
| 167 |
+
│ │ └── state_store.py # State file I/O
|
| 168 |
+
│ ├── ui/
|
| 169 |
+
│ │ ├── __init__.py
|
| 170 |
+
│ │ ├── tui.py # Interface Textual (TUI)
|
| 171 |
+
│ │ ├── progress_renderer.py # Barra de progresso rich
|
| 172 |
+
│ │ └── prompts.py # Diálogos e inputs
|
| 173 |
+
│ ├── logger/
|
| 174 |
+
│ │ ├── __init__.py
|
| 175 |
+
│ │ ├── structured_logger.py # Logs estruturados JSONL
|
| 176 |
+
│ │ └── telemetry.py # Eventos de telemetria
|
| 177 |
+
│ └── utils/
|
| 178 |
+
│ ├── __init__.py
|
| 179 |
+
│ ├── file_utils.py # Operações de arquivo
|
| 180 |
+
│ ├── hash_utils.py # Computação de hash
|
| 181 |
+
│ └── error_handler.py # Tratamento de erros
|
| 182 |
+
├── build/
|
| 183 |
+
│ ├── pyinstaller_config.spec # Config do PyInstaller
|
| 184 |
+
│ └── build.py # Script de build
|
| 185 |
+
├── tests/
|
| 186 |
+
│ ├── __init__.py
|
| 187 |
+
│ ├── test_auth.py
|
| 188 |
+
│ ├── test_uploader.py
|
| 189 |
+
│ ├── test_session.py
|
| 190 |
+
│ └── test_csv_matcher.py
|
| 191 |
+
├── README.md
|
| 192 |
+
├── requirements.txt
|
| 193 |
+
└── setup.py
|
| 194 |
+
```
|
| 195 |
+
|
| 196 |
+
### Dependências Principais
|
| 197 |
+
|
| 198 |
+
```
|
| 199 |
+
# requirements.txt
|
| 200 |
+
huggingface_hub>=0.28.0 # HF Hub API
|
| 201 |
+
click>=8.0.0 # CLI framework
|
| 202 |
+
textual>=0.30.0 # TUI moderna
|
| 203 |
+
rich>=13.0.0 # Terminal formatting
|
| 204 |
+
pydantic>=2.0.0 # Validação de dados
|
| 205 |
+
pandas>=2.0.0 # CSV parsing
|
| 206 |
+
pyarrow>=14.0.0 # Parquet para shards
|
| 207 |
+
keyring>=24.0.0 # Armazenamento seguro
|
| 208 |
+
cryptography>=41.0.0 # Criptografia local
|
| 209 |
+
```
|
| 210 |
+
|
| 211 |
+
---
|
| 212 |
+
|
| 213 |
+
## 3. COMPATIBILIDADE COM ESTRUTURA DO VALIDADOR
|
| 214 |
+
|
| 215 |
+
### Estrutura Esperada do Dataset
|
| 216 |
+
|
| 217 |
+
```
|
| 218 |
+
dataset_repo_id = "username/project-dataset"
|
| 219 |
+
|
| 220 |
+
audio/
|
| 221 |
+
├── {project_slug}/
|
| 222 |
+
│ ├── Specie_A/
|
| 223 |
+
│ │ ├── segment_001.wav
|
| 224 |
+
│ │ ├── segment_002.wav
|
| 225 |
+
│ │ └── ...
|
| 226 |
+
│ ├── Specie_B/
|
| 227 |
+
│ │ └── ...
|
| 228 |
+
│ └── ...
|
| 229 |
+
|
| 230 |
+
index/
|
| 231 |
+
├── shards/
|
| 232 |
+
│ ├── shard-00000.parquet
|
| 233 |
+
│ ├── shard-00001.parquet
|
| 234 |
+
│ └── ...
|
| 235 |
+
└── manifest.json
|
| 236 |
+
|
| 237 |
+
validations/
|
| 238 |
+
├── results-20250419.parquet
|
| 239 |
+
|
| 240 |
+
audit/
|
| 241 |
+
├── ingestion-runs/
|
| 242 |
+
│ └── 20250419T100000Z.json
|
| 243 |
+
```
|
| 244 |
+
|
| 245 |
+
### Manifest Format (Mantém compatibilidade)
|
| 246 |
+
|
| 247 |
+
```json
|
| 248 |
+
{
|
| 249 |
+
"schema_version": "1.0.0",
|
| 250 |
+
"project_slug": "my-birds",
|
| 251 |
+
"dataset_repo_id": "alice/my-birds-2026",
|
| 252 |
+
"updated_at": "2025-04-19T10:45:30Z",
|
| 253 |
+
"index": {
|
| 254 |
+
"total_detections": 1227,
|
| 255 |
+
"total_audio_files": 1227,
|
| 256 |
+
"shard_size": 10000,
|
| 257 |
+
"shards": [
|
| 258 |
+
{
|
| 259 |
+
"path": "index/shards/shard-00000.parquet",
|
| 260 |
+
"rows": 10000,
|
| 261 |
+
"sha256": "abc123...",
|
| 262 |
+
"size_bytes": 5242880
|
| 263 |
+
}
|
| 264 |
+
]
|
| 265 |
+
}
|
| 266 |
+
}
|
| 267 |
+
```
|
| 268 |
+
|
| 269 |
+
---
|
| 270 |
+
|
| 271 |
+
## 4. MÓDULO POR MÓDULO
|
| 272 |
+
|
| 273 |
+
### 4.1 Auth Service (`src/auth/auth_service.py`)
|
| 274 |
+
|
| 275 |
+
```python
|
| 276 |
+
from keyring import get_password, set_password
|
| 277 |
+
from huggingface_hub import HfApi, login
|
| 278 |
+
|
| 279 |
+
class AuthService:
|
| 280 |
+
KEYRING_SERVICE = "birdnet-uploader"
|
| 281 |
+
|
| 282 |
+
def authenticate(self, token: str) -> dict:
|
| 283 |
+
"""Valida token e retorna context do usuário."""
|
| 284 |
+
try:
|
| 285 |
+
api = HfApi(token=token)
|
| 286 |
+
user_info = api.whoami()
|
| 287 |
+
|
| 288 |
+
# Salvar em keyring seguro
|
| 289 |
+
set_password(self.KEYRING_SERVICE, "hf_token", token)
|
| 290 |
+
|
| 291 |
+
return {
|
| 292 |
+
"username": user_info["name"],
|
| 293 |
+
"email": user_info.get("email", ""),
|
| 294 |
+
"user_id": user_info["user_id"],
|
| 295 |
+
"token_expiry": self._estimate_expiry()
|
| 296 |
+
}
|
| 297 |
+
except Exception as e:
|
| 298 |
+
raise AuthenticationError(f"Token inválido: {str(e)}")
|
| 299 |
+
|
| 300 |
+
def get_token(self) -> str:
|
| 301 |
+
"""Recupera token do keyring."""
|
| 302 |
+
token = get_password(self.KEYRING_SERVICE, "hf_token")
|
| 303 |
+
if not token:
|
| 304 |
+
raise TokenNotFoundError("Token não encontrado. Execute 'birdnet auth login'")
|
| 305 |
+
return token
|
| 306 |
+
```
|
| 307 |
+
|
| 308 |
+
### 4.2 Repository Service (`src/repository/repo_service.py`)
|
| 309 |
+
|
| 310 |
+
```python
|
| 311 |
+
class RepositoryService:
|
| 312 |
+
def create_dataset(self,
|
| 313 |
+
repo_name: str,
|
| 314 |
+
private: bool = True) -> str:
|
| 315 |
+
"""Cria dataset no HF com estrutura padrão."""
|
| 316 |
+
api = HfApi(token=self.get_token())
|
| 317 |
+
|
| 318 |
+
repo_id = f"{username}/{repo_name}"
|
| 319 |
+
|
| 320 |
+
# Criar repo
|
| 321 |
+
repo = api.create_repo(
|
| 322 |
+
repo_id=repo_id,
|
| 323 |
+
repo_type="dataset",
|
| 324 |
+
private=private,
|
| 325 |
+
exist_ok=True
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
# Inicializar estrutura padrão
|
| 329 |
+
self._init_structure(repo_id)
|
| 330 |
+
|
| 331 |
+
return repo_id
|
| 332 |
+
|
| 333 |
+
def _init_structure(self, repo_id: str):
|
| 334 |
+
"""Inicializa pastas e arquivos padrão."""
|
| 335 |
+
folders = [
|
| 336 |
+
"audio/{project_slug}",
|
| 337 |
+
"index/shards",
|
| 338 |
+
"validations",
|
| 339 |
+
"audit/ingestion-runs"
|
| 340 |
+
]
|
| 341 |
+
|
| 342 |
+
for folder in folders:
|
| 343 |
+
# Criar .gitkeep
|
| 344 |
+
api.upload_file(
|
| 345 |
+
path_or_fileobj=BytesIO(b""),
|
| 346 |
+
path_in_repo=f"{folder}/.gitkeep",
|
| 347 |
+
repo_id=repo_id,
|
| 348 |
+
repo_type="dataset"
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
# Criar manifest.json inicial
|
| 352 |
+
manifest = {
|
| 353 |
+
"schema_version": "1.0.0",
|
| 354 |
+
"project_slug": project_slug,
|
| 355 |
+
"dataset_repo_id": repo_id,
|
| 356 |
+
"index": {
|
| 357 |
+
"total_detections": 0,
|
| 358 |
+
"total_audio_files": 0,
|
| 359 |
+
"shard_size": 10000,
|
| 360 |
+
"shards": []
|
| 361 |
+
}
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
api.upload_file(
|
| 365 |
+
path_or_fileobj=BytesIO(json.dumps(manifest).encode()),
|
| 366 |
+
path_in_repo="index/manifest.json",
|
| 367 |
+
repo_id=repo_id,
|
| 368 |
+
repo_type="dataset"
|
| 369 |
+
)
|
| 370 |
+
```
|
| 371 |
+
|
| 372 |
+
### 4.3 Scanner (`src/upload/scanner.py`)
|
| 373 |
+
|
| 374 |
+
```python
|
| 375 |
+
class LocalScanner:
|
| 376 |
+
def scan_folder(self, folder_path: str) -> dict:
|
| 377 |
+
"""Varre pasta recursiva e retorna estrutura."""
|
| 378 |
+
structure = {}
|
| 379 |
+
total_size = 0
|
| 380 |
+
file_count = 0
|
| 381 |
+
|
| 382 |
+
for root, dirs, files in os.walk(folder_path):
|
| 383 |
+
for file in files:
|
| 384 |
+
if file.lower().endswith(('.wav', '.mp3', '.flac', '.ogg')):
|
| 385 |
+
full_path = os.path.join(root, file)
|
| 386 |
+
|
| 387 |
+
# Inferir espécie do caminho ou nome
|
| 388 |
+
specie = self._infer_specie(root, file)
|
| 389 |
+
|
| 390 |
+
if specie not in structure:
|
| 391 |
+
structure[specie] = []
|
| 392 |
+
|
| 393 |
+
size = os.path.getsize(full_path)
|
| 394 |
+
structure[specie].append({
|
| 395 |
+
"name": file,
|
| 396 |
+
"full_path": full_path,
|
| 397 |
+
"size": size,
|
| 398 |
+
"local_hash": compute_hash(full_path)
|
| 399 |
+
})
|
| 400 |
+
|
| 401 |
+
total_size += size
|
| 402 |
+
file_count += 1
|
| 403 |
+
|
| 404 |
+
return {
|
| 405 |
+
"total_files": file_count,
|
| 406 |
+
"total_size": total_size,
|
| 407 |
+
"by_specie": structure,
|
| 408 |
+
"scan_timestamp": datetime.now()
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
def _infer_specie(self, root: str, file: str) -> str:
|
| 412 |
+
"""Inferir espécie do caminho da pasta."""
|
| 413 |
+
# Estratégia 1: última pasta em root é a espécie
|
| 414 |
+
parts = root.split(os.sep)
|
| 415 |
+
if len(parts) > 0:
|
| 416 |
+
return parts[-1]
|
| 417 |
+
|
| 418 |
+
# Fallback: usar nome do arquivo
|
| 419 |
+
return file.replace(".wav", "").split("_")[0]
|
| 420 |
+
```
|
| 421 |
+
|
| 422 |
+
### 4.4 Uploader (`src/upload/uploader.py`)
|
| 423 |
+
|
| 424 |
+
```python
|
| 425 |
+
class BatchUploader:
|
| 426 |
+
def __init__(self, repo_id: str, project_slug: str, token: str):
|
| 427 |
+
self.repo_id = repo_id
|
| 428 |
+
self.project_slug = project_slug
|
| 429 |
+
self.api = HfApi(token=token)
|
| 430 |
+
self.session = SessionManager(repo_id)
|
| 431 |
+
|
| 432 |
+
async def upload_files(self,
|
| 433 |
+
structure: dict,
|
| 434 |
+
csv_matcher: Optional[CSVMatcher] = None,
|
| 435 |
+
batch_size: int = 10,
|
| 436 |
+
on_progress: Optional[Callable] = None) -> dict:
|
| 437 |
+
"""Upload em lotes com retomada."""
|
| 438 |
+
checkpoint = self.session.load_checkpoint()
|
| 439 |
+
|
| 440 |
+
results = {
|
| 441 |
+
"total": 0,
|
| 442 |
+
"uploaded": 0,
|
| 443 |
+
"skipped": 0,
|
| 444 |
+
"failed": 0,
|
| 445 |
+
"errors": []
|
| 446 |
+
}
|
| 447 |
+
|
| 448 |
+
for specie, files in structure["by_specie"].items():
|
| 449 |
+
# Processar em lotes
|
| 450 |
+
for i in range(0, len(files), batch_size):
|
| 451 |
+
batch = files[i:i + batch_size]
|
| 452 |
+
|
| 453 |
+
# Verificar checkpoint
|
| 454 |
+
if self.session.is_batch_done(specie, i):
|
| 455 |
+
results["skipped"] += len(batch)
|
| 456 |
+
continue
|
| 457 |
+
|
| 458 |
+
# Enviar batch
|
| 459 |
+
for file_info in batch:
|
| 460 |
+
try:
|
| 461 |
+
if self._should_skip(file_info):
|
| 462 |
+
results["skipped"] += 1
|
| 463 |
+
self.session.mark_skipped(file_info)
|
| 464 |
+
continue
|
| 465 |
+
|
| 466 |
+
remote_path = f"audio/{self.project_slug}/{specie}/{file_info['name']}"
|
| 467 |
+
|
| 468 |
+
await self._upload_file_with_retry(
|
| 469 |
+
file_info["full_path"],
|
| 470 |
+
remote_path
|
| 471 |
+
)
|
| 472 |
+
|
| 473 |
+
results["uploaded"] += 1
|
| 474 |
+
self.session.mark_done(file_info)
|
| 475 |
+
|
| 476 |
+
if on_progress:
|
| 477 |
+
on_progress({
|
| 478 |
+
"total": structure["total_files"],
|
| 479 |
+
"uploaded": results["uploaded"],
|
| 480 |
+
"skipped": results["skipped"],
|
| 481 |
+
"failed": results["failed"],
|
| 482 |
+
"current_file": file_info["name"],
|
| 483 |
+
"current_size": file_info["size"]
|
| 484 |
+
})
|
| 485 |
+
|
| 486 |
+
except Exception as e:
|
| 487 |
+
results["failed"] += 1
|
| 488 |
+
results["errors"].append({
|
| 489 |
+
"file": file_info["name"],
|
| 490 |
+
"error": str(e)
|
| 491 |
+
})
|
| 492 |
+
self.session.mark_failed(file_info)
|
| 493 |
+
|
| 494 |
+
# Checkpoint por batch
|
| 495 |
+
self.session.save_checkpoint()
|
| 496 |
+
|
| 497 |
+
return results
|
| 498 |
+
|
| 499 |
+
async def _upload_file_with_retry(self,
|
| 500 |
+
local_path: str,
|
| 501 |
+
remote_path: str,
|
| 502 |
+
max_retries: int = 3):
|
| 503 |
+
"""Upload com retries e backoff exponencial."""
|
| 504 |
+
for attempt in range(max_retries):
|
| 505 |
+
try:
|
| 506 |
+
self.api.upload_file(
|
| 507 |
+
path_or_fileobj=local_path,
|
| 508 |
+
path_in_repo=remote_path,
|
| 509 |
+
repo_id=self.repo_id,
|
| 510 |
+
repo_type="dataset"
|
| 511 |
+
)
|
| 512 |
+
return
|
| 513 |
+
except Exception as e:
|
| 514 |
+
if attempt < max_retries - 1:
|
| 515 |
+
wait_time = 2 ** attempt + random.uniform(0, 1)
|
| 516 |
+
await asyncio.sleep(wait_time)
|
| 517 |
+
else:
|
| 518 |
+
raise
|
| 519 |
+
|
| 520 |
+
def _should_skip(self, file_info: dict) -> bool:
|
| 521 |
+
"""Verifica se arquivo já existe remotamente."""
|
| 522 |
+
try:
|
| 523 |
+
remote_path = f"audio/{self.project_slug}/{specie}/{file_info['name']}"
|
| 524 |
+
file_info_remote = self.api.get_file_info(
|
| 525 |
+
repo_id=self.repo_id,
|
| 526 |
+
filename=remote_path
|
| 527 |
+
)
|
| 528 |
+
return file_info_remote.size == file_info["size"]
|
| 529 |
+
except:
|
| 530 |
+
return False
|
| 531 |
+
```
|
| 532 |
+
|
| 533 |
+
### 4.5 Session Manager (`src/session/session_manager.py`)
|
| 534 |
+
|
| 535 |
+
```python
|
| 536 |
+
class SessionManager:
|
| 537 |
+
def __init__(self, repo_id: str):
|
| 538 |
+
self.session_id = f"upload-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
|
| 539 |
+
self.session_dir = Path(f"~/.birdnet-uploader/sessions/{self.session_id}")
|
| 540 |
+
self.session_dir.mkdir(parents=True, exist_ok=True)
|
| 541 |
+
|
| 542 |
+
def save_checkpoint(self, data: dict):
|
| 543 |
+
"""Salva checkpoint atomicamente."""
|
| 544 |
+
checkpoint_file = self.session_dir / "checkpoint.json"
|
| 545 |
+
temp_file = checkpoint_file.with_suffix(".tmp")
|
| 546 |
+
|
| 547 |
+
temp_file.write_text(json.dumps(data, indent=2))
|
| 548 |
+
temp_file.replace(checkpoint_file)
|
| 549 |
+
|
| 550 |
+
def load_checkpoint(self) -> dict:
|
| 551 |
+
"""Carrega último checkpoint."""
|
| 552 |
+
checkpoint_file = self.session_dir / "checkpoint.json"
|
| 553 |
+
if checkpoint_file.exists():
|
| 554 |
+
return json.loads(checkpoint_file.read_text())
|
| 555 |
+
return {}
|
| 556 |
+
```
|
| 557 |
+
|
| 558 |
+
### 4.6 CSV Matcher (`src/csv/csv_matcher.py`)
|
| 559 |
+
|
| 560 |
+
```python
|
| 561 |
+
class CSVMatcher:
|
| 562 |
+
def match_and_generate_index(self,
|
| 563 |
+
csv_path: str,
|
| 564 |
+
files_structure: dict) -> pd.DataFrame:
|
| 565 |
+
"""Faz matching entre CSV e segmentos, gera índice."""
|
| 566 |
+
df = pd.read_csv(csv_path)
|
| 567 |
+
|
| 568 |
+
# Validar colunas esperadas
|
| 569 |
+
required_cols = ["source_file", "scientific_name", "confidence",
|
| 570 |
+
"start_time", "end_time"]
|
| 571 |
+
if not all(col in df.columns for col in required_cols):
|
| 572 |
+
raise ValueError(f"CSV deve conter colunas: {required_cols}")
|
| 573 |
+
|
| 574 |
+
# Matching lógica
|
| 575 |
+
matched = []
|
| 576 |
+
for _, row in df.iterrows():
|
| 577 |
+
source_stem = Path(row["source_file"]).stem
|
| 578 |
+
|
| 579 |
+
# Procurar arquivo correspondente
|
| 580 |
+
for specie, files in files_structure["by_specie"].items():
|
| 581 |
+
for file_info in files:
|
| 582 |
+
if source_stem in file_info["name"]:
|
| 583 |
+
matched.append({
|
| 584 |
+
**row,
|
| 585 |
+
"segment_path": f"audio/{self.project_slug}/{specie}/{file_info['name']}"
|
| 586 |
+
})
|
| 587 |
+
|
| 588 |
+
return pd.DataFrame(matched)
|
| 589 |
+
|
| 590 |
+
def generate_shards(self,
|
| 591 |
+
index_df: pd.DataFrame,
|
| 592 |
+
output_dir: str,
|
| 593 |
+
shard_size: int = 10000) -> list:
|
| 594 |
+
"""Gera shards parquet para índice."""
|
| 595 |
+
shards = []
|
| 596 |
+
|
| 597 |
+
for i in range(0, len(index_df), shard_size):
|
| 598 |
+
shard_df = index_df.iloc[i:i+shard_size]
|
| 599 |
+
shard_file = f"{output_dir}/shard-{i//shard_size:05d}.parquet"
|
| 600 |
+
|
| 601 |
+
shard_df.to_parquet(shard_file, index=False)
|
| 602 |
+
shards.append(shard_file)
|
| 603 |
+
|
| 604 |
+
return shards
|
| 605 |
+
```
|
| 606 |
+
|
| 607 |
+
---
|
| 608 |
+
|
| 609 |
+
## 5. FLUXO DE COMANDOS CLI
|
| 610 |
+
|
| 611 |
+
```bash
|
| 612 |
+
# Autenticar
|
| 613 |
+
$ birdnet auth login
|
| 614 |
+
> Enter HF token: hf_...
|
| 615 |
+
|
| 616 |
+
# Upload simples (interactive)
|
| 617 |
+
$ birdnet upload
|
| 618 |
+
|
| 619 |
+
# Upload com flags (não-interativo)
|
| 620 |
+
$ birdnet upload \
|
| 621 |
+
--repo alice/my-birds \
|
| 622 |
+
--segments /path/to/segments \
|
| 623 |
+
--csv /path/to/metadata.csv \
|
| 624 |
+
--private
|
| 625 |
+
|
| 626 |
+
# Listar sessões
|
| 627 |
+
$ birdnet sessions list
|
| 628 |
+
|
| 629 |
+
# Retomar sessão
|
| 630 |
+
$ birdnet upload resume <session-id>
|
| 631 |
+
|
| 632 |
+
# Ver status
|
| 633 |
+
$ birdnet upload status <session-id>
|
| 634 |
+
|
| 635 |
+
# Exportar relatório
|
| 636 |
+
$ birdnet upload report <session-id>
|
| 637 |
+
```
|
| 638 |
+
|
| 639 |
+
---
|
| 640 |
+
|
| 641 |
+
## 6. CRITÉRIOS DE ACEITE DO MVP
|
| 642 |
+
|
| 643 |
+
- [ ] Usuário baixa executável portátil, sem Python pré-instalado
|
| 644 |
+
- [ ] Login com token HF funciona e não vaza em logs
|
| 645 |
+
- [ ] Criar dataset público/privado com estrutura padrão
|
| 646 |
+
- [ ] Varrer pasta local corretamente (recursivo, agrupa por espécie)
|
| 647 |
+
- [ ] Upload de 50GB+ com progresso contínuo e ETA coerente
|
| 648 |
+
- [ ] Pausar/fechar app/retomar sem duplicar upload
|
| 649 |
+
- [ ] Queda de rede + 429 recuperam com retries
|
| 650 |
+
- [ ] CSV opcional com matching integrado
|
| 651 |
+
- [ ] Relatório final estruturado (JSONL/JSON)
|
| 652 |
+
- [ ] Estrutura de arquivos compatível com validador
|
| 653 |
+
- [ ] Compatibilidade 100% com manifest/index esperado
|
| 654 |
+
- [ ] Sem regressão do fluxo atual do Space
|
| 655 |
+
|
| 656 |
+
---
|
| 657 |
+
|
| 658 |
+
## 7. PRÓXIMOS PASSOS
|
| 659 |
+
|
| 660 |
+
1. **Implementar módulos base** (Auth, Repo, Scanner, Uploader)
|
| 661 |
+
2. **Construir UI** (Textual TUI ou Rich + Click)
|
| 662 |
+
3. **Integrar Session Manager** e Checkpoint
|
| 663 |
+
4. **Testes de integração** (50GB+, retomada, rede)
|
| 664 |
+
5. **Construir executável** (PyInstaller)
|
| 665 |
+
6. **Validação end-to-end** com usuários reais
|
README.md
CHANGED
|
@@ -12,20 +12,52 @@ pinned: false
|
|
| 12 |
|
| 13 |
# BirdNET-Validator-App
|
| 14 |
|
| 15 |
-
BirdNET-Validator-App is a
|
| 16 |
|
| 17 |
-
|
|
|
|
| 18 |
|
| 19 |
-
|
| 20 |
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
- controlled access by user and project
|
| 24 |
- rapid decision flow for detections
|
| 25 |
- auditability through append-only events
|
| 26 |
- conflict-aware updates for concurrent validators
|
| 27 |
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
1. Multi-project login with role-based access
|
| 31 |
2. Project selection for authorized datasets
|
|
@@ -35,7 +67,7 @@ This project focuses on high-confidence validation operations by combining:
|
|
| 35 |
6. Concurrency-safe writes with optimistic lock feedback
|
| 36 |
7. Conflict resolution support and validation reporting
|
| 37 |
|
| 38 |
-
## Quick Start (Local)
|
| 39 |
|
| 40 |
1. Create and activate a Python 3.11+ virtual environment.
|
| 41 |
2. Install dependencies:
|
|
@@ -50,7 +82,9 @@ pip install -r requirements.txt
|
|
| 50 |
python app.py
|
| 51 |
```
|
| 52 |
|
| 53 |
-
Default port is 7860.
|
|
|
|
|
|
|
| 54 |
|
| 55 |
Optional runtime configuration:
|
| 56 |
|
|
@@ -79,20 +113,22 @@ Invite email settings (EmailJS):
|
|
| 79 |
- `BIRDNET_EMAILJS_ENDPOINT`: EmailJS API endpoint (default: `https://api.emailjs.com/api/v1.0/email/send`).
|
| 80 |
- `BIRDNET_EMAILJS_TIMEOUT_SECONDS`: request timeout in seconds (default: `20`).
|
| 81 |
|
| 82 |
-
|
|
|
|
|
|
|
| 83 |
|
| 84 |
```json
|
| 85 |
{
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
}
|
| 97 |
```
|
| 98 |
|
|
@@ -122,7 +158,7 @@ User access bootstrap file example (`BIRDNET_USER_ACCESS_FILE`):
|
|
| 122 |
}
|
| 123 |
```
|
| 124 |
|
| 125 |
-
## Typical Validation Flow
|
| 126 |
|
| 127 |
1. Login with a valid user.
|
| 128 |
2. Select an authorized project.
|
|
@@ -131,88 +167,7 @@ User access bootstrap file example (`BIRDNET_USER_ACCESS_FILE`):
|
|
| 131 |
5. Resolve conflicts when concurrent updates occur.
|
| 132 |
6. Export or inspect project validation report.
|
| 133 |
|
| 134 |
-
##
|
| 135 |
-
|
| 136 |
-
The repository now includes a basic project bootstrap CLI:
|
| 137 |
-
|
| 138 |
-
```bash
|
| 139 |
-
python -m src.cli.project_cli --help
|
| 140 |
-
```
|
| 141 |
-
|
| 142 |
-
Available commands:
|
| 143 |
-
|
| 144 |
-
- `create-project`: adds a project entry to the projects bootstrap JSON.
|
| 145 |
-
- `init-dataset`: creates the initial local dataset folder structure.
|
| 146 |
-
- `build-index`: builds a confidence-sorted index from `detections.jsonl`.
|
| 147 |
-
- `verify-project`: verifies project config and required local scaffold files.
|
| 148 |
-
- Use `--dry-run` to print findings without failing the command exit code.
|
| 149 |
-
|
| 150 |
-
Example flow:
|
| 151 |
-
|
| 152 |
-
```bash
|
| 153 |
-
python -m src.cli.project_cli create-project \
|
| 154 |
-
--projects-file docs/spaces/projects.sample.json \
|
| 155 |
-
--user-access-file docs/spaces/user_access.sample.json \
|
| 156 |
-
--slug amazonia-2026 \
|
| 157 |
-
--name "Amazonia 2026" \
|
| 158 |
-
--dataset-repo-id birdnet/amazonia-2026 \
|
| 159 |
-
--owner admin_user
|
| 160 |
-
|
| 161 |
-
python -m src.cli.project_cli init-dataset \
|
| 162 |
-
--dataset-root data/projects \
|
| 163 |
-
--slug amazonia-2026 \
|
| 164 |
-
--dataset-repo-id birdnet/amazonia-2026 \
|
| 165 |
-
--name "Amazonia 2026"
|
| 166 |
-
|
| 167 |
-
python -m src.cli.project_cli build-index \
|
| 168 |
-
--dataset-root data/projects \
|
| 169 |
-
--slug amazonia-2026
|
| 170 |
-
|
| 171 |
-
python -m src.cli.project_cli verify-project \
|
| 172 |
-
--projects-file docs/spaces/projects.sample.json \
|
| 173 |
-
--dataset-root data/projects \
|
| 174 |
-
--slug amazonia-2026
|
| 175 |
-
|
| 176 |
-
python -m src.cli.project_cli verify-project \
|
| 177 |
-
--projects-file docs/spaces/projects.sample.json \
|
| 178 |
-
--dataset-root data/projects \
|
| 179 |
-
--slug amazonia-2026 \
|
| 180 |
-
--dry-run
|
| 181 |
-
```
|
| 182 |
-
|
| 183 |
-
## Repository Structure
|
| 184 |
-
|
| 185 |
-
- `app.py`: validator app entrypoint
|
| 186 |
-
- `src/auth`: authentication/session and ACL logic
|
| 187 |
-
- `src/domain`: domain models
|
| 188 |
-
- `src/repositories`: in-memory and append-only persistence
|
| 189 |
-
- `src/services`: queue, validation, and audio fetch services
|
| 190 |
-
- `src/ui`: Gradio interface composition and callbacks
|
| 191 |
-
- `tests`: unit and integration test suites
|
| 192 |
-
|
| 193 |
-
## Project Status
|
| 194 |
-
|
| 195 |
-
Validator workflow is active and under continuous development, with emphasis on reliability, auditability, and operator productivity.
|
| 196 |
-
|
| 197 |
-
## FAQ
|
| 198 |
-
|
| 199 |
-
### Is this app intended for production use?
|
| 200 |
-
|
| 201 |
-
Yes, for validator workflows. It is already structured for multi-project access control, audit-friendly writes, and collaborative validation.
|
| 202 |
-
|
| 203 |
-
### Does the app preload all audio files?
|
| 204 |
-
|
| 205 |
-
No. Audio is fetched on demand for the selected detection, helping keep memory and network usage under control.
|
| 206 |
-
|
| 207 |
-
### Can multiple validators work at the same time?
|
| 208 |
-
|
| 209 |
-
Yes. The validation flow uses optimistic concurrency checks to detect and handle conflicting updates.
|
| 210 |
-
|
| 211 |
-
### Can this run on Hugging Face Spaces?
|
| 212 |
-
|
| 213 |
-
Yes. The project is designed for Gradio deployment on Hugging Face Spaces.
|
| 214 |
-
|
| 215 |
-
## Deploy on Hugging Face Spaces
|
| 216 |
|
| 217 |
1. Create a new Space with:
|
| 218 |
- SDK: `Gradio`
|
|
@@ -223,10 +178,12 @@ Yes. The project is designed for Gradio deployment on Hugging Face Spaces.
|
|
| 223 |
3. Configure Variables/Secrets in the Space settings.
|
| 224 |
|
| 225 |
Required for production bootstrap:
|
|
|
|
| 226 |
- `BIRDNET_PROJECTS_FILE` (example: `docs/spaces/projects.sample.json`)
|
| 227 |
- `BIRDNET_USER_ACCESS_FILE` (example: `docs/spaces/user_access.sample.json`)
|
| 228 |
|
| 229 |
Optional runtime settings:
|
|
|
|
| 230 |
- `BIRDNET_DETECTIONS_FILE` (seed detections JSON)
|
| 231 |
- `BIRDNET_VALIDATIONS_DIR` (recommended: `/data/validations`)
|
| 232 |
- `BIRDNET_BOOTSTRAP_DIR` (recommended: `/data/bootstrap`)
|
|
@@ -236,6 +193,7 @@ Optional runtime settings:
|
|
| 236 |
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP` (`false` in production)
|
| 237 |
|
| 238 |
Optional invite email settings:
|
|
|
|
| 239 |
- `BIRDNET_INVITE_EMAIL_ENABLED=true`
|
| 240 |
- `BIRDNET_INVITE_EMAIL_SENDER`
|
| 241 |
- `BIRDNET_INVITE_EMAIL_LOGIN_URL`
|
|
@@ -247,23 +205,305 @@ Optional invite email settings:
|
|
| 247 |
- `BIRDNET_EMAILJS_TIMEOUT_SECONDS`
|
| 248 |
|
| 249 |
4. For first smoke test only, you may temporarily set:
|
|
|
|
| 250 |
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP=true`
|
| 251 |
|
| 252 |
Then log in with one of the demo users:
|
|
|
|
| 253 |
- `admin_user`
|
| 254 |
- `demo_user`
|
| 255 |
- `validator_demo`
|
| 256 |
|
| 257 |
5. After validation, switch to production bootstrap:
|
|
|
|
| 258 |
- Set `BIRDNET_ENABLE_DEMO_BOOTSTRAP=false`
|
| 259 |
- Provide real `BIRDNET_PROJECTS_FILE` and `BIRDNET_USER_ACCESS_FILE`
|
| 260 |
|
| 261 |
Notes:
|
|
|
|
| 262 |
- The app entrypoint reads `PORT` automatically in Spaces.
|
| 263 |
- Keep user/project bootstrap files private if they contain sensitive assignments.
|
| 264 |
- Use `/data` paths in Spaces to keep projects, invites, ACL, and validations across redeploys.
|
| 265 |
- Collaborative access is token-per-user: each collaborator logs in with their own Hugging Face token.
|
| 266 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
## Troubleshooting
|
| 268 |
|
| 269 |
### The app does not start locally
|
|
@@ -301,6 +541,18 @@ Check that:
|
|
| 301 |
|
| 302 |
This means another validator updated the same detection first. Refresh and reapply your decision on the newest version.
|
| 303 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 304 |
## Recent Updates
|
| 305 |
|
| 306 |
- Integrated Validation tab flow in the multi-project app (no placeholder stage).
|
|
@@ -308,3 +560,10 @@ This means another validator updated the same detection first. Refresh and reapp
|
|
| 308 |
- Added seed-file warning banner with actionable remediation guidance in Validation.
|
| 309 |
- Standardized all user-facing UI text and feedback messages in English.
|
| 310 |
- Updated authentication/session timestamps to timezone-aware UTC handling.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
# BirdNET-Validator-App
|
| 14 |
|
| 15 |
+
BirdNET-Validator-App is a multi-purpose BirdNET workflow repository with two complementary entry points:
|
| 16 |
|
| 17 |
+
1. A Gradio-based web application for human validation of BirdNET detections in multi-project workflows.
|
| 18 |
+
2. A local command-line uploader for resumable Hugging Face dataset uploads with checkpoints, deduplication, and a Windows-friendly interactive flow.
|
| 19 |
|
| 20 |
+
The validator app focuses on review, authorization, and auditability. The uploader CLI focuses on reliable dataset ingestion for large audio collections.
|
| 21 |
|
| 22 |
+
## Contents
|
| 23 |
+
|
| 24 |
+
- [Overview](#overview)
|
| 25 |
+
- [Validator App](#validator-app)
|
| 26 |
+
- [Local Uploader CLI](#local-uploader-cli)
|
| 27 |
+
- [Quick Start](#quick-start)
|
| 28 |
+
- [Configuration](#configuration)
|
| 29 |
+
- [Typical Flows](#typical-flows)
|
| 30 |
+
- [Repository Structure](#repository-structure)
|
| 31 |
+
- [Troubleshooting](#troubleshooting)
|
| 32 |
+
- [Development Notes](#development-notes)
|
| 33 |
+
|
| 34 |
+
## Overview
|
| 35 |
+
|
| 36 |
+
This repository is designed for two high-confidence operations:
|
| 37 |
+
|
| 38 |
+
- controlled, multi-project human validation of detections
|
| 39 |
+
- reliable, resumable upload of large audio datasets to Hugging Face
|
| 40 |
+
|
| 41 |
+
The validator workflow emphasizes:
|
| 42 |
|
| 43 |
- controlled access by user and project
|
| 44 |
- rapid decision flow for detections
|
| 45 |
- auditability through append-only events
|
| 46 |
- conflict-aware updates for concurrent validators
|
| 47 |
|
| 48 |
+
The uploader workflow emphasizes:
|
| 49 |
+
|
| 50 |
+
- single-flow login → repo → scan → upload
|
| 51 |
+
- resumable sessions stored locally
|
| 52 |
+
- remote deduplication with a local cache
|
| 53 |
+
- progress tracking and checkpoint recovery
|
| 54 |
+
- compatibility with the validator app's expected dataset structure
|
| 55 |
+
|
| 56 |
+
## Validator App
|
| 57 |
+
|
| 58 |
+
The validator is a Gradio application for human review of BirdNET detections in multi-project workflows.
|
| 59 |
+
|
| 60 |
+
### Key Usability Features
|
| 61 |
|
| 62 |
1. Multi-project login with role-based access
|
| 63 |
2. Project selection for authorized datasets
|
|
|
|
| 67 |
6. Concurrency-safe writes with optimistic lock feedback
|
| 68 |
7. Conflict resolution support and validation reporting
|
| 69 |
|
| 70 |
+
### Quick Start (Local)
|
| 71 |
|
| 72 |
1. Create and activate a Python 3.11+ virtual environment.
|
| 73 |
2. Install dependencies:
|
|
|
|
| 82 |
python app.py
|
| 83 |
```
|
| 84 |
|
| 85 |
+
Default port is `7860`.
|
| 86 |
+
|
| 87 |
+
### Runtime Configuration
|
| 88 |
|
| 89 |
Optional runtime configuration:
|
| 90 |
|
|
|
|
| 113 |
- `BIRDNET_EMAILJS_ENDPOINT`: EmailJS API endpoint (default: `https://api.emailjs.com/api/v1.0/email/send`).
|
| 114 |
- `BIRDNET_EMAILJS_TIMEOUT_SECONDS`: request timeout in seconds (default: `20`).
|
| 115 |
|
| 116 |
+
### Validation Data Shapes
|
| 117 |
+
|
| 118 |
+
Detection seed file example:
|
| 119 |
|
| 120 |
```json
|
| 121 |
{
|
| 122 |
+
"kenya-2024": [
|
| 123 |
+
{
|
| 124 |
+
"detection_key": "0000000000001001",
|
| 125 |
+
"audio_id": "audio_1001",
|
| 126 |
+
"scientific_name": "Cyanocorax cyanopogon",
|
| 127 |
+
"confidence": 0.91,
|
| 128 |
+
"start_time": 0.0,
|
| 129 |
+
"end_time": 1.0
|
| 130 |
+
}
|
| 131 |
+
]
|
| 132 |
}
|
| 133 |
```
|
| 134 |
|
|
|
|
| 158 |
}
|
| 159 |
```
|
| 160 |
|
| 161 |
+
### Typical Validation Flow
|
| 162 |
|
| 163 |
1. Login with a valid user.
|
| 164 |
2. Select an authorized project.
|
|
|
|
| 167 |
5. Resolve conflicts when concurrent updates occur.
|
| 168 |
6. Export or inspect project validation report.
|
| 169 |
|
| 170 |
+
### Deploy on Hugging Face Spaces
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
|
| 172 |
1. Create a new Space with:
|
| 173 |
- SDK: `Gradio`
|
|
|
|
| 178 |
3. Configure Variables/Secrets in the Space settings.
|
| 179 |
|
| 180 |
Required for production bootstrap:
|
| 181 |
+
|
| 182 |
- `BIRDNET_PROJECTS_FILE` (example: `docs/spaces/projects.sample.json`)
|
| 183 |
- `BIRDNET_USER_ACCESS_FILE` (example: `docs/spaces/user_access.sample.json`)
|
| 184 |
|
| 185 |
Optional runtime settings:
|
| 186 |
+
|
| 187 |
- `BIRDNET_DETECTIONS_FILE` (seed detections JSON)
|
| 188 |
- `BIRDNET_VALIDATIONS_DIR` (recommended: `/data/validations`)
|
| 189 |
- `BIRDNET_BOOTSTRAP_DIR` (recommended: `/data/bootstrap`)
|
|
|
|
| 193 |
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP` (`false` in production)
|
| 194 |
|
| 195 |
Optional invite email settings:
|
| 196 |
+
|
| 197 |
- `BIRDNET_INVITE_EMAIL_ENABLED=true`
|
| 198 |
- `BIRDNET_INVITE_EMAIL_SENDER`
|
| 199 |
- `BIRDNET_INVITE_EMAIL_LOGIN_URL`
|
|
|
|
| 205 |
- `BIRDNET_EMAILJS_TIMEOUT_SECONDS`
|
| 206 |
|
| 207 |
4. For first smoke test only, you may temporarily set:
|
| 208 |
+
|
| 209 |
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP=true`
|
| 210 |
|
| 211 |
Then log in with one of the demo users:
|
| 212 |
+
|
| 213 |
- `admin_user`
|
| 214 |
- `demo_user`
|
| 215 |
- `validator_demo`
|
| 216 |
|
| 217 |
5. After validation, switch to production bootstrap:
|
| 218 |
+
|
| 219 |
- Set `BIRDNET_ENABLE_DEMO_BOOTSTRAP=false`
|
| 220 |
- Provide real `BIRDNET_PROJECTS_FILE` and `BIRDNET_USER_ACCESS_FILE`
|
| 221 |
|
| 222 |
Notes:
|
| 223 |
+
|
| 224 |
- The app entrypoint reads `PORT` automatically in Spaces.
|
| 225 |
- Keep user/project bootstrap files private if they contain sensitive assignments.
|
| 226 |
- Use `/data` paths in Spaces to keep projects, invites, ACL, and validations across redeploys.
|
| 227 |
- Collaborative access is token-per-user: each collaborator logs in with their own Hugging Face token.
|
| 228 |
|
| 229 |
+
## Local Uploader CLI
|
| 230 |
+
|
| 231 |
+
The repository also includes a local uploader CLI designed for large dataset ingest and resume support.
|
| 232 |
+
|
| 233 |
+
### What It Does
|
| 234 |
+
|
| 235 |
+
- authenticates with a Hugging Face token stored in the OS keyring
|
| 236 |
+
- creates or validates dataset repositories
|
| 237 |
+
- scans a local audio folder recursively
|
| 238 |
+
- computes per-file SHA-256 hashes
|
| 239 |
+
- deduplicates against a cached remote index
|
| 240 |
+
- uploads files with retries and session checkpoints
|
| 241 |
+
- resumes from the last checkpoint after close or crash
|
| 242 |
+
|
| 243 |
+
### Main Commands
|
| 244 |
+
|
| 245 |
+
The CLI entrypoint is:
|
| 246 |
+
|
| 247 |
+
```bash
|
| 248 |
+
python -m src.uploader_cli.main --help
|
| 249 |
+
```
|
| 250 |
+
|
| 251 |
+
Commands:
|
| 252 |
+
|
| 253 |
+
- `login`: validate and store a Hugging Face token
|
| 254 |
+
- `init-repo`: create and initialize a dataset repository
|
| 255 |
+
- `scan`: scan a local folder and summarize audio files
|
| 256 |
+
- `start`: create a local upload session checkpoint
|
| 257 |
+
- `resume`: print the saved checkpoint for a session
|
| 258 |
+
|
| 259 |
+
### Example Workflow
|
| 260 |
+
|
| 261 |
+
```bash
|
| 262 |
+
# Authenticate once
|
| 263 |
+
python -m src.uploader_cli.main login
|
| 264 |
+
|
| 265 |
+
# Create or initialize the dataset repository
|
| 266 |
+
python -m src.uploader_cli.main init-repo --repo-id alice/birdnet-2026 --private
|
| 267 |
+
|
| 268 |
+
# Inspect the local folder before upload
|
| 269 |
+
python -m src.uploader_cli.main scan --segments C:\audio\segments
|
| 270 |
+
|
| 271 |
+
# Create a resumable local session checkpoint
|
| 272 |
+
python -m src.uploader_cli.main start --repo-id alice/birdnet-2026 --segments C:\audio\segments
|
| 273 |
+
|
| 274 |
+
# Resume and inspect the checkpoint later
|
| 275 |
+
python -m src.uploader_cli.main resume upload-20260429T120000Z
|
| 276 |
+
```
|
| 277 |
+
|
| 278 |
+
### Upload Engine Building Blocks
|
| 279 |
+
|
| 280 |
+
The uploader CLI is built from the following modules:
|
| 281 |
+
|
| 282 |
+
- `src/uploader_cli/auth_service.py`: token validation and keyring storage
|
| 283 |
+
- `src/uploader_cli/repo_service.py`: dataset creation and structure initialization
|
| 284 |
+
- `src/uploader_cli/scanner.py`: recursive audio discovery and metadata collection
|
| 285 |
+
- `src/uploader_cli/hash_utils.py`: streaming SHA-256 helpers
|
| 286 |
+
- `src/uploader_cli/deduplicator.py`: remote index cache and skip/upload decisions
|
| 287 |
+
- `src/uploader_cli/session_manager.py`: checkpoint and session metadata persistence
|
| 288 |
+
- `src/uploader_cli/batch_uploader.py`: retrying upload orchestration
|
| 289 |
+
- `src/uploader_cli/error_handler.py`: user-friendly error formatting
|
| 290 |
+
|
| 291 |
+
### Local Data Layout
|
| 292 |
+
|
| 293 |
+
The uploader stores local state under the user's profile directory by default:
|
| 294 |
+
|
| 295 |
+
```text
|
| 296 |
+
~/.birdnet-uploader/
|
| 297 |
+
sessions/
|
| 298 |
+
upload-YYYYMMDDTHHMMSSZ/
|
| 299 |
+
metadata.json
|
| 300 |
+
checkpoint.json
|
| 301 |
+
cache/
|
| 302 |
+
dedup/
|
| 303 |
+
<repo-hash>.json
|
| 304 |
+
logs/
|
| 305 |
+
```
|
| 306 |
+
|
| 307 |
+
Environment overrides are supported:
|
| 308 |
+
|
| 309 |
+
- `BIRDNET_UPLOADER_SESSION_DIR`
|
| 310 |
+
- `BIRDNET_UPLOADER_CACHE_DIR`
|
| 311 |
+
- `BIRDNET_UPLOADER_LOG_DIR`
|
| 312 |
+
|
| 313 |
+
### Dataset Layout Expected by the Validator
|
| 314 |
+
|
| 315 |
+
The uploader is designed to preserve the validator's expected structure:
|
| 316 |
+
|
| 317 |
+
```text
|
| 318 |
+
audio/{project_slug}/... # uploaded audio files
|
| 319 |
+
index/shards/... # parquet shard files
|
| 320 |
+
index/manifest.json # index manifest
|
| 321 |
+
validations/... # validation outputs
|
| 322 |
+
audit/ingestion-runs/... # ingestion run history
|
| 323 |
+
```
|
| 324 |
+
|
| 325 |
+
### Uploader Configuration
|
| 326 |
+
|
| 327 |
+
Runtime constants are centralized in `src/uploader_cli/config.py`:
|
| 328 |
+
|
| 329 |
+
- `APP_NAME`
|
| 330 |
+
- `SCHEMA_VERSION`
|
| 331 |
+
- `AUDIO_EXTENSIONS`
|
| 332 |
+
- `INDEX_SHARD_SIZE`
|
| 333 |
+
- `MAX_BATCH_SIZE`
|
| 334 |
+
- `RETRY_MAX_ATTEMPTS`
|
| 335 |
+
- `RETRY_INITIAL_BACKOFF_SECONDS`
|
| 336 |
+
- `RETRY_MAX_BACKOFF_SECONDS`
|
| 337 |
+
- `KEYRING_SERVICE`
|
| 338 |
+
- `KEYRING_ACCOUNT`
|
| 339 |
+
|
| 340 |
+
### Resumability Guarantees
|
| 341 |
+
|
| 342 |
+
The uploader uses local checkpoints so it can recover from:
|
| 343 |
+
|
| 344 |
+
- process crashes
|
| 345 |
+
- app close
|
| 346 |
+
- transient network failures
|
| 347 |
+
- partial progress during large uploads
|
| 348 |
+
|
| 349 |
+
The checkpoint stores session metadata and cumulative counters such as:
|
| 350 |
+
|
| 351 |
+
- uploaded count
|
| 352 |
+
- skipped count
|
| 353 |
+
- failed count
|
| 354 |
+
- last completed file
|
| 355 |
+
- last failed file
|
| 356 |
+
- total uploaded bytes
|
| 357 |
+
|
| 358 |
+
## Quick Start
|
| 359 |
+
|
| 360 |
+
### Local Development
|
| 361 |
+
|
| 362 |
+
1. Create and activate a Python 3.11+ virtual environment.
|
| 363 |
+
2. Install dependencies:
|
| 364 |
+
|
| 365 |
+
```bash
|
| 366 |
+
pip install -r requirements.txt
|
| 367 |
+
```
|
| 368 |
+
|
| 369 |
+
3. Run the validator app:
|
| 370 |
+
|
| 371 |
+
```bash
|
| 372 |
+
python app.py
|
| 373 |
+
```
|
| 374 |
+
|
| 375 |
+
4. Run the uploader CLI help:
|
| 376 |
+
|
| 377 |
+
```bash
|
| 378 |
+
python -m src.uploader_cli.main --help
|
| 379 |
+
```
|
| 380 |
+
|
| 381 |
+
### First-Time Uploader Setup
|
| 382 |
+
|
| 383 |
+
1. Log in with a Hugging Face token:
|
| 384 |
+
|
| 385 |
+
```bash
|
| 386 |
+
python -m src.uploader_cli.main login
|
| 387 |
+
```
|
| 388 |
+
|
| 389 |
+
2. Validate your target repository exists or create it:
|
| 390 |
+
|
| 391 |
+
```bash
|
| 392 |
+
python -m src.uploader_cli.main init-repo --repo-id alice/birdnet-2026 --private
|
| 393 |
+
```
|
| 394 |
+
|
| 395 |
+
3. Scan your local dataset folder:
|
| 396 |
+
|
| 397 |
+
```bash
|
| 398 |
+
python -m src.uploader_cli.main scan --segments C:\audio\segments
|
| 399 |
+
```
|
| 400 |
+
|
| 401 |
+
4. Start a resumable session:
|
| 402 |
+
|
| 403 |
+
```bash
|
| 404 |
+
python -m src.uploader_cli.main start --repo-id alice/birdnet-2026 --segments C:\audio\segments
|
| 405 |
+
```
|
| 406 |
+
|
| 407 |
+
## Configuration
|
| 408 |
+
|
| 409 |
+
### Validator App Environment Variables
|
| 410 |
+
|
| 411 |
+
- `BIRDNET_DETECTIONS_FILE`
|
| 412 |
+
- `BIRDNET_VALIDATIONS_DIR`
|
| 413 |
+
- `BIRDNET_PAGE_SIZE`
|
| 414 |
+
- `BIRDNET_PROJECTS_FILE`
|
| 415 |
+
- `BIRDNET_USER_ACCESS_FILE`
|
| 416 |
+
- `BIRDNET_INVITES_FILE`
|
| 417 |
+
- `BIRDNET_INVITE_TTL_HOURS`
|
| 418 |
+
- `BIRDNET_BOOTSTRAP_DIR`
|
| 419 |
+
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP`
|
| 420 |
+
- `BIRDNET_EMAILJS_ENABLED`
|
| 421 |
+
- `BIRDNET_INVITE_EMAIL_ENABLED`
|
| 422 |
+
- `BIRDNET_INVITE_EMAIL_SENDER`
|
| 423 |
+
- `BIRDNET_INVITE_EMAIL_LOGIN_URL`
|
| 424 |
+
- `BIRDNET_EMAILJS_SERVICE_ID`
|
| 425 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID`
|
| 426 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_USERNAME_ONLY`
|
| 427 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_EMAIL_ONLY`
|
| 428 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_DUAL`
|
| 429 |
+
- `BIRDNET_EMAILJS_PUBLIC_KEY`
|
| 430 |
+
- `BIRDNET_EMAILJS_ENDPOINT`
|
| 431 |
+
- `BIRDNET_EMAILJS_TIMEOUT_SECONDS`
|
| 432 |
+
|
| 433 |
+
### Uploader CLI Environment Variables
|
| 434 |
+
|
| 435 |
+
- `BIRDNET_UPLOADER_SESSION_DIR`
|
| 436 |
+
- `BIRDNET_UPLOADER_CACHE_DIR`
|
| 437 |
+
- `BIRDNET_UPLOADER_LOG_DIR`
|
| 438 |
+
|
| 439 |
+
## Typical Flows
|
| 440 |
+
|
| 441 |
+
### Validation Flow
|
| 442 |
+
|
| 443 |
+
1. Login with a valid user.
|
| 444 |
+
2. Select an authorized project.
|
| 445 |
+
3. Load queue items and apply filters when needed.
|
| 446 |
+
4. Listen to selected audio and submit validation status.
|
| 447 |
+
5. Resolve conflicts when concurrent updates occur.
|
| 448 |
+
6. Export or inspect project validation report.
|
| 449 |
+
|
| 450 |
+
### Upload Flow
|
| 451 |
+
|
| 452 |
+
1. Authenticate once with a Hugging Face token.
|
| 453 |
+
2. Create or validate a dataset repository.
|
| 454 |
+
3. Scan the local folder recursively.
|
| 455 |
+
4. Deduplicate against remote files.
|
| 456 |
+
5. Upload with retries and checkpoints.
|
| 457 |
+
6. Resume later from the saved session metadata if needed.
|
| 458 |
+
|
| 459 |
+
## Repository Structure
|
| 460 |
+
|
| 461 |
+
- `app.py`: validator app entrypoint
|
| 462 |
+
- `src/auth`: authentication/session and ACL logic
|
| 463 |
+
- `src/domain`: domain models
|
| 464 |
+
- `src/repositories`: in-memory and append-only persistence
|
| 465 |
+
- `src/services`: queue, validation, and audio fetch services
|
| 466 |
+
- `src/ui`: Gradio interface composition and callbacks
|
| 467 |
+
- `src/uploader_cli`: local uploader CLI, session persistence, deduplication, and retrying uploads
|
| 468 |
+
- `tests`: unit and integration test suites
|
| 469 |
+
- `docs`: collaborative workflow and QA checklists
|
| 470 |
+
|
| 471 |
+
## Project Status
|
| 472 |
+
|
| 473 |
+
Validator workflow is active and under continuous development, with emphasis on reliability, auditability, and operator productivity.
|
| 474 |
+
|
| 475 |
+
The uploader CLI is being built as a companion local workflow for large ingest tasks, with a focus on resumability and validator compatibility.
|
| 476 |
+
|
| 477 |
+
## FAQ
|
| 478 |
+
|
| 479 |
+
### Is this app intended for production use?
|
| 480 |
+
|
| 481 |
+
Yes, for validator workflows. It is structured for multi-project access control, audit-friendly writes, and collaborative validation.
|
| 482 |
+
|
| 483 |
+
### Is the uploader CLI meant to replace the validator app?
|
| 484 |
+
|
| 485 |
+
No. It is a companion tool for large dataset uploads and repository initialization.
|
| 486 |
+
|
| 487 |
+
### Does the app preload all audio files?
|
| 488 |
+
|
| 489 |
+
No. Audio is fetched on demand for the selected detection, helping keep memory and network usage under control.
|
| 490 |
+
|
| 491 |
+
### Can multiple validators work at the same time?
|
| 492 |
+
|
| 493 |
+
Yes. The validation flow uses optimistic concurrency checks to detect and handle conflicting updates.
|
| 494 |
+
|
| 495 |
+
### Can this run on Hugging Face Spaces?
|
| 496 |
+
|
| 497 |
+
Yes. The validator app is designed for Gradio deployment on Hugging Face Spaces.
|
| 498 |
+
|
| 499 |
+
### Can the uploader resume after closing the app?
|
| 500 |
+
|
| 501 |
+
Yes. It persists checkpoints and session metadata locally and can be resumed later.
|
| 502 |
+
|
| 503 |
+
### Does the uploader support deduplication?
|
| 504 |
+
|
| 505 |
+
Yes. It checks the remote dataset index, caches the remote file list locally, and skips files that are already present.
|
| 506 |
+
|
| 507 |
## Troubleshooting
|
| 508 |
|
| 509 |
### The app does not start locally
|
|
|
|
| 541 |
|
| 542 |
This means another validator updated the same detection first. Refresh and reapply your decision on the newest version.
|
| 543 |
|
| 544 |
+
### The uploader says the token cannot be saved
|
| 545 |
+
|
| 546 |
+
Make sure your OS keyring backend is available. On Windows, the default credential store should work automatically.
|
| 547 |
+
|
| 548 |
+
### The uploader skips files unexpectedly
|
| 549 |
+
|
| 550 |
+
It may have found them in the remote dataset index cache. Clear the uploader cache directory if you want to force a fresh remote listing.
|
| 551 |
+
|
| 552 |
+
### Resume shows an empty checkpoint
|
| 553 |
+
|
| 554 |
+
Check that you ran the uploader `start` command at least once and that the session directory still exists.
|
| 555 |
+
|
| 556 |
## Recent Updates
|
| 557 |
|
| 558 |
- Integrated Validation tab flow in the multi-project app (no placeholder stage).
|
|
|
|
| 560 |
- Added seed-file warning banner with actionable remediation guidance in Validation.
|
| 561 |
- Standardized all user-facing UI text and feedback messages in English.
|
| 562 |
- Updated authentication/session timestamps to timezone-aware UTC handling.
|
| 563 |
+
- Added a local uploader CLI with login, repository initialization, scan, deduplication, checkpointing, and resumable upload foundations.
|
| 564 |
+
|
| 565 |
+
## Related Documents
|
| 566 |
+
|
| 567 |
+
- `BIRDNET_UPLOADER_REDESIGN.md`: uploader UX and architecture notes
|
| 568 |
+
- `SPRINT_ROADMAP.md`: implementation roadmap
|
| 569 |
+
- `SPRINT1_KICKOFF.md`: sprint kickoff details
|
SPRINT1_KICKOFF.md
ADDED
|
@@ -0,0 +1,769 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Sprint 1 Kickoff: Primeiros Passos (HOJE)
|
| 2 |
+
|
| 3 |
+
## Objetivo
|
| 4 |
+
Estrutura base pronta, dependências instaladas, todos os módulos esqueléticos criados, testes rodando.
|
| 5 |
+
|
| 6 |
+
**Tempo estimado**: 4-5 horas para 1-2 pessoas
|
| 7 |
+
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
## Step 1: Criar Branch de Trabalho
|
| 11 |
+
|
| 12 |
+
```bash
|
| 13 |
+
cd c:\Users\jonat\Documents\Python\BirdNET-validator-App
|
| 14 |
+
git checkout -b feature/birdnet-uploader-cli
|
| 15 |
+
git push -u origin feature/birdnet-uploader-cli
|
| 16 |
+
```
|
| 17 |
+
|
| 18 |
+
---
|
| 19 |
+
|
| 20 |
+
## Step 2: Criar Estrutura de Diretórios
|
| 21 |
+
|
| 22 |
+
```bash
|
| 23 |
+
# Criar estrutura completa
|
| 24 |
+
mkdir -p src\auth
|
| 25 |
+
mkdir -p src\repository
|
| 26 |
+
mkdir -p src\upload
|
| 27 |
+
mkdir -p src\csv
|
| 28 |
+
mkdir -p src\session
|
| 29 |
+
mkdir -p src\ui\screens
|
| 30 |
+
mkdir -p src\logger
|
| 31 |
+
mkdir -p src\utils
|
| 32 |
+
mkdir -p tests
|
| 33 |
+
mkdir -p build
|
| 34 |
+
mkdir -p docs
|
| 35 |
+
|
| 36 |
+
# Criar __init__.py em todos
|
| 37 |
+
echo "" > src\__init__.py
|
| 38 |
+
echo "" > src\auth\__init__.py
|
| 39 |
+
echo "" > src\repository\__init__.py
|
| 40 |
+
echo "" > src\upload\__init__.py
|
| 41 |
+
echo "" > src\csv\__init__.py
|
| 42 |
+
echo "" > src\session\__init__.py
|
| 43 |
+
echo "" > src\ui\__init__.py
|
| 44 |
+
echo "" > src\ui\screens\__init__.py
|
| 45 |
+
echo "" > src\logger\__init__.py
|
| 46 |
+
echo "" > src\utils\__init__.py
|
| 47 |
+
echo "" > tests\__init__.py
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
---
|
| 51 |
+
|
| 52 |
+
## Step 3: Criar `requirements.txt`
|
| 53 |
+
|
| 54 |
+
Criar arquivo: `requirements.txt`
|
| 55 |
+
|
| 56 |
+
```
|
| 57 |
+
# Core CLI
|
| 58 |
+
click>=8.0.0
|
| 59 |
+
typer>=0.9.0
|
| 60 |
+
|
| 61 |
+
# UI/TUI
|
| 62 |
+
textual>=0.30.0
|
| 63 |
+
rich>=13.0.0
|
| 64 |
+
|
| 65 |
+
# HF Integration
|
| 66 |
+
huggingface_hub>=0.28.0
|
| 67 |
+
|
| 68 |
+
# Data Processing
|
| 69 |
+
pandas>=2.0.0
|
| 70 |
+
pyarrow>=14.0.0
|
| 71 |
+
|
| 72 |
+
# Security
|
| 73 |
+
keyring>=24.0.0
|
| 74 |
+
cryptography>=41.0.0
|
| 75 |
+
|
| 76 |
+
# Validation
|
| 77 |
+
pydantic>=2.0.0
|
| 78 |
+
|
| 79 |
+
# Development & Testing
|
| 80 |
+
pytest>=7.0.0
|
| 81 |
+
pytest-cov>=4.0.0
|
| 82 |
+
black>=23.0.0
|
| 83 |
+
flake8>=6.0.0
|
| 84 |
+
mypy>=1.0.0
|
| 85 |
+
|
| 86 |
+
# Optional: async
|
| 87 |
+
asyncio>=3.4.3
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
Instalar:
|
| 91 |
+
```bash
|
| 92 |
+
pip install -r requirements.txt
|
| 93 |
+
```
|
| 94 |
+
|
| 95 |
+
---
|
| 96 |
+
|
| 97 |
+
## Step 4: Criar `pyproject.toml`
|
| 98 |
+
|
| 99 |
+
Criar arquivo: `pyproject.toml`
|
| 100 |
+
|
| 101 |
+
```toml
|
| 102 |
+
[build-system]
|
| 103 |
+
requires = ["setuptools>=65.0", "wheel"]
|
| 104 |
+
build-backend = "setuptools.build_meta"
|
| 105 |
+
|
| 106 |
+
[project]
|
| 107 |
+
name = "birdnet-uploader"
|
| 108 |
+
version = "0.1.0"
|
| 109 |
+
description = "Local CLI for resumable HuggingFace dataset uploads (BirdNET)"
|
| 110 |
+
authors = [
|
| 111 |
+
{name = "BirdNET Team", email = "team@example.com"}
|
| 112 |
+
]
|
| 113 |
+
requires-python = ">=3.9"
|
| 114 |
+
dependencies = [
|
| 115 |
+
"click>=8.0.0",
|
| 116 |
+
"textual>=0.30.0",
|
| 117 |
+
"rich>=13.0.0",
|
| 118 |
+
"huggingface_hub>=0.28.0",
|
| 119 |
+
"pandas>=2.0.0",
|
| 120 |
+
"pyarrow>=14.0.0",
|
| 121 |
+
"keyring>=24.0.0",
|
| 122 |
+
"pydantic>=2.0.0",
|
| 123 |
+
]
|
| 124 |
+
|
| 125 |
+
[project.optional-dependencies]
|
| 126 |
+
dev = [
|
| 127 |
+
"pytest>=7.0.0",
|
| 128 |
+
"pytest-cov>=4.0.0",
|
| 129 |
+
"black>=23.0.0",
|
| 130 |
+
"flake8>=6.0.0",
|
| 131 |
+
"mypy>=1.0.0",
|
| 132 |
+
]
|
| 133 |
+
|
| 134 |
+
[project.scripts]
|
| 135 |
+
birdnet = "src.main:app"
|
| 136 |
+
|
| 137 |
+
[tool.black]
|
| 138 |
+
line-length = 100
|
| 139 |
+
target-version = ['py39']
|
| 140 |
+
|
| 141 |
+
[tool.isort]
|
| 142 |
+
profile = "black"
|
| 143 |
+
line_length = 100
|
| 144 |
+
|
| 145 |
+
[tool.mypy]
|
| 146 |
+
python_version = "3.9"
|
| 147 |
+
warn_return_any = true
|
| 148 |
+
warn_unused_configs = true
|
| 149 |
+
|
| 150 |
+
[tool.pytest.ini_options]
|
| 151 |
+
testpaths = ["tests"]
|
| 152 |
+
addopts = "-v --cov=src --cov-report=html --cov-report=term-missing"
|
| 153 |
+
```
|
| 154 |
+
|
| 155 |
+
---
|
| 156 |
+
|
| 157 |
+
## Step 5: Criar Estrutura Base de Módulos (Skeletons)
|
| 158 |
+
|
| 159 |
+
### 5.1: `src/main.py`
|
| 160 |
+
|
| 161 |
+
```python
|
| 162 |
+
"""BirdNET Uploader - Main Entry Point"""
|
| 163 |
+
|
| 164 |
+
import click
|
| 165 |
+
from src.ui.tui import BirdNETApp
|
| 166 |
+
from src.auth.auth_service import AuthService
|
| 167 |
+
from src.config import get_config
|
| 168 |
+
|
| 169 |
+
__version__ = "0.1.0"
|
| 170 |
+
|
| 171 |
+
@click.group()
|
| 172 |
+
@click.version_option(version=__version__)
|
| 173 |
+
def app():
|
| 174 |
+
"""BirdNET Uploader - Upload audio to HuggingFace Datasets"""
|
| 175 |
+
pass
|
| 176 |
+
|
| 177 |
+
@app.command()
|
| 178 |
+
def upload():
|
| 179 |
+
"""Start the upload wizard (interactive TUI)"""
|
| 180 |
+
try:
|
| 181 |
+
app_ui = BirdNETApp()
|
| 182 |
+
app_ui.run()
|
| 183 |
+
except Exception as e:
|
| 184 |
+
click.echo(f"Error: {e}", err=True)
|
| 185 |
+
|
| 186 |
+
@app.command()
|
| 187 |
+
def login():
|
| 188 |
+
"""Authenticate with HuggingFace token"""
|
| 189 |
+
try:
|
| 190 |
+
token = click.prompt("Enter your HF token", hide_input=True)
|
| 191 |
+
auth = AuthService()
|
| 192 |
+
user_info = auth.authenticate(token)
|
| 193 |
+
click.echo(f"✓ Logged in as {user_info['username']}")
|
| 194 |
+
except Exception as e:
|
| 195 |
+
click.echo(f"Error: {e}", err=True)
|
| 196 |
+
|
| 197 |
+
@app.command()
|
| 198 |
+
@click.argument("session_id")
|
| 199 |
+
def resume(session_id: str):
|
| 200 |
+
"""Resume a previous upload session"""
|
| 201 |
+
click.echo(f"Resuming session: {session_id}")
|
| 202 |
+
# TODO: Implement
|
| 203 |
+
|
| 204 |
+
@app.command()
|
| 205 |
+
def list_sessions():
|
| 206 |
+
"""List all upload sessions"""
|
| 207 |
+
click.echo("Sessions:")
|
| 208 |
+
# TODO: Implement
|
| 209 |
+
|
| 210 |
+
if __name__ == "__main__":
|
| 211 |
+
app()
|
| 212 |
+
```
|
| 213 |
+
|
| 214 |
+
### 5.2: `src/config.py`
|
| 215 |
+
|
| 216 |
+
```python
|
| 217 |
+
"""Configuration and Constants"""
|
| 218 |
+
|
| 219 |
+
from pathlib import Path
|
| 220 |
+
import os
|
| 221 |
+
|
| 222 |
+
# App
|
| 223 |
+
APP_NAME = "BirdNET Uploader"
|
| 224 |
+
APP_VERSION = "0.1.0"
|
| 225 |
+
|
| 226 |
+
# Directories
|
| 227 |
+
SESSION_DIR = Path.home() / ".birdnet-uploader" / "sessions"
|
| 228 |
+
CACHE_DIR = Path.home() / ".birdnet-uploader" / "cache"
|
| 229 |
+
LOG_DIR = Path.home() / ".birdnet-uploader" / "logs"
|
| 230 |
+
|
| 231 |
+
# Create directories if not exist
|
| 232 |
+
SESSION_DIR.mkdir(parents=True, exist_ok=True)
|
| 233 |
+
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
| 234 |
+
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
| 235 |
+
|
| 236 |
+
# Audio
|
| 237 |
+
AUDIO_EXTENSIONS = ['.wav', '.mp3', '.flac', '.ogg', '.m4a']
|
| 238 |
+
|
| 239 |
+
# Upload
|
| 240 |
+
MAX_BATCH_SIZE = 10
|
| 241 |
+
MAX_RETRIES = 3
|
| 242 |
+
RETRY_INITIAL_BACKOFF = 1.0 # seconds
|
| 243 |
+
RETRY_MAX_BACKOFF = 30.0 # seconds
|
| 244 |
+
|
| 245 |
+
# Manifest
|
| 246 |
+
MANIFEST_SCHEMA_VERSION = "1.0.0"
|
| 247 |
+
SHARD_SIZE = 10000
|
| 248 |
+
|
| 249 |
+
# Keyring
|
| 250 |
+
KEYRING_SERVICE = "birdnet-uploader"
|
| 251 |
+
KEYRING_ACCOUNT = "hf_token"
|
| 252 |
+
|
| 253 |
+
def get_config():
|
| 254 |
+
"""Get runtime configuration from environment or defaults"""
|
| 255 |
+
return {
|
| 256 |
+
"session_dir": os.getenv("BIRDNET_SESSION_DIR", str(SESSION_DIR)),
|
| 257 |
+
"cache_dir": os.getenv("BIRDNET_CACHE_DIR", str(CACHE_DIR)),
|
| 258 |
+
"log_dir": os.getenv("BIRDNET_LOG_DIR", str(LOG_DIR)),
|
| 259 |
+
}
|
| 260 |
+
```
|
| 261 |
+
|
| 262 |
+
### 5.3: `src/auth/auth_service.py`
|
| 263 |
+
|
| 264 |
+
```python
|
| 265 |
+
"""Authentication Service"""
|
| 266 |
+
|
| 267 |
+
from typing import Optional, Dict
|
| 268 |
+
import keyring
|
| 269 |
+
from huggingface_hub import HfApi, HfFolder
|
| 270 |
+
from src.config import KEYRING_SERVICE, KEYRING_ACCOUNT
|
| 271 |
+
|
| 272 |
+
class AuthenticationError(Exception):
|
| 273 |
+
"""Authentication failed"""
|
| 274 |
+
pass
|
| 275 |
+
|
| 276 |
+
class AuthService:
|
| 277 |
+
"""Manage authentication with HuggingFace"""
|
| 278 |
+
|
| 279 |
+
def __init__(self):
|
| 280 |
+
self.api = None
|
| 281 |
+
|
| 282 |
+
def authenticate(self, token: str) -> Dict[str, str]:
|
| 283 |
+
"""
|
| 284 |
+
Validate token and store it securely.
|
| 285 |
+
|
| 286 |
+
Args:
|
| 287 |
+
token: HF API token
|
| 288 |
+
|
| 289 |
+
Returns:
|
| 290 |
+
User info dict with username, email, user_id
|
| 291 |
+
|
| 292 |
+
Raises:
|
| 293 |
+
AuthenticationError: If token is invalid
|
| 294 |
+
"""
|
| 295 |
+
try:
|
| 296 |
+
api = HfApi(token=token)
|
| 297 |
+
user_info = api.whoami()
|
| 298 |
+
|
| 299 |
+
# Store token in keyring
|
| 300 |
+
keyring.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, token)
|
| 301 |
+
|
| 302 |
+
self.api = api
|
| 303 |
+
return {
|
| 304 |
+
"username": user_info.get("name"),
|
| 305 |
+
"email": user_info.get("email", ""),
|
| 306 |
+
"user_id": user_info.get("user_id"),
|
| 307 |
+
}
|
| 308 |
+
except Exception as e:
|
| 309 |
+
raise AuthenticationError(f"Invalid token: {str(e)}")
|
| 310 |
+
|
| 311 |
+
def get_token(self) -> Optional[str]:
|
| 312 |
+
"""Get stored token from keyring"""
|
| 313 |
+
return keyring.get_password(KEYRING_SERVICE, KEYRING_ACCOUNT)
|
| 314 |
+
|
| 315 |
+
def clear_token(self):
|
| 316 |
+
"""Clear stored token"""
|
| 317 |
+
keyring.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT)
|
| 318 |
+
```
|
| 319 |
+
|
| 320 |
+
### 5.4: `src/repository/repo_service.py`
|
| 321 |
+
|
| 322 |
+
```python
|
| 323 |
+
"""Repository Service"""
|
| 324 |
+
|
| 325 |
+
from typing import Optional
|
| 326 |
+
from huggingface_hub import HfApi
|
| 327 |
+
import json
|
| 328 |
+
from io import BytesIO
|
| 329 |
+
|
| 330 |
+
class RepositoryError(Exception):
|
| 331 |
+
"""Repository operation failed"""
|
| 332 |
+
pass
|
| 333 |
+
|
| 334 |
+
class RepositoryService:
|
| 335 |
+
"""Manage HuggingFace datasets"""
|
| 336 |
+
|
| 337 |
+
def __init__(self, token: str):
|
| 338 |
+
self.api = HfApi(token=token)
|
| 339 |
+
|
| 340 |
+
def create_dataset(self, repo_name: str, private: bool = True) -> str:
|
| 341 |
+
"""
|
| 342 |
+
Create a new dataset on HuggingFace.
|
| 343 |
+
|
| 344 |
+
Args:
|
| 345 |
+
repo_name: Dataset name
|
| 346 |
+
private: Whether dataset should be private
|
| 347 |
+
|
| 348 |
+
Returns:
|
| 349 |
+
Full repo_id (username/repo_name)
|
| 350 |
+
"""
|
| 351 |
+
try:
|
| 352 |
+
# Create repo
|
| 353 |
+
repo = self.api.create_repo(
|
| 354 |
+
repo_id=repo_name,
|
| 355 |
+
repo_type="dataset",
|
| 356 |
+
private=private,
|
| 357 |
+
exist_ok=True
|
| 358 |
+
)
|
| 359 |
+
repo_id = repo.repo_id
|
| 360 |
+
|
| 361 |
+
# Initialize structure
|
| 362 |
+
self._init_structure(repo_id)
|
| 363 |
+
|
| 364 |
+
return repo_id
|
| 365 |
+
except Exception as e:
|
| 366 |
+
raise RepositoryError(f"Failed to create dataset: {str(e)}")
|
| 367 |
+
|
| 368 |
+
def _init_structure(self, repo_id: str):
|
| 369 |
+
"""Initialize default folder structure"""
|
| 370 |
+
folders = [
|
| 371 |
+
"audio/project",
|
| 372 |
+
"index/shards",
|
| 373 |
+
"validations",
|
| 374 |
+
"audit/ingestion-runs"
|
| 375 |
+
]
|
| 376 |
+
|
| 377 |
+
for folder in folders:
|
| 378 |
+
# Upload placeholder file to create folder
|
| 379 |
+
self.api.upload_file(
|
| 380 |
+
path_or_fileobj=BytesIO(b""),
|
| 381 |
+
path_in_repo=f"{folder}/.gitkeep",
|
| 382 |
+
repo_id=repo_id,
|
| 383 |
+
repo_type="dataset"
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
def validate_repo(self, repo_id: str) -> bool:
|
| 387 |
+
"""Validate dataset has expected structure"""
|
| 388 |
+
# TODO: Implement
|
| 389 |
+
return True
|
| 390 |
+
```
|
| 391 |
+
|
| 392 |
+
### 5.5: `src/upload/scanner.py`
|
| 393 |
+
|
| 394 |
+
```python
|
| 395 |
+
"""Local Folder Scanner"""
|
| 396 |
+
|
| 397 |
+
import os
|
| 398 |
+
from pathlib import Path
|
| 399 |
+
from typing import Dict, List
|
| 400 |
+
from src.config import AUDIO_EXTENSIONS
|
| 401 |
+
|
| 402 |
+
class LocalScanner:
|
| 403 |
+
"""Scan local folders for audio files"""
|
| 404 |
+
|
| 405 |
+
def scan_folder(self, folder_path: str) -> Dict:
|
| 406 |
+
"""
|
| 407 |
+
Recursively scan folder for audio files.
|
| 408 |
+
|
| 409 |
+
Args:
|
| 410 |
+
folder_path: Path to scan
|
| 411 |
+
|
| 412 |
+
Returns:
|
| 413 |
+
Structure dict with files grouped by species
|
| 414 |
+
"""
|
| 415 |
+
structure = {}
|
| 416 |
+
total_size = 0
|
| 417 |
+
file_count = 0
|
| 418 |
+
|
| 419 |
+
for root, dirs, files in os.walk(folder_path):
|
| 420 |
+
for file in files:
|
| 421 |
+
if any(file.lower().endswith(ext) for ext in AUDIO_EXTENSIONS):
|
| 422 |
+
full_path = os.path.join(root, file)
|
| 423 |
+
size = os.path.getsize(full_path)
|
| 424 |
+
|
| 425 |
+
# Infer species
|
| 426 |
+
specie = self._infer_species(root, file)
|
| 427 |
+
|
| 428 |
+
if specie not in structure:
|
| 429 |
+
structure[specie] = []
|
| 430 |
+
|
| 431 |
+
structure[specie].append({
|
| 432 |
+
"name": file,
|
| 433 |
+
"full_path": full_path,
|
| 434 |
+
"size": size,
|
| 435 |
+
})
|
| 436 |
+
|
| 437 |
+
total_size += size
|
| 438 |
+
file_count += 1
|
| 439 |
+
|
| 440 |
+
return {
|
| 441 |
+
"total_files": file_count,
|
| 442 |
+
"total_size": total_size,
|
| 443 |
+
"by_species": structure,
|
| 444 |
+
}
|
| 445 |
+
|
| 446 |
+
def _infer_species(self, root: str, filename: str) -> str:
|
| 447 |
+
"""Infer species from path or filename"""
|
| 448 |
+
parts = root.split(os.sep)
|
| 449 |
+
if len(parts) > 0:
|
| 450 |
+
return parts[-1]
|
| 451 |
+
return "unknown"
|
| 452 |
+
```
|
| 453 |
+
|
| 454 |
+
### 5.6: `src/session/session_manager.py`
|
| 455 |
+
|
| 456 |
+
```python
|
| 457 |
+
"""Session Management"""
|
| 458 |
+
|
| 459 |
+
import json
|
| 460 |
+
from pathlib import Path
|
| 461 |
+
from datetime import datetime
|
| 462 |
+
from src.config import SESSION_DIR
|
| 463 |
+
|
| 464 |
+
class SessionManager:
|
| 465 |
+
"""Manage upload sessions"""
|
| 466 |
+
|
| 467 |
+
def __init__(self, session_id: str = None):
|
| 468 |
+
if session_id is None:
|
| 469 |
+
session_id = f"upload-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
|
| 470 |
+
|
| 471 |
+
self.session_id = session_id
|
| 472 |
+
self.session_dir = Path(SESSION_DIR) / session_id
|
| 473 |
+
self.session_dir.mkdir(parents=True, exist_ok=True)
|
| 474 |
+
|
| 475 |
+
def save_checkpoint(self, data: dict):
|
| 476 |
+
"""Save checkpoint atomically"""
|
| 477 |
+
checkpoint_file = self.session_dir / "checkpoint.json"
|
| 478 |
+
temp_file = checkpoint_file.with_suffix(".tmp")
|
| 479 |
+
|
| 480 |
+
with open(temp_file, "w") as f:
|
| 481 |
+
json.dump(data, f, indent=2)
|
| 482 |
+
|
| 483 |
+
temp_file.replace(checkpoint_file)
|
| 484 |
+
|
| 485 |
+
def load_checkpoint(self) -> dict:
|
| 486 |
+
"""Load last checkpoint"""
|
| 487 |
+
checkpoint_file = self.session_dir / "checkpoint.json"
|
| 488 |
+
if checkpoint_file.exists():
|
| 489 |
+
with open(checkpoint_file) as f:
|
| 490 |
+
return json.load(f)
|
| 491 |
+
return {}
|
| 492 |
+
```
|
| 493 |
+
|
| 494 |
+
### 5.7: `src/ui/tui.py`
|
| 495 |
+
|
| 496 |
+
```python
|
| 497 |
+
"""TUI Application (Textual)"""
|
| 498 |
+
|
| 499 |
+
from textual.app import ComposeResult, App
|
| 500 |
+
from textual.containers import Container
|
| 501 |
+
from textual.widgets import Static, Button
|
| 502 |
+
|
| 503 |
+
class BirdNETApp(App):
|
| 504 |
+
"""Main TUI Application"""
|
| 505 |
+
|
| 506 |
+
def compose(self) -> ComposeResult:
|
| 507 |
+
"""Create child widgets for the app."""
|
| 508 |
+
yield Container(
|
| 509 |
+
Static("BirdNET Uploader", id="header"),
|
| 510 |
+
Button("Start Upload", id="btn_upload"),
|
| 511 |
+
id="container"
|
| 512 |
+
)
|
| 513 |
+
|
| 514 |
+
def on_button_pressed(self) -> None:
|
| 515 |
+
"""Handle button press."""
|
| 516 |
+
self.exit()
|
| 517 |
+
|
| 518 |
+
def run():
|
| 519 |
+
"""Run the TUI app"""
|
| 520 |
+
app = BirdNETApp()
|
| 521 |
+
app.run()
|
| 522 |
+
```
|
| 523 |
+
|
| 524 |
+
### 5.8: `src/utils/error_handler.py`
|
| 525 |
+
|
| 526 |
+
```python
|
| 527 |
+
"""Error Handling"""
|
| 528 |
+
|
| 529 |
+
class BirdNETError(Exception):
|
| 530 |
+
"""Base exception"""
|
| 531 |
+
pass
|
| 532 |
+
|
| 533 |
+
class AuthenticationError(BirdNETError):
|
| 534 |
+
"""Authentication failed"""
|
| 535 |
+
pass
|
| 536 |
+
|
| 537 |
+
class RepositoryError(BirdNETError):
|
| 538 |
+
"""Repository operation failed"""
|
| 539 |
+
pass
|
| 540 |
+
|
| 541 |
+
class UploadError(BirdNETError):
|
| 542 |
+
"""Upload operation failed"""
|
| 543 |
+
pass
|
| 544 |
+
|
| 545 |
+
class ValidationError(BirdNETError):
|
| 546 |
+
"""Validation failed"""
|
| 547 |
+
pass
|
| 548 |
+
```
|
| 549 |
+
|
| 550 |
+
---
|
| 551 |
+
|
| 552 |
+
## Step 6: Criar Test Infrastructure
|
| 553 |
+
|
| 554 |
+
### 6.1: `pytest.ini`
|
| 555 |
+
|
| 556 |
+
```ini
|
| 557 |
+
[pytest]
|
| 558 |
+
testpaths = tests
|
| 559 |
+
addopts = -v --tb=short
|
| 560 |
+
python_files = test_*.py
|
| 561 |
+
python_classes = Test*
|
| 562 |
+
python_functions = test_*
|
| 563 |
+
```
|
| 564 |
+
|
| 565 |
+
### 6.2: `tests/conftest.py`
|
| 566 |
+
|
| 567 |
+
```python
|
| 568 |
+
"""Test Configuration and Fixtures"""
|
| 569 |
+
|
| 570 |
+
import pytest
|
| 571 |
+
from pathlib import Path
|
| 572 |
+
import tempfile
|
| 573 |
+
import json
|
| 574 |
+
|
| 575 |
+
@pytest.fixture
|
| 576 |
+
def temp_folder():
|
| 577 |
+
"""Create temporary folder with test audio files"""
|
| 578 |
+
with tempfile.TemporaryDirectory() as tmpdir:
|
| 579 |
+
# Create test structure
|
| 580 |
+
audio_dir = Path(tmpdir) / "audio" / "Parrot"
|
| 581 |
+
audio_dir.mkdir(parents=True)
|
| 582 |
+
|
| 583 |
+
# Create dummy audio files
|
| 584 |
+
for i in range(5):
|
| 585 |
+
(audio_dir / f"segment_{i:03d}.wav").write_bytes(b"fake audio data")
|
| 586 |
+
|
| 587 |
+
yield tmpdir
|
| 588 |
+
|
| 589 |
+
@pytest.fixture
|
| 590 |
+
def mock_hf_token():
|
| 591 |
+
"""Mock HF token"""
|
| 592 |
+
return "hf_test_token_12345"
|
| 593 |
+
|
| 594 |
+
@pytest.fixture
|
| 595 |
+
def session_dir(tmp_path):
|
| 596 |
+
"""Temporary session directory"""
|
| 597 |
+
session_dir = tmp_path / "sessions" / "test-session"
|
| 598 |
+
session_dir.mkdir(parents=True)
|
| 599 |
+
return session_dir
|
| 600 |
+
```
|
| 601 |
+
|
| 602 |
+
### 6.3: `tests/test_auth.py`
|
| 603 |
+
|
| 604 |
+
```python
|
| 605 |
+
"""Tests for Auth Service"""
|
| 606 |
+
|
| 607 |
+
import pytest
|
| 608 |
+
from src.auth.auth_service import AuthService, AuthenticationError
|
| 609 |
+
|
| 610 |
+
def test_auth_invalid_token(mock_hf_token):
|
| 611 |
+
"""Test that invalid token raises error"""
|
| 612 |
+
auth = AuthService()
|
| 613 |
+
with pytest.raises(AuthenticationError):
|
| 614 |
+
auth.authenticate("invalid_token_xyz")
|
| 615 |
+
|
| 616 |
+
# More tests to be implemented
|
| 617 |
+
```
|
| 618 |
+
|
| 619 |
+
### 6.4: `tests/test_scanner.py`
|
| 620 |
+
|
| 621 |
+
```python
|
| 622 |
+
"""Tests for File Scanner"""
|
| 623 |
+
|
| 624 |
+
import pytest
|
| 625 |
+
from src.upload.scanner import LocalScanner
|
| 626 |
+
|
| 627 |
+
def test_scan_folder_empty(tmp_path):
|
| 628 |
+
"""Test scanning empty folder"""
|
| 629 |
+
scanner = LocalScanner()
|
| 630 |
+
result = scanner.scan_folder(str(tmp_path))
|
| 631 |
+
assert result["total_files"] == 0
|
| 632 |
+
assert result["total_size"] == 0
|
| 633 |
+
|
| 634 |
+
def test_scan_folder_with_audio(temp_folder):
|
| 635 |
+
"""Test scanning folder with audio files"""
|
| 636 |
+
scanner = LocalScanner()
|
| 637 |
+
result = scanner.scan_folder(temp_folder)
|
| 638 |
+
assert result["total_files"] == 5
|
| 639 |
+
assert result["total_size"] > 0
|
| 640 |
+
|
| 641 |
+
# More tests to be implemented
|
| 642 |
+
```
|
| 643 |
+
|
| 644 |
+
---
|
| 645 |
+
|
| 646 |
+
## Step 7: Configurar CI/CD (GitHub Actions)
|
| 647 |
+
|
| 648 |
+
Criar arquivo: `.github/workflows/test.yml`
|
| 649 |
+
|
| 650 |
+
```yaml
|
| 651 |
+
name: Tests
|
| 652 |
+
|
| 653 |
+
on: [push, pull_request]
|
| 654 |
+
|
| 655 |
+
jobs:
|
| 656 |
+
test:
|
| 657 |
+
runs-on: ubuntu-latest
|
| 658 |
+
strategy:
|
| 659 |
+
matrix:
|
| 660 |
+
python-version: ["3.9", "3.10", "3.11"]
|
| 661 |
+
|
| 662 |
+
steps:
|
| 663 |
+
- uses: actions/checkout@v3
|
| 664 |
+
|
| 665 |
+
- name: Set up Python
|
| 666 |
+
uses: actions/setup-python@v4
|
| 667 |
+
with:
|
| 668 |
+
python-version: ${{ matrix.python-version }}
|
| 669 |
+
|
| 670 |
+
- name: Install dependencies
|
| 671 |
+
run: |
|
| 672 |
+
pip install -r requirements.txt
|
| 673 |
+
|
| 674 |
+
- name: Lint
|
| 675 |
+
run: |
|
| 676 |
+
flake8 src tests
|
| 677 |
+
|
| 678 |
+
- name: Run tests
|
| 679 |
+
run: |
|
| 680 |
+
pytest
|
| 681 |
+
|
| 682 |
+
- name: Upload coverage
|
| 683 |
+
uses: codecov/codecov-action@v3
|
| 684 |
+
```
|
| 685 |
+
|
| 686 |
+
---
|
| 687 |
+
|
| 688 |
+
## Step 8: Testar Instalação
|
| 689 |
+
|
| 690 |
+
```bash
|
| 691 |
+
# Instalar dependências
|
| 692 |
+
pip install -r requirements.txt
|
| 693 |
+
|
| 694 |
+
# Rodar testes
|
| 695 |
+
pytest -v
|
| 696 |
+
|
| 697 |
+
# Testar import
|
| 698 |
+
python -c "import src; print('✓ Imports OK')"
|
| 699 |
+
|
| 700 |
+
# Rodar CLI help
|
| 701 |
+
python -m src.main --help
|
| 702 |
+
```
|
| 703 |
+
|
| 704 |
+
**Esperado**: Todos os comandos rodarem sem erro.
|
| 705 |
+
|
| 706 |
+
---
|
| 707 |
+
|
| 708 |
+
## Step 9: Commit Initial
|
| 709 |
+
|
| 710 |
+
```bash
|
| 711 |
+
git add -A
|
| 712 |
+
git commit -m "feat: Sprint 1 - Initial project structure and scaffolding"
|
| 713 |
+
git push origin feature/birdnet-uploader-cli
|
| 714 |
+
```
|
| 715 |
+
|
| 716 |
+
---
|
| 717 |
+
|
| 718 |
+
## Checklist: Sprint 1, Day 1 - DONE ✓
|
| 719 |
+
|
| 720 |
+
- [ ] Branch `feature/birdnet-uploader-cli` criada
|
| 721 |
+
- [ ] Estrutura de diretórios completa
|
| 722 |
+
- [ ] `requirements.txt` instalado
|
| 723 |
+
- [ ] `pyproject.toml` criado
|
| 724 |
+
- [ ] Todos os módulos esqueléticos criados (8+ arquivos)
|
| 725 |
+
- [ ] `pytest.ini` e `conftest.py` configurados
|
| 726 |
+
- [ ] Testes básicos criados
|
| 727 |
+
- [ ] CI/CD workflow criado
|
| 728 |
+
- [ ] Primeira execução sem erro: `pytest -v`
|
| 729 |
+
- [ ] Commit feito e pushed
|
| 730 |
+
|
| 731 |
+
**Tempo**: ~2-3 horas
|
| 732 |
+
|
| 733 |
+
---
|
| 734 |
+
|
| 735 |
+
## Próximos Passos (Task 1.2 - Design Review)
|
| 736 |
+
|
| 737 |
+
Quando tudo acima estiver **DONE**, fazer:
|
| 738 |
+
|
| 739 |
+
1. **Meeting de Design Review** com o time (30min)
|
| 740 |
+
- Confirmar Tech Stack (Textual vs Rich+Click)
|
| 741 |
+
- Confirmar persistência (JSON vs SQLite)
|
| 742 |
+
- Validar estrutura de repo HF
|
| 743 |
+
- Assinar decision log
|
| 744 |
+
|
| 745 |
+
2. **Começar Task 2.1** (Auth Service Implementation)
|
| 746 |
+
- Implementar `AuthService.authenticate()` completo
|
| 747 |
+
- Implementar keyring integration
|
| 748 |
+
- Escrever testes com mocks
|
| 749 |
+
|
| 750 |
+
---
|
| 751 |
+
|
| 752 |
+
## Troubleshooting
|
| 753 |
+
|
| 754 |
+
**Problema**: `pip install` falha com erro de dependência
|
| 755 |
+
**Solução**: Upgrade pip: `pip install --upgrade pip`
|
| 756 |
+
|
| 757 |
+
**Problema**: Imports falhando
|
| 758 |
+
**Solução**: Adicionar `export PYTHONPATH="${PYTHONPATH}:$(pwd)"`
|
| 759 |
+
|
| 760 |
+
**Problema**: Pytest não encontra testes
|
| 761 |
+
**Solução**: Verificar que todos os `__init__.py` existem
|
| 762 |
+
|
| 763 |
+
---
|
| 764 |
+
|
| 765 |
+
## Documentação de Referência
|
| 766 |
+
|
| 767 |
+
- BIRDNET_UPLOADER_REDESIGN.md (arquitetura e UX)
|
| 768 |
+
- SPRINT_ROADMAP.md (timeline e tasks)
|
| 769 |
+
- Este arquivo (Sprint 1 Day 1 setup)
|
SPRINT_ROADMAP.md
ADDED
|
@@ -0,0 +1,649 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BirdNET Uploader: Sprint Roadmap (MVP 6-8 semanas)
|
| 2 |
+
|
| 3 |
+
## Overview
|
| 4 |
+
**Objetivo**: Entregar executável local moderno, um fluxo integrado, pausa/retomada, compatibilidade com validador.
|
| 5 |
+
|
| 6 |
+
**Timeline**:
|
| 7 |
+
- **Semanas 1-2**: Arquitetura + Setup (Foundation)
|
| 8 |
+
- **Semanas 3-4**: Módulos Core (Auth, Repo, Upload)
|
| 9 |
+
- **Semanas 5-6**: UI + Session Manager (Integration)
|
| 10 |
+
- **Semanas 7-8**: Build + Testes + Release (Polish)
|
| 11 |
+
|
| 12 |
+
**Team**: 2-3 engineers (1 backend, 1 frontend/UI, 1 QA/DevOps)
|
| 13 |
+
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
## SPRINT 1: Foundation & Architecture (Semanas 1-2)
|
| 17 |
+
|
| 18 |
+
### Semana 1: Setup Inicial + Design Review
|
| 19 |
+
|
| 20 |
+
#### Task 1.1: Setup do Projeto (2 dias, 3 pts)
|
| 21 |
+
**Objetivo**: Estrutura base, dependências, CI/CD
|
| 22 |
+
|
| 23 |
+
**Tarefas**:
|
| 24 |
+
- [ ] Criar branch `feature/birdnet-uploader-cli`
|
| 25 |
+
- [ ] Criar estrutura de diretórios (`src/`, `tests/`, `build/`)
|
| 26 |
+
- [ ] Setup `pyproject.toml` e `requirements.txt`
|
| 27 |
+
- huggingface_hub>=0.28.0
|
| 28 |
+
- click>=8.0.0
|
| 29 |
+
- textual>=0.30.0 (TUI)
|
| 30 |
+
- rich>=13.0.0
|
| 31 |
+
- pydantic>=2.0.0
|
| 32 |
+
- pandas>=2.0.0
|
| 33 |
+
- pyarrow>=14.0.0
|
| 34 |
+
- keyring>=24.0.0
|
| 35 |
+
- [ ] Setup GitHub Actions para lint/test
|
| 36 |
+
- [ ] Criar `src/__init__.py` e `src/main.py` (skeleton)
|
| 37 |
+
|
| 38 |
+
**Critério de Pronto**:
|
| 39 |
+
- `pip install -r requirements.txt` sem erro
|
| 40 |
+
- `python -m src.main --help` roda e exibe help text
|
| 41 |
+
|
| 42 |
+
---
|
| 43 |
+
|
| 44 |
+
#### Task 1.2: Design Review + Arquitetura Finalizada (2 dias, 2 pts)
|
| 45 |
+
**Objetivo**: Alinhamento com stakeholders, decisões arquiteturais bloqueantes
|
| 46 |
+
|
| 47 |
+
**Tarefas**:
|
| 48 |
+
- [ ] Revisar documento REDESIGN com time
|
| 49 |
+
- [ ] Decisão final: Textual vs Rich+Click vs Typer (recomendação: Textual para melhor UX)
|
| 50 |
+
- [ ] Decidir sobre persistência: SQLite vs JSON files (recomendação: JSON + estrutura de diretórios)
|
| 51 |
+
- [ ] Validar estrutura de repositório HF (audio/index/validations/audit)
|
| 52 |
+
- [ ] Documentar todas as decisões em `docs/architecture-decisions.md`
|
| 53 |
+
|
| 54 |
+
**Critério de Pronto**:
|
| 55 |
+
- Documento de decisões assinado
|
| 56 |
+
- Time alinhado em tecnologia stack
|
| 57 |
+
- Nenhuma bloqueante técnica identificada
|
| 58 |
+
|
| 59 |
+
---
|
| 60 |
+
|
| 61 |
+
#### Task 1.3: Estrutura Base de Módulos (2 dias, 3 pts)
|
| 62 |
+
**Objetivo**: Criar interfaces/contracts de todos os módulos
|
| 63 |
+
|
| 64 |
+
**Tarefas**:
|
| 65 |
+
- [ ] Criar `src/auth/__init__.py` + `auth_service.py` (skeleton com docstrings)
|
| 66 |
+
- [ ] Criar `src/repository/__init__.py` + `repo_service.py` (skeleton)
|
| 67 |
+
- [ ] Criar `src/upload/__init__.py` + `scanner.py`, `uploader.py`, `deduplicator.py` (skeleton)
|
| 68 |
+
- [ ] Criar `src/csv/__init__.py` + `csv_matcher.py`, `indexer.py` (skeleton)
|
| 69 |
+
- [ ] Criar `src/session/__init__.py` + `session_manager.py`, `checkpoint.py` (skeleton)
|
| 70 |
+
- [ ] Criar `src/ui/__init__.py` + `tui.py`, `progress_renderer.py` (skeleton)
|
| 71 |
+
- [ ] Criar `src/logger/__init__.py` + `structured_logger.py` (skeleton)
|
| 72 |
+
- [ ] Criar `src/utils/__init__.py` + `file_utils.py`, `hash_utils.py`, `error_handler.py` (skeleton)
|
| 73 |
+
|
| 74 |
+
**Critério de Pronto**:
|
| 75 |
+
- Todos os módulos existem com `pass` ou docstrings
|
| 76 |
+
- Nenhum erro de import circular
|
| 77 |
+
- `python -c "import src; print('OK')"` funciona
|
| 78 |
+
|
| 79 |
+
---
|
| 80 |
+
|
| 81 |
+
#### Task 1.4: Testes Base + Test Fixtures (1 dia, 2 pts)
|
| 82 |
+
**Objetivo**: Infraestrutura de testes pronta
|
| 83 |
+
|
| 84 |
+
**Tarefas**:
|
| 85 |
+
- [ ] Setup `pytest.ini` e `conftest.py`
|
| 86 |
+
- [ ] Criar test fixtures para:
|
| 87 |
+
- Mock HfApi
|
| 88 |
+
- Pasta local temporária com arquivos de teste
|
| 89 |
+
- Session mock
|
| 90 |
+
- Token mock
|
| 91 |
+
- [ ] Criar `tests/test_auth.py` (skeleton)
|
| 92 |
+
- [ ] Criar `tests/test_uploader.py` (skeleton)
|
| 93 |
+
- [ ] Criar `tests/test_session.py` (skeleton)
|
| 94 |
+
|
| 95 |
+
**Critério de Pronto**:
|
| 96 |
+
- `pytest` executa sem erro (mesmo que tests sejam vazios)
|
| 97 |
+
- Fixtures estão disponíveis para todos os testes
|
| 98 |
+
|
| 99 |
+
---
|
| 100 |
+
|
| 101 |
+
### Semana 2: Auth + Repository Services
|
| 102 |
+
|
| 103 |
+
#### Task 2.1: Auth Service - Implementação Completa (2.5 dias, 5 pts)
|
| 104 |
+
**Objetivo**: Login com token, validação, armazenamento seguro
|
| 105 |
+
|
| 106 |
+
**Tarefas**:
|
| 107 |
+
- [ ] Implementar `AuthService.authenticate(token)`:
|
| 108 |
+
- Validar token contra HF API
|
| 109 |
+
- Recuperar info do usuário (username, email, user_id)
|
| 110 |
+
- Salvar em keyring (Windows DPAPI)
|
| 111 |
+
- Retornar user context
|
| 112 |
+
- [ ] Implementar `AuthService.get_stored_token()`:
|
| 113 |
+
- Recuperar token do keyring
|
| 114 |
+
- Lançar erro claro se não encontrado
|
| 115 |
+
- [ ] Implementar `AuthService.clear_token()`:
|
| 116 |
+
- Remover token do keyring
|
| 117 |
+
- [ ] Implementar `TokenManager`:
|
| 118 |
+
- Gerenciar TTL de token
|
| 119 |
+
- Detectar expiração
|
| 120 |
+
- Forçar re-autenticação se necessário
|
| 121 |
+
- [ ] Tests:
|
| 122 |
+
- [ ] `test_auth_valid_token()`
|
| 123 |
+
- [ ] `test_auth_invalid_token()`
|
| 124 |
+
- [ ] `test_auth_token_stored_in_keyring()`
|
| 125 |
+
- [ ] `test_auth_token_retrieval()`
|
| 126 |
+
|
| 127 |
+
**Critério de Pronto**:
|
| 128 |
+
- Todos os testes de auth passando
|
| 129 |
+
- Token nunca é logado/printed
|
| 130 |
+
- Falha de auth retorna mensagem clara
|
| 131 |
+
|
| 132 |
+
---
|
| 133 |
+
|
| 134 |
+
#### Task 2.2: Repository Service - Criar Dataset (2 dias, 4 pts)
|
| 135 |
+
**Objetivo**: Criar dataset HF e inicializar estrutura padrão
|
| 136 |
+
|
| 137 |
+
**Tarefas**:
|
| 138 |
+
- [ ] Implementar `RepositoryService.create_dataset()`:
|
| 139 |
+
- Validar nome do repositório (slug format)
|
| 140 |
+
- Criar dataset no HF (público/privado)
|
| 141 |
+
- Inicializar pastas padrão (audio/, index/shards/, validations/, audit/)
|
| 142 |
+
- Criar manifest.json inicial
|
| 143 |
+
- Retornar repo_id
|
| 144 |
+
- [ ] Implementar `RepositoryService.validate_repo()`:
|
| 145 |
+
- Verificar permissão de escrita
|
| 146 |
+
- Verificar estrutura esperada
|
| 147 |
+
- Retornar validation result
|
| 148 |
+
- [ ] Tests:
|
| 149 |
+
- [ ] `test_create_dataset_public()`
|
| 150 |
+
- [ ] `test_create_dataset_private()`
|
| 151 |
+
- [ ] `test_create_dataset_name_validation()`
|
| 152 |
+
- [ ] `test_validate_repo_structure()`
|
| 153 |
+
|
| 154 |
+
**Critério de Pronto**:
|
| 155 |
+
- Dataset criado no HF com structure correta
|
| 156 |
+
- Manifest.json com schema_version, project_slug, index
|
| 157 |
+
- Testes mockeados passando
|
| 158 |
+
|
| 159 |
+
---
|
| 160 |
+
|
| 161 |
+
#### Task 2.3: Config + Constants Finalizadas (1 dia, 2 pts)
|
| 162 |
+
**Objetivo**: Configuração centralizada do app
|
| 163 |
+
|
| 164 |
+
**Tarefas**:
|
| 165 |
+
- [ ] Criar `src/config.py`:
|
| 166 |
+
- `AUDIO_EXTENSIONS = ['.wav', '.mp3', '.flac', '.ogg']`
|
| 167 |
+
- `SESSION_DIR = ~/.birdnet-uploader/sessions/`
|
| 168 |
+
- `CACHE_DIR = ~/.birdnet-uploader/cache/`
|
| 169 |
+
- `MAX_BATCH_SIZE = 10`
|
| 170 |
+
- `RETRY_MAX_ATTEMPTS = 3`
|
| 171 |
+
- `RETRY_INITIAL_BACKOFF = 1.0`
|
| 172 |
+
- `MANIFEST_SCHEMA_VERSION = "1.0.0"`
|
| 173 |
+
- [ ] Implementar config via `.env` se necessário
|
| 174 |
+
|
| 175 |
+
**Critério de Pronto**:
|
| 176 |
+
- Todas as constantes centralizadas
|
| 177 |
+
- Nenhum hardcoded value em módulos
|
| 178 |
+
|
| 179 |
+
---
|
| 180 |
+
|
| 181 |
+
#### Task 2.4: Error Handling Framework (1.5 dias, 3 pts)
|
| 182 |
+
**Objetivo**: Exceções customizadas e mensagens claras
|
| 183 |
+
|
| 184 |
+
**Tarefas**:
|
| 185 |
+
- [ ] Criar `src/exceptions.py`:
|
| 186 |
+
- `AuthenticationError`
|
| 187 |
+
- `RepositoryError`
|
| 188 |
+
- `UploadError`
|
| 189 |
+
- `SessionError`
|
| 190 |
+
- `ValidationError`
|
| 191 |
+
- [ ] Implementar `src/utils/error_handler.py`:
|
| 192 |
+
- Converter exceções em mensagens user-friendly
|
| 193 |
+
- Log estruturado de erros
|
| 194 |
+
- Sugestões de ação
|
| 195 |
+
|
| 196 |
+
**Critério de Pronto**:
|
| 197 |
+
- Todas as exceções possuem mensagem clara
|
| 198 |
+
- Tests cobrem cenários de erro
|
| 199 |
+
|
| 200 |
+
---
|
| 201 |
+
|
| 202 |
+
## SPRINT 2: Upload & CSV Matching (Semanas 3-4)
|
| 203 |
+
|
| 204 |
+
### Semana 3: Scanner + File Operations
|
| 205 |
+
|
| 206 |
+
#### Task 3.1: Local Scanner - Varrer Pasta (2 dias, 4 pts)
|
| 207 |
+
**Objetivo**: Descobrir arquivos locais, agrupar por espécie
|
| 208 |
+
|
| 209 |
+
**Tarefas**:
|
| 210 |
+
- [ ] Implementar `LocalScanner.scan_folder()`:
|
| 211 |
+
- Varrer recursivamente
|
| 212 |
+
- Filtrar por extensões de áudio
|
| 213 |
+
- Inferir espécie do caminho/nome
|
| 214 |
+
- Calcular tamanho total e contagem
|
| 215 |
+
- Retornar estrutura normalizada
|
| 216 |
+
- [ ] Implementar estratégia de inferência de espécie:
|
| 217 |
+
- Strategy 1: Usar última pasta como specie
|
| 218 |
+
- Strategy 2: Usar prefixo do arquivo
|
| 219 |
+
- Strategy 3: Permitir mapeamento manual
|
| 220 |
+
- [ ] Tests:
|
| 221 |
+
- [ ] `test_scan_folder_empty()`
|
| 222 |
+
- [ ] `test_scan_folder_multiple_species()`
|
| 223 |
+
- [ ] `test_scan_folder_specie_inference()`
|
| 224 |
+
- [ ] `test_scan_folder_with_nested_dirs()`
|
| 225 |
+
|
| 226 |
+
**Critério de Pronto**:
|
| 227 |
+
- Escaneia pasta com 1000+ arquivos em <5 segundos
|
| 228 |
+
- Estrutura normalizada e validada
|
| 229 |
+
|
| 230 |
+
---
|
| 231 |
+
|
| 232 |
+
#### Task 3.2: Hash Utils + File Operations (1.5 dias, 3 pts)
|
| 233 |
+
**Objetivo**: Computação de hash segura, operações de arquivo
|
| 234 |
+
|
| 235 |
+
**Tarefas**:
|
| 236 |
+
- [ ] Implementar `compute_file_hash(path) -> str`:
|
| 237 |
+
- SHA256 eficiente (streaming para arquivos grandes)
|
| 238 |
+
- Cache local de hashes
|
| 239 |
+
- [ ] Implementar `verify_file_integrity(path, hash)`:
|
| 240 |
+
- Validar arquivo não corrompido
|
| 241 |
+
- [ ] Tests:
|
| 242 |
+
- [ ] `test_compute_hash_small_file()`
|
| 243 |
+
- [ ] `test_compute_hash_large_file()`
|
| 244 |
+
- [ ] `test_hash_consistency()`
|
| 245 |
+
|
| 246 |
+
**Critério de Pronto**:
|
| 247 |
+
- Hash de arquivo 1GB < 2 segundos
|
| 248 |
+
- Testes passando
|
| 249 |
+
|
| 250 |
+
---
|
| 251 |
+
|
| 252 |
+
#### Task 3.3: Deduplicator - Verificar Existência (2 dias, 3 pts)
|
| 253 |
+
**Objetivo**: Detectar arquivos já no destino
|
| 254 |
+
|
| 255 |
+
**Tarefas**:
|
| 256 |
+
- [ ] Implementar `Deduplicator.check_remote()`:
|
| 257 |
+
- Query HF API para verificar existência por caminho
|
| 258 |
+
- Comparar tamanho + hash opcional
|
| 259 |
+
- Retornar status (SKIP, UPLOAD, CONFLICT)
|
| 260 |
+
- [ ] Implementar dedup local cache (evitar queries repetidas)
|
| 261 |
+
- [ ] Tests:
|
| 262 |
+
- [ ] `test_check_remote_new_file()`
|
| 263 |
+
- [ ] `test_check_remote_existing_file()`
|
| 264 |
+
- [ ] `test_check_remote_conflict()`
|
| 265 |
+
|
| 266 |
+
**Critério de Pronto**:
|
| 267 |
+
- Dedup funciona sem re-query excessiva
|
| 268 |
+
- Cache valida corretamente
|
| 269 |
+
|
| 270 |
+
---
|
| 271 |
+
|
| 272 |
+
#### Task 3.4: Session + Checkpoint - State Persistence (2.5 dias, 5 pts)
|
| 273 |
+
**Objetivo**: Persistência atomática de estado para retomada
|
| 274 |
+
|
| 275 |
+
**Tarefas**:
|
| 276 |
+
- [ ] Implementar `SessionManager.create_session()`:
|
| 277 |
+
- Gerar session_id
|
| 278 |
+
- Criar diretório de sessão
|
| 279 |
+
- Inicializar metadata.json
|
| 280 |
+
- [ ] Implementar `CheckpointManager.save_checkpoint()`:
|
| 281 |
+
- Escrita atômica (temp + rename)
|
| 282 |
+
- Validação de JSON
|
| 283 |
+
- Flush para disco (fsync)
|
| 284 |
+
- [ ] Implementar `CheckpointManager.load_checkpoint()`:
|
| 285 |
+
- Recuperar último checkpoint
|
| 286 |
+
- Validação de integridade
|
| 287 |
+
- [ ] Implementar `SessionManager.mark_file_done()`:
|
| 288 |
+
- Atualizar manifest por arquivo
|
| 289 |
+
- [ ] Tests:
|
| 290 |
+
- [ ] `test_session_creation()`
|
| 291 |
+
- [ ] `test_checkpoint_atomic_write()`
|
| 292 |
+
- [ ] `test_checkpoint_recovery_after_crash()`
|
| 293 |
+
- [ ] `test_session_resume_same_id()`
|
| 294 |
+
|
| 295 |
+
**Critério de Pronto**:
|
| 296 |
+
- Checkpoint atomático garante consistência
|
| 297 |
+
- Recuperação de crash funciona
|
| 298 |
+
- Testes cobrem falha de escrita
|
| 299 |
+
|
| 300 |
+
---
|
| 301 |
+
|
| 302 |
+
### Semana 4: Upload Engine + CSV Matching
|
| 303 |
+
|
| 304 |
+
#### Task 4.1: Batch Uploader - Motor Principal (3 dias, 6 pts)
|
| 305 |
+
**Objetivo**: Upload resiliente com retry/dedup/pause
|
| 306 |
+
|
| 307 |
+
**Tarefas**:
|
| 308 |
+
- [ ] Implementar `BatchUploader.upload_files()`:
|
| 309 |
+
- Iterar sobre estrutura de arquivos
|
| 310 |
+
- Aplicar dedup check
|
| 311 |
+
- Upload com retry automático
|
| 312 |
+
- Atualizar checkpoint por lote
|
| 313 |
+
- Callback de progresso
|
| 314 |
+
- [ ] Implementar retry com exponential backoff:
|
| 315 |
+
- Backoff: 1s → 2s → 5s → 10s → 30s
|
| 316 |
+
- Tratamento de 429 (rate limit)
|
| 317 |
+
- Max 3 retries por padrão
|
| 318 |
+
- [ ] Implementar pause/resume:
|
| 319 |
+
- Signal SIGTERM/SIGINT
|
| 320 |
+
- Salvar state
|
| 321 |
+
- Resume com checkpoint
|
| 322 |
+
- [ ] Tests:
|
| 323 |
+
- [ ] `test_upload_batch_success()`
|
| 324 |
+
- [ ] `test_upload_with_retry()`
|
| 325 |
+
- [ ] `test_upload_skip_existing()`
|
| 326 |
+
- [ ] `test_upload_pause_resume()`
|
| 327 |
+
|
| 328 |
+
**Critério de Pronto**:
|
| 329 |
+
- Upload funciona end-to-end
|
| 330 |
+
- Retry logic testado
|
| 331 |
+
- Pause/resume funciona
|
| 332 |
+
|
| 333 |
+
---
|
| 334 |
+
|
| 335 |
+
#### Task 4.2: CSV Matcher + Index Generator (2.5 dias, 5 pts)
|
| 336 |
+
**Objetivo**: Matching CSV com segmentos e geração de índice
|
| 337 |
+
|
| 338 |
+
**Tarefas**:
|
| 339 |
+
- [ ] Implementar `CSVMatcher.match_detections()`:
|
| 340 |
+
- Ler CSV com validação de schema
|
| 341 |
+
- Fazer matching com segmentos (source_file → arquivo local)
|
| 342 |
+
- Gerar detection_key determinístico
|
| 343 |
+
- Retornar matched + unmatched + ambiguous
|
| 344 |
+
- [ ] Implementar `IndexGenerator.generate_shards()`:
|
| 345 |
+
- Particionar dados em shards (tamanho configurável)
|
| 346 |
+
- Salvar como parquet
|
| 347 |
+
- Gerar metadata de shards
|
| 348 |
+
- [ ] Implementar `ManifestBuilder.build()`:
|
| 349 |
+
- Construir manifest.json com schema_version
|
| 350 |
+
- Incluir metadados de index
|
| 351 |
+
- Serializar JSON
|
| 352 |
+
- [ ] Tests:
|
| 353 |
+
- [ ] `test_csv_matching_success()`
|
| 354 |
+
- [ ] `test_csv_missing_columns()`
|
| 355 |
+
- [ ] `test_shard_generation()`
|
| 356 |
+
- [ ] `test_manifest_format()`
|
| 357 |
+
|
| 358 |
+
**Critério de Pronto**:
|
| 359 |
+
- CSV+segmentos matchados corretamente
|
| 360 |
+
- Shards gerados com formato correto
|
| 361 |
+
- Manifest válido e legível
|
| 362 |
+
|
| 363 |
+
---
|
| 364 |
+
|
| 365 |
+
## SPRINT 3: UI + Integration (Semanas 5-6)
|
| 366 |
+
|
| 367 |
+
### Semana 5: UI TUI Implementation
|
| 368 |
+
|
| 369 |
+
#### Task 5.1: Textual TUI - Fluxo Principal (3 dias, 6 pts)
|
| 370 |
+
**Objetivo**: Interface TUI moderna para guiar usuário
|
| 371 |
+
|
| 372 |
+
**Tarefas**:
|
| 373 |
+
- [ ] Criar `src/ui/screens/login_screen.py`:
|
| 374 |
+
- Input para token HF
|
| 375 |
+
- Validação em tempo real
|
| 376 |
+
- Mensagens de erro claras
|
| 377 |
+
- [ ] Criar `src/ui/screens/config_screen.py`:
|
| 378 |
+
- Form com campos: repo name, visibility, segments path, csv path
|
| 379 |
+
- Verificação de pasta
|
| 380 |
+
- Validação de formato
|
| 381 |
+
- [ ] Criar `src/ui/screens/summary_screen.py`:
|
| 382 |
+
- Exibir configuração resumida
|
| 383 |
+
- Botão de confirmação
|
| 384 |
+
- [ ] Criar `src/ui/screens/upload_screen.py`:
|
| 385 |
+
- Barra de progresso principal
|
| 386 |
+
- Progresso por arquivo
|
| 387 |
+
- Arquivo atual
|
| 388 |
+
- Último evento
|
| 389 |
+
- Botões pause/cancel
|
| 390 |
+
- [ ] Implementar navegação entre screens
|
| 391 |
+
|
| 392 |
+
**Critério de Pronto**:
|
| 393 |
+
- Fluxo completo funcionando (login → config → summary → upload)
|
| 394 |
+
- Inputs validados
|
| 395 |
+
- Navegação intuitiva
|
| 396 |
+
|
| 397 |
+
---
|
| 398 |
+
|
| 399 |
+
#### Task 5.2: Progress Renderer (1.5 dias, 3 pts)
|
| 400 |
+
**Objetivo**: Visualização rich de progresso
|
| 401 |
+
|
| 402 |
+
**Tarefas**:
|
| 403 |
+
- [ ] Implementar progress bar customizada:
|
| 404 |
+
- Percentual + barra visual
|
| 405 |
+
- ETA com update em tempo real
|
| 406 |
+
- Arquivo atual
|
| 407 |
+
- Contadores (uploaded/pending/failed/skipped)
|
| 408 |
+
- [ ] Implementar event log:
|
| 409 |
+
- Últimos 10 eventos
|
| 410 |
+
- Código de status/cor por tipo
|
| 411 |
+
|
| 412 |
+
**Critério de Pronto**:
|
| 413 |
+
- Progress bar atualiza sem flicker
|
| 414 |
+
- ETA coerente
|
| 415 |
+
- Eventos visíveis
|
| 416 |
+
|
| 417 |
+
---
|
| 418 |
+
|
| 419 |
+
### Semana 6: Integration + Testes End-to-End
|
| 420 |
+
|
| 421 |
+
#### Task 6.1: Integration Testing + Mocks (2 dias, 4 pts)
|
| 422 |
+
**Objetivo**: Teste de fluxo completo (mockeado)
|
| 423 |
+
|
| 424 |
+
**Tarefas**:
|
| 425 |
+
- [ ] Criar `tests/test_integration.py`:
|
| 426 |
+
- Teste login + config + upload (mockeado)
|
| 427 |
+
- Teste pause/resume
|
| 428 |
+
- Teste com CSV
|
| 429 |
+
- [ ] Mockar HfApi completamente
|
| 430 |
+
- [ ] Rodar testes de carga (1000+ arquivos mockeados)
|
| 431 |
+
|
| 432 |
+
**Critério de Pronto**:
|
| 433 |
+
- Testes de integração passando
|
| 434 |
+
- Sem vazamento de recursos
|
| 435 |
+
|
| 436 |
+
---
|
| 437 |
+
|
| 438 |
+
#### Task 6.2: Real Integration Test (pequena escala) (2 dias, 4 pts)
|
| 439 |
+
**Objetivo**: Teste real com dataset HF de verdade
|
| 440 |
+
|
| 441 |
+
**Tarefas**:
|
| 442 |
+
- [ ] Criar dataset de teste no HF (alice/test-uploader-small)
|
| 443 |
+
- [ ] Teste com 100 arquivos reais (~500MB)
|
| 444 |
+
- [ ] Validar estrutura no HF pós-upload
|
| 445 |
+
- [ ] Validar manifest.json e shards
|
| 446 |
+
|
| 447 |
+
**Critério de Pronto**:
|
| 448 |
+
- Arquivos aparecem no HF
|
| 449 |
+
- Estrutura correta
|
| 450 |
+
- Manifest válido
|
| 451 |
+
|
| 452 |
+
---
|
| 453 |
+
|
| 454 |
+
## SPRINT 4: Build + Polish (Semanas 7-8)
|
| 455 |
+
|
| 456 |
+
### Semana 7: Build + Release
|
| 457 |
+
|
| 458 |
+
#### Task 7.1: PyInstaller Setup (2 dias, 4 pts)
|
| 459 |
+
**Objetivo**: Gerar executável portátil Windows
|
| 460 |
+
|
| 461 |
+
**Tarefas**:
|
| 462 |
+
- [ ] Criar `build/pyinstaller_config.spec`:
|
| 463 |
+
- One-file executable
|
| 464 |
+
- Hidden imports para textual, rich, pandas, pyarrow
|
| 465 |
+
- Resource bundling
|
| 466 |
+
- [ ] Criar `build/build.py` script
|
| 467 |
+
- [ ] Testar executável gerado localmente
|
| 468 |
+
- [ ] Validar tamanho (~100-150MB)
|
| 469 |
+
|
| 470 |
+
**Critério de Pronto**:
|
| 471 |
+
- Executável funciona sem Python instalado
|
| 472 |
+
- Tamanho razoável
|
| 473 |
+
- Sem erros de runtime
|
| 474 |
+
|
| 475 |
+
---
|
| 476 |
+
|
| 477 |
+
#### Task 7.2: Documentação + README (1.5 dias, 2 pts)
|
| 478 |
+
**Objetivo**: Docs para usuários finais
|
| 479 |
+
|
| 480 |
+
**Tarefas**:
|
| 481 |
+
- [ ] Criar README.md com:
|
| 482 |
+
- Quick start (download, run, autenticar)
|
| 483 |
+
- Screenshots
|
| 484 |
+
- Troubleshooting
|
| 485 |
+
- FAQ
|
| 486 |
+
- [ ] Criar INSTALLATION.md
|
| 487 |
+
- [ ] Criar USAGE.md (exemplos de comando)
|
| 488 |
+
|
| 489 |
+
**Critério de Pronto**:
|
| 490 |
+
- Usuário não-técnico consegue rodar pelo README
|
| 491 |
+
|
| 492 |
+
---
|
| 493 |
+
|
| 494 |
+
#### Task 7.3: Testes de Carga + Stress (2 dias, 4 pts)
|
| 495 |
+
**Objetivo**: Validar com dados reais maiores
|
| 496 |
+
|
| 497 |
+
**Tarefas**:
|
| 498 |
+
- [ ] Teste com pasta 5GB (1000+ arquivos)
|
| 499 |
+
- [ ] Teste com CSV 5000+ linhas
|
| 500 |
+
- [ ] Teste pause/resume 5x
|
| 501 |
+
- [ ] Monitorar memória e CPU
|
| 502 |
+
- [ ] Documentar performance (tempo/MB/s)
|
| 503 |
+
|
| 504 |
+
**Critério de Pronto**:
|
| 505 |
+
- Memória < 500MB durante upload
|
| 506 |
+
- CPU < 30% média
|
| 507 |
+
- Upload rate > 5 MB/s (com rede OK)
|
| 508 |
+
|
| 509 |
+
---
|
| 510 |
+
|
| 511 |
+
### Semana 8: Release + Smoke Tests
|
| 512 |
+
|
| 513 |
+
#### Task 8.1: Release Pipeline (1.5 dias, 2 pts)
|
| 514 |
+
**Objetivo**: Setup de distribuição
|
| 515 |
+
|
| 516 |
+
**Tarefas**:
|
| 517 |
+
- [ ] Criar release no GitHub com:
|
| 518 |
+
- Executável (`.exe`)
|
| 519 |
+
- SHA256 hash
|
| 520 |
+
- Release notes
|
| 521 |
+
- Link para docs
|
| 522 |
+
- [ ] Testar download + execução do exe
|
| 523 |
+
- [ ] Validar assinatura (opcional nesta fase)
|
| 524 |
+
|
| 525 |
+
**Critério de Pronto**:
|
| 526 |
+
- Release publicado
|
| 527 |
+
- Download funciona
|
| 528 |
+
- Exe roda após download/extração
|
| 529 |
+
|
| 530 |
+
---
|
| 531 |
+
|
| 532 |
+
#### Task 8.2: Smoke Tests + Final QA (1.5 dias, 3 pts)
|
| 533 |
+
**Objetivo**: Validação final antes do release
|
| 534 |
+
|
| 535 |
+
**Tarefas**:
|
| 536 |
+
- [ ] Checklist final:
|
| 537 |
+
- [ ] Login funciona
|
| 538 |
+
- [ ] Criar repo funciona
|
| 539 |
+
- [ ] Upload de 10GB completa
|
| 540 |
+
- [ ] Pause/resume sem perda
|
| 541 |
+
- [ ] CSV matching correto
|
| 542 |
+
- [ ] Manifest válido no destino
|
| 543 |
+
- [ ] Estrutura compatível com validador
|
| 544 |
+
- [ ] Teste com 3+ usuários externos (team)
|
| 545 |
+
- [ ] Coleta de feedback
|
| 546 |
+
|
| 547 |
+
**Critério de Pronto**:
|
| 548 |
+
- Zero bugs críticos
|
| 549 |
+
- Feedback incorporado
|
| 550 |
+
- Pronto para release
|
| 551 |
+
|
| 552 |
+
---
|
| 553 |
+
|
| 554 |
+
## BACKLOG: Priorização de Tasks
|
| 555 |
+
|
| 556 |
+
| Sprint | Semana | Task | Pts | Status |
|
| 557 |
+
|--------|--------|------|-----|--------|
|
| 558 |
+
| 1 | 1 | Setup + Design | 5 | Não iniciado |
|
| 559 |
+
| 1 | 1 | Arquitetura de Módulos | 3 | Não iniciado |
|
| 560 |
+
| 1 | 1 | Test Infrastructure | 2 | Não iniciado |
|
| 561 |
+
| 1 | 2 | Auth Service | 5 | Bloqueado por 1.1 |
|
| 562 |
+
| 1 | 2 | Repository Service | 4 | Bloqueado por 1.1 |
|
| 563 |
+
| 1 | 2 | Config + Constants | 2 | Bloqueado por 1.1 |
|
| 564 |
+
| 1 | 2 | Error Handling | 3 | Bloqueado por 1.1 |
|
| 565 |
+
| 2 | 3 | Local Scanner | 4 | Bloqueado por Sprint 1 |
|
| 566 |
+
| 2 | 3 | Hash Utils | 3 | Bloqueado por Sprint 1 |
|
| 567 |
+
| 2 | 3 | Deduplicator | 3 | Bloqueado por Sprint 1 |
|
| 568 |
+
| 2 | 3 | Session + Checkpoint | 5 | Bloqueado por Sprint 1 |
|
| 569 |
+
| 2 | 4 | Batch Uploader | 6 | Bloqueado por Sprint 1 |
|
| 570 |
+
| 2 | 4 | CSV Matcher | 5 | Bloqueado por Sprint 1 |
|
| 571 |
+
| 3 | 5 | Textual TUI | 6 | Bloqueado por Sprint 2 |
|
| 572 |
+
| 3 | 5 | Progress Renderer | 3 | Bloqueado por Sprint 2 |
|
| 573 |
+
| 3 | 6 | Integration Tests | 4 | Bloqueado por Sprint 2 |
|
| 574 |
+
| 3 | 6 | Real Integration Test | 4 | Bloqueado por Sprint 2 |
|
| 575 |
+
| 4 | 7 | PyInstaller | 4 | Bloqueado por Sprint 3 |
|
| 576 |
+
| 4 | 7 | Documentation | 2 | Bloqueado por Sprint 3 |
|
| 577 |
+
| 4 | 7 | Load Testing | 4 | Bloqueado por Sprint 3 |
|
| 578 |
+
| 4 | 8 | Release Pipeline | 2 | Bloqueado por Sprint 3 |
|
| 579 |
+
| 4 | 8 | Final QA | 3 | Bloqueado por Sprint 3 |
|
| 580 |
+
|
| 581 |
+
**Total: ~84 story points (6-8 semanas com 2-3 eng)**
|
| 582 |
+
|
| 583 |
+
---
|
| 584 |
+
|
| 585 |
+
## Velocity + Alocação de Equipe
|
| 586 |
+
|
| 587 |
+
### Cenário 1: Full Team (2 eng)
|
| 588 |
+
- Sprint de 2 semanas, 2 eng = ~40 pts/sprint
|
| 589 |
+
- Sprint 1-2: 26 pts (semana 1 paralelo, semana 2 paralelo) ✓ cabe
|
| 590 |
+
- Sprint 2 continuation: ~20 pts (cabe)
|
| 591 |
+
- Sprint 3: ~13 pts (cabe)
|
| 592 |
+
- Sprint 4: ~15 pts (cabe)
|
| 593 |
+
|
| 594 |
+
### Cenário 2: 3 Eng (Backend + Frontend + QA)
|
| 595 |
+
- Paralelizar mais agressivamente
|
| 596 |
+
- Timeline pode reduzir de 8 para 6-7 semanas
|
| 597 |
+
|
| 598 |
+
---
|
| 599 |
+
|
| 600 |
+
## Critérios de "Done" por Sprint
|
| 601 |
+
|
| 602 |
+
### Sprint 1 Done ✓
|
| 603 |
+
- [ ] Todos os módulos base existem (importáveis)
|
| 604 |
+
- [ ] Auth funciona com keyring
|
| 605 |
+
- [ ] Repo pode ser criado no HF
|
| 606 |
+
- [ ] Testes de auth e repo passando
|
| 607 |
+
- [ ] Setup pronto para Sprint 2
|
| 608 |
+
|
| 609 |
+
### Sprint 2 Done ✓
|
| 610 |
+
- [ ] Scanner varre pasta corretamente
|
| 611 |
+
- [ ] Uploader faz batch com retry
|
| 612 |
+
- [ ] CSV matcher funciona end-to-end
|
| 613 |
+
- [ ] Checkpoint salva e recupera estado
|
| 614 |
+
- [ ] Testes cobrem 80%+ de lógica
|
| 615 |
+
|
| 616 |
+
### Sprint 3 Done ✓
|
| 617 |
+
- [ ] Fluxo TUI completo (login → config → upload)
|
| 618 |
+
- [ ] Progresso visual atualiza sem flicker
|
| 619 |
+
- [ ] Integração completa mockeada testada
|
| 620 |
+
- [ ] Real integration test com 500MB+ passou
|
| 621 |
+
- [ ] Pronto para build
|
| 622 |
+
|
| 623 |
+
### Sprint 4 Done ✓
|
| 624 |
+
- [ ] Executável portátil gerado e testado
|
| 625 |
+
- [ ] Release pipeline setup
|
| 626 |
+
- [ ] Documentação completa
|
| 627 |
+
- [ ] Smoke tests passando
|
| 628 |
+
- [ ] Pronto para distribuição
|
| 629 |
+
|
| 630 |
+
---
|
| 631 |
+
|
| 632 |
+
## Next Steps
|
| 633 |
+
|
| 634 |
+
1. **Aprovação**: Confirmar timeline, team size, prioridades
|
| 635 |
+
2. **Sprint 1 Kickoff**: Semana que vem
|
| 636 |
+
3. **Daily Standups**: 15min todo dia weekday
|
| 637 |
+
4. **Sprint Reviews**: Sexta-feira cada semana
|
| 638 |
+
5. **Retrospectives**: Fim de cada sprint
|
| 639 |
+
|
| 640 |
+
---
|
| 641 |
+
|
| 642 |
+
## Notas & Riscos
|
| 643 |
+
|
| 644 |
+
- **Risco 1**: PyInstaller não empacotar corretamente → Mitigar: spike de 1 dia em week 7
|
| 645 |
+
- **Risco 2**: Performance ruim em 50GB+ → Mitigar: load test cedo (week 4)
|
| 646 |
+
- **Risco 3**: Incompatibilidade com validador → Mitigar: validação de manifest early (week 4)
|
| 647 |
+
- **Risco 4**: UX TUI confusa → Mitigar: user testing em week 6
|
| 648 |
+
|
| 649 |
+
Todas as risks têm mitigation strategy atrelada.
|
pyproject.toml
CHANGED
|
@@ -5,5 +5,8 @@ description = "BirdNET-Validator-App for multi-project bird detection review wor
|
|
| 5 |
readme = "README.md"
|
| 6 |
requires-python = ">=3.11"
|
| 7 |
|
|
|
|
|
|
|
|
|
|
| 8 |
[tool.pytest.ini_options]
|
| 9 |
testpaths = ["tests"]
|
|
|
|
| 5 |
readme = "README.md"
|
| 6 |
requires-python = ">=3.11"
|
| 7 |
|
| 8 |
+
[project.scripts]
|
| 9 |
+
birdnet-uploader = "src.uploader_cli.main:cli"
|
| 10 |
+
|
| 11 |
[tool.pytest.ini_options]
|
| 12 |
testpaths = ["tests"]
|
requirements.txt
CHANGED
|
@@ -6,3 +6,7 @@ Pillow>=10.4.0
|
|
| 6 |
pyarrow>=16.0.0
|
| 7 |
pydantic>=2.8.0
|
| 8 |
pytest>=8.3.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
pyarrow>=16.0.0
|
| 7 |
pydantic>=2.8.0
|
| 8 |
pytest>=8.3.0
|
| 9 |
+
click>=8.1.7
|
| 10 |
+
rich>=13.7.1
|
| 11 |
+
textual>=0.58.1
|
| 12 |
+
keyring>=25.2.1
|
src/uploader_cli/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""BirdNET local uploader CLI package."""
|
| 2 |
+
|
| 3 |
+
__all__ = [
|
| 4 |
+
"__version__",
|
| 5 |
+
]
|
| 6 |
+
|
| 7 |
+
__version__ = "0.1.0"
|
src/uploader_cli/auth_service.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
import keyring
|
| 6 |
+
from huggingface_hub import HfApi
|
| 7 |
+
|
| 8 |
+
from .config import KEYRING_ACCOUNT, KEYRING_SERVICE
|
| 9 |
+
from .exceptions import AuthenticationError
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AuthService:
|
| 13 |
+
"""Authenticate and securely store Hugging Face tokens."""
|
| 14 |
+
|
| 15 |
+
KEYRING_SERVICE = KEYRING_SERVICE
|
| 16 |
+
KEYRING_ACCOUNT = KEYRING_ACCOUNT
|
| 17 |
+
|
| 18 |
+
def authenticate(self, token: str) -> dict[str, Any]:
|
| 19 |
+
token = token.strip()
|
| 20 |
+
if not token:
|
| 21 |
+
raise AuthenticationError("Token is required")
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
api = HfApi(token=token)
|
| 25 |
+
whoami = api.whoami()
|
| 26 |
+
except Exception as exc: # pragma: no cover - external API behavior
|
| 27 |
+
raise AuthenticationError(f"Token validation failed: {exc}") from exc
|
| 28 |
+
|
| 29 |
+
try:
|
| 30 |
+
keyring.set_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT, token)
|
| 31 |
+
except Exception as exc: # pragma: no cover - backend-specific behavior
|
| 32 |
+
raise AuthenticationError(f"Token validated, but could not be saved securely: {exc}") from exc
|
| 33 |
+
|
| 34 |
+
return {
|
| 35 |
+
"username": whoami.get("name") or "unknown",
|
| 36 |
+
"email": whoami.get("email", ""),
|
| 37 |
+
"user_id": whoami.get("id") or whoami.get("user_id", ""),
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
def get_token(self) -> str | None:
|
| 41 |
+
try:
|
| 42 |
+
return keyring.get_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
|
| 43 |
+
except Exception as exc: # pragma: no cover - backend-specific behavior
|
| 44 |
+
raise AuthenticationError(f"Could not read stored token: {exc}") from exc
|
| 45 |
+
|
| 46 |
+
def require_token(self) -> str:
|
| 47 |
+
token = self.get_token()
|
| 48 |
+
if not token:
|
| 49 |
+
raise AuthenticationError("No stored token found. Run 'birdnet-uploader login' first.")
|
| 50 |
+
return token
|
| 51 |
+
|
| 52 |
+
def clear_token(self) -> None:
|
| 53 |
+
try:
|
| 54 |
+
keyring.delete_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
|
| 55 |
+
except keyring.errors.PasswordDeleteError:
|
| 56 |
+
return
|
| 57 |
+
except Exception as exc: # pragma: no cover - backend-specific behavior
|
| 58 |
+
raise AuthenticationError(f"Could not clear stored token: {exc}") from exc
|
| 59 |
+
else:
|
| 60 |
+
return
|
src/uploader_cli/batch_uploader.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import time
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any, Callable, Iterable
|
| 6 |
+
|
| 7 |
+
from huggingface_hub import HfApi
|
| 8 |
+
|
| 9 |
+
from .config import MAX_BATCH_SIZE, RETRY_MAX_ATTEMPTS, RETRY_INITIAL_BACKOFF_SECONDS
|
| 10 |
+
from .deduplicator import Deduplicator
|
| 11 |
+
from .session_manager import SessionManager
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class BatchUploader:
|
| 15 |
+
"""Orchestrate uploads with deduplication, retries and session checkpoints."""
|
| 16 |
+
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
api: HfApi,
|
| 20 |
+
repo_id: str,
|
| 21 |
+
deduplicator: Deduplicator,
|
| 22 |
+
session: SessionManager | None = None,
|
| 23 |
+
*,
|
| 24 |
+
max_retries: int | None = None,
|
| 25 |
+
initial_backoff: float | None = None,
|
| 26 |
+
) -> None:
|
| 27 |
+
self._api = api
|
| 28 |
+
self.repo_id = repo_id
|
| 29 |
+
self.deduplicator = deduplicator
|
| 30 |
+
self.session = session or SessionManager.create_session()
|
| 31 |
+
self.max_retries = max_retries or RETRY_MAX_ATTEMPTS
|
| 32 |
+
self.initial_backoff = initial_backoff or RETRY_INITIAL_BACKOFF_SECONDS
|
| 33 |
+
|
| 34 |
+
def _upload_file_with_retry(self, local_path: str | Path, remote_path: str) -> None:
|
| 35 |
+
attempts = 0
|
| 36 |
+
last_exc: Exception | None = None
|
| 37 |
+
while attempts <= self.max_retries:
|
| 38 |
+
try:
|
| 39 |
+
# The HfApi.upload_file signature varies across versions; callers/tests
|
| 40 |
+
# should accept the positional usage shown here and monkeypatch in tests.
|
| 41 |
+
self._api.upload_file(path_or_file=local_path, path_in_repo=remote_path, repo_id=self.repo_id)
|
| 42 |
+
return
|
| 43 |
+
except Exception as exc: # pragma: no cover - network behavior
|
| 44 |
+
last_exc = exc
|
| 45 |
+
attempts += 1
|
| 46 |
+
if attempts > self.max_retries:
|
| 47 |
+
raise
|
| 48 |
+
backoff = self.initial_backoff * (2 ** (attempts - 1))
|
| 49 |
+
time.sleep(backoff)
|
| 50 |
+
|
| 51 |
+
def upload_files(
|
| 52 |
+
self,
|
| 53 |
+
file_infos: Iterable[dict[str, Any]],
|
| 54 |
+
*,
|
| 55 |
+
remote_base: str = "",
|
| 56 |
+
batch_size: int | None = None,
|
| 57 |
+
on_progress: Callable[[dict[str, Any]], None] | None = None,
|
| 58 |
+
) -> dict[str, Any]:
|
| 59 |
+
"""Upload an iterable of file info dictionaries.
|
| 60 |
+
|
| 61 |
+
Each file_info must include `full_path` and `relative_path`. `size` is optional.
|
| 62 |
+
"""
|
| 63 |
+
batch_size = batch_size or MAX_BATCH_SIZE
|
| 64 |
+
uploaded = 0
|
| 65 |
+
skipped = 0
|
| 66 |
+
failed = 0
|
| 67 |
+
|
| 68 |
+
for info in file_infos:
|
| 69 |
+
full_path = str(info["full_path"])
|
| 70 |
+
relative = info["relative_path"].lstrip("/")
|
| 71 |
+
remote_path = f"{remote_base.rstrip('/')}/{relative}" if remote_base else relative
|
| 72 |
+
remote_path = remote_path.lstrip("/")
|
| 73 |
+
|
| 74 |
+
decision = self.deduplicator.check_remote(remote_path, file_path=full_path)
|
| 75 |
+
if decision.get("status") == "skip":
|
| 76 |
+
skipped += 1
|
| 77 |
+
self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=0, status="skipped")
|
| 78 |
+
if on_progress:
|
| 79 |
+
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 80 |
+
continue
|
| 81 |
+
|
| 82 |
+
try:
|
| 83 |
+
self._upload_file_with_retry(full_path, remote_path)
|
| 84 |
+
# mark remote as present in local dedup cache
|
| 85 |
+
try:
|
| 86 |
+
self.deduplicator.mark_uploaded(remote_path)
|
| 87 |
+
except Exception:
|
| 88 |
+
# non-fatal cache update failure should not break overall flow
|
| 89 |
+
pass
|
| 90 |
+
|
| 91 |
+
uploaded += 1
|
| 92 |
+
bytes_uploaded = int(info.get("size", 0))
|
| 93 |
+
self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=bytes_uploaded)
|
| 94 |
+
if on_progress:
|
| 95 |
+
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 96 |
+
except Exception as exc:
|
| 97 |
+
failed += 1
|
| 98 |
+
self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
|
| 99 |
+
if on_progress:
|
| 100 |
+
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 101 |
+
|
| 102 |
+
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
src/uploader_cli/config.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
APP_NAME = "BirdNET Uploader"
|
| 9 |
+
SCHEMA_VERSION = "1.0.0"
|
| 10 |
+
AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
|
| 11 |
+
INDEX_SHARD_SIZE = 10_000
|
| 12 |
+
|
| 13 |
+
MAX_BATCH_SIZE = 10
|
| 14 |
+
RETRY_MAX_ATTEMPTS = 3
|
| 15 |
+
RETRY_INITIAL_BACKOFF_SECONDS = 1.0
|
| 16 |
+
RETRY_MAX_BACKOFF_SECONDS = 30.0
|
| 17 |
+
|
| 18 |
+
KEYRING_SERVICE = "birdnet-uploader"
|
| 19 |
+
KEYRING_ACCOUNT = "hf_token"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _resolve_path_from_env(env_name: str, default: Path) -> Path:
|
| 23 |
+
raw = os.getenv(env_name)
|
| 24 |
+
value = Path(raw).expanduser() if raw else default
|
| 25 |
+
value.mkdir(parents=True, exist_ok=True)
|
| 26 |
+
return value
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def get_session_root() -> Path:
|
| 30 |
+
"""Resolve and create the session storage root directory."""
|
| 31 |
+
return _resolve_path_from_env(
|
| 32 |
+
"BIRDNET_UPLOADER_SESSION_DIR",
|
| 33 |
+
Path.home() / ".birdnet-uploader" / "sessions",
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def get_cache_root() -> Path:
|
| 38 |
+
return _resolve_path_from_env(
|
| 39 |
+
"BIRDNET_UPLOADER_CACHE_DIR",
|
| 40 |
+
Path.home() / ".birdnet-uploader" / "cache",
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def get_log_root() -> Path:
|
| 45 |
+
return _resolve_path_from_env(
|
| 46 |
+
"BIRDNET_UPLOADER_LOG_DIR",
|
| 47 |
+
Path.home() / ".birdnet-uploader" / "logs",
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def get_runtime_config() -> dict[str, Any]:
|
| 52 |
+
return {
|
| 53 |
+
"session_root": str(get_session_root()),
|
| 54 |
+
"cache_root": str(get_cache_root()),
|
| 55 |
+
"log_root": str(get_log_root()),
|
| 56 |
+
"max_batch_size": MAX_BATCH_SIZE,
|
| 57 |
+
"retry_max_attempts": RETRY_MAX_ATTEMPTS,
|
| 58 |
+
"retry_initial_backoff_seconds": RETRY_INITIAL_BACKOFF_SECONDS,
|
| 59 |
+
"retry_max_backoff_seconds": RETRY_MAX_BACKOFF_SECONDS,
|
| 60 |
+
"index_shard_size": INDEX_SHARD_SIZE,
|
| 61 |
+
}
|
src/uploader_cli/deduplicator.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import hashlib
|
| 5 |
+
from datetime import UTC, datetime
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from huggingface_hub import HfApi
|
| 10 |
+
|
| 11 |
+
from .config import get_cache_root
|
| 12 |
+
from .hash_utils import compute_file_hash
|
| 13 |
+
from .exceptions import ValidationError
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Deduplicator:
|
| 17 |
+
"""Check remote file presence and cache the remote path index locally."""
|
| 18 |
+
|
| 19 |
+
def __init__(self, api: HfApi, repo_id: str, *, repo_type: str = "dataset") -> None:
|
| 20 |
+
self._api = api
|
| 21 |
+
self.repo_id = repo_id
|
| 22 |
+
self.repo_type = repo_type
|
| 23 |
+
self._remote_paths: set[str] | None = None
|
| 24 |
+
self._decision_cache: dict[str, dict[str, Any]] = {}
|
| 25 |
+
self.cache_path = self._build_cache_path()
|
| 26 |
+
|
| 27 |
+
def _build_cache_path(self) -> Path:
|
| 28 |
+
cache_root = get_cache_root() / "dedup"
|
| 29 |
+
cache_root.mkdir(parents=True, exist_ok=True)
|
| 30 |
+
digest = hashlib.sha256(f"{self.repo_type}:{self.repo_id}".encode("utf-8")).hexdigest()[:16]
|
| 31 |
+
return cache_root / f"{digest}.json"
|
| 32 |
+
|
| 33 |
+
def load_cached_index(self) -> set[str]:
|
| 34 |
+
if self._remote_paths is not None:
|
| 35 |
+
return self._remote_paths
|
| 36 |
+
|
| 37 |
+
if self.cache_path.exists():
|
| 38 |
+
raw = self.cache_path.read_text(encoding="utf-8")
|
| 39 |
+
try:
|
| 40 |
+
payload = json.loads(raw)
|
| 41 |
+
except json.JSONDecodeError as exc:
|
| 42 |
+
raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc
|
| 43 |
+
|
| 44 |
+
remote_paths = payload.get("remote_paths", [])
|
| 45 |
+
if not isinstance(remote_paths, list):
|
| 46 |
+
raise ValidationError(f"Invalid dedup cache format at {self.cache_path}")
|
| 47 |
+
self._remote_paths = {str(path) for path in remote_paths}
|
| 48 |
+
return self._remote_paths
|
| 49 |
+
|
| 50 |
+
self._remote_paths = self.refresh_remote_index()
|
| 51 |
+
return self._remote_paths
|
| 52 |
+
|
| 53 |
+
def refresh_remote_index(self) -> set[str]:
|
| 54 |
+
try:
|
| 55 |
+
repo_files = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
|
| 56 |
+
except Exception as exc: # pragma: no cover - external API behavior
|
| 57 |
+
raise ValidationError(f"Could not read remote file listing for {self.repo_id}: {exc}") from exc
|
| 58 |
+
|
| 59 |
+
remote_paths = {str(path) for path in repo_files}
|
| 60 |
+
self._remote_paths = remote_paths
|
| 61 |
+
self._write_cache(remote_paths)
|
| 62 |
+
return remote_paths
|
| 63 |
+
|
| 64 |
+
def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
|
| 65 |
+
cache_key = self._build_decision_key(remote_path, file_path=file_path)
|
| 66 |
+
cached = self._decision_cache.get(cache_key)
|
| 67 |
+
if cached is not None:
|
| 68 |
+
return dict(cached)
|
| 69 |
+
|
| 70 |
+
remote_paths = self.load_cached_index()
|
| 71 |
+
exists = remote_path in remote_paths
|
| 72 |
+
|
| 73 |
+
payload = {
|
| 74 |
+
"remote_path": remote_path,
|
| 75 |
+
"status": "skip" if exists else "upload",
|
| 76 |
+
"cached_remote_index": True,
|
| 77 |
+
}
|
| 78 |
+
if file_path is not None:
|
| 79 |
+
file_path_obj = Path(file_path)
|
| 80 |
+
payload["local_size"] = file_path_obj.stat().st_size
|
| 81 |
+
payload["local_sha256"] = compute_file_hash(file_path_obj)
|
| 82 |
+
|
| 83 |
+
self._decision_cache[cache_key] = dict(payload)
|
| 84 |
+
return payload
|
| 85 |
+
|
| 86 |
+
def mark_uploaded(self, remote_path: str) -> None:
|
| 87 |
+
remote_paths = self.load_cached_index()
|
| 88 |
+
if remote_path not in remote_paths:
|
| 89 |
+
remote_paths.add(remote_path)
|
| 90 |
+
self._write_cache(remote_paths)
|
| 91 |
+
|
| 92 |
+
keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path]
|
| 93 |
+
for key in keys_to_update:
|
| 94 |
+
self._decision_cache[key]["status"] = "skip"
|
| 95 |
+
|
| 96 |
+
def _write_cache(self, remote_paths: set[str]) -> None:
|
| 97 |
+
payload = {
|
| 98 |
+
"repo_id": self.repo_id,
|
| 99 |
+
"repo_type": self.repo_type,
|
| 100 |
+
"updated_at": datetime.now(UTC).isoformat(),
|
| 101 |
+
"remote_paths": sorted(remote_paths),
|
| 102 |
+
}
|
| 103 |
+
temp_path = self.cache_path.with_suffix(".json.tmp")
|
| 104 |
+
temp_path.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
|
| 105 |
+
temp_path.replace(self.cache_path)
|
| 106 |
+
|
| 107 |
+
def _build_decision_key(self, remote_path: str, *, file_path: str | Path | None) -> str:
|
| 108 |
+
if file_path is None:
|
| 109 |
+
return remote_path
|
| 110 |
+
file_path_obj = Path(file_path)
|
| 111 |
+
try:
|
| 112 |
+
stat = file_path_obj.stat()
|
| 113 |
+
except FileNotFoundError as exc:
|
| 114 |
+
raise ValidationError(f"Local file not found: {file_path_obj}") from exc
|
| 115 |
+
return f"{remote_path}:{file_path_obj.resolve()}:{stat.st_size}:{stat.st_mtime_ns}"
|
src/uploader_cli/error_handler.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from .exceptions import (
|
| 4 |
+
AuthenticationError,
|
| 5 |
+
RepositoryError,
|
| 6 |
+
SessionError,
|
| 7 |
+
UploaderCliError,
|
| 8 |
+
UploadError,
|
| 9 |
+
ValidationError,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
ERROR_HINTS: dict[type[UploaderCliError], str] = {
|
| 14 |
+
AuthenticationError: "Run 'birdnet-uploader login' and verify your Hugging Face token.",
|
| 15 |
+
RepositoryError: "Check repository id format (owner/name) and write permissions on Hugging Face.",
|
| 16 |
+
SessionError: "Verify session id and local checkpoint files under .birdnet-uploader/sessions.",
|
| 17 |
+
ValidationError: "Fix invalid paths or inputs and retry the command.",
|
| 18 |
+
UploadError: "Check your network and run the command again to resume from checkpoint.",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def build_error_message(error: Exception) -> str:
|
| 23 |
+
if isinstance(error, UploaderCliError):
|
| 24 |
+
hint = error.suggestion or ERROR_HINTS.get(type(error), "")
|
| 25 |
+
if hint:
|
| 26 |
+
return f"{error} Hint: {hint}"
|
| 27 |
+
return str(error)
|
| 28 |
+
|
| 29 |
+
return f"Unexpected error: {error}"
|
src/uploader_cli/exceptions.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class UploaderCliError(Exception):
|
| 2 |
+
"""Base exception for uploader CLI."""
|
| 3 |
+
|
| 4 |
+
def __init__(self, message: str, *, suggestion: str = ""):
|
| 5 |
+
super().__init__(message)
|
| 6 |
+
self.suggestion = suggestion
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class AuthenticationError(UploaderCliError):
|
| 10 |
+
"""Raised when Hugging Face authentication fails."""
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class RepositoryError(UploaderCliError):
|
| 14 |
+
"""Raised for dataset repository failures."""
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class SessionError(UploaderCliError):
|
| 18 |
+
"""Raised when session state cannot be read or written."""
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ValidationError(UploaderCliError):
|
| 22 |
+
"""Raised when user input or local resources are invalid."""
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class UploadError(UploaderCliError):
|
| 26 |
+
"""Raised when upload operations fail after retries."""
|
src/uploader_cli/hash_utils.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def compute_file_hash(path: str | Path, *, chunk_size: int = 1024 * 1024) -> str:
|
| 8 |
+
"""Compute a streaming SHA-256 hash for a file."""
|
| 9 |
+
digest = hashlib.sha256()
|
| 10 |
+
file_path = Path(path)
|
| 11 |
+
|
| 12 |
+
with file_path.open("rb") as handle:
|
| 13 |
+
for chunk in iter(lambda: handle.read(chunk_size), b""):
|
| 14 |
+
digest.update(chunk)
|
| 15 |
+
|
| 16 |
+
return digest.hexdigest()
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def verify_file_integrity(path: str | Path, expected_hash: str, *, chunk_size: int = 1024 * 1024) -> bool:
|
| 20 |
+
"""Return True when a file's SHA-256 matches the expected hash."""
|
| 21 |
+
return compute_file_hash(path, chunk_size=chunk_size) == expected_hash
|
src/uploader_cli/main.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from functools import wraps
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Any, Callable
|
| 7 |
+
|
| 8 |
+
import click
|
| 9 |
+
|
| 10 |
+
from .auth_service import AuthService
|
| 11 |
+
from .error_handler import build_error_message
|
| 12 |
+
from .repo_service import RepositoryService
|
| 13 |
+
from .scanner import LocalScanner
|
| 14 |
+
from .session_manager import SessionManager
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def handle_cli_errors(fn: Callable[..., Any]) -> Callable[..., Any]:
|
| 18 |
+
@wraps(fn)
|
| 19 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 20 |
+
try:
|
| 21 |
+
return fn(*args, **kwargs)
|
| 22 |
+
except click.ClickException:
|
| 23 |
+
raise
|
| 24 |
+
except Exception as exc:
|
| 25 |
+
raise click.ClickException(build_error_message(exc)) from exc
|
| 26 |
+
|
| 27 |
+
return wrapper
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@click.group(help="BirdNET local uploader CLI")
|
| 31 |
+
def cli() -> None:
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@cli.command("login")
|
| 36 |
+
@click.option("--token", prompt=True, hide_input=True, help="Hugging Face token")
|
| 37 |
+
@handle_cli_errors
|
| 38 |
+
def login_cmd(token: str) -> None:
|
| 39 |
+
service = AuthService()
|
| 40 |
+
user = service.authenticate(token)
|
| 41 |
+
click.echo(f"OK: authenticated as {user['username']}")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
@cli.command("init-repo")
|
| 45 |
+
@click.option("--repo-id", required=True, help="Dataset repo id in owner/name format")
|
| 46 |
+
@click.option("--private/--public", "private_repo", default=True, show_default=True)
|
| 47 |
+
@handle_cli_errors
|
| 48 |
+
def init_repo_cmd(repo_id: str, private_repo: bool) -> None:
|
| 49 |
+
token = AuthService().require_token()
|
| 50 |
+
|
| 51 |
+
created = RepositoryService(token).create_dataset(repo_id, private=private_repo)
|
| 52 |
+
click.echo(f"OK: dataset ready at {created}")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@cli.command("scan")
|
| 56 |
+
@click.option("--segments", "segments_dir", required=True, type=click.Path(exists=True, file_okay=False))
|
| 57 |
+
@handle_cli_errors
|
| 58 |
+
def scan_cmd(segments_dir: str) -> None:
|
| 59 |
+
summary = LocalScanner().scan_folder(segments_dir)
|
| 60 |
+
click.echo(
|
| 61 |
+
json.dumps(
|
| 62 |
+
{
|
| 63 |
+
"total_files": summary["total_files"],
|
| 64 |
+
"total_size": summary["total_size"],
|
| 65 |
+
"species_count": len(summary["by_species"]),
|
| 66 |
+
},
|
| 67 |
+
ensure_ascii=True,
|
| 68 |
+
indent=2,
|
| 69 |
+
)
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@cli.command("start")
|
| 74 |
+
@click.option("--repo-id", required=True)
|
| 75 |
+
@click.option("--segments", "segments_dir", required=True, type=click.Path(exists=True, file_okay=False))
|
| 76 |
+
@handle_cli_errors
|
| 77 |
+
def start_cmd(repo_id: str, segments_dir: str) -> None:
|
| 78 |
+
scanner = LocalScanner()
|
| 79 |
+
summary = scanner.scan_folder(segments_dir)
|
| 80 |
+
session = SessionManager()
|
| 81 |
+
payload = {
|
| 82 |
+
"repo_id": repo_id,
|
| 83 |
+
"segments_dir": str(Path(segments_dir).resolve()),
|
| 84 |
+
"total_files": summary["total_files"],
|
| 85 |
+
"total_size": summary["total_size"],
|
| 86 |
+
"uploaded": 0,
|
| 87 |
+
"failed": 0,
|
| 88 |
+
"status": "ready",
|
| 89 |
+
}
|
| 90 |
+
session.save_checkpoint(payload)
|
| 91 |
+
click.echo(f"OK: session created: {session.session_id}")
|
| 92 |
+
click.echo(f"Checkpoint: {session.checkpoint_path}")
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
@cli.command("resume")
|
| 96 |
+
@click.argument("session_id")
|
| 97 |
+
@handle_cli_errors
|
| 98 |
+
def resume_cmd(session_id: str) -> None:
|
| 99 |
+
session = SessionManager(session_id=session_id)
|
| 100 |
+
payload = session.load_checkpoint()
|
| 101 |
+
if not payload:
|
| 102 |
+
raise click.ClickException(f"Session has no checkpoint: {session_id}")
|
| 103 |
+
click.echo(json.dumps(payload, ensure_ascii=True, indent=2))
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
if __name__ == "__main__":
|
| 107 |
+
cli()
|
src/uploader_cli/repo_service.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from io import BytesIO
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from huggingface_hub import HfApi
|
| 8 |
+
|
| 9 |
+
from .config import INDEX_SHARD_SIZE, SCHEMA_VERSION
|
| 10 |
+
from .exceptions import RepositoryError
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class RepositoryService:
|
| 14 |
+
"""Create and initialize dataset repositories for uploader sessions."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, token: str):
|
| 17 |
+
self._api = HfApi(token=token)
|
| 18 |
+
|
| 19 |
+
def create_dataset(self, repo_id: str, *, private: bool = True) -> str:
|
| 20 |
+
if not self._is_valid_repo_id(repo_id):
|
| 21 |
+
raise RepositoryError("Repository id must be in the form 'owner/name'")
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
self._api.create_repo(repo_id=repo_id, repo_type="dataset", private=private, exist_ok=True)
|
| 25 |
+
self._init_structure(repo_id)
|
| 26 |
+
except Exception as exc: # pragma: no cover - external API behavior
|
| 27 |
+
raise RepositoryError(f"Could not create dataset: {exc}") from exc
|
| 28 |
+
|
| 29 |
+
return repo_id
|
| 30 |
+
|
| 31 |
+
def _init_structure(self, repo_id: str) -> None:
|
| 32 |
+
project_slug = self._project_slug_from_repo_id(repo_id)
|
| 33 |
+
placeholder_dirs = [
|
| 34 |
+
f"audio/{project_slug}/.gitkeep",
|
| 35 |
+
"index/shards/.gitkeep",
|
| 36 |
+
"validations/.gitkeep",
|
| 37 |
+
"audit/ingestion-runs/.gitkeep",
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
for path_in_repo in placeholder_dirs:
|
| 41 |
+
self._api.upload_file(
|
| 42 |
+
path_or_fileobj=BytesIO(b""),
|
| 43 |
+
path_in_repo=path_in_repo,
|
| 44 |
+
repo_id=repo_id,
|
| 45 |
+
repo_type="dataset",
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
manifest = {
|
| 49 |
+
"schema_version": SCHEMA_VERSION,
|
| 50 |
+
"project_slug": project_slug,
|
| 51 |
+
"dataset_repo_id": repo_id,
|
| 52 |
+
"index": {
|
| 53 |
+
"total_detections": 0,
|
| 54 |
+
"total_audio_files": 0,
|
| 55 |
+
"shard_size": INDEX_SHARD_SIZE,
|
| 56 |
+
"shards": [],
|
| 57 |
+
},
|
| 58 |
+
}
|
| 59 |
+
self._api.upload_file(
|
| 60 |
+
path_or_fileobj=BytesIO(json.dumps(manifest, ensure_ascii=True, indent=2).encode("utf-8")),
|
| 61 |
+
path_in_repo="index/manifest.json",
|
| 62 |
+
repo_id=repo_id,
|
| 63 |
+
repo_type="dataset",
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
def validate_repo(self, repo_id: str) -> dict[str, Any]:
|
| 67 |
+
if not self._is_valid_repo_id(repo_id):
|
| 68 |
+
raise RepositoryError("Repository id must be in the form 'owner/name'")
|
| 69 |
+
|
| 70 |
+
try:
|
| 71 |
+
repo_files = self._api.list_repo_files(repo_id=repo_id, repo_type="dataset")
|
| 72 |
+
except Exception as exc: # pragma: no cover - external API behavior
|
| 73 |
+
raise RepositoryError(f"Could not list dataset files: {exc}") from exc
|
| 74 |
+
|
| 75 |
+
project_slug = self._project_slug_from_repo_id(repo_id)
|
| 76 |
+
required_prefixes = [
|
| 77 |
+
f"audio/{project_slug}/",
|
| 78 |
+
"index/shards/",
|
| 79 |
+
"validations/",
|
| 80 |
+
"audit/ingestion-runs/",
|
| 81 |
+
]
|
| 82 |
+
missing_prefixes = [
|
| 83 |
+
prefix for prefix in required_prefixes if not any(path.startswith(prefix) for path in repo_files)
|
| 84 |
+
]
|
| 85 |
+
|
| 86 |
+
has_manifest = "index/manifest.json" in repo_files
|
| 87 |
+
manifest_ok = has_manifest
|
| 88 |
+
manifest_error = ""
|
| 89 |
+
|
| 90 |
+
is_valid = not missing_prefixes and has_manifest
|
| 91 |
+
return {
|
| 92 |
+
"repo_id": repo_id,
|
| 93 |
+
"is_valid": is_valid,
|
| 94 |
+
"project_slug": project_slug,
|
| 95 |
+
"missing_prefixes": missing_prefixes,
|
| 96 |
+
"has_manifest": has_manifest,
|
| 97 |
+
"manifest_ok": manifest_ok if has_manifest else False,
|
| 98 |
+
"manifest_error": manifest_error,
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
@staticmethod
|
| 102 |
+
def _is_valid_repo_id(repo_id: str) -> bool:
|
| 103 |
+
if not repo_id or repo_id.count("/") != 1:
|
| 104 |
+
return False
|
| 105 |
+
owner, name = repo_id.split("/", maxsplit=1)
|
| 106 |
+
return bool(owner.strip()) and bool(name.strip()) and " " not in repo_id
|
| 107 |
+
|
| 108 |
+
@staticmethod
|
| 109 |
+
def _project_slug_from_repo_id(repo_id: str) -> str:
|
| 110 |
+
return repo_id.split("/", maxsplit=1)[1]
|
src/uploader_cli/scanner.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from .config import AUDIO_EXTENSIONS
|
| 8 |
+
from .hash_utils import compute_file_hash
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class LocalScanner:
|
| 12 |
+
"""Scan local directories and group files by inferred species."""
|
| 13 |
+
|
| 14 |
+
def scan_folder(self, folder_path: str) -> dict[str, Any]:
|
| 15 |
+
root_path = Path(folder_path).resolve()
|
| 16 |
+
by_species: dict[str, list[dict[str, Any]]] = {}
|
| 17 |
+
total_files = 0
|
| 18 |
+
total_size = 0
|
| 19 |
+
|
| 20 |
+
for root, _, files in os.walk(root_path):
|
| 21 |
+
current_root = Path(root)
|
| 22 |
+
for filename in files:
|
| 23 |
+
ext = Path(filename).suffix.lower()
|
| 24 |
+
if ext not in AUDIO_EXTENSIONS:
|
| 25 |
+
continue
|
| 26 |
+
|
| 27 |
+
full_path = current_root / filename
|
| 28 |
+
relative_path = full_path.relative_to(root_path)
|
| 29 |
+
species = self._infer_species(root_path, full_path)
|
| 30 |
+
item = {
|
| 31 |
+
"name": filename,
|
| 32 |
+
"full_path": str(full_path),
|
| 33 |
+
"relative_path": relative_path.as_posix(),
|
| 34 |
+
"species": species,
|
| 35 |
+
"size": full_path.stat().st_size,
|
| 36 |
+
"sha256": compute_file_hash(full_path),
|
| 37 |
+
}
|
| 38 |
+
by_species.setdefault(species, []).append(item)
|
| 39 |
+
total_files += 1
|
| 40 |
+
total_size += item["size"]
|
| 41 |
+
|
| 42 |
+
return {
|
| 43 |
+
"total_files": total_files,
|
| 44 |
+
"total_size": total_size,
|
| 45 |
+
"by_species": by_species,
|
| 46 |
+
"root_path": str(root_path),
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
def _infer_species(self, root_path: Path, file_path: Path) -> str:
|
| 50 |
+
try:
|
| 51 |
+
relative_parts = file_path.relative_to(root_path).parts
|
| 52 |
+
except ValueError:
|
| 53 |
+
relative_parts = file_path.parts
|
| 54 |
+
|
| 55 |
+
if len(relative_parts) >= 2:
|
| 56 |
+
return relative_parts[0]
|
| 57 |
+
|
| 58 |
+
if len(relative_parts) == 1:
|
| 59 |
+
return Path(relative_parts[0]).stem.split("_")[0] or "unknown"
|
| 60 |
+
|
| 61 |
+
stem = file_path.stem
|
| 62 |
+
return stem.split("_")[0] if "_" in stem else "unknown"
|
src/uploader_cli/session_manager.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import json
|
| 5 |
+
from datetime import UTC, datetime
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from .config import get_session_root
|
| 10 |
+
from .exceptions import SessionError
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class SessionManager:
|
| 14 |
+
"""Persist and recover upload checkpoints."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, session_id: str | None = None):
|
| 17 |
+
if session_id is None:
|
| 18 |
+
now = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
| 19 |
+
session_id = f"upload-{now}"
|
| 20 |
+
self.session_id = session_id
|
| 21 |
+
self.session_dir = get_session_root() / session_id
|
| 22 |
+
self.session_dir.mkdir(parents=True, exist_ok=True)
|
| 23 |
+
|
| 24 |
+
@classmethod
|
| 25 |
+
def create_session(cls, metadata: dict[str, Any] | None = None) -> "SessionManager":
|
| 26 |
+
session = cls()
|
| 27 |
+
if metadata:
|
| 28 |
+
session.save_metadata(metadata)
|
| 29 |
+
return session
|
| 30 |
+
|
| 31 |
+
@property
|
| 32 |
+
def checkpoint_path(self) -> Path:
|
| 33 |
+
return self.session_dir / "checkpoint.json"
|
| 34 |
+
|
| 35 |
+
@property
|
| 36 |
+
def metadata_path(self) -> Path:
|
| 37 |
+
return self.session_dir / "metadata.json"
|
| 38 |
+
|
| 39 |
+
def _atomic_write_json(self, path: Path, payload: dict[str, Any]) -> None:
|
| 40 |
+
temp_path = path.with_suffix(path.suffix + ".tmp")
|
| 41 |
+
serialized = json.dumps(payload, ensure_ascii=True, indent=2) + "\n"
|
| 42 |
+
temp_path.write_text(serialized, encoding="utf-8")
|
| 43 |
+
with temp_path.open("a", encoding="utf-8") as handle:
|
| 44 |
+
handle.flush()
|
| 45 |
+
os.fsync(handle.fileno())
|
| 46 |
+
temp_path.replace(path)
|
| 47 |
+
|
| 48 |
+
def save_metadata(self, payload: dict[str, Any]) -> None:
|
| 49 |
+
self._atomic_write_json(self.metadata_path, payload)
|
| 50 |
+
|
| 51 |
+
def load_metadata(self) -> dict[str, Any]:
|
| 52 |
+
if not self.metadata_path.exists():
|
| 53 |
+
return {}
|
| 54 |
+
raw = self.metadata_path.read_text(encoding="utf-8")
|
| 55 |
+
try:
|
| 56 |
+
return json.loads(raw)
|
| 57 |
+
except json.JSONDecodeError as exc:
|
| 58 |
+
raise SessionError(f"Invalid metadata JSON for session '{self.session_id}'") from exc
|
| 59 |
+
|
| 60 |
+
def save_checkpoint(self, payload: dict[str, Any]) -> None:
|
| 61 |
+
self._atomic_write_json(self.checkpoint_path, payload)
|
| 62 |
+
|
| 63 |
+
def load_checkpoint(self) -> dict[str, Any]:
|
| 64 |
+
if not self.checkpoint_path.exists():
|
| 65 |
+
return {}
|
| 66 |
+
raw = self.checkpoint_path.read_text(encoding="utf-8")
|
| 67 |
+
try:
|
| 68 |
+
return json.loads(raw)
|
| 69 |
+
except json.JSONDecodeError as exc:
|
| 70 |
+
raise SessionError(f"Invalid checkpoint JSON for session '{self.session_id}'") from exc
|
| 71 |
+
|
| 72 |
+
def mark_file_done(self, *, remote_path: str, bytes_uploaded: int, status: str = "uploaded") -> dict[str, Any]:
|
| 73 |
+
checkpoint = self.load_checkpoint()
|
| 74 |
+
checkpoint.setdefault("uploaded_files", [])
|
| 75 |
+
checkpoint.setdefault("uploaded", 0)
|
| 76 |
+
checkpoint.setdefault("failed", 0)
|
| 77 |
+
checkpoint["uploaded"] = int(checkpoint["uploaded"]) + 1
|
| 78 |
+
checkpoint["status"] = status
|
| 79 |
+
checkpoint["last_completed_file"] = remote_path
|
| 80 |
+
checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
|
| 81 |
+
checkpoint["bytes_uploaded"] = int(checkpoint.get("bytes_uploaded", 0)) + int(bytes_uploaded)
|
| 82 |
+
checkpoint["uploaded_files"].append(remote_path)
|
| 83 |
+
self.save_checkpoint(checkpoint)
|
| 84 |
+
return checkpoint
|
| 85 |
+
|
| 86 |
+
def mark_file_failed(self, *, remote_path: str, error: str) -> dict[str, Any]:
|
| 87 |
+
checkpoint = self.load_checkpoint()
|
| 88 |
+
checkpoint.setdefault("failed_files", [])
|
| 89 |
+
checkpoint.setdefault("uploaded", 0)
|
| 90 |
+
checkpoint.setdefault("failed", 0)
|
| 91 |
+
checkpoint["failed"] = int(checkpoint["failed"]) + 1
|
| 92 |
+
checkpoint["status"] = "failed"
|
| 93 |
+
checkpoint["last_failed_file"] = remote_path
|
| 94 |
+
checkpoint["last_error"] = error
|
| 95 |
+
checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
|
| 96 |
+
checkpoint["failed_files"].append({"remote_path": remote_path, "error": error})
|
| 97 |
+
self.save_checkpoint(checkpoint)
|
| 98 |
+
return checkpoint
|
src/uploader_cli/tui.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from textual.app import App, ComposeResult
|
| 4 |
+
from textual.containers import Container
|
| 5 |
+
from textual.widgets import Button, Static
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class BirdNetUploaderTui(App[None]):
|
| 9 |
+
"""Minimal shell used during sprint scaffolding."""
|
| 10 |
+
|
| 11 |
+
TITLE = "BirdNET Uploader"
|
| 12 |
+
|
| 13 |
+
def compose(self) -> ComposeResult:
|
| 14 |
+
yield Container(
|
| 15 |
+
Static("BirdNET Uploader\nLogin -> Repo -> Upload", id="title"),
|
| 16 |
+
Button("Exit", id="exit"),
|
| 17 |
+
id="main",
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
| 21 |
+
if event.button.id == "exit":
|
| 22 |
+
self.exit()
|
tests/unit/test_batch_uploader.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
from src.uploader_cli.batch_uploader import BatchUploader
|
| 6 |
+
from src.uploader_cli.session_manager import SessionManager
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class FakeAPI:
|
| 10 |
+
def __init__(self):
|
| 11 |
+
self.calls = []
|
| 12 |
+
self.raise_on = 0
|
| 13 |
+
|
| 14 |
+
def upload_file(self, path_or_file, path_in_repo, repo_id):
|
| 15 |
+
self.calls.append((path_or_file, path_in_repo, repo_id))
|
| 16 |
+
if self.raise_on:
|
| 17 |
+
self.raise_on -= 1
|
| 18 |
+
raise RuntimeError("transient error")
|
| 19 |
+
return {"file": path_in_repo}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class FakeDedup:
|
| 23 |
+
def __init__(self, to_skip=None):
|
| 24 |
+
self.to_skip = set(to_skip or [])
|
| 25 |
+
self.marked = []
|
| 26 |
+
|
| 27 |
+
def check_remote(self, remote_path, file_path=None):
|
| 28 |
+
return {"remote_path": remote_path, "status": "skip" if remote_path in self.to_skip else "upload"}
|
| 29 |
+
|
| 30 |
+
def mark_uploaded(self, remote_path):
|
| 31 |
+
self.marked.append(remote_path)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def test_batch_uploader_retry_and_dedup(tmp_path, monkeypatch):
|
| 35 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_SESSION_DIR", str(tmp_path / "sessions"))
|
| 36 |
+
|
| 37 |
+
f1 = tmp_path / "a.wav"
|
| 38 |
+
f1.write_bytes(b"one")
|
| 39 |
+
f2 = tmp_path / "b.wav"
|
| 40 |
+
f2.write_bytes(b"two")
|
| 41 |
+
|
| 42 |
+
infos = [
|
| 43 |
+
{"full_path": str(f1), "relative_path": "proj/a.wav", "size": f1.stat().st_size},
|
| 44 |
+
{"full_path": str(f2), "relative_path": "proj/b.wav", "size": f2.stat().st_size},
|
| 45 |
+
]
|
| 46 |
+
|
| 47 |
+
api = FakeAPI()
|
| 48 |
+
api.raise_on = 1
|
| 49 |
+
dedup = FakeDedup(to_skip={"audio/proj/b.wav"})
|
| 50 |
+
session = SessionManager(session_id="test-batch")
|
| 51 |
+
|
| 52 |
+
uploader = BatchUploader(api=api, repo_id="me/repo", deduplicator=dedup, session=session, max_retries=2, initial_backoff=0)
|
| 53 |
+
result = uploader.upload_files(infos, remote_base="audio")
|
| 54 |
+
|
| 55 |
+
assert result["uploaded"] == 1
|
| 56 |
+
assert result["skipped"] == 1
|
| 57 |
+
assert result["failed"] == 0
|
| 58 |
+
assert dedup.marked == ["audio/proj/a.wav"]
|
tests/unit/test_uploader_cli_auth_service.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pytest
|
| 2 |
+
|
| 3 |
+
from src.uploader_cli.auth_service import AuthService
|
| 4 |
+
from src.uploader_cli.exceptions import AuthenticationError
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def test_authenticate_success_stores_token(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 8 |
+
store: dict[tuple[str, str], str] = {}
|
| 9 |
+
|
| 10 |
+
class FakeApi:
|
| 11 |
+
def whoami(self) -> dict[str, str]:
|
| 12 |
+
return {"name": "alice", "email": "alice@example.com", "user_id": "u123"}
|
| 13 |
+
|
| 14 |
+
monkeypatch.setattr("src.uploader_cli.auth_service.HfApi", lambda token: FakeApi())
|
| 15 |
+
monkeypatch.setattr(
|
| 16 |
+
"src.uploader_cli.auth_service.keyring.set_password",
|
| 17 |
+
lambda service, account, token: store.__setitem__((service, account), token),
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
service = AuthService()
|
| 21 |
+
user = service.authenticate(" hf_test_token ")
|
| 22 |
+
|
| 23 |
+
assert user["username"] == "alice"
|
| 24 |
+
assert user["email"] == "alice@example.com"
|
| 25 |
+
assert user["user_id"] == "u123"
|
| 26 |
+
assert store[(service.KEYRING_SERVICE, service.KEYRING_ACCOUNT)] == "hf_test_token"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def test_authenticate_rejects_blank_token() -> None:
|
| 30 |
+
with pytest.raises(AuthenticationError, match="Token is required"):
|
| 31 |
+
AuthService().authenticate(" ")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def test_authenticate_invalid_token_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 35 |
+
class FakeApi:
|
| 36 |
+
def whoami(self) -> dict[str, str]:
|
| 37 |
+
raise RuntimeError("401 Unauthorized")
|
| 38 |
+
|
| 39 |
+
monkeypatch.setattr("src.uploader_cli.auth_service.HfApi", lambda token: FakeApi())
|
| 40 |
+
|
| 41 |
+
with pytest.raises(AuthenticationError, match="Token validation failed"):
|
| 42 |
+
AuthService().authenticate("hf_invalid")
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def test_require_token_returns_stored_value(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 46 |
+
monkeypatch.setattr(
|
| 47 |
+
"src.uploader_cli.auth_service.keyring.get_password",
|
| 48 |
+
lambda service, account: "hf_saved_token",
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
assert AuthService().require_token() == "hf_saved_token"
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def test_require_token_raises_when_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 55 |
+
monkeypatch.setattr(
|
| 56 |
+
"src.uploader_cli.auth_service.keyring.get_password",
|
| 57 |
+
lambda service, account: None,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
with pytest.raises(AuthenticationError, match="No stored token found"):
|
| 61 |
+
AuthService().require_token()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def test_clear_token_ignores_missing_password(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 65 |
+
class FakePasswordDeleteError(Exception):
|
| 66 |
+
pass
|
| 67 |
+
|
| 68 |
+
class FakeErrors:
|
| 69 |
+
PasswordDeleteError = FakePasswordDeleteError
|
| 70 |
+
|
| 71 |
+
def raise_delete_error(service: str, account: str) -> None:
|
| 72 |
+
raise FakePasswordDeleteError("not found")
|
| 73 |
+
|
| 74 |
+
monkeypatch.setattr("src.uploader_cli.auth_service.keyring.errors", FakeErrors)
|
| 75 |
+
monkeypatch.setattr(
|
| 76 |
+
"src.uploader_cli.auth_service.keyring.delete_password",
|
| 77 |
+
raise_delete_error,
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
AuthService().clear_token()
|
tests/unit/test_uploader_cli_config.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
from src.uploader_cli.config import get_runtime_config, get_session_root
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def test_get_session_root_uses_env_override(tmp_path: Path, monkeypatch) -> None:
|
| 7 |
+
custom_root = tmp_path / "custom-sessions"
|
| 8 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_SESSION_DIR", str(custom_root))
|
| 9 |
+
|
| 10 |
+
resolved = get_session_root()
|
| 11 |
+
|
| 12 |
+
assert resolved == custom_root
|
| 13 |
+
assert resolved.exists()
|
| 14 |
+
assert resolved.is_dir()
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def test_get_runtime_config_contains_expected_keys() -> None:
|
| 18 |
+
config = get_runtime_config()
|
| 19 |
+
|
| 20 |
+
assert "session_root" in config
|
| 21 |
+
assert "cache_root" in config
|
| 22 |
+
assert "log_root" in config
|
| 23 |
+
assert config["max_batch_size"] == 10
|
| 24 |
+
assert config["retry_max_attempts"] == 3
|
| 25 |
+
assert config["index_shard_size"] == 10000
|
tests/unit/test_uploader_cli_deduplicator.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from src.uploader_cli.deduplicator import Deduplicator
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class FakeApi:
|
| 9 |
+
def __init__(self, files: list[str]) -> None:
|
| 10 |
+
self.files = files
|
| 11 |
+
self.calls = 0
|
| 12 |
+
|
| 13 |
+
def list_repo_files(self, repo_id: str, repo_type: str = "dataset") -> list[str]:
|
| 14 |
+
_ = repo_id
|
| 15 |
+
_ = repo_type
|
| 16 |
+
self.calls += 1
|
| 17 |
+
return list(self.files)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def test_check_remote_uses_cached_remote_index(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 21 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_CACHE_DIR", str(tmp_path / "cache"))
|
| 22 |
+
api = FakeApi(["audio/parrots-2026/existing.wav"])
|
| 23 |
+
local_file = tmp_path / "local.wav"
|
| 24 |
+
local_file.write_bytes(b"birdnet")
|
| 25 |
+
|
| 26 |
+
dedup = Deduplicator(api, "alice/parrots-2026")
|
| 27 |
+
first = dedup.check_remote("audio/parrots-2026/existing.wav", file_path=local_file)
|
| 28 |
+
second = dedup.check_remote("audio/parrots-2026/missing.wav", file_path=local_file)
|
| 29 |
+
|
| 30 |
+
assert first["status"] == "skip"
|
| 31 |
+
assert second["status"] == "upload"
|
| 32 |
+
assert api.calls == 1
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def test_check_remote_persists_remote_index_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 36 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_CACHE_DIR", str(tmp_path / "cache"))
|
| 37 |
+
api = FakeApi(["audio/parrots-2026/existing.wav"])
|
| 38 |
+
|
| 39 |
+
dedup = Deduplicator(api, "alice/parrots-2026")
|
| 40 |
+
dedup.load_cached_index()
|
| 41 |
+
|
| 42 |
+
assert dedup.cache_path.exists()
|
| 43 |
+
assert "existing.wav" in dedup.cache_path.read_text(encoding="utf-8")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def test_mark_uploaded_updates_local_remote_index(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 47 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_CACHE_DIR", str(tmp_path / "cache"))
|
| 48 |
+
api = FakeApi([])
|
| 49 |
+
|
| 50 |
+
dedup = Deduplicator(api, "alice/parrots-2026")
|
| 51 |
+
assert dedup.check_remote("audio/parrots-2026/new.wav")["status"] == "upload"
|
| 52 |
+
|
| 53 |
+
dedup.mark_uploaded("audio/parrots-2026/new.wav")
|
| 54 |
+
|
| 55 |
+
assert dedup.check_remote("audio/parrots-2026/new.wav")["status"] == "skip"
|
tests/unit/test_uploader_cli_error_handler.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from src.uploader_cli.error_handler import build_error_message
|
| 2 |
+
from src.uploader_cli.exceptions import AuthenticationError, UploadError
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def test_build_error_message_uses_default_hint() -> None:
|
| 6 |
+
message = build_error_message(AuthenticationError("Invalid token"))
|
| 7 |
+
|
| 8 |
+
assert "Invalid token" in message
|
| 9 |
+
assert "Hint:" in message
|
| 10 |
+
assert "birdnet-uploader login" in message
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def test_build_error_message_uses_exception_suggestion() -> None:
|
| 14 |
+
message = build_error_message(
|
| 15 |
+
UploadError("Upload interrupted", suggestion="Run resume with your last session id.")
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
assert "Upload interrupted" in message
|
| 19 |
+
assert "Run resume with your last session id." in message
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def test_build_error_message_for_unexpected_exception() -> None:
|
| 23 |
+
message = build_error_message(RuntimeError("boom"))
|
| 24 |
+
|
| 25 |
+
assert message == "Unexpected error: boom"
|
tests/unit/test_uploader_cli_hash_utils.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
from src.uploader_cli.hash_utils import compute_file_hash, verify_file_integrity
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def test_compute_file_hash_matches_known_sha256(tmp_path: Path) -> None:
|
| 7 |
+
file_path = tmp_path / "sample.txt"
|
| 8 |
+
file_path.write_bytes(b"hello world")
|
| 9 |
+
|
| 10 |
+
assert compute_file_hash(file_path) == "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def test_verify_file_integrity_returns_true_for_matching_hash(tmp_path: Path) -> None:
|
| 14 |
+
file_path = tmp_path / "sample.bin"
|
| 15 |
+
file_path.write_bytes(b"birdnet")
|
| 16 |
+
expected = compute_file_hash(file_path)
|
| 17 |
+
|
| 18 |
+
assert verify_file_integrity(file_path, expected) is True
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def test_verify_file_integrity_returns_false_for_mismatch(tmp_path: Path) -> None:
|
| 22 |
+
file_path = tmp_path / "sample.bin"
|
| 23 |
+
file_path.write_bytes(b"birdnet")
|
| 24 |
+
|
| 25 |
+
assert verify_file_integrity(file_path, "0" * 64) is False
|
tests/unit/test_uploader_cli_repo_service.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from src.uploader_cli.exceptions import RepositoryError
|
| 6 |
+
from src.uploader_cli.repo_service import RepositoryService
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class _FakeApi:
|
| 10 |
+
def __init__(self, token: str | None = None) -> None:
|
| 11 |
+
self.token = token
|
| 12 |
+
self.created_repo_calls: list[dict[str, object]] = []
|
| 13 |
+
self.upload_calls: list[dict[str, object]] = []
|
| 14 |
+
self.repo_files: list[str] = []
|
| 15 |
+
|
| 16 |
+
def create_repo(self, repo_id: str, repo_type: str, private: bool, exist_ok: bool) -> None:
|
| 17 |
+
self.created_repo_calls.append(
|
| 18 |
+
{
|
| 19 |
+
"repo_id": repo_id,
|
| 20 |
+
"repo_type": repo_type,
|
| 21 |
+
"private": private,
|
| 22 |
+
"exist_ok": exist_ok,
|
| 23 |
+
}
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
def upload_file(self, path_or_fileobj, path_in_repo: str, repo_id: str, repo_type: str) -> None:
|
| 27 |
+
payload_bytes = b""
|
| 28 |
+
if hasattr(path_or_fileobj, "read"):
|
| 29 |
+
payload_bytes = path_or_fileobj.read()
|
| 30 |
+
elif isinstance(path_or_fileobj, (bytes, bytearray)):
|
| 31 |
+
payload_bytes = bytes(path_or_fileobj)
|
| 32 |
+
|
| 33 |
+
self.upload_calls.append(
|
| 34 |
+
{
|
| 35 |
+
"path_in_repo": path_in_repo,
|
| 36 |
+
"repo_id": repo_id,
|
| 37 |
+
"repo_type": repo_type,
|
| 38 |
+
"payload": payload_bytes,
|
| 39 |
+
}
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
def list_repo_files(self, repo_id: str, repo_type: str = "dataset") -> list[str]:
|
| 43 |
+
_ = repo_id
|
| 44 |
+
_ = repo_type
|
| 45 |
+
return list(self.repo_files)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def test_create_dataset_initializes_expected_structure(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 49 |
+
fake_api = _FakeApi(token="hf_test")
|
| 50 |
+
monkeypatch.setattr("src.uploader_cli.repo_service.HfApi", lambda token: fake_api)
|
| 51 |
+
|
| 52 |
+
service = RepositoryService(token="hf_test")
|
| 53 |
+
repo_id = service.create_dataset("alice/parrots-2026", private=True)
|
| 54 |
+
|
| 55 |
+
assert repo_id == "alice/parrots-2026"
|
| 56 |
+
assert len(fake_api.created_repo_calls) == 1
|
| 57 |
+
assert fake_api.created_repo_calls[0]["repo_type"] == "dataset"
|
| 58 |
+
assert fake_api.created_repo_calls[0]["private"] is True
|
| 59 |
+
|
| 60 |
+
uploaded_paths = [call["path_in_repo"] for call in fake_api.upload_calls]
|
| 61 |
+
assert "audio/parrots-2026/.gitkeep" in uploaded_paths
|
| 62 |
+
assert "index/shards/.gitkeep" in uploaded_paths
|
| 63 |
+
assert "validations/.gitkeep" in uploaded_paths
|
| 64 |
+
assert "audit/ingestion-runs/.gitkeep" in uploaded_paths
|
| 65 |
+
assert "index/manifest.json" in uploaded_paths
|
| 66 |
+
|
| 67 |
+
manifest_payload = next(call for call in fake_api.upload_calls if call["path_in_repo"] == "index/manifest.json")
|
| 68 |
+
manifest = json.loads(manifest_payload["payload"].decode("utf-8"))
|
| 69 |
+
assert manifest["schema_version"] == "1.0.0"
|
| 70 |
+
assert manifest["project_slug"] == "parrots-2026"
|
| 71 |
+
assert manifest["dataset_repo_id"] == "alice/parrots-2026"
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def test_create_dataset_rejects_invalid_repo_id(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 75 |
+
monkeypatch.setattr("src.uploader_cli.repo_service.HfApi", lambda token: _FakeApi(token=token))
|
| 76 |
+
|
| 77 |
+
service = RepositoryService(token="hf_test")
|
| 78 |
+
with pytest.raises(RepositoryError, match="owner/name"):
|
| 79 |
+
service.create_dataset("invalid_repo")
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def test_validate_repo_returns_valid_for_expected_structure(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 83 |
+
fake_api = _FakeApi(token="hf_test")
|
| 84 |
+
fake_api.repo_files = [
|
| 85 |
+
"audio/parrots-2026/.gitkeep",
|
| 86 |
+
"index/shards/.gitkeep",
|
| 87 |
+
"validations/.gitkeep",
|
| 88 |
+
"audit/ingestion-runs/.gitkeep",
|
| 89 |
+
"index/manifest.json",
|
| 90 |
+
]
|
| 91 |
+
monkeypatch.setattr("src.uploader_cli.repo_service.HfApi", lambda token: fake_api)
|
| 92 |
+
|
| 93 |
+
service = RepositoryService(token="hf_test")
|
| 94 |
+
result = service.validate_repo("alice/parrots-2026")
|
| 95 |
+
|
| 96 |
+
assert result["is_valid"] is True
|
| 97 |
+
assert result["missing_prefixes"] == []
|
| 98 |
+
assert result["has_manifest"] is True
|
| 99 |
+
assert result["project_slug"] == "parrots-2026"
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def test_validate_repo_reports_missing_prefixes(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 103 |
+
fake_api = _FakeApi(token="hf_test")
|
| 104 |
+
fake_api.repo_files = [
|
| 105 |
+
"audio/parrots-2026/.gitkeep",
|
| 106 |
+
"index/manifest.json",
|
| 107 |
+
]
|
| 108 |
+
monkeypatch.setattr("src.uploader_cli.repo_service.HfApi", lambda token: fake_api)
|
| 109 |
+
|
| 110 |
+
service = RepositoryService(token="hf_test")
|
| 111 |
+
result = service.validate_repo("alice/parrots-2026")
|
| 112 |
+
|
| 113 |
+
assert result["is_valid"] is False
|
| 114 |
+
assert "index/shards/" in result["missing_prefixes"]
|
| 115 |
+
assert "validations/" in result["missing_prefixes"]
|
| 116 |
+
assert "audit/ingestion-runs/" in result["missing_prefixes"]
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def test_validate_repo_rejects_invalid_repo_id(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 120 |
+
monkeypatch.setattr("src.uploader_cli.repo_service.HfApi", lambda token: _FakeApi(token=token))
|
| 121 |
+
service = RepositoryService(token="hf_test")
|
| 122 |
+
|
| 123 |
+
with pytest.raises(RepositoryError, match="owner/name"):
|
| 124 |
+
service.validate_repo("bad-format")
|
tests/unit/test_uploader_cli_scanner.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
from src.uploader_cli.scanner import LocalScanner
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def test_scan_folder_counts_audio_files(tmp_path: Path) -> None:
|
| 7 |
+
species_dir = tmp_path / "Toucan"
|
| 8 |
+
species_dir.mkdir(parents=True)
|
| 9 |
+
(species_dir / "sample_001.wav").write_bytes(b"abc")
|
| 10 |
+
(species_dir / "sample_002.mp3").write_bytes(b"abcdef")
|
| 11 |
+
(species_dir / "ignore.txt").write_text("x", encoding="utf-8")
|
| 12 |
+
|
| 13 |
+
result = LocalScanner().scan_folder(str(tmp_path))
|
| 14 |
+
|
| 15 |
+
assert result["total_files"] == 2
|
| 16 |
+
assert result["total_size"] == 9
|
| 17 |
+
assert "Toucan" in result["by_species"]
|
| 18 |
+
assert result["by_species"]["Toucan"][0]["relative_path"].endswith("sample_001.wav")
|
| 19 |
+
assert len(result["by_species"]["Toucan"][0]["sha256"]) == 64
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def test_scan_folder_uses_first_directory_as_species_for_nested_layout(tmp_path: Path) -> None:
|
| 23 |
+
nested_dir = tmp_path / "Falcon" / "session_a" / "day_01"
|
| 24 |
+
nested_dir.mkdir(parents=True)
|
| 25 |
+
(nested_dir / "capture_001.wav").write_bytes(b"abc123")
|
| 26 |
+
|
| 27 |
+
result = LocalScanner().scan_folder(str(tmp_path))
|
| 28 |
+
|
| 29 |
+
assert result["total_files"] == 1
|
| 30 |
+
assert result["total_size"] == 6
|
| 31 |
+
assert "Falcon" in result["by_species"]
|
| 32 |
+
item = result["by_species"]["Falcon"][0]
|
| 33 |
+
assert item["relative_path"].startswith("Falcon/")
|
| 34 |
+
assert item["species"] == "Falcon"
|
tests/unit/test_uploader_cli_session.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
from src.uploader_cli.session_manager import SessionManager
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def test_session_checkpoint_roundtrip(tmp_path: Path, monkeypatch) -> None:
|
| 7 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_SESSION_DIR", str(tmp_path))
|
| 8 |
+
|
| 9 |
+
manager = SessionManager(session_id="upload-test-001")
|
| 10 |
+
payload = {"repo_id": "alice/demo", "uploaded": 5, "status": "uploading"}
|
| 11 |
+
manager.save_checkpoint(payload)
|
| 12 |
+
|
| 13 |
+
reloaded = manager.load_checkpoint()
|
| 14 |
+
assert reloaded == payload
|
| 15 |
+
assert manager.checkpoint_path.exists()
|
tests/unit/test_uploader_cli_session_manager.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from src.uploader_cli.exceptions import SessionError
|
| 6 |
+
from src.uploader_cli.session_manager import SessionManager
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def test_create_session_persists_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 10 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_SESSION_DIR", str(tmp_path))
|
| 11 |
+
|
| 12 |
+
session = SessionManager.create_session({"repo_id": "alice/parrots-2026", "segments_dir": "C:/data"})
|
| 13 |
+
|
| 14 |
+
assert session.metadata_path.exists()
|
| 15 |
+
metadata = session.load_metadata()
|
| 16 |
+
assert metadata["repo_id"] == "alice/parrots-2026"
|
| 17 |
+
assert metadata["segments_dir"] == "C:/data"
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def test_save_checkpoint_is_atomic_and_loads_back(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 21 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_SESSION_DIR", str(tmp_path))
|
| 22 |
+
|
| 23 |
+
session = SessionManager(session_id="upload-test-001")
|
| 24 |
+
checkpoint = {"repo_id": "alice/parrots-2026", "uploaded": 1, "failed": 0, "status": "ready"}
|
| 25 |
+
session.save_checkpoint(checkpoint)
|
| 26 |
+
|
| 27 |
+
assert session.checkpoint_path.exists()
|
| 28 |
+
assert session.load_checkpoint() == checkpoint
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def test_mark_file_done_updates_checkpoint_counters(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 32 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_SESSION_DIR", str(tmp_path))
|
| 33 |
+
|
| 34 |
+
session = SessionManager(session_id="upload-test-002")
|
| 35 |
+
session.save_checkpoint({"repo_id": "alice/parrots-2026", "uploaded": 0, "failed": 0, "status": "ready"})
|
| 36 |
+
|
| 37 |
+
updated = session.mark_file_done(remote_path="audio/parrots-2026/song.wav", bytes_uploaded=512)
|
| 38 |
+
|
| 39 |
+
assert updated["uploaded"] == 1
|
| 40 |
+
assert updated["bytes_uploaded"] == 512
|
| 41 |
+
assert updated["last_completed_file"] == "audio/parrots-2026/song.wav"
|
| 42 |
+
assert updated["status"] == "uploaded"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def test_mark_file_failed_updates_checkpoint(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 46 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_SESSION_DIR", str(tmp_path))
|
| 47 |
+
|
| 48 |
+
session = SessionManager(session_id="upload-test-003")
|
| 49 |
+
session.save_checkpoint({"repo_id": "alice/parrots-2026", "uploaded": 0, "failed": 0, "status": "ready"})
|
| 50 |
+
|
| 51 |
+
updated = session.mark_file_failed(remote_path="audio/parrots-2026/broken.wav", error="network error")
|
| 52 |
+
|
| 53 |
+
assert updated["failed"] == 1
|
| 54 |
+
assert updated["last_failed_file"] == "audio/parrots-2026/broken.wav"
|
| 55 |
+
assert updated["last_error"] == "network error"
|
| 56 |
+
assert updated["status"] == "failed"
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def test_load_checkpoint_rejects_invalid_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 60 |
+
monkeypatch.setenv("BIRDNET_UPLOADER_SESSION_DIR", str(tmp_path))
|
| 61 |
+
|
| 62 |
+
session = SessionManager(session_id="upload-test-004")
|
| 63 |
+
session.checkpoint_path.write_text("not-json", encoding="utf-8")
|
| 64 |
+
|
| 65 |
+
with pytest.raises(SessionError, match="Invalid checkpoint JSON"):
|
| 66 |
+
session.load_checkpoint()
|