jrrribeiro commited on
Commit
2f1a5fb
·
1 Parent(s): 3ab4a64

Forçar atualização total do app

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .github/workflows/check-file-size.yml +0 -15
  2. .github/workflows/ci.yml +29 -0
  3. .github/workflows/release-uploader.yml +79 -0
  4. .github/workflows/sync-hf-space.yml +42 -8
  5. .gitignore +11 -0
  6. BIRDNET_UPLOADER_REDESIGN.md +665 -0
  7. README.md +579 -86
  8. SPRINT1_KICKOFF.md +769 -0
  9. SPRINT_ROADMAP.md +649 -0
  10. app.py +5 -4
  11. build/publish_release_to_hf.py +50 -0
  12. build/pyinstaller_uploader.spec +73 -0
  13. build/release_pipeline.ps1 +79 -0
  14. build/release_uploader.py +75 -0
  15. build/validate_release.ps1 +80 -0
  16. cli/__pycache__/__init__.cpython-314.pyc +0 -0
  17. cli/__pycache__/hf_dataset_cli.cpython-314.pyc +0 -0
  18. cli/hf_dataset_cli.py +0 -1314
  19. docs/COLLABORATIVE_WORKFLOW_GUIDE.md +424 -0
  20. docs/END_TO_END_QA_CHECKLIST_SPRINT4.md +102 -0
  21. docs/spaces/projects.sample.json +14 -0
  22. docs/spaces/user_access.sample.json +12 -0
  23. pyproject.toml +5 -2
  24. requirements.txt +11 -6
  25. src/__pycache__/__init__.cpython-313.pyc +0 -0
  26. src/__pycache__/__init__.cpython-314.pyc +0 -0
  27. {cli → src/auth}/__init__.py +0 -0
  28. src/auth/auth_service.py +684 -0
  29. src/cache/__init__.py +0 -0
  30. src/cache/ephemeral_cache_manager.py +82 -0
  31. src/cli/__init__.py +0 -0
  32. src/cli/project_cli.py +339 -0
  33. src/config/__init__.py +1 -0
  34. src/config/runtime_config.py +119 -0
  35. src/domain/__init__.py +0 -0
  36. src/domain/models.py +51 -0
  37. src/repositories/__init__.py +0 -0
  38. src/repositories/append_only_validation_repository.py +108 -0
  39. src/repositories/contracts.py +35 -0
  40. src/repositories/in_memory_detection_repository.py +73 -0
  41. src/repositories/in_memory_validation_repository.py +25 -0
  42. src/services/__init__.py +0 -0
  43. src/services/audio_fetch_service.py +170 -0
  44. src/services/detection_queue_service.py +64 -0
  45. src/services/invite_email_notifier.py +122 -0
  46. src/services/validation_service.py +39 -0
  47. src/telemetry/__init__.py +0 -0
  48. src/ui/__pycache__/__init__.cpython-313.pyc +0 -0
  49. src/ui/__pycache__/__init__.cpython-314.pyc +0 -0
  50. src/ui/__pycache__/upload_app_factory.cpython-313.pyc +0 -0
.github/workflows/check-file-size.yml DELETED
@@ -1,15 +0,0 @@
1
- name: Check file size
2
-
3
- on:
4
- pull_request:
5
- branches: [main]
6
- workflow_dispatch:
7
-
8
- jobs:
9
- check-large-files:
10
- runs-on: ubuntu-latest
11
- steps:
12
- - name: Check large files
13
- uses: ActionsDesk/lfs-warning@v2.0
14
- with:
15
- filesizelimit: 10485760
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/ci.yml ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+
9
+ jobs:
10
+ test:
11
+ runs-on: ubuntu-latest
12
+ env:
13
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
14
+ steps:
15
+ - name: Checkout
16
+ uses: actions/checkout@v4
17
+
18
+ - name: Setup Python
19
+ uses: actions/setup-python@v5
20
+ with:
21
+ python-version: '3.11'
22
+
23
+ - name: Install dependencies
24
+ run: |
25
+ python -m pip install --upgrade pip
26
+ pip install -r requirements.txt
27
+
28
+ - name: Run tests
29
+ run: pytest -q
.github/workflows/release-uploader.yml ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Release BirdNET Uploader
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ version:
7
+ description: "Release version (for example 0.1.0)"
8
+ required: true
9
+ type: string
10
+ hf_repo_id:
11
+ description: "Optional Hugging Face repo id to publish artifacts to (for example jrrribeiro/birdnet-uploader-releases)"
12
+ required: false
13
+ type: string
14
+ hf_repo_type:
15
+ description: "Hugging Face repo type for uploads"
16
+ required: false
17
+ default: "dataset"
18
+ type: choice
19
+ options:
20
+ - dataset
21
+ - model
22
+ publish_to_hf:
23
+ description: "Upload release bundle to Hugging Face"
24
+ required: false
25
+ default: false
26
+ type: boolean
27
+
28
+ jobs:
29
+ build-release:
30
+ runs-on: windows-latest
31
+ permissions:
32
+ contents: read
33
+ env:
34
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
35
+ steps:
36
+ - name: Checkout
37
+ uses: actions/checkout@v4
38
+
39
+ - name: Setup Python
40
+ uses: actions/setup-python@v5
41
+ with:
42
+ python-version: "3.11"
43
+
44
+ - name: Install dependencies
45
+ shell: pwsh
46
+ run: |
47
+ python -m pip install --upgrade pip
48
+ pip install -r requirements.txt
49
+ pip install pyinstaller
50
+
51
+ - name: Build release bundle
52
+ shell: pwsh
53
+ run: |
54
+ python build/release_uploader.py --version "${{ inputs.version }}"
55
+
56
+ - name: Upload GitHub artifact
57
+ uses: actions/upload-artifact@v4
58
+ with:
59
+ name: birdnet-uploader-${{ inputs.version }}
60
+ path: |
61
+ build/release/*
62
+ if-no-files-found: error
63
+
64
+ - name: Publish to Hugging Face
65
+ if: ${{ inputs.publish_to_hf == true }}
66
+ shell: pwsh
67
+ env:
68
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
69
+ HF_REPO_ID: ${{ inputs.hf_repo_id }}
70
+ HF_REPO_TYPE: ${{ inputs.hf_repo_type }}
71
+ RELEASE_VERSION: ${{ inputs.version }}
72
+ run: |
73
+ if ([string]::IsNullOrWhiteSpace($env:HF_TOKEN)) { throw "HF_TOKEN secret is required when publish_to_hf is true" }
74
+ if ([string]::IsNullOrWhiteSpace($env:HF_REPO_ID)) { throw "hf_repo_id input is required when publish_to_hf is true" }
75
+ python -m pip install huggingface_hub
76
+ $bundle = Get-ChildItem build/release/*.zip | Select-Object -First 1
77
+ $checksum = Get-ChildItem build/release/*.sha256 | Select-Object -First 1
78
+ python build/publish_release_to_hf.py --repo-id "$env:HF_REPO_ID" --repo-type "$env:HF_REPO_TYPE" --bundle "$($bundle.FullName)" --checksum "$($checksum.FullName)" --version "$env:RELEASE_VERSION"
79
+
.github/workflows/sync-hf-space.yml CHANGED
@@ -1,4 +1,4 @@
1
- name: Sync app to Hugging Face Space
2
 
3
  on:
4
  push:
@@ -10,6 +10,8 @@ jobs:
10
  runs-on: ubuntu-latest
11
  permissions:
12
  contents: read
 
 
13
  steps:
14
  - name: Checkout repository
15
  uses: actions/checkout@v4
@@ -17,16 +19,48 @@ jobs:
17
  fetch-depth: 0
18
  lfs: true
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  - name: Push to Hugging Face Space
21
  env:
22
  HF_TOKEN: ${{ secrets.HF_TOKEN }}
23
- HF_USERNAME: jrrribeiro
24
- HF_SPACE_NAME: BirdNET-Uploader-App
25
  run: |
 
 
 
 
26
  git config --global user.name "github-actions"
27
  git config --global user.email "actions@github.com"
28
- if [ -z "$HF_TOKEN" ]; then
29
- echo "HF_TOKEN secret is required"
30
- exit 1
31
- fi
32
- git push "https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_USERNAME}/${HF_SPACE_NAME}" HEAD:main --force
 
1
+ name: Sync app to Hugging Face Space
2
 
3
  on:
4
  push:
 
10
  runs-on: ubuntu-latest
11
  permissions:
12
  contents: read
13
+ env:
14
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
15
  steps:
16
  - name: Checkout repository
17
  uses: actions/checkout@v4
 
19
  fetch-depth: 0
20
  lfs: true
21
 
22
+
23
+ - name: Ensure Hugging Face Space exists
24
+ env:
25
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
26
+ HF_USERNAME: ${{ secrets.HF_USERNAME }}
27
+ HF_SPACE_NAME: ${{ secrets.HF_SPACE_NAME }}
28
+ run: |
29
+ if [ -z "$HF_TOKEN" ] || [ -z "$HF_USERNAME" ] || [ -z "$HF_SPACE_NAME" ]; then
30
+ echo "Skipping: missing secrets"
31
+ exit 0
32
+ fi
33
+ python - <<'PY'
34
+ import os
35
+ from huggingface_hub import HfApi
36
+
37
+ token = os.environ["HF_TOKEN"]
38
+ username = os.environ["HF_USERNAME"]
39
+ space_name = os.environ["HF_SPACE_NAME"]
40
+
41
+ api = HfApi(token=token)
42
+ api.create_repo(
43
+ repo_id=f"{username}/{space_name}",
44
+ repo_type="space",
45
+ exist_ok=True,
46
+ space_sdk="gradio",
47
+ )
48
+ PY
49
+
50
+
51
  - name: Push to Hugging Face Space
52
  env:
53
  HF_TOKEN: ${{ secrets.HF_TOKEN }}
54
+ HF_USERNAME: ${{ secrets.HF_USERNAME }}
55
+ HF_SPACE_NAME: ${{ secrets.HF_SPACE_NAME }}
56
  run: |
57
+ if [ -z "$HF_TOKEN" ] || [ -z "$HF_USERNAME" ] || [ -z "$HF_SPACE_NAME" ]; then
58
+ echo "Skipping: missing secrets"
59
+ exit 0
60
+ fi
61
  git config --global user.name "github-actions"
62
  git config --global user.email "actions@github.com"
63
+ git push "https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_USERNAME}/${HF_SPACE_NAME}.git" HEAD:main --force
64
+
65
+
66
+ # Step de skip removido pois a checagem agora é interna nos steps acima
 
.gitignore ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.pyc
3
+ .venv/
4
+ .pytest_cache/
5
+ .env
6
+ .env.*
7
+ .DS_Store
8
+ .vscode/
9
+ dist/
10
+ build/release/
11
+ build/pyinstaller_uploader/
BIRDNET_UPLOADER_REDESIGN.md ADDED
@@ -0,0 +1,665 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # BirdNET Uploader: Redesign CLI Moderno
2
+
3
+ ## Visão Geral
4
+
5
+ Executável local moderno com **um único fluxo integrado**: login → criar repositório → apontar pasta → fazer upload.
6
+
7
+ Interface limpa, baseada em TUI (Textual) ou rich terminal output, compatível com a estrutura esperada pelo BirdNET Validator.
8
+
9
+ ---
10
+
11
+ ## 1. FLUXO DE UX ÚNICO INTEGRADO
12
+
13
+ ### Tela 1: Autenticação
14
+ ```
15
+ ┌─────────────────────────────────────────────────┐
16
+ │ BirdNET Uploader │
17
+ │ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
18
+ │ │
19
+ │ 🐦 Autentique-se no Hugging Face │
20
+ │ │
21
+ │ [i] Cole seu token HF (Settings → Access │
22
+ │ Tokens em huggingface.co) │
23
+ │ │
24
+ │ Token: [●●●●●●●●●●●●●●●●●●●●●●] │
25
+ │ │
26
+ │ [ Continuar ] [ Sair ] │
27
+ │ │
28
+ └─────────────────────────────────────────────────┘
29
+ ```
30
+
31
+ ### Tela 2: Configuração (Um formulário integrado)
32
+ ```
33
+ ┌─────────────────────────────────────────────────┐
34
+ │ BirdNET Uploader - Configuração │
35
+ │ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
36
+ │ │
37
+ │ Conectado como: alice@example.com │
38
+ │ │
39
+ │ REPOSITÓRIO │
40
+ │ Nome do repo: [my-birds-2026 ] │
41
+ │ Visibilidade: ( ) Público (●) Privado │
42
+ │ │
43
+ │ ARQUIVOS │
44
+ │ Pasta segmentos: [C:\audio\segments] │
45
+ │ Arquivo CSV: [C:\audio\metadata.csv] (opt)
46
+ │ [ Verificar ] │
47
+ │ ✓ 1.247 arquivos encontrados │
48
+ │ ✓ Tamanho total: 53 GB │
49
+ │ ✓ CSV compatível: 1.200 linhas │
50
+ │ │
51
+ │ [ Criar Repo e Fazer Upload ] │
52
+ │ [ Voltar ] │
53
+ │ │
54
+ └─────────────────────────────────────────────────┘
55
+ ```
56
+
57
+ ### Tela 3: Resumo e Confirmação
58
+ ```
59
+ ┌─────────────────────────────────────────────────┐
60
+ │ Pronto para Começar │
61
+ │ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
62
+ │ │
63
+ │ Repositório: alice/my-birds-2026 │
64
+ │ Visibilidade: Privado │
65
+ │ URL: https://huggingface.co/datasets/alice/.. │
66
+ │ │
67
+ │ Origem: │
68
+ │ Pasta: C:\audio\segments │
69
+ │ Arquivos: 1.247 │
70
+ │ Tamanho: 53 GB │
71
+ │ CSV: metadata.csv (1.200 linhas) │
72
+ │ │
73
+ │ Duração estimada: ~4 horas (com conexão 10MB/s)
74
+ │ │
75
+ │ [ Iniciar Upload ] [ Cancelar ] │
76
+ │ │
77
+ └─────────────────────────────────────────────────┘
78
+ ```
79
+
80
+ ### Tela 4: Upload em Andamento
81
+ ```
82
+ ┌─────────────────────────────────────────────────┐
83
+ │ Upload em Andamento │
84
+ │ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
85
+ │ │
86
+ │ Progresso Global │
87
+ │ ████████████████░░░░░░░░░░░░░░ 62.3% │
88
+ │ │
89
+ │ Arquivos: 775/1.247 (skipped: 18, erro: 2) │
90
+ │ Dados: 33 GB / 53 GB │
91
+ │ Velocidade: 9.6 MB/s | ETA: 01h 15m │
92
+ │ │
93
+ │ Arquivo Atual: Parrot_Ara_001.wav │
94
+ │ ████████████░░░░░░░░░░░░░░░░░░░ 45.2% │
95
+ │ 3.2 / 7.1 MB │
96
+ │ │
97
+ │ Último evento: ✓ Thrush_042.wav enviado (2s) │
98
+ │ │
99
+ │ [ Pausar ] [ Cancelar ] [ Detalhes ] │
100
+ │ │
101
+ └─────────────────────────────────────────────────┘
102
+ ```
103
+
104
+ ### Tela 5: Resumo Final
105
+ ```
106
+ ┌─────────────────────────────────────────────────┐
107
+ │ ✓ Upload Concluído! │
108
+ │ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
109
+ │ │
110
+ │ Repositório: alice/my-birds-2026 │
111
+ │ URL: https://huggingface.co/datasets/alice/.. │
112
+ │ │
113
+ │ Resumo: │
114
+ │ ✓ Enviados: 1.227 arquivos │
115
+ │ → Pulados: 18 (já existiam) │
116
+ │ ✗ Erros: 2 (revisar detalhes) │
117
+ │ ⏱ Tempo total: 4h 23min │
118
+ │ 📊 Velocidade média: 9.8 MB/s │
119
+ │ │
120
+ │ Próximos passos: │
121
+ │ 1. Validar dados com BirdNET Validator │
122
+ │ 2. Verificar relatório de ingestão │
123
+ │ │
124
+ │ Relatório salvo em: ~/.birdnet-uploader/ │
125
+ │ sessions/upload-xxx/ │
126
+ │ report.json │
127
+ │ │
128
+ │ [ Ver Relatório Completo ] [ Sair ] │
129
+ │ │
130
+ └─────────────────────────────────────────────────┘
131
+ ```
132
+
133
+ ---
134
+
135
+ ## 2. ARQUITETURA LIMPA E MODULAR
136
+
137
+ ### Estrutura de Diretórios do Projeto
138
+
139
+ ```
140
+ birdnet-uploader/
141
+ ├── src/
142
+ │ ├── __init__.py
143
+ │ ├── main.py # Entrypoint principal
144
+ │ ├── config.py # Config app e constantes
145
+ │ ├── auth/
146
+ │ │ ├── __init__.py
147
+ │ │ ├── auth_service.py # Login, token, keyring
148
+ │ │ └── token_manager.py # Gerenciamento de token
149
+ │ ├── repository/
150
+ │ │ ├── __init__.py
151
+ │ │ ├── repo_service.py # Criar/validar dataset HF
152
+ │ │ └── structure_initializer.py # Init estrutura padrão
153
+ │ ├── upload/
154
+ │ │ ├── __init__.py
155
+ │ │ ├── scanner.py # Varrer pasta local
156
+ │ │ ├── uploader.py # Motor de upload por lotes
157
+ │ │ ├── deduplicator.py # Checagem de duplicação
158
+ │ │ └── batch_processor.py # Processamento de lotes
159
+ │ ├── csv/
160
+ │ │ ├── __init__.py
161
+ │ │ ├── csv_matcher.py # Matching CSV + segmentos
162
+ │ │ └── indexer.py # Geração de índice/shards
163
+ │ ├── session/
164
+ │ │ ├── __init__.py
165
+ │ │ ├── session_manager.py # Gerenciamento de sessão
166
+ │ │ ├── checkpoint.py # Persistência de checkpoint
167
+ │ │ └── state_store.py # State file I/O
168
+ │ ├── ui/
169
+ │ │ ├── __init__.py
170
+ │ │ ├── tui.py # Interface Textual (TUI)
171
+ │ │ ├── progress_renderer.py # Barra de progresso rich
172
+ │ │ └── prompts.py # Diálogos e inputs
173
+ │ ├── logger/
174
+ │ │ ├── __init__.py
175
+ │ │ ├── structured_logger.py # Logs estruturados JSONL
176
+ │ │ └── telemetry.py # Eventos de telemetria
177
+ │ └── utils/
178
+ │ ├── __init__.py
179
+ │ ├── file_utils.py # Operações de arquivo
180
+ │ ├── hash_utils.py # Computação de hash
181
+ │ └── error_handler.py # Tratamento de erros
182
+ ├── build/
183
+ │ ├── pyinstaller_config.spec # Config do PyInstaller
184
+ │ └── build.py # Script de build
185
+ ├── tests/
186
+ │ ├── __init__.py
187
+ │ ├── test_auth.py
188
+ │ ├── test_uploader.py
189
+ │ ├── test_session.py
190
+ │ └── test_csv_matcher.py
191
+ ├── README.md
192
+ ├── requirements.txt
193
+ └── setup.py
194
+ ```
195
+
196
+ ### Dependências Principais
197
+
198
+ ```
199
+ # requirements.txt
200
+ huggingface_hub>=0.28.0 # HF Hub API
201
+ click>=8.0.0 # CLI framework
202
+ textual>=0.30.0 # TUI moderna
203
+ rich>=13.0.0 # Terminal formatting
204
+ pydantic>=2.0.0 # Validação de dados
205
+ pandas>=2.0.0 # CSV parsing
206
+ pyarrow>=14.0.0 # Parquet para shards
207
+ keyring>=24.0.0 # Armazenamento seguro
208
+ cryptography>=41.0.0 # Criptografia local
209
+ ```
210
+
211
+ ---
212
+
213
+ ## 3. COMPATIBILIDADE COM ESTRUTURA DO VALIDADOR
214
+
215
+ ### Estrutura Esperada do Dataset
216
+
217
+ ```
218
+ dataset_repo_id = "username/project-dataset"
219
+
220
+ audio/
221
+ ├── {project_slug}/
222
+ │ ├── Specie_A/
223
+ │ │ ├── segment_001.wav
224
+ │ │ ├── segment_002.wav
225
+ │ │ └── ...
226
+ │ ├── Specie_B/
227
+ │ │ └── ...
228
+ │ └── ...
229
+
230
+ index/
231
+ ├── shards/
232
+ │ ├── shard-00000.parquet
233
+ │ ├── shard-00001.parquet
234
+ │ └── ...
235
+ └── manifest.json
236
+
237
+ validations/
238
+ ├── results-20250419.parquet
239
+
240
+ audit/
241
+ ├── ingestion-runs/
242
+ │ └── 20250419T100000Z.json
243
+ ```
244
+
245
+ ### Manifest Format (Mantém compatibilidade)
246
+
247
+ ```json
248
+ {
249
+ "schema_version": "1.0.0",
250
+ "project_slug": "my-birds",
251
+ "dataset_repo_id": "alice/my-birds-2026",
252
+ "updated_at": "2025-04-19T10:45:30Z",
253
+ "index": {
254
+ "total_detections": 1227,
255
+ "total_audio_files": 1227,
256
+ "shard_size": 10000,
257
+ "shards": [
258
+ {
259
+ "path": "index/shards/shard-00000.parquet",
260
+ "rows": 10000,
261
+ "sha256": "abc123...",
262
+ "size_bytes": 5242880
263
+ }
264
+ ]
265
+ }
266
+ }
267
+ ```
268
+
269
+ ---
270
+
271
+ ## 4. MÓDULO POR MÓDULO
272
+
273
+ ### 4.1 Auth Service (`src/auth/auth_service.py`)
274
+
275
+ ```python
276
+ from keyring import get_password, set_password
277
+ from huggingface_hub import HfApi, login
278
+
279
+ class AuthService:
280
+ KEYRING_SERVICE = "birdnet-uploader"
281
+
282
+ def authenticate(self, token: str) -> dict:
283
+ """Valida token e retorna context do usuário."""
284
+ try:
285
+ api = HfApi(token=token)
286
+ user_info = api.whoami()
287
+
288
+ # Salvar em keyring seguro
289
+ set_password(self.KEYRING_SERVICE, "hf_token", token)
290
+
291
+ return {
292
+ "username": user_info["name"],
293
+ "email": user_info.get("email", ""),
294
+ "user_id": user_info["user_id"],
295
+ "token_expiry": self._estimate_expiry()
296
+ }
297
+ except Exception as e:
298
+ raise AuthenticationError(f"Token inválido: {str(e)}")
299
+
300
+ def get_token(self) -> str:
301
+ """Recupera token do keyring."""
302
+ token = get_password(self.KEYRING_SERVICE, "hf_token")
303
+ if not token:
304
+ raise TokenNotFoundError("Token não encontrado. Execute 'birdnet auth login'")
305
+ return token
306
+ ```
307
+
308
+ ### 4.2 Repository Service (`src/repository/repo_service.py`)
309
+
310
+ ```python
311
+ class RepositoryService:
312
+ def create_dataset(self,
313
+ repo_name: str,
314
+ private: bool = True) -> str:
315
+ """Cria dataset no HF com estrutura padrão."""
316
+ api = HfApi(token=self.get_token())
317
+
318
+ repo_id = f"{username}/{repo_name}"
319
+
320
+ # Criar repo
321
+ repo = api.create_repo(
322
+ repo_id=repo_id,
323
+ repo_type="dataset",
324
+ private=private,
325
+ exist_ok=True
326
+ )
327
+
328
+ # Inicializar estrutura padrão
329
+ self._init_structure(repo_id)
330
+
331
+ return repo_id
332
+
333
+ def _init_structure(self, repo_id: str):
334
+ """Inicializa pastas e arquivos padrão."""
335
+ folders = [
336
+ "audio/{project_slug}",
337
+ "index/shards",
338
+ "validations",
339
+ "audit/ingestion-runs"
340
+ ]
341
+
342
+ for folder in folders:
343
+ # Criar .gitkeep
344
+ api.upload_file(
345
+ path_or_fileobj=BytesIO(b""),
346
+ path_in_repo=f"{folder}/.gitkeep",
347
+ repo_id=repo_id,
348
+ repo_type="dataset"
349
+ )
350
+
351
+ # Criar manifest.json inicial
352
+ manifest = {
353
+ "schema_version": "1.0.0",
354
+ "project_slug": project_slug,
355
+ "dataset_repo_id": repo_id,
356
+ "index": {
357
+ "total_detections": 0,
358
+ "total_audio_files": 0,
359
+ "shard_size": 10000,
360
+ "shards": []
361
+ }
362
+ }
363
+
364
+ api.upload_file(
365
+ path_or_fileobj=BytesIO(json.dumps(manifest).encode()),
366
+ path_in_repo="index/manifest.json",
367
+ repo_id=repo_id,
368
+ repo_type="dataset"
369
+ )
370
+ ```
371
+
372
+ ### 4.3 Scanner (`src/upload/scanner.py`)
373
+
374
+ ```python
375
+ class LocalScanner:
376
+ def scan_folder(self, folder_path: str) -> dict:
377
+ """Varre pasta recursiva e retorna estrutura."""
378
+ structure = {}
379
+ total_size = 0
380
+ file_count = 0
381
+
382
+ for root, dirs, files in os.walk(folder_path):
383
+ for file in files:
384
+ if file.lower().endswith(('.wav', '.mp3', '.flac', '.ogg')):
385
+ full_path = os.path.join(root, file)
386
+
387
+ # Inferir espécie do caminho ou nome
388
+ specie = self._infer_specie(root, file)
389
+
390
+ if specie not in structure:
391
+ structure[specie] = []
392
+
393
+ size = os.path.getsize(full_path)
394
+ structure[specie].append({
395
+ "name": file,
396
+ "full_path": full_path,
397
+ "size": size,
398
+ "local_hash": compute_hash(full_path)
399
+ })
400
+
401
+ total_size += size
402
+ file_count += 1
403
+
404
+ return {
405
+ "total_files": file_count,
406
+ "total_size": total_size,
407
+ "by_specie": structure,
408
+ "scan_timestamp": datetime.now()
409
+ }
410
+
411
+ def _infer_specie(self, root: str, file: str) -> str:
412
+ """Inferir espécie do caminho da pasta."""
413
+ # Estratégia 1: última pasta em root é a espécie
414
+ parts = root.split(os.sep)
415
+ if len(parts) > 0:
416
+ return parts[-1]
417
+
418
+ # Fallback: usar nome do arquivo
419
+ return file.replace(".wav", "").split("_")[0]
420
+ ```
421
+
422
+ ### 4.4 Uploader (`src/upload/uploader.py`)
423
+
424
+ ```python
425
+ class BatchUploader:
426
+ def __init__(self, repo_id: str, project_slug: str, token: str):
427
+ self.repo_id = repo_id
428
+ self.project_slug = project_slug
429
+ self.api = HfApi(token=token)
430
+ self.session = SessionManager(repo_id)
431
+
432
+ async def upload_files(self,
433
+ structure: dict,
434
+ csv_matcher: Optional[CSVMatcher] = None,
435
+ batch_size: int = 10,
436
+ on_progress: Optional[Callable] = None) -> dict:
437
+ """Upload em lotes com retomada."""
438
+ checkpoint = self.session.load_checkpoint()
439
+
440
+ results = {
441
+ "total": 0,
442
+ "uploaded": 0,
443
+ "skipped": 0,
444
+ "failed": 0,
445
+ "errors": []
446
+ }
447
+
448
+ for specie, files in structure["by_specie"].items():
449
+ # Processar em lotes
450
+ for i in range(0, len(files), batch_size):
451
+ batch = files[i:i + batch_size]
452
+
453
+ # Verificar checkpoint
454
+ if self.session.is_batch_done(specie, i):
455
+ results["skipped"] += len(batch)
456
+ continue
457
+
458
+ # Enviar batch
459
+ for file_info in batch:
460
+ try:
461
+ if self._should_skip(file_info):
462
+ results["skipped"] += 1
463
+ self.session.mark_skipped(file_info)
464
+ continue
465
+
466
+ remote_path = f"audio/{self.project_slug}/{specie}/{file_info['name']}"
467
+
468
+ await self._upload_file_with_retry(
469
+ file_info["full_path"],
470
+ remote_path
471
+ )
472
+
473
+ results["uploaded"] += 1
474
+ self.session.mark_done(file_info)
475
+
476
+ if on_progress:
477
+ on_progress({
478
+ "total": structure["total_files"],
479
+ "uploaded": results["uploaded"],
480
+ "skipped": results["skipped"],
481
+ "failed": results["failed"],
482
+ "current_file": file_info["name"],
483
+ "current_size": file_info["size"]
484
+ })
485
+
486
+ except Exception as e:
487
+ results["failed"] += 1
488
+ results["errors"].append({
489
+ "file": file_info["name"],
490
+ "error": str(e)
491
+ })
492
+ self.session.mark_failed(file_info)
493
+
494
+ # Checkpoint por batch
495
+ self.session.save_checkpoint()
496
+
497
+ return results
498
+
499
+ async def _upload_file_with_retry(self,
500
+ local_path: str,
501
+ remote_path: str,
502
+ max_retries: int = 3):
503
+ """Upload com retries e backoff exponencial."""
504
+ for attempt in range(max_retries):
505
+ try:
506
+ self.api.upload_file(
507
+ path_or_fileobj=local_path,
508
+ path_in_repo=remote_path,
509
+ repo_id=self.repo_id,
510
+ repo_type="dataset"
511
+ )
512
+ return
513
+ except Exception as e:
514
+ if attempt < max_retries - 1:
515
+ wait_time = 2 ** attempt + random.uniform(0, 1)
516
+ await asyncio.sleep(wait_time)
517
+ else:
518
+ raise
519
+
520
+ def _should_skip(self, file_info: dict) -> bool:
521
+ """Verifica se arquivo já existe remotamente."""
522
+ try:
523
+ remote_path = f"audio/{self.project_slug}/{specie}/{file_info['name']}"
524
+ file_info_remote = self.api.get_file_info(
525
+ repo_id=self.repo_id,
526
+ filename=remote_path
527
+ )
528
+ return file_info_remote.size == file_info["size"]
529
+ except:
530
+ return False
531
+ ```
532
+
533
+ ### 4.5 Session Manager (`src/session/session_manager.py`)
534
+
535
+ ```python
536
+ class SessionManager:
537
+ def __init__(self, repo_id: str):
538
+ self.session_id = f"upload-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
539
+ self.session_dir = Path(f"~/.birdnet-uploader/sessions/{self.session_id}")
540
+ self.session_dir.mkdir(parents=True, exist_ok=True)
541
+
542
+ def save_checkpoint(self, data: dict):
543
+ """Salva checkpoint atomicamente."""
544
+ checkpoint_file = self.session_dir / "checkpoint.json"
545
+ temp_file = checkpoint_file.with_suffix(".tmp")
546
+
547
+ temp_file.write_text(json.dumps(data, indent=2))
548
+ temp_file.replace(checkpoint_file)
549
+
550
+ def load_checkpoint(self) -> dict:
551
+ """Carrega último checkpoint."""
552
+ checkpoint_file = self.session_dir / "checkpoint.json"
553
+ if checkpoint_file.exists():
554
+ return json.loads(checkpoint_file.read_text())
555
+ return {}
556
+ ```
557
+
558
+ ### 4.6 CSV Matcher (`src/csv/csv_matcher.py`)
559
+
560
+ ```python
561
+ class CSVMatcher:
562
+ def match_and_generate_index(self,
563
+ csv_path: str,
564
+ files_structure: dict) -> pd.DataFrame:
565
+ """Faz matching entre CSV e segmentos, gera índice."""
566
+ df = pd.read_csv(csv_path)
567
+
568
+ # Validar colunas esperadas
569
+ required_cols = ["source_file", "scientific_name", "confidence",
570
+ "start_time", "end_time"]
571
+ if not all(col in df.columns for col in required_cols):
572
+ raise ValueError(f"CSV deve conter colunas: {required_cols}")
573
+
574
+ # Matching lógica
575
+ matched = []
576
+ for _, row in df.iterrows():
577
+ source_stem = Path(row["source_file"]).stem
578
+
579
+ # Procurar arquivo correspondente
580
+ for specie, files in files_structure["by_specie"].items():
581
+ for file_info in files:
582
+ if source_stem in file_info["name"]:
583
+ matched.append({
584
+ **row,
585
+ "segment_path": f"audio/{self.project_slug}/{specie}/{file_info['name']}"
586
+ })
587
+
588
+ return pd.DataFrame(matched)
589
+
590
+ def generate_shards(self,
591
+ index_df: pd.DataFrame,
592
+ output_dir: str,
593
+ shard_size: int = 10000) -> list:
594
+ """Gera shards parquet para índice."""
595
+ shards = []
596
+
597
+ for i in range(0, len(index_df), shard_size):
598
+ shard_df = index_df.iloc[i:i+shard_size]
599
+ shard_file = f"{output_dir}/shard-{i//shard_size:05d}.parquet"
600
+
601
+ shard_df.to_parquet(shard_file, index=False)
602
+ shards.append(shard_file)
603
+
604
+ return shards
605
+ ```
606
+
607
+ ---
608
+
609
+ ## 5. FLUXO DE COMANDOS CLI
610
+
611
+ ```bash
612
+ # Autenticar
613
+ $ birdnet auth login
614
+ > Enter HF token: hf_...
615
+
616
+ # Upload simples (interactive)
617
+ $ birdnet upload
618
+
619
+ # Upload com flags (não-interativo)
620
+ $ birdnet upload \
621
+ --repo alice/my-birds \
622
+ --segments /path/to/segments \
623
+ --csv /path/to/metadata.csv \
624
+ --private
625
+
626
+ # Listar sessões
627
+ $ birdnet sessions list
628
+
629
+ # Retomar sessão
630
+ $ birdnet upload resume <session-id>
631
+
632
+ # Ver status
633
+ $ birdnet upload status <session-id>
634
+
635
+ # Exportar relatório
636
+ $ birdnet upload report <session-id>
637
+ ```
638
+
639
+ ---
640
+
641
+ ## 6. CRITÉRIOS DE ACEITE DO MVP
642
+
643
+ - [ ] Usuário baixa executável portátil, sem Python pré-instalado
644
+ - [ ] Login com token HF funciona e não vaza em logs
645
+ - [ ] Criar dataset público/privado com estrutura padrão
646
+ - [ ] Varrer pasta local corretamente (recursivo, agrupa por espécie)
647
+ - [ ] Upload de 50GB+ com progresso contínuo e ETA coerente
648
+ - [ ] Pausar/fechar app/retomar sem duplicar upload
649
+ - [ ] Queda de rede + 429 recuperam com retries
650
+ - [ ] CSV opcional com matching integrado
651
+ - [ ] Relatório final estruturado (JSONL/JSON)
652
+ - [ ] Estrutura de arquivos compatível com validador
653
+ - [ ] Compatibilidade 100% com manifest/index esperado
654
+ - [ ] Sem regressão do fluxo atual do Space
655
+
656
+ ---
657
+
658
+ ## 7. PRÓXIMOS PASSOS
659
+
660
+ 1. **Implementar módulos base** (Auth, Repo, Scanner, Uploader)
661
+ 2. **Construir UI** (Textual TUI ou Rich + Click)
662
+ 3. **Integrar Session Manager** e Checkpoint
663
+ 4. **Testes de integração** (50GB+, retomada, rede)
664
+ 5. **Construir executável** (PyInstaller)
665
+ 6. **Validação end-to-end** com usuários reais
README.md CHANGED
@@ -1,5 +1,5 @@
1
  ---
2
- title: BirdNET Uploader App
3
  emoji: "🐦"
4
  colorFrom: green
5
  colorTo: blue
@@ -10,76 +10,430 @@ app_file: app.py
10
  pinned: false
11
  ---
12
 
13
- # BirdNET-Uploader-App
14
 
15
- BirdNET-Uploader-App is a Gradio application for structured ingestion of BirdNET detections in multi-project workflows.
16
 
17
- It provides an operator-focused upload interface with deterministic key generation, resumable batch processing, index artifact creation, and audit-ready run reporting for Hugging Face Datasets.
 
18
 
19
- ## Objective
20
- This app centralizes ingestion for bioacoustics projects by:
21
 
22
- - matching BirdNET detection rows with segment files
23
- - creating deterministic detection keys
24
- - uploading data in resumable batches
25
- - generating index shards and manifest metadata
26
- - producing audit-friendly ingestion reports
27
 
28
- ## Usability
 
 
 
 
 
 
 
 
29
 
30
- The app supports a setup-first workflow:
31
 
32
- 1. Dataset Repo (HF)
33
- - initialize or repair the dataset repository structure
34
- - choose repository visibility (Public by default, Private optional)
35
- - prefill Dataset Repo (HF) and Project Slug for the next tabs
36
 
37
- 2. Option A - Upload Segments Only
38
- - upload compressed segments archive to the dataset repo
39
- - supports pause/resume/cancel with progress tracking
40
 
41
- 3. Option B - Ingest CSV + Segments
42
- - ingest detections CSV + segments archive in one run
43
- - updates index and manifest artifacts for downstream consumers
44
 
45
- This flow reduces manual errors and keeps project setup consistent across teams.
 
 
 
46
 
47
- ## Upload Strategies
48
 
49
- ### Strategy 1: Upload Segments via Hugging Face Space (Recommended for Teams)
 
 
 
 
50
 
51
- 1. Open the deployed uploader app on Hugging Face Spaces
52
- 2. In "Upload de Segmentos" section:
53
- - **Locally**: Create compressed file of segments:
54
- ```bash
55
- tar -czf segments.tar.gz /path/to/BirdNET\ Segments/
56
- ```
57
- - **Upload**: Drag & drop `segments.tar.gz` to the file input
58
- - Specify dataset repo (e.g., `username/ppbio-dataset`)
59
- - Click "Enviar Segmentos para Dataset"
60
- 3. Once segments are uploaded, proceed to dry-run and upload detections
61
 
62
- ### Strategy 2: Upload Segments via Local CLI Script
63
 
64
- If you prefer to run locally with full control:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
 
66
  ```bash
67
- python upload_segments_local.py "/path/to/BirdNET Segments" "username/ppbio-dataset"
68
  ```
69
 
70
- Optional arguments:
 
 
 
 
 
 
 
 
 
71
  ```bash
72
- python upload_segments_local.py "/path" "user/repo" --token YOUR_HF_TOKEN --batch-size 100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  ```
74
 
75
- ## Input Requirements
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
 
77
- - **BirdNET detections CSV** with required columns: `source_file`, `scientific_name`, `confidence`, `start_time`, `end_time`
78
- - **Segments folder or archive** (.tar, .tar.gz, .zip) containing .wav files organized by species
79
- - **Hugging Face dataset repo** in format `owner/repo`
80
- - **HF token** (optional if already authenticated in environment)
 
 
 
 
 
 
81
 
82
- ## Quick Start (Local)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
83
 
84
  1. Create and activate a Python 3.11+ virtual environment.
85
  2. Install dependencies:
@@ -88,65 +442,204 @@ python upload_segments_local.py "/path" "user/repo" --token YOUR_HF_TOKEN --batc
88
  pip install -r requirements.txt
89
  ```
90
 
91
- 3. Run the app:
92
 
93
  ```bash
94
  python app.py
95
  ```
96
 
97
- Default port is 7862. You can change it using environment variable `BIRDNET_UPLOADER_PORT`.
98
 
99
- ## Typical Workflow
 
 
100
 
101
- 1. Fill `project_slug` and `dataset_repo`.
102
- 2. Select detections CSV and segments root folder.
103
- 3. Run Dry-run and inspect output JSON.
104
- 4. Fix data issues if unmatched/ambiguous rows are high.
105
- 5. Run Upload mode when quality is acceptable.
106
- 6. Save run report for audit and reproducibility.
107
 
108
- ## CLI Commands (Optional)
109
 
110
- The same ingestion pipeline is available by CLI:
 
 
 
 
111
 
112
  ```bash
113
- python -m cli.hf_dataset_cli ingest-segments \
114
- --project-slug ppbio-rabeca \
115
- --dataset-repo USER/birdnet-ppbio-rabeca-dataset \
116
- --detections-csv detections.csv \
117
- --segments-root "C:\\BirdNET Segments" \
118
- --dry-run \
119
- --report-file .ingest-dry-run-report.json
120
  ```
121
 
 
 
122
  ```bash
123
- python -m cli.hf_dataset_cli ingest-segments \
124
- --project-slug ppbio-rabeca \
125
- --dataset-repo USER/birdnet-ppbio-rabeca-dataset \
126
- --detections-csv detections.csv \
127
- --segments-root "C:\\BirdNET Segments" \
128
- --batch-size 200 \
129
- --shard-size 10000 \
130
- --max-retries 3 \
131
- --retry-backoff-seconds 1.0 \
132
- --resume-state-file .ingest-segments-state.json \
133
- --report-file .ingest-run-report.json
134
  ```
135
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
  ## Repository Structure
137
 
138
- - `app.py`: uploader app entrypoint
139
- - `src/ui/upload_app_factory.py`: Gradio UI and actions
140
- - `cli/hf_dataset_cli.py`: ingestion engine and commands
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
 
142
- ## Operational Notes
143
 
144
- - Use dry-run first for every new project or new data source.
145
- - Keep run reports for traceability.
146
- - Prefer project-level dataset repos for cleaner access control.
147
- - For public deployment, use Hugging Face Spaces with repository sync from GitHub.
148
 
149
- ## Status
 
 
 
 
 
150
 
151
- Uploader workflow is implemented and ready for staged validation in real project datasets.
152
 
 
 
 
 
1
  ---
2
+ title: BirdNET-Validator-App
3
  emoji: "🐦"
4
  colorFrom: green
5
  colorTo: blue
 
10
  pinned: false
11
  ---
12
 
13
+ # BirdNET-Validator-App
14
 
15
+ BirdNET-Validator-App is a multi-purpose BirdNET workflow repository with two complementary entry points:
16
 
17
+ 1. A Gradio-based web application for human validation of BirdNET detections in multi-project workflows.
18
+ 2. A local command-line uploader for resumable Hugging Face dataset uploads with checkpoints, deduplication, and a Windows-friendly interactive flow.
19
 
20
+ The validator app focuses on review, authorization, and auditability. The uploader CLI focuses on reliable dataset ingestion for large audio collections.
 
21
 
22
+ ## Contents
 
 
 
 
23
 
24
+ - [Overview](#overview)
25
+ - [Validator App](#validator-app)
26
+ - [Local Uploader CLI](#local-uploader-cli)
27
+ - [Quick Start](#quick-start)
28
+ - [Configuration](#configuration)
29
+ - [Typical Flows](#typical-flows)
30
+ - [Repository Structure](#repository-structure)
31
+ - [Troubleshooting](#troubleshooting)
32
+ - [Development Notes](#development-notes)
33
 
34
+ ## Overview
35
 
36
+ This repository is designed for two high-confidence operations:
 
 
 
37
 
38
+ - controlled, multi-project human validation of detections
39
+ - reliable, resumable upload of large audio datasets to Hugging Face
 
40
 
41
+ The validator workflow emphasizes:
 
 
42
 
43
+ - controlled access by user and project
44
+ - rapid decision flow for detections
45
+ - auditability through append-only events
46
+ - conflict-aware updates for concurrent validators
47
 
48
+ The uploader workflow emphasizes:
49
 
50
+ - single-flow login → repo → scan → upload
51
+ - resumable sessions stored locally
52
+ - remote deduplication with a local cache
53
+ - progress tracking and checkpoint recovery
54
+ - compatibility with the validator app's expected dataset structure
55
 
56
+ ## Validator App
 
 
 
 
 
 
 
 
 
57
 
58
+ The validator is a Gradio application for human review of BirdNET detections in multi-project workflows.
59
 
60
+ ### Key Usability Features
61
+
62
+ 1. Multi-project login with role-based access
63
+ 2. Project selection for authorized datasets
64
+ 3. Detection queue with filters and pagination
65
+ 4. Audio loading on demand with ephemeral cache
66
+ 5. Fast validation actions (positive, negative, uncertain, skip)
67
+ 6. Concurrency-safe writes with optimistic lock feedback
68
+ 7. Conflict resolution support and validation reporting
69
+
70
+ ### Quick Start (Local)
71
+
72
+ 1. Create and activate a Python 3.11+ virtual environment.
73
+ 2. Install dependencies:
74
+
75
+ ```bash
76
+ pip install -r requirements.txt
77
+ ```
78
+
79
+ 3. Run the app:
80
+
81
+ ```bash
82
+ python app.py
83
+ ```
84
+
85
+ Default port is `7860`.
86
+
87
+ ### Runtime Configuration
88
+
89
+ Optional runtime configuration:
90
+
91
+ - `BIRDNET_DETECTIONS_FILE`: path to JSON file with detections grouped by project slug.
92
+ - `BIRDNET_VALIDATIONS_DIR`: custom directory for append-only validation events and snapshot files.
93
+ - `BIRDNET_PAGE_SIZE`: queue page size (default: `25`).
94
+ - `BIRDNET_PROJECTS_FILE`: JSON file with project catalog used at app startup.
95
+ - `BIRDNET_USER_ACCESS_FILE`: JSON file mapping users to project roles (`admin`/`validator`).
96
+ - `BIRDNET_INVITES_FILE`: JSON file storing pending invites.
97
+ - `BIRDNET_INVITE_TTL_HOURS`: invite expiration in hours (default: `72`).
98
+ - `BIRDNET_BOOTSTRAP_DIR`: base directory for bootstrap state files (`projects.json`, `user_access.json`, `invites.json`).
99
+ - `BIRDNET_ENABLE_DEMO_BOOTSTRAP`: set to `true` only for local/demo mode to load built-in sample users/projects.
100
+
101
+ Invite email settings (EmailJS):
102
+
103
+ - `BIRDNET_EMAILJS_ENABLED`: set to `true` to send invite emails via EmailJS.
104
+ - `BIRDNET_INVITE_EMAIL_ENABLED`: set to `true` to enable invite delivery.
105
+ - `BIRDNET_INVITE_EMAIL_SENDER`: sender label shown in the invite email.
106
+ - `BIRDNET_INVITE_EMAIL_LOGIN_URL`: login URL included in invitation instructions.
107
+ - `BIRDNET_EMAILJS_SERVICE_ID`: EmailJS service ID.
108
+ - `BIRDNET_EMAILJS_TEMPLATE_ID`: default/fallback EmailJS template ID.
109
+ - `BIRDNET_EMAILJS_TEMPLATE_ID_USERNAME_ONLY`: optional template for username-only mode.
110
+ - `BIRDNET_EMAILJS_TEMPLATE_ID_EMAIL_ONLY`: optional template for email-only mode.
111
+ - `BIRDNET_EMAILJS_TEMPLATE_ID_DUAL`: optional template for dual mode (username + email).
112
+ - `BIRDNET_EMAILJS_PUBLIC_KEY`: EmailJS public key.
113
+ - `BIRDNET_EMAILJS_ENDPOINT`: EmailJS API endpoint (default: `https://api.emailjs.com/api/v1.0/email/send`).
114
+ - `BIRDNET_EMAILJS_TIMEOUT_SECONDS`: request timeout in seconds (default: `20`).
115
+
116
+ ### Validation Data Shapes
117
+
118
+ Detection seed file example:
119
+
120
+ ```json
121
+ {
122
+ "kenya-2024": [
123
+ {
124
+ "detection_key": "0000000000001001",
125
+ "audio_id": "audio_1001",
126
+ "scientific_name": "Cyanocorax cyanopogon",
127
+ "confidence": 0.91,
128
+ "start_time": 0.0,
129
+ "end_time": 1.0
130
+ }
131
+ ]
132
+ }
133
+ ```
134
+
135
+ Projects bootstrap file example (`BIRDNET_PROJECTS_FILE`):
136
+
137
+ ```json
138
+ [
139
+ {
140
+ "project_slug": "kenya-2024",
141
+ "name": "Kenya Survey 2024",
142
+ "dataset_repo_id": "org/kenya-2024-dataset",
143
+ "active": true
144
+ }
145
+ ]
146
+ ```
147
+
148
+ User access bootstrap file example (`BIRDNET_USER_ACCESS_FILE`):
149
+
150
+ ```json
151
+ {
152
+ "admin_user": {
153
+ "kenya-2024": "admin"
154
+ },
155
+ "validator_a": {
156
+ "kenya-2024": "validator"
157
+ }
158
+ }
159
+ ```
160
+
161
+ ### Typical Validation Flow
162
+
163
+ 1. Login with a valid user.
164
+ 2. Select an authorized project.
165
+ 3. Load queue items and apply filters when needed.
166
+ 4. Listen to selected audio and submit validation status.
167
+ 5. Resolve conflicts when concurrent updates occur.
168
+ 6. Export or inspect project validation report.
169
+
170
+ ### Deploy on Hugging Face Spaces
171
+
172
+ 1. Create a new Space with:
173
+ - SDK: `Gradio`
174
+ - Python: `3.11`
175
+
176
+ 2. Push this repository to the Space.
177
+
178
+ 3. Configure Variables/Secrets in the Space settings.
179
+
180
+ Required for production bootstrap:
181
+
182
+ - `BIRDNET_PROJECTS_FILE` (example: `docs/spaces/projects.sample.json`)
183
+ - `BIRDNET_USER_ACCESS_FILE` (example: `docs/spaces/user_access.sample.json`)
184
+
185
+ Optional runtime settings:
186
+
187
+ - `BIRDNET_DETECTIONS_FILE` (seed detections JSON)
188
+ - `BIRDNET_VALIDATIONS_DIR` (recommended: `/data/validations`)
189
+ - `BIRDNET_BOOTSTRAP_DIR` (recommended: `/data/bootstrap`)
190
+ - `BIRDNET_INVITES_FILE` (recommended: `/data/bootstrap/invites.json`)
191
+ - `BIRDNET_INVITE_TTL_HOURS` (default `72`)
192
+ - `BIRDNET_PAGE_SIZE` (default `25`)
193
+ - `BIRDNET_ENABLE_DEMO_BOOTSTRAP` (`false` in production)
194
+
195
+ Optional invite email settings:
196
+
197
+ - `BIRDNET_INVITE_EMAIL_ENABLED=true`
198
+ - `BIRDNET_INVITE_EMAIL_SENDER`
199
+ - `BIRDNET_INVITE_EMAIL_LOGIN_URL`
200
+ - `BIRDNET_EMAILJS_ENABLED`
201
+ - `BIRDNET_EMAILJS_SERVICE_ID`
202
+ - `BIRDNET_EMAILJS_TEMPLATE_ID`
203
+ - `BIRDNET_EMAILJS_PUBLIC_KEY`
204
+ - `BIRDNET_EMAILJS_ENDPOINT`
205
+ - `BIRDNET_EMAILJS_TIMEOUT_SECONDS`
206
+
207
+ 4. For first smoke test only, you may temporarily set:
208
+
209
+ - `BIRDNET_ENABLE_DEMO_BOOTSTRAP=true`
210
+
211
+ Then log in with one of the demo users:
212
+
213
+ - `admin_user`
214
+ - `demo_user`
215
+ - `validator_demo`
216
+
217
+ 5. After validation, switch to production bootstrap:
218
+
219
+ - Set `BIRDNET_ENABLE_DEMO_BOOTSTRAP=false`
220
+ - Provide real `BIRDNET_PROJECTS_FILE` and `BIRDNET_USER_ACCESS_FILE`
221
+
222
+ Notes:
223
+
224
+ - The app entrypoint reads `PORT` automatically in Spaces.
225
+ - Keep user/project bootstrap files private if they contain sensitive assignments.
226
+ - Use `/data` paths in Spaces to keep projects, invites, ACL, and validations across redeploys.
227
+ - Collaborative access is token-per-user: each collaborator logs in with their own Hugging Face token.
228
+
229
+ ## Local Uploader CLI
230
+
231
+ The repository also includes a local uploader CLI designed for large dataset ingest and resume support.
232
+
233
+ ### Release Pipeline
234
+
235
+ The repository includes a release pipeline that produces a portable uploader bundle and can publish it to Hugging Face.
236
+
237
+ One-command Windows pipeline:
238
+
239
+ ```powershell
240
+ .\build\release_pipeline.ps1 -Version 0.1.0
241
+ ```
242
+
243
+ Build and publish in one step:
244
+
245
+ ```powershell
246
+ $env:HF_TOKEN = "hf_xxx"
247
+ .\build\release_pipeline.ps1 -Version 0.1.0 -PublishToHf -RepoId jrrribeiro/birdnet-uploader-releases -RepoType dataset
248
+ ```
249
+
250
+ Validation-only step (recommended before publish):
251
+
252
+ ```powershell
253
+ .\build\validate_release.ps1 -Version 0.1.0
254
+ ```
255
+
256
+ Local build steps:
257
+
258
+ ```bash
259
+ python -m pip install --upgrade pip
260
+ pip install -r requirements.txt
261
+ pip install pyinstaller
262
+ python build/release_uploader.py --version 0.1.0
263
+ ```
264
+
265
+ The build produces:
266
+
267
+ - a zipped release bundle under `build/release/`
268
+ - a `.sha256` checksum file next to the zip
269
+
270
+ To publish a release bundle to Hugging Face from a local machine:
271
+
272
+ ```bash
273
+ set HF_TOKEN=hf_xxx
274
+ python build/publish_release_to_hf.py ^
275
+ --repo-id jrrribeiro/birdnet-uploader-releases ^
276
+ --repo-type dataset ^
277
+ --bundle build\release\birdnet-uploader-0.1.0-windows.zip ^
278
+ --checksum build\release\birdnet-uploader-0.1.0-windows.zip.sha256 ^
279
+ --version 0.1.0
280
+ ```
281
+
282
+ Recommended Hugging Face release repo layout:
283
+
284
+ ```text
285
+ birdnet-uploader-releases/
286
+ releases/
287
+ v0.1.0/
288
+ birdnet-uploader-0.1.0-windows.zip
289
+ birdnet-uploader-0.1.0-windows.zip.sha256
290
+ ```
291
+
292
+ Workflow on GitHub Actions:
293
+
294
+ - run the release workflow manually
295
+ - build the Windows executable with PyInstaller
296
+ - archive the bundle as a workflow artifact
297
+ - optionally upload the bundle and checksum to a Hugging Face repo
298
+
299
+ Recommended user-facing validation after publishing:
300
+
301
+ 1. Download the zip from Hugging Face.
302
+ 2. Unzip into a clean folder.
303
+ 3. Run the executable without Python installed.
304
+ 4. Authenticate with a Hugging Face token.
305
+ 5. Create a session and confirm resume/recovery works.
306
+
307
+ ### What It Does
308
+
309
+ - authenticates with a Hugging Face token stored in the OS keyring
310
+ - creates or validates dataset repositories
311
+ - scans a local audio folder recursively
312
+ - computes per-file SHA-256 hashes
313
+ - deduplicates against a cached remote index
314
+ - uploads files with retries and session checkpoints
315
+ - resumes from the last checkpoint after close or crash
316
+
317
+ ### Main Commands
318
+
319
+
320
+ The CLI entrypoint is:
321
 
322
  ```bash
323
+ python -m src.uploader_cli.main --help
324
  ```
325
 
326
+ Commands:
327
+
328
+ - `login`: validate and store a Hugging Face token
329
+ - `init-repo`: create and initialize a dataset repository
330
+ - `scan`: scan a local folder and summarize audio files
331
+ - `start`: create a local upload session checkpoint
332
+ - `resume`: print the saved checkpoint for a session
333
+
334
+ ### Example Workflow
335
+
336
  ```bash
337
+ # Authenticate once
338
+ python -m src.uploader_cli.main login
339
+
340
+ # Create or initialize the dataset repository
341
+ python -m src.uploader_cli.main init-repo --repo-id alice/birdnet-2026 --private
342
+
343
+ # Inspect the local folder before upload
344
+ python -m src.uploader_cli.main scan --segments C:\audio\segments
345
+
346
+ # Create a resumable local session checkpoint
347
+ python -m src.uploader_cli.main start --repo-id alice/birdnet-2026 --segments C:\audio\segments
348
+
349
+
350
+ # Resume and inspect the checkpoint later
351
+ python -m src.uploader_cli.main resume upload-20260429T120000Z
352
  ```
353
 
354
+ ### Upload Engine Building Blocks
355
+
356
+ The uploader CLI is built from the following modules:
357
+
358
+ - `src/uploader_cli/auth_service.py`: token validation and keyring storage
359
+ - `src/uploader_cli/repo_service.py`: dataset creation and structure initialization
360
+ - `src/uploader_cli/scanner.py`: recursive audio discovery and metadata collection
361
+ - `src/uploader_cli/hash_utils.py`: streaming SHA-256 helpers
362
+ - `src/uploader_cli/deduplicator.py`: remote index cache and skip/upload decisions
363
+ - `src/uploader_cli/session_manager.py`: checkpoint and session metadata persistence
364
+ - `src/uploader_cli/batch_uploader.py`: retrying upload orchestration
365
+ - `src/uploader_cli/error_handler.py`: user-friendly error formatting
366
+
367
+ ### Local Data Layout
368
+
369
+ The uploader stores local state under the user's profile directory by default:
370
+
371
+ ```text
372
+ ~/.birdnet-uploader/
373
+ sessions/
374
+ upload-YYYYMMDDTHHMMSSZ/
375
+ metadata.json
376
+ checkpoint.json
377
+ cache/
378
+ dedup/
379
+ <repo-hash>.json
380
+ logs/
381
+ ```
382
+
383
+ Environment overrides are supported:
384
+
385
+ - `BIRDNET_UPLOADER_SESSION_DIR`
386
+ - `BIRDNET_UPLOADER_CACHE_DIR`
387
+ - `BIRDNET_UPLOADER_LOG_DIR`
388
+
389
+ ### Dataset Layout Expected by the Validator
390
+
391
+ The uploader is designed to preserve the validator's expected structure:
392
+
393
+ ```text
394
+ audio/{project_slug}/... # uploaded audio files
395
+ index/shards/... # parquet shard files
396
+ index/manifest.json # index manifest
397
+ validations/... # validation outputs
398
+ audit/ingestion-runs/... # ingestion run history
399
+ ```
400
+
401
+ ### Uploader Configuration
402
+
403
+ Runtime constants are centralized in `src/uploader_cli/config.py`:
404
 
405
+ - `APP_NAME`
406
+ - `SCHEMA_VERSION`
407
+ - `AUDIO_EXTENSIONS`
408
+ - `INDEX_SHARD_SIZE`
409
+ - `MAX_BATCH_SIZE`
410
+ - `RETRY_MAX_ATTEMPTS`
411
+ - `RETRY_INITIAL_BACKOFF_SECONDS`
412
+ - `RETRY_MAX_BACKOFF_SECONDS`
413
+ - `KEYRING_SERVICE`
414
+ - `KEYRING_ACCOUNT`
415
 
416
+ ### Resumability Guarantees
417
+
418
+ The uploader uses local checkpoints so it can recover from:
419
+
420
+ - process crashes
421
+ - app close
422
+ - transient network failures
423
+ - partial progress during large uploads
424
+
425
+ The checkpoint stores session metadata and cumulative counters such as:
426
+
427
+ - uploaded count
428
+ - skipped count
429
+ - failed count
430
+ - last completed file
431
+ - last failed file
432
+ - total uploaded bytes
433
+
434
+ ## Quick Start
435
+
436
+ ### Local Development
437
 
438
  1. Create and activate a Python 3.11+ virtual environment.
439
  2. Install dependencies:
 
442
  pip install -r requirements.txt
443
  ```
444
 
445
+ 3. Run the validator app:
446
 
447
  ```bash
448
  python app.py
449
  ```
450
 
451
+ 4. Run the uploader CLI help:
452
 
453
+ ```bash
454
+ python -m src.uploader_cli.main --help
455
+ ```
456
 
457
+ ### First-Time Uploader Setup
 
 
 
 
 
458
 
459
+ 1. Log in with a Hugging Face token:
460
 
461
+ ```bash
462
+ python -m src.uploader_cli.main login
463
+ ```
464
+
465
+ 2. Validate your target repository exists or create it:
466
 
467
  ```bash
468
+ python -m src.uploader_cli.main init-repo --repo-id alice/birdnet-2026 --private
 
 
 
 
 
 
469
  ```
470
 
471
+ 3. Scan your local dataset folder:
472
+
473
  ```bash
474
+ python -m src.uploader_cli.main scan --segments C:\audio\segments
 
 
 
 
 
 
 
 
 
 
475
  ```
476
 
477
+ 4. Start a resumable session:
478
+
479
+ ```bash
480
+ python -m src.uploader_cli.main start --repo-id alice/birdnet-2026 --segments C:\audio\segments
481
+ ```
482
+
483
+ ## Configuration
484
+
485
+ ### Validator App Environment Variables
486
+
487
+ - `BIRDNET_DETECTIONS_FILE`
488
+ - `BIRDNET_VALIDATIONS_DIR`
489
+ - `BIRDNET_PAGE_SIZE`
490
+ - `BIRDNET_PROJECTS_FILE`
491
+ - `BIRDNET_USER_ACCESS_FILE`
492
+ - `BIRDNET_INVITES_FILE`
493
+ - `BIRDNET_INVITE_TTL_HOURS`
494
+ - `BIRDNET_BOOTSTRAP_DIR`
495
+ - `BIRDNET_ENABLE_DEMO_BOOTSTRAP`
496
+ - `BIRDNET_EMAILJS_ENABLED`
497
+ - `BIRDNET_INVITE_EMAIL_ENABLED`
498
+ - `BIRDNET_INVITE_EMAIL_SENDER`
499
+ - `BIRDNET_INVITE_EMAIL_LOGIN_URL`
500
+ - `BIRDNET_EMAILJS_SERVICE_ID`
501
+ - `BIRDNET_EMAILJS_TEMPLATE_ID`
502
+ - `BIRDNET_EMAILJS_TEMPLATE_ID_USERNAME_ONLY`
503
+ - `BIRDNET_EMAILJS_TEMPLATE_ID_EMAIL_ONLY`
504
+ - `BIRDNET_EMAILJS_TEMPLATE_ID_DUAL`
505
+ - `BIRDNET_EMAILJS_PUBLIC_KEY`
506
+ - `BIRDNET_EMAILJS_ENDPOINT`
507
+ - `BIRDNET_EMAILJS_TIMEOUT_SECONDS`
508
+
509
+ ### Uploader CLI Environment Variables
510
+
511
+ - `BIRDNET_UPLOADER_SESSION_DIR`
512
+ - `BIRDNET_UPLOADER_CACHE_DIR`
513
+ - `BIRDNET_UPLOADER_LOG_DIR`
514
+
515
+ ## Typical Flows
516
+
517
+ ### Validation Flow
518
+
519
+ 1. Login with a valid user.
520
+ 2. Select an authorized project.
521
+ 3. Load queue items and apply filters when needed.
522
+ 4. Listen to selected audio and submit validation status.
523
+ 5. Resolve conflicts when concurrent updates occur.
524
+ 6. Export or inspect project validation report.
525
+
526
+ ### Upload Flow
527
+
528
+ 1. Authenticate once with a Hugging Face token.
529
+ 2. Create or validate a dataset repository.
530
+ 3. Scan the local folder recursively.
531
+ 4. Deduplicate against remote files.
532
+ 5. Upload with retries and checkpoints.
533
+ 6. Resume later from the saved session metadata if needed.
534
+
535
  ## Repository Structure
536
 
537
+ - `app.py`: validator app entrypoint
538
+ - `src/auth`: authentication/session and ACL logic
539
+ - `src/domain`: domain models
540
+ - `src/repositories`: in-memory and append-only persistence
541
+ - `src/services`: queue, validation, and audio fetch services
542
+ - `src/ui`: Gradio interface composition and callbacks
543
+ - `src/uploader_cli`: local uploader CLI, session persistence, deduplication, and retrying uploads
544
+ - `tests`: unit and integration test suites
545
+ - `docs`: collaborative workflow and QA checklists
546
+
547
+ ## Project Status
548
+
549
+ Validator workflow is active and under continuous development, with emphasis on reliability, auditability, and operator productivity.
550
+
551
+ The uploader CLI is being built as a companion local workflow for large ingest tasks, with a focus on resumability and validator compatibility.
552
+
553
+ ## FAQ
554
+
555
+ ### Is this app intended for production use?
556
+
557
+ Yes, for validator workflows. It is structured for multi-project access control, audit-friendly writes, and collaborative validation.
558
+
559
+ ### Is the uploader CLI meant to replace the validator app?
560
+
561
+ No. It is a companion tool for large dataset uploads and repository initialization.
562
+
563
+ ### Does the app preload all audio files?
564
+
565
+ No. Audio is fetched on demand for the selected detection, helping keep memory and network usage under control.
566
+
567
+ ### Can multiple validators work at the same time?
568
+
569
+ Yes. The validation flow uses optimistic concurrency checks to detect and handle conflicting updates.
570
+
571
+ ### Can this run on Hugging Face Spaces?
572
+
573
+ Yes. The validator app is designed for Gradio deployment on Hugging Face Spaces.
574
+
575
+ ### Can the uploader resume after closing the app?
576
+
577
+ Yes. It persists checkpoints and session metadata locally and can be resumed later.
578
+
579
+ ### Does the uploader support deduplication?
580
+
581
+ Yes. It checks the remote dataset index, caches the remote file list locally, and skips files that are already present.
582
+
583
+ ## Troubleshooting
584
+
585
+ ### The app does not start locally
586
+
587
+ 1. Confirm Python 3.11+ is active.
588
+ 2. Reinstall dependencies with `pip install -r requirements.txt`.
589
+ 3. Run `python app.py` from the repository root.
590
+
591
+ ### Port 7860 is already in use
592
+
593
+ Stop the existing process using that port, or launch after setting a different `GRADIO_SERVER_PORT` environment variable.
594
+
595
+ ### Login works but no project appears
596
+
597
+ Your user likely has no project assignment. Verify `BIRDNET_PROJECTS_FILE` and `BIRDNET_USER_ACCESS_FILE`, or add project access through the admin flow.
598
+
599
+ ### Seed warning appears in Validation tab
600
+
601
+ If you see a seed warning banner:
602
+
603
+ 1. Verify `BIRDNET_DETECTIONS_FILE` points to an existing file.
604
+ 2. Confirm the file is valid UTF-8 JSON.
605
+ 3. Confirm each project maps to a list of detections, or each list item includes `project_slug`.
606
+ 4. If needed, unset `BIRDNET_DETECTIONS_FILE` to fall back to default demo detections.
607
+
608
+ ### Audio does not load for a detection
609
+
610
+ Check that:
611
+
612
+ 1. `audio_id` exists and is valid for the selected project.
613
+ 2. The dataset repository is reachable.
614
+ 3. You have permission to read the project dataset.
615
+
616
+ ### I get conflict messages while validating
617
+
618
+ This means another validator updated the same detection first. Refresh and reapply your decision on the newest version.
619
+
620
+ ### The uploader says the token cannot be saved
621
+
622
+ Make sure your OS keyring backend is available. On Windows, the default credential store should work automatically.
623
+
624
+ ### The uploader skips files unexpectedly
625
+
626
+ It may have found them in the remote dataset index cache. Clear the uploader cache directory if you want to force a fresh remote listing.
627
+
628
+ ### Resume shows an empty checkpoint
629
 
630
+ Check that you ran the uploader `start` command at least once and that the session directory still exists.
631
 
632
+ ## Recent Updates
 
 
 
633
 
634
+ - Integrated Validation tab flow in the multi-project app (no placeholder stage).
635
+ - Added project-scoped queue badge (`Queue: N`) and improved queue context feedback.
636
+ - Added seed-file warning banner with actionable remediation guidance in Validation.
637
+ - Standardized all user-facing UI text and feedback messages in English.
638
+ - Updated authentication/session timestamps to timezone-aware UTC handling.
639
+ - Added a local uploader CLI with login, repository initialization, scan, deduplication, checkpointing, and resumable upload foundations.
640
 
641
+ ## Related Documents
642
 
643
+ - `BIRDNET_UPLOADER_REDESIGN.md`: uploader UX and architecture notes
644
+ - `SPRINT_ROADMAP.md`: implementation roadmap
645
+ - `SPRINT1_KICKOFF.md`: sprint kickoff details
SPRINT1_KICKOFF.md ADDED
@@ -0,0 +1,769 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Sprint 1 Kickoff: Primeiros Passos (HOJE)
2
+
3
+ ## Objetivo
4
+ Estrutura base pronta, dependências instaladas, todos os módulos esqueléticos criados, testes rodando.
5
+
6
+ **Tempo estimado**: 4-5 horas para 1-2 pessoas
7
+
8
+ ---
9
+
10
+ ## Step 1: Criar Branch de Trabalho
11
+
12
+ ```bash
13
+ cd c:\Users\jonat\Documents\Python\BirdNET-validator-App
14
+ git checkout -b feature/birdnet-uploader-cli
15
+ git push -u origin feature/birdnet-uploader-cli
16
+ ```
17
+
18
+ ---
19
+
20
+ ## Step 2: Criar Estrutura de Diretórios
21
+
22
+ ```bash
23
+ # Criar estrutura completa
24
+ mkdir -p src\auth
25
+ mkdir -p src\repository
26
+ mkdir -p src\upload
27
+ mkdir -p src\csv
28
+ mkdir -p src\session
29
+ mkdir -p src\ui\screens
30
+ mkdir -p src\logger
31
+ mkdir -p src\utils
32
+ mkdir -p tests
33
+ mkdir -p build
34
+ mkdir -p docs
35
+
36
+ # Criar __init__.py em todos
37
+ echo "" > src\__init__.py
38
+ echo "" > src\auth\__init__.py
39
+ echo "" > src\repository\__init__.py
40
+ echo "" > src\upload\__init__.py
41
+ echo "" > src\csv\__init__.py
42
+ echo "" > src\session\__init__.py
43
+ echo "" > src\ui\__init__.py
44
+ echo "" > src\ui\screens\__init__.py
45
+ echo "" > src\logger\__init__.py
46
+ echo "" > src\utils\__init__.py
47
+ echo "" > tests\__init__.py
48
+ ```
49
+
50
+ ---
51
+
52
+ ## Step 3: Criar `requirements.txt`
53
+
54
+ Criar arquivo: `requirements.txt`
55
+
56
+ ```
57
+ # Core CLI
58
+ click>=8.0.0
59
+ typer>=0.9.0
60
+
61
+ # UI/TUI
62
+ textual>=0.30.0
63
+ rich>=13.0.0
64
+
65
+ # HF Integration
66
+ huggingface_hub>=0.28.0
67
+
68
+ # Data Processing
69
+ pandas>=2.0.0
70
+ pyarrow>=14.0.0
71
+
72
+ # Security
73
+ keyring>=24.0.0
74
+ cryptography>=41.0.0
75
+
76
+ # Validation
77
+ pydantic>=2.0.0
78
+
79
+ # Development & Testing
80
+ pytest>=7.0.0
81
+ pytest-cov>=4.0.0
82
+ black>=23.0.0
83
+ flake8>=6.0.0
84
+ mypy>=1.0.0
85
+
86
+ # Optional: async
87
+ asyncio>=3.4.3
88
+ ```
89
+
90
+ Instalar:
91
+ ```bash
92
+ pip install -r requirements.txt
93
+ ```
94
+
95
+ ---
96
+
97
+ ## Step 4: Criar `pyproject.toml`
98
+
99
+ Criar arquivo: `pyproject.toml`
100
+
101
+ ```toml
102
+ [build-system]
103
+ requires = ["setuptools>=65.0", "wheel"]
104
+ build-backend = "setuptools.build_meta"
105
+
106
+ [project]
107
+ name = "birdnet-uploader"
108
+ version = "0.1.0"
109
+ description = "Local CLI for resumable HuggingFace dataset uploads (BirdNET)"
110
+ authors = [
111
+ {name = "BirdNET Team", email = "team@example.com"}
112
+ ]
113
+ requires-python = ">=3.9"
114
+ dependencies = [
115
+ "click>=8.0.0",
116
+ "textual>=0.30.0",
117
+ "rich>=13.0.0",
118
+ "huggingface_hub>=0.28.0",
119
+ "pandas>=2.0.0",
120
+ "pyarrow>=14.0.0",
121
+ "keyring>=24.0.0",
122
+ "pydantic>=2.0.0",
123
+ ]
124
+
125
+ [project.optional-dependencies]
126
+ dev = [
127
+ "pytest>=7.0.0",
128
+ "pytest-cov>=4.0.0",
129
+ "black>=23.0.0",
130
+ "flake8>=6.0.0",
131
+ "mypy>=1.0.0",
132
+ ]
133
+
134
+ [project.scripts]
135
+ birdnet = "src.main:app"
136
+
137
+ [tool.black]
138
+ line-length = 100
139
+ target-version = ['py39']
140
+
141
+ [tool.isort]
142
+ profile = "black"
143
+ line_length = 100
144
+
145
+ [tool.mypy]
146
+ python_version = "3.9"
147
+ warn_return_any = true
148
+ warn_unused_configs = true
149
+
150
+ [tool.pytest.ini_options]
151
+ testpaths = ["tests"]
152
+ addopts = "-v --cov=src --cov-report=html --cov-report=term-missing"
153
+ ```
154
+
155
+ ---
156
+
157
+ ## Step 5: Criar Estrutura Base de Módulos (Skeletons)
158
+
159
+ ### 5.1: `src/main.py`
160
+
161
+ ```python
162
+ """BirdNET Uploader - Main Entry Point"""
163
+
164
+ import click
165
+ from src.ui.tui import BirdNETApp
166
+ from src.auth.auth_service import AuthService
167
+ from src.config import get_config
168
+
169
+ __version__ = "0.1.0"
170
+
171
+ @click.group()
172
+ @click.version_option(version=__version__)
173
+ def app():
174
+ """BirdNET Uploader - Upload audio to HuggingFace Datasets"""
175
+ pass
176
+
177
+ @app.command()
178
+ def upload():
179
+ """Start the upload wizard (interactive TUI)"""
180
+ try:
181
+ app_ui = BirdNETApp()
182
+ app_ui.run()
183
+ except Exception as e:
184
+ click.echo(f"Error: {e}", err=True)
185
+
186
+ @app.command()
187
+ def login():
188
+ """Authenticate with HuggingFace token"""
189
+ try:
190
+ token = click.prompt("Enter your HF token", hide_input=True)
191
+ auth = AuthService()
192
+ user_info = auth.authenticate(token)
193
+ click.echo(f"✓ Logged in as {user_info['username']}")
194
+ except Exception as e:
195
+ click.echo(f"Error: {e}", err=True)
196
+
197
+ @app.command()
198
+ @click.argument("session_id")
199
+ def resume(session_id: str):
200
+ """Resume a previous upload session"""
201
+ click.echo(f"Resuming session: {session_id}")
202
+ # TODO: Implement
203
+
204
+ @app.command()
205
+ def list_sessions():
206
+ """List all upload sessions"""
207
+ click.echo("Sessions:")
208
+ # TODO: Implement
209
+
210
+ if __name__ == "__main__":
211
+ app()
212
+ ```
213
+
214
+ ### 5.2: `src/config.py`
215
+
216
+ ```python
217
+ """Configuration and Constants"""
218
+
219
+ from pathlib import Path
220
+ import os
221
+
222
+ # App
223
+ APP_NAME = "BirdNET Uploader"
224
+ APP_VERSION = "0.1.0"
225
+
226
+ # Directories
227
+ SESSION_DIR = Path.home() / ".birdnet-uploader" / "sessions"
228
+ CACHE_DIR = Path.home() / ".birdnet-uploader" / "cache"
229
+ LOG_DIR = Path.home() / ".birdnet-uploader" / "logs"
230
+
231
+ # Create directories if not exist
232
+ SESSION_DIR.mkdir(parents=True, exist_ok=True)
233
+ CACHE_DIR.mkdir(parents=True, exist_ok=True)
234
+ LOG_DIR.mkdir(parents=True, exist_ok=True)
235
+
236
+ # Audio
237
+ AUDIO_EXTENSIONS = ['.wav', '.mp3', '.flac', '.ogg', '.m4a']
238
+
239
+ # Upload
240
+ MAX_BATCH_SIZE = 10
241
+ MAX_RETRIES = 3
242
+ RETRY_INITIAL_BACKOFF = 1.0 # seconds
243
+ RETRY_MAX_BACKOFF = 30.0 # seconds
244
+
245
+ # Manifest
246
+ MANIFEST_SCHEMA_VERSION = "1.0.0"
247
+ SHARD_SIZE = 10000
248
+
249
+ # Keyring
250
+ KEYRING_SERVICE = "birdnet-uploader"
251
+ KEYRING_ACCOUNT = "hf_token"
252
+
253
+ def get_config():
254
+ """Get runtime configuration from environment or defaults"""
255
+ return {
256
+ "session_dir": os.getenv("BIRDNET_SESSION_DIR", str(SESSION_DIR)),
257
+ "cache_dir": os.getenv("BIRDNET_CACHE_DIR", str(CACHE_DIR)),
258
+ "log_dir": os.getenv("BIRDNET_LOG_DIR", str(LOG_DIR)),
259
+ }
260
+ ```
261
+
262
+ ### 5.3: `src/auth/auth_service.py`
263
+
264
+ ```python
265
+ """Authentication Service"""
266
+
267
+ from typing import Optional, Dict
268
+ import keyring
269
+ from huggingface_hub import HfApi, HfFolder
270
+ from src.config import KEYRING_SERVICE, KEYRING_ACCOUNT
271
+
272
+ class AuthenticationError(Exception):
273
+ """Authentication failed"""
274
+ pass
275
+
276
+ class AuthService:
277
+ """Manage authentication with HuggingFace"""
278
+
279
+ def __init__(self):
280
+ self.api = None
281
+
282
+ def authenticate(self, token: str) -> Dict[str, str]:
283
+ """
284
+ Validate token and store it securely.
285
+
286
+ Args:
287
+ token: HF API token
288
+
289
+ Returns:
290
+ User info dict with username, email, user_id
291
+
292
+ Raises:
293
+ AuthenticationError: If token is invalid
294
+ """
295
+ try:
296
+ api = HfApi(token=token)
297
+ user_info = api.whoami()
298
+
299
+ # Store token in keyring
300
+ keyring.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, token)
301
+
302
+ self.api = api
303
+ return {
304
+ "username": user_info.get("name"),
305
+ "email": user_info.get("email", ""),
306
+ "user_id": user_info.get("user_id"),
307
+ }
308
+ except Exception as e:
309
+ raise AuthenticationError(f"Invalid token: {str(e)}")
310
+
311
+ def get_token(self) -> Optional[str]:
312
+ """Get stored token from keyring"""
313
+ return keyring.get_password(KEYRING_SERVICE, KEYRING_ACCOUNT)
314
+
315
+ def clear_token(self):
316
+ """Clear stored token"""
317
+ keyring.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT)
318
+ ```
319
+
320
+ ### 5.4: `src/repository/repo_service.py`
321
+
322
+ ```python
323
+ """Repository Service"""
324
+
325
+ from typing import Optional
326
+ from huggingface_hub import HfApi
327
+ import json
328
+ from io import BytesIO
329
+
330
+ class RepositoryError(Exception):
331
+ """Repository operation failed"""
332
+ pass
333
+
334
+ class RepositoryService:
335
+ """Manage HuggingFace datasets"""
336
+
337
+ def __init__(self, token: str):
338
+ self.api = HfApi(token=token)
339
+
340
+ def create_dataset(self, repo_name: str, private: bool = True) -> str:
341
+ """
342
+ Create a new dataset on HuggingFace.
343
+
344
+ Args:
345
+ repo_name: Dataset name
346
+ private: Whether dataset should be private
347
+
348
+ Returns:
349
+ Full repo_id (username/repo_name)
350
+ """
351
+ try:
352
+ # Create repo
353
+ repo = self.api.create_repo(
354
+ repo_id=repo_name,
355
+ repo_type="dataset",
356
+ private=private,
357
+ exist_ok=True
358
+ )
359
+ repo_id = repo.repo_id
360
+
361
+ # Initialize structure
362
+ self._init_structure(repo_id)
363
+
364
+ return repo_id
365
+ except Exception as e:
366
+ raise RepositoryError(f"Failed to create dataset: {str(e)}")
367
+
368
+ def _init_structure(self, repo_id: str):
369
+ """Initialize default folder structure"""
370
+ folders = [
371
+ "audio/project",
372
+ "index/shards",
373
+ "validations",
374
+ "audit/ingestion-runs"
375
+ ]
376
+
377
+ for folder in folders:
378
+ # Upload placeholder file to create folder
379
+ self.api.upload_file(
380
+ path_or_fileobj=BytesIO(b""),
381
+ path_in_repo=f"{folder}/.gitkeep",
382
+ repo_id=repo_id,
383
+ repo_type="dataset"
384
+ )
385
+
386
+ def validate_repo(self, repo_id: str) -> bool:
387
+ """Validate dataset has expected structure"""
388
+ # TODO: Implement
389
+ return True
390
+ ```
391
+
392
+ ### 5.5: `src/upload/scanner.py`
393
+
394
+ ```python
395
+ """Local Folder Scanner"""
396
+
397
+ import os
398
+ from pathlib import Path
399
+ from typing import Dict, List
400
+ from src.config import AUDIO_EXTENSIONS
401
+
402
+ class LocalScanner:
403
+ """Scan local folders for audio files"""
404
+
405
+ def scan_folder(self, folder_path: str) -> Dict:
406
+ """
407
+ Recursively scan folder for audio files.
408
+
409
+ Args:
410
+ folder_path: Path to scan
411
+
412
+ Returns:
413
+ Structure dict with files grouped by species
414
+ """
415
+ structure = {}
416
+ total_size = 0
417
+ file_count = 0
418
+
419
+ for root, dirs, files in os.walk(folder_path):
420
+ for file in files:
421
+ if any(file.lower().endswith(ext) for ext in AUDIO_EXTENSIONS):
422
+ full_path = os.path.join(root, file)
423
+ size = os.path.getsize(full_path)
424
+
425
+ # Infer species
426
+ specie = self._infer_species(root, file)
427
+
428
+ if specie not in structure:
429
+ structure[specie] = []
430
+
431
+ structure[specie].append({
432
+ "name": file,
433
+ "full_path": full_path,
434
+ "size": size,
435
+ })
436
+
437
+ total_size += size
438
+ file_count += 1
439
+
440
+ return {
441
+ "total_files": file_count,
442
+ "total_size": total_size,
443
+ "by_species": structure,
444
+ }
445
+
446
+ def _infer_species(self, root: str, filename: str) -> str:
447
+ """Infer species from path or filename"""
448
+ parts = root.split(os.sep)
449
+ if len(parts) > 0:
450
+ return parts[-1]
451
+ return "unknown"
452
+ ```
453
+
454
+ ### 5.6: `src/session/session_manager.py`
455
+
456
+ ```python
457
+ """Session Management"""
458
+
459
+ import json
460
+ from pathlib import Path
461
+ from datetime import datetime
462
+ from src.config import SESSION_DIR
463
+
464
+ class SessionManager:
465
+ """Manage upload sessions"""
466
+
467
+ def __init__(self, session_id: str = None):
468
+ if session_id is None:
469
+ session_id = f"upload-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
470
+
471
+ self.session_id = session_id
472
+ self.session_dir = Path(SESSION_DIR) / session_id
473
+ self.session_dir.mkdir(parents=True, exist_ok=True)
474
+
475
+ def save_checkpoint(self, data: dict):
476
+ """Save checkpoint atomically"""
477
+ checkpoint_file = self.session_dir / "checkpoint.json"
478
+ temp_file = checkpoint_file.with_suffix(".tmp")
479
+
480
+ with open(temp_file, "w") as f:
481
+ json.dump(data, f, indent=2)
482
+
483
+ temp_file.replace(checkpoint_file)
484
+
485
+ def load_checkpoint(self) -> dict:
486
+ """Load last checkpoint"""
487
+ checkpoint_file = self.session_dir / "checkpoint.json"
488
+ if checkpoint_file.exists():
489
+ with open(checkpoint_file) as f:
490
+ return json.load(f)
491
+ return {}
492
+ ```
493
+
494
+ ### 5.7: `src/ui/tui.py`
495
+
496
+ ```python
497
+ """TUI Application (Textual)"""
498
+
499
+ from textual.app import ComposeResult, App
500
+ from textual.containers import Container
501
+ from textual.widgets import Static, Button
502
+
503
+ class BirdNETApp(App):
504
+ """Main TUI Application"""
505
+
506
+ def compose(self) -> ComposeResult:
507
+ """Create child widgets for the app."""
508
+ yield Container(
509
+ Static("BirdNET Uploader", id="header"),
510
+ Button("Start Upload", id="btn_upload"),
511
+ id="container"
512
+ )
513
+
514
+ def on_button_pressed(self) -> None:
515
+ """Handle button press."""
516
+ self.exit()
517
+
518
+ def run():
519
+ """Run the TUI app"""
520
+ app = BirdNETApp()
521
+ app.run()
522
+ ```
523
+
524
+ ### 5.8: `src/utils/error_handler.py`
525
+
526
+ ```python
527
+ """Error Handling"""
528
+
529
+ class BirdNETError(Exception):
530
+ """Base exception"""
531
+ pass
532
+
533
+ class AuthenticationError(BirdNETError):
534
+ """Authentication failed"""
535
+ pass
536
+
537
+ class RepositoryError(BirdNETError):
538
+ """Repository operation failed"""
539
+ pass
540
+
541
+ class UploadError(BirdNETError):
542
+ """Upload operation failed"""
543
+ pass
544
+
545
+ class ValidationError(BirdNETError):
546
+ """Validation failed"""
547
+ pass
548
+ ```
549
+
550
+ ---
551
+
552
+ ## Step 6: Criar Test Infrastructure
553
+
554
+ ### 6.1: `pytest.ini`
555
+
556
+ ```ini
557
+ [pytest]
558
+ testpaths = tests
559
+ addopts = -v --tb=short
560
+ python_files = test_*.py
561
+ python_classes = Test*
562
+ python_functions = test_*
563
+ ```
564
+
565
+ ### 6.2: `tests/conftest.py`
566
+
567
+ ```python
568
+ """Test Configuration and Fixtures"""
569
+
570
+ import pytest
571
+ from pathlib import Path
572
+ import tempfile
573
+ import json
574
+
575
+ @pytest.fixture
576
+ def temp_folder():
577
+ """Create temporary folder with test audio files"""
578
+ with tempfile.TemporaryDirectory() as tmpdir:
579
+ # Create test structure
580
+ audio_dir = Path(tmpdir) / "audio" / "Parrot"
581
+ audio_dir.mkdir(parents=True)
582
+
583
+ # Create dummy audio files
584
+ for i in range(5):
585
+ (audio_dir / f"segment_{i:03d}.wav").write_bytes(b"fake audio data")
586
+
587
+ yield tmpdir
588
+
589
+ @pytest.fixture
590
+ def mock_hf_token():
591
+ """Mock HF token"""
592
+ return "hf_test_token_12345"
593
+
594
+ @pytest.fixture
595
+ def session_dir(tmp_path):
596
+ """Temporary session directory"""
597
+ session_dir = tmp_path / "sessions" / "test-session"
598
+ session_dir.mkdir(parents=True)
599
+ return session_dir
600
+ ```
601
+
602
+ ### 6.3: `tests/test_auth.py`
603
+
604
+ ```python
605
+ """Tests for Auth Service"""
606
+
607
+ import pytest
608
+ from src.auth.auth_service import AuthService, AuthenticationError
609
+
610
+ def test_auth_invalid_token(mock_hf_token):
611
+ """Test that invalid token raises error"""
612
+ auth = AuthService()
613
+ with pytest.raises(AuthenticationError):
614
+ auth.authenticate("invalid_token_xyz")
615
+
616
+ # More tests to be implemented
617
+ ```
618
+
619
+ ### 6.4: `tests/test_scanner.py`
620
+
621
+ ```python
622
+ """Tests for File Scanner"""
623
+
624
+ import pytest
625
+ from src.upload.scanner import LocalScanner
626
+
627
+ def test_scan_folder_empty(tmp_path):
628
+ """Test scanning empty folder"""
629
+ scanner = LocalScanner()
630
+ result = scanner.scan_folder(str(tmp_path))
631
+ assert result["total_files"] == 0
632
+ assert result["total_size"] == 0
633
+
634
+ def test_scan_folder_with_audio(temp_folder):
635
+ """Test scanning folder with audio files"""
636
+ scanner = LocalScanner()
637
+ result = scanner.scan_folder(temp_folder)
638
+ assert result["total_files"] == 5
639
+ assert result["total_size"] > 0
640
+
641
+ # More tests to be implemented
642
+ ```
643
+
644
+ ---
645
+
646
+ ## Step 7: Configurar CI/CD (GitHub Actions)
647
+
648
+ Criar arquivo: `.github/workflows/test.yml`
649
+
650
+ ```yaml
651
+ name: Tests
652
+
653
+ on: [push, pull_request]
654
+
655
+ jobs:
656
+ test:
657
+ runs-on: ubuntu-latest
658
+ strategy:
659
+ matrix:
660
+ python-version: ["3.9", "3.10", "3.11"]
661
+
662
+ steps:
663
+ - uses: actions/checkout@v3
664
+
665
+ - name: Set up Python
666
+ uses: actions/setup-python@v4
667
+ with:
668
+ python-version: ${{ matrix.python-version }}
669
+
670
+ - name: Install dependencies
671
+ run: |
672
+ pip install -r requirements.txt
673
+
674
+ - name: Lint
675
+ run: |
676
+ flake8 src tests
677
+
678
+ - name: Run tests
679
+ run: |
680
+ pytest
681
+
682
+ - name: Upload coverage
683
+ uses: codecov/codecov-action@v3
684
+ ```
685
+
686
+ ---
687
+
688
+ ## Step 8: Testar Instalação
689
+
690
+ ```bash
691
+ # Instalar dependências
692
+ pip install -r requirements.txt
693
+
694
+ # Rodar testes
695
+ pytest -v
696
+
697
+ # Testar import
698
+ python -c "import src; print('✓ Imports OK')"
699
+
700
+ # Rodar CLI help
701
+ python -m src.main --help
702
+ ```
703
+
704
+ **Esperado**: Todos os comandos rodarem sem erro.
705
+
706
+ ---
707
+
708
+ ## Step 9: Commit Initial
709
+
710
+ ```bash
711
+ git add -A
712
+ git commit -m "feat: Sprint 1 - Initial project structure and scaffolding"
713
+ git push origin feature/birdnet-uploader-cli
714
+ ```
715
+
716
+ ---
717
+
718
+ ## Checklist: Sprint 1, Day 1 - DONE ✓
719
+
720
+ - [ ] Branch `feature/birdnet-uploader-cli` criada
721
+ - [ ] Estrutura de diretórios completa
722
+ - [ ] `requirements.txt` instalado
723
+ - [ ] `pyproject.toml` criado
724
+ - [ ] Todos os módulos esqueléticos criados (8+ arquivos)
725
+ - [ ] `pytest.ini` e `conftest.py` configurados
726
+ - [ ] Testes básicos criados
727
+ - [ ] CI/CD workflow criado
728
+ - [ ] Primeira execução sem erro: `pytest -v`
729
+ - [ ] Commit feito e pushed
730
+
731
+ **Tempo**: ~2-3 horas
732
+
733
+ ---
734
+
735
+ ## Próximos Passos (Task 1.2 - Design Review)
736
+
737
+ Quando tudo acima estiver **DONE**, fazer:
738
+
739
+ 1. **Meeting de Design Review** com o time (30min)
740
+ - Confirmar Tech Stack (Textual vs Rich+Click)
741
+ - Confirmar persistência (JSON vs SQLite)
742
+ - Validar estrutura de repo HF
743
+ - Assinar decision log
744
+
745
+ 2. **Começar Task 2.1** (Auth Service Implementation)
746
+ - Implementar `AuthService.authenticate()` completo
747
+ - Implementar keyring integration
748
+ - Escrever testes com mocks
749
+
750
+ ---
751
+
752
+ ## Troubleshooting
753
+
754
+ **Problema**: `pip install` falha com erro de dependência
755
+ **Solução**: Upgrade pip: `pip install --upgrade pip`
756
+
757
+ **Problema**: Imports falhando
758
+ **Solução**: Adicionar `export PYTHONPATH="${PYTHONPATH}:$(pwd)"`
759
+
760
+ **Problema**: Pytest não encontra testes
761
+ **Solução**: Verificar que todos os `__init__.py` existem
762
+
763
+ ---
764
+
765
+ ## Documentação de Referência
766
+
767
+ - BIRDNET_UPLOADER_REDESIGN.md (arquitetura e UX)
768
+ - SPRINT_ROADMAP.md (timeline e tasks)
769
+ - Este arquivo (Sprint 1 Day 1 setup)
SPRINT_ROADMAP.md ADDED
@@ -0,0 +1,649 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # BirdNET Uploader: Sprint Roadmap (MVP 6-8 semanas)
2
+
3
+ ## Overview
4
+ **Objetivo**: Entregar executável local moderno, um fluxo integrado, pausa/retomada, compatibilidade com validador.
5
+
6
+ **Timeline**:
7
+ - **Semanas 1-2**: Arquitetura + Setup (Foundation)
8
+ - **Semanas 3-4**: Módulos Core (Auth, Repo, Upload)
9
+ - **Semanas 5-6**: UI + Session Manager (Integration)
10
+ - **Semanas 7-8**: Build + Testes + Release (Polish)
11
+
12
+ **Team**: 2-3 engineers (1 backend, 1 frontend/UI, 1 QA/DevOps)
13
+
14
+ ---
15
+
16
+ ## SPRINT 1: Foundation & Architecture (Semanas 1-2)
17
+
18
+ ### Semana 1: Setup Inicial + Design Review
19
+
20
+ #### Task 1.1: Setup do Projeto (2 dias, 3 pts)
21
+ **Objetivo**: Estrutura base, dependências, CI/CD
22
+
23
+ **Tarefas**:
24
+ - [ ] Criar branch `feature/birdnet-uploader-cli`
25
+ - [ ] Criar estrutura de diretórios (`src/`, `tests/`, `build/`)
26
+ - [ ] Setup `pyproject.toml` e `requirements.txt`
27
+ - huggingface_hub>=0.28.0
28
+ - click>=8.0.0
29
+ - textual>=0.30.0 (TUI)
30
+ - rich>=13.0.0
31
+ - pydantic>=2.0.0
32
+ - pandas>=2.0.0
33
+ - pyarrow>=14.0.0
34
+ - keyring>=24.0.0
35
+ - [ ] Setup GitHub Actions para lint/test
36
+ - [ ] Criar `src/__init__.py` e `src/main.py` (skeleton)
37
+
38
+ **Critério de Pronto**:
39
+ - `pip install -r requirements.txt` sem erro
40
+ - `python -m src.main --help` roda e exibe help text
41
+
42
+ ---
43
+
44
+ #### Task 1.2: Design Review + Arquitetura Finalizada (2 dias, 2 pts)
45
+ **Objetivo**: Alinhamento com stakeholders, decisões arquiteturais bloqueantes
46
+
47
+ **Tarefas**:
48
+ - [ ] Revisar documento REDESIGN com time
49
+ - [ ] Decisão final: Textual vs Rich+Click vs Typer (recomendação: Textual para melhor UX)
50
+ - [ ] Decidir sobre persistência: SQLite vs JSON files (recomendação: JSON + estrutura de diretórios)
51
+ - [ ] Validar estrutura de repositório HF (audio/index/validations/audit)
52
+ - [ ] Documentar todas as decisões em `docs/architecture-decisions.md`
53
+
54
+ **Critério de Pronto**:
55
+ - Documento de decisões assinado
56
+ - Time alinhado em tecnologia stack
57
+ - Nenhuma bloqueante técnica identificada
58
+
59
+ ---
60
+
61
+ #### Task 1.3: Estrutura Base de Módulos (2 dias, 3 pts)
62
+ **Objetivo**: Criar interfaces/contracts de todos os módulos
63
+
64
+ **Tarefas**:
65
+ - [ ] Criar `src/auth/__init__.py` + `auth_service.py` (skeleton com docstrings)
66
+ - [ ] Criar `src/repository/__init__.py` + `repo_service.py` (skeleton)
67
+ - [ ] Criar `src/upload/__init__.py` + `scanner.py`, `uploader.py`, `deduplicator.py` (skeleton)
68
+ - [ ] Criar `src/csv/__init__.py` + `csv_matcher.py`, `indexer.py` (skeleton)
69
+ - [ ] Criar `src/session/__init__.py` + `session_manager.py`, `checkpoint.py` (skeleton)
70
+ - [ ] Criar `src/ui/__init__.py` + `tui.py`, `progress_renderer.py` (skeleton)
71
+ - [ ] Criar `src/logger/__init__.py` + `structured_logger.py` (skeleton)
72
+ - [ ] Criar `src/utils/__init__.py` + `file_utils.py`, `hash_utils.py`, `error_handler.py` (skeleton)
73
+
74
+ **Critério de Pronto**:
75
+ - Todos os módulos existem com `pass` ou docstrings
76
+ - Nenhum erro de import circular
77
+ - `python -c "import src; print('OK')"` funciona
78
+
79
+ ---
80
+
81
+ #### Task 1.4: Testes Base + Test Fixtures (1 dia, 2 pts)
82
+ **Objetivo**: Infraestrutura de testes pronta
83
+
84
+ **Tarefas**:
85
+ - [ ] Setup `pytest.ini` e `conftest.py`
86
+ - [ ] Criar test fixtures para:
87
+ - Mock HfApi
88
+ - Pasta local temporária com arquivos de teste
89
+ - Session mock
90
+ - Token mock
91
+ - [ ] Criar `tests/test_auth.py` (skeleton)
92
+ - [ ] Criar `tests/test_uploader.py` (skeleton)
93
+ - [ ] Criar `tests/test_session.py` (skeleton)
94
+
95
+ **Critério de Pronto**:
96
+ - `pytest` executa sem erro (mesmo que tests sejam vazios)
97
+ - Fixtures estão disponíveis para todos os testes
98
+
99
+ ---
100
+
101
+ ### Semana 2: Auth + Repository Services
102
+
103
+ #### Task 2.1: Auth Service - Implementação Completa (2.5 dias, 5 pts)
104
+ **Objetivo**: Login com token, validação, armazenamento seguro
105
+
106
+ **Tarefas**:
107
+ - [ ] Implementar `AuthService.authenticate(token)`:
108
+ - Validar token contra HF API
109
+ - Recuperar info do usuário (username, email, user_id)
110
+ - Salvar em keyring (Windows DPAPI)
111
+ - Retornar user context
112
+ - [ ] Implementar `AuthService.get_stored_token()`:
113
+ - Recuperar token do keyring
114
+ - Lançar erro claro se não encontrado
115
+ - [ ] Implementar `AuthService.clear_token()`:
116
+ - Remover token do keyring
117
+ - [ ] Implementar `TokenManager`:
118
+ - Gerenciar TTL de token
119
+ - Detectar expiração
120
+ - Forçar re-autenticação se necessário
121
+ - [ ] Tests:
122
+ - [ ] `test_auth_valid_token()`
123
+ - [ ] `test_auth_invalid_token()`
124
+ - [ ] `test_auth_token_stored_in_keyring()`
125
+ - [ ] `test_auth_token_retrieval()`
126
+
127
+ **Critério de Pronto**:
128
+ - Todos os testes de auth passando
129
+ - Token nunca é logado/printed
130
+ - Falha de auth retorna mensagem clara
131
+
132
+ ---
133
+
134
+ #### Task 2.2: Repository Service - Criar Dataset (2 dias, 4 pts)
135
+ **Objetivo**: Criar dataset HF e inicializar estrutura padrão
136
+
137
+ **Tarefas**:
138
+ - [ ] Implementar `RepositoryService.create_dataset()`:
139
+ - Validar nome do repositório (slug format)
140
+ - Criar dataset no HF (público/privado)
141
+ - Inicializar pastas padrão (audio/, index/shards/, validations/, audit/)
142
+ - Criar manifest.json inicial
143
+ - Retornar repo_id
144
+ - [ ] Implementar `RepositoryService.validate_repo()`:
145
+ - Verificar permissão de escrita
146
+ - Verificar estrutura esperada
147
+ - Retornar validation result
148
+ - [ ] Tests:
149
+ - [ ] `test_create_dataset_public()`
150
+ - [ ] `test_create_dataset_private()`
151
+ - [ ] `test_create_dataset_name_validation()`
152
+ - [ ] `test_validate_repo_structure()`
153
+
154
+ **Critério de Pronto**:
155
+ - Dataset criado no HF com structure correta
156
+ - Manifest.json com schema_version, project_slug, index
157
+ - Testes mockeados passando
158
+
159
+ ---
160
+
161
+ #### Task 2.3: Config + Constants Finalizadas (1 dia, 2 pts)
162
+ **Objetivo**: Configuração centralizada do app
163
+
164
+ **Tarefas**:
165
+ - [ ] Criar `src/config.py`:
166
+ - `AUDIO_EXTENSIONS = ['.wav', '.mp3', '.flac', '.ogg']`
167
+ - `SESSION_DIR = ~/.birdnet-uploader/sessions/`
168
+ - `CACHE_DIR = ~/.birdnet-uploader/cache/`
169
+ - `MAX_BATCH_SIZE = 10`
170
+ - `RETRY_MAX_ATTEMPTS = 3`
171
+ - `RETRY_INITIAL_BACKOFF = 1.0`
172
+ - `MANIFEST_SCHEMA_VERSION = "1.0.0"`
173
+ - [ ] Implementar config via `.env` se necessário
174
+
175
+ **Critério de Pronto**:
176
+ - Todas as constantes centralizadas
177
+ - Nenhum hardcoded value em módulos
178
+
179
+ ---
180
+
181
+ #### Task 2.4: Error Handling Framework (1.5 dias, 3 pts)
182
+ **Objetivo**: Exceções customizadas e mensagens claras
183
+
184
+ **Tarefas**:
185
+ - [ ] Criar `src/exceptions.py`:
186
+ - `AuthenticationError`
187
+ - `RepositoryError`
188
+ - `UploadError`
189
+ - `SessionError`
190
+ - `ValidationError`
191
+ - [ ] Implementar `src/utils/error_handler.py`:
192
+ - Converter exceções em mensagens user-friendly
193
+ - Log estruturado de erros
194
+ - Sugestões de ação
195
+
196
+ **Critério de Pronto**:
197
+ - Todas as exceções possuem mensagem clara
198
+ - Tests cobrem cenários de erro
199
+
200
+ ---
201
+
202
+ ## SPRINT 2: Upload & CSV Matching (Semanas 3-4)
203
+
204
+ ### Semana 3: Scanner + File Operations
205
+
206
+ #### Task 3.1: Local Scanner - Varrer Pasta (2 dias, 4 pts)
207
+ **Objetivo**: Descobrir arquivos locais, agrupar por espécie
208
+
209
+ **Tarefas**:
210
+ - [ ] Implementar `LocalScanner.scan_folder()`:
211
+ - Varrer recursivamente
212
+ - Filtrar por extensões de áudio
213
+ - Inferir espécie do caminho/nome
214
+ - Calcular tamanho total e contagem
215
+ - Retornar estrutura normalizada
216
+ - [ ] Implementar estratégia de inferência de espécie:
217
+ - Strategy 1: Usar última pasta como specie
218
+ - Strategy 2: Usar prefixo do arquivo
219
+ - Strategy 3: Permitir mapeamento manual
220
+ - [ ] Tests:
221
+ - [ ] `test_scan_folder_empty()`
222
+ - [ ] `test_scan_folder_multiple_species()`
223
+ - [ ] `test_scan_folder_specie_inference()`
224
+ - [ ] `test_scan_folder_with_nested_dirs()`
225
+
226
+ **Critério de Pronto**:
227
+ - Escaneia pasta com 1000+ arquivos em <5 segundos
228
+ - Estrutura normalizada e validada
229
+
230
+ ---
231
+
232
+ #### Task 3.2: Hash Utils + File Operations (1.5 dias, 3 pts)
233
+ **Objetivo**: Computação de hash segura, operações de arquivo
234
+
235
+ **Tarefas**:
236
+ - [ ] Implementar `compute_file_hash(path) -> str`:
237
+ - SHA256 eficiente (streaming para arquivos grandes)
238
+ - Cache local de hashes
239
+ - [ ] Implementar `verify_file_integrity(path, hash)`:
240
+ - Validar arquivo não corrompido
241
+ - [ ] Tests:
242
+ - [ ] `test_compute_hash_small_file()`
243
+ - [ ] `test_compute_hash_large_file()`
244
+ - [ ] `test_hash_consistency()`
245
+
246
+ **Critério de Pronto**:
247
+ - Hash de arquivo 1GB < 2 segundos
248
+ - Testes passando
249
+
250
+ ---
251
+
252
+ #### Task 3.3: Deduplicator - Verificar Existência (2 dias, 3 pts)
253
+ **Objetivo**: Detectar arquivos já no destino
254
+
255
+ **Tarefas**:
256
+ - [ ] Implementar `Deduplicator.check_remote()`:
257
+ - Query HF API para verificar existência por caminho
258
+ - Comparar tamanho + hash opcional
259
+ - Retornar status (SKIP, UPLOAD, CONFLICT)
260
+ - [ ] Implementar dedup local cache (evitar queries repetidas)
261
+ - [ ] Tests:
262
+ - [ ] `test_check_remote_new_file()`
263
+ - [ ] `test_check_remote_existing_file()`
264
+ - [ ] `test_check_remote_conflict()`
265
+
266
+ **Critério de Pronto**:
267
+ - Dedup funciona sem re-query excessiva
268
+ - Cache valida corretamente
269
+
270
+ ---
271
+
272
+ #### Task 3.4: Session + Checkpoint - State Persistence (2.5 dias, 5 pts)
273
+ **Objetivo**: Persistência atomática de estado para retomada
274
+
275
+ **Tarefas**:
276
+ - [ ] Implementar `SessionManager.create_session()`:
277
+ - Gerar session_id
278
+ - Criar diretório de sessão
279
+ - Inicializar metadata.json
280
+ - [ ] Implementar `CheckpointManager.save_checkpoint()`:
281
+ - Escrita atômica (temp + rename)
282
+ - Validação de JSON
283
+ - Flush para disco (fsync)
284
+ - [ ] Implementar `CheckpointManager.load_checkpoint()`:
285
+ - Recuperar último checkpoint
286
+ - Validação de integridade
287
+ - [ ] Implementar `SessionManager.mark_file_done()`:
288
+ - Atualizar manifest por arquivo
289
+ - [ ] Tests:
290
+ - [ ] `test_session_creation()`
291
+ - [ ] `test_checkpoint_atomic_write()`
292
+ - [ ] `test_checkpoint_recovery_after_crash()`
293
+ - [ ] `test_session_resume_same_id()`
294
+
295
+ **Critério de Pronto**:
296
+ - Checkpoint atomático garante consistência
297
+ - Recuperação de crash funciona
298
+ - Testes cobrem falha de escrita
299
+
300
+ ---
301
+
302
+ ### Semana 4: Upload Engine + CSV Matching
303
+
304
+ #### Task 4.1: Batch Uploader - Motor Principal (3 dias, 6 pts)
305
+ **Objetivo**: Upload resiliente com retry/dedup/pause
306
+
307
+ **Tarefas**:
308
+ - [ ] Implementar `BatchUploader.upload_files()`:
309
+ - Iterar sobre estrutura de arquivos
310
+ - Aplicar dedup check
311
+ - Upload com retry automático
312
+ - Atualizar checkpoint por lote
313
+ - Callback de progresso
314
+ - [ ] Implementar retry com exponential backoff:
315
+ - Backoff: 1s → 2s → 5s → 10s → 30s
316
+ - Tratamento de 429 (rate limit)
317
+ - Max 3 retries por padrão
318
+ - [ ] Implementar pause/resume:
319
+ - Signal SIGTERM/SIGINT
320
+ - Salvar state
321
+ - Resume com checkpoint
322
+ - [ ] Tests:
323
+ - [ ] `test_upload_batch_success()`
324
+ - [ ] `test_upload_with_retry()`
325
+ - [ ] `test_upload_skip_existing()`
326
+ - [ ] `test_upload_pause_resume()`
327
+
328
+ **Critério de Pronto**:
329
+ - Upload funciona end-to-end
330
+ - Retry logic testado
331
+ - Pause/resume funciona
332
+
333
+ ---
334
+
335
+ #### Task 4.2: CSV Matcher + Index Generator (2.5 dias, 5 pts)
336
+ **Objetivo**: Matching CSV com segmentos e geração de índice
337
+
338
+ **Tarefas**:
339
+ - [ ] Implementar `CSVMatcher.match_detections()`:
340
+ - Ler CSV com validação de schema
341
+ - Fazer matching com segmentos (source_file → arquivo local)
342
+ - Gerar detection_key determinístico
343
+ - Retornar matched + unmatched + ambiguous
344
+ - [ ] Implementar `IndexGenerator.generate_shards()`:
345
+ - Particionar dados em shards (tamanho configurável)
346
+ - Salvar como parquet
347
+ - Gerar metadata de shards
348
+ - [ ] Implementar `ManifestBuilder.build()`:
349
+ - Construir manifest.json com schema_version
350
+ - Incluir metadados de index
351
+ - Serializar JSON
352
+ - [ ] Tests:
353
+ - [ ] `test_csv_matching_success()`
354
+ - [ ] `test_csv_missing_columns()`
355
+ - [ ] `test_shard_generation()`
356
+ - [ ] `test_manifest_format()`
357
+
358
+ **Critério de Pronto**:
359
+ - CSV+segmentos matchados corretamente
360
+ - Shards gerados com formato correto
361
+ - Manifest válido e legível
362
+
363
+ ---
364
+
365
+ ## SPRINT 3: UI + Integration (Semanas 5-6)
366
+
367
+ ### Semana 5: UI TUI Implementation
368
+
369
+ #### Task 5.1: Textual TUI - Fluxo Principal (3 dias, 6 pts)
370
+ **Objetivo**: Interface TUI moderna para guiar usuário
371
+
372
+ **Tarefas**:
373
+ - [ ] Criar `src/ui/screens/login_screen.py`:
374
+ - Input para token HF
375
+ - Validação em tempo real
376
+ - Mensagens de erro claras
377
+ - [ ] Criar `src/ui/screens/config_screen.py`:
378
+ - Form com campos: repo name, visibility, segments path, csv path
379
+ - Verificação de pasta
380
+ - Validação de formato
381
+ - [ ] Criar `src/ui/screens/summary_screen.py`:
382
+ - Exibir configuração resumida
383
+ - Botão de confirmação
384
+ - [ ] Criar `src/ui/screens/upload_screen.py`:
385
+ - Barra de progresso principal
386
+ - Progresso por arquivo
387
+ - Arquivo atual
388
+ - Último evento
389
+ - Botões pause/cancel
390
+ - [ ] Implementar navegação entre screens
391
+
392
+ **Critério de Pronto**:
393
+ - Fluxo completo funcionando (login → config → summary → upload)
394
+ - Inputs validados
395
+ - Navegação intuitiva
396
+
397
+ ---
398
+
399
+ #### Task 5.2: Progress Renderer (1.5 dias, 3 pts)
400
+ **Objetivo**: Visualização rich de progresso
401
+
402
+ **Tarefas**:
403
+ - [ ] Implementar progress bar customizada:
404
+ - Percentual + barra visual
405
+ - ETA com update em tempo real
406
+ - Arquivo atual
407
+ - Contadores (uploaded/pending/failed/skipped)
408
+ - [ ] Implementar event log:
409
+ - Últimos 10 eventos
410
+ - Código de status/cor por tipo
411
+
412
+ **Critério de Pronto**:
413
+ - Progress bar atualiza sem flicker
414
+ - ETA coerente
415
+ - Eventos visíveis
416
+
417
+ ---
418
+
419
+ ### Semana 6: Integration + Testes End-to-End
420
+
421
+ #### Task 6.1: Integration Testing + Mocks (2 dias, 4 pts)
422
+ **Objetivo**: Teste de fluxo completo (mockeado)
423
+
424
+ **Tarefas**:
425
+ - [ ] Criar `tests/test_integration.py`:
426
+ - Teste login + config + upload (mockeado)
427
+ - Teste pause/resume
428
+ - Teste com CSV
429
+ - [ ] Mockar HfApi completamente
430
+ - [ ] Rodar testes de carga (1000+ arquivos mockeados)
431
+
432
+ **Critério de Pronto**:
433
+ - Testes de integração passando
434
+ - Sem vazamento de recursos
435
+
436
+ ---
437
+
438
+ #### Task 6.2: Real Integration Test (pequena escala) (2 dias, 4 pts)
439
+ **Objetivo**: Teste real com dataset HF de verdade
440
+
441
+ **Tarefas**:
442
+ - [ ] Criar dataset de teste no HF (alice/test-uploader-small)
443
+ - [ ] Teste com 100 arquivos reais (~500MB)
444
+ - [ ] Validar estrutura no HF pós-upload
445
+ - [ ] Validar manifest.json e shards
446
+
447
+ **Critério de Pronto**:
448
+ - Arquivos aparecem no HF
449
+ - Estrutura correta
450
+ - Manifest válido
451
+
452
+ ---
453
+
454
+ ## SPRINT 4: Build + Polish (Semanas 7-8)
455
+
456
+ ### Semana 7: Build + Release
457
+
458
+ #### Task 7.1: PyInstaller Setup (2 dias, 4 pts)
459
+ **Objetivo**: Gerar executável portátil Windows
460
+
461
+ **Tarefas**:
462
+ - [ ] Criar `build/pyinstaller_config.spec`:
463
+ - One-file executable
464
+ - Hidden imports para textual, rich, pandas, pyarrow
465
+ - Resource bundling
466
+ - [ ] Criar `build/build.py` script
467
+ - [ ] Testar executável gerado localmente
468
+ - [ ] Validar tamanho (~100-150MB)
469
+
470
+ **Critério de Pronto**:
471
+ - Executável funciona sem Python instalado
472
+ - Tamanho razoável
473
+ - Sem erros de runtime
474
+
475
+ ---
476
+
477
+ #### Task 7.2: Documentação + README (1.5 dias, 2 pts)
478
+ **Objetivo**: Docs para usuários finais
479
+
480
+ **Tarefas**:
481
+ - [ ] Criar README.md com:
482
+ - Quick start (download, run, autenticar)
483
+ - Screenshots
484
+ - Troubleshooting
485
+ - FAQ
486
+ - [ ] Criar INSTALLATION.md
487
+ - [ ] Criar USAGE.md (exemplos de comando)
488
+
489
+ **Critério de Pronto**:
490
+ - Usuário não-técnico consegue rodar pelo README
491
+
492
+ ---
493
+
494
+ #### Task 7.3: Testes de Carga + Stress (2 dias, 4 pts)
495
+ **Objetivo**: Validar com dados reais maiores
496
+
497
+ **Tarefas**:
498
+ - [ ] Teste com pasta 5GB (1000+ arquivos)
499
+ - [ ] Teste com CSV 5000+ linhas
500
+ - [ ] Teste pause/resume 5x
501
+ - [ ] Monitorar memória e CPU
502
+ - [ ] Documentar performance (tempo/MB/s)
503
+
504
+ **Critério de Pronto**:
505
+ - Memória < 500MB durante upload
506
+ - CPU < 30% média
507
+ - Upload rate > 5 MB/s (com rede OK)
508
+
509
+ ---
510
+
511
+ ### Semana 8: Release + Smoke Tests
512
+
513
+ #### Task 8.1: Release Pipeline (1.5 dias, 2 pts)
514
+ **Objetivo**: Setup de distribuição
515
+
516
+ **Tarefas**:
517
+ - [ ] Criar release no GitHub com:
518
+ - Executável (`.exe`)
519
+ - SHA256 hash
520
+ - Release notes
521
+ - Link para docs
522
+ - [ ] Testar download + execução do exe
523
+ - [ ] Validar assinatura (opcional nesta fase)
524
+
525
+ **Critério de Pronto**:
526
+ - Release publicado
527
+ - Download funciona
528
+ - Exe roda após download/extração
529
+
530
+ ---
531
+
532
+ #### Task 8.2: Smoke Tests + Final QA (1.5 dias, 3 pts)
533
+ **Objetivo**: Validação final antes do release
534
+
535
+ **Tarefas**:
536
+ - [ ] Checklist final:
537
+ - [ ] Login funciona
538
+ - [ ] Criar repo funciona
539
+ - [ ] Upload de 10GB completa
540
+ - [ ] Pause/resume sem perda
541
+ - [ ] CSV matching correto
542
+ - [ ] Manifest válido no destino
543
+ - [ ] Estrutura compatível com validador
544
+ - [ ] Teste com 3+ usuários externos (team)
545
+ - [ ] Coleta de feedback
546
+
547
+ **Critério de Pronto**:
548
+ - Zero bugs críticos
549
+ - Feedback incorporado
550
+ - Pronto para release
551
+
552
+ ---
553
+
554
+ ## BACKLOG: Priorização de Tasks
555
+
556
+ | Sprint | Semana | Task | Pts | Status |
557
+ |--------|--------|------|-----|--------|
558
+ | 1 | 1 | Setup + Design | 5 | Não iniciado |
559
+ | 1 | 1 | Arquitetura de Módulos | 3 | Não iniciado |
560
+ | 1 | 1 | Test Infrastructure | 2 | Não iniciado |
561
+ | 1 | 2 | Auth Service | 5 | Bloqueado por 1.1 |
562
+ | 1 | 2 | Repository Service | 4 | Bloqueado por 1.1 |
563
+ | 1 | 2 | Config + Constants | 2 | Bloqueado por 1.1 |
564
+ | 1 | 2 | Error Handling | 3 | Bloqueado por 1.1 |
565
+ | 2 | 3 | Local Scanner | 4 | Bloqueado por Sprint 1 |
566
+ | 2 | 3 | Hash Utils | 3 | Bloqueado por Sprint 1 |
567
+ | 2 | 3 | Deduplicator | 3 | Bloqueado por Sprint 1 |
568
+ | 2 | 3 | Session + Checkpoint | 5 | Bloqueado por Sprint 1 |
569
+ | 2 | 4 | Batch Uploader | 6 | Bloqueado por Sprint 1 |
570
+ | 2 | 4 | CSV Matcher | 5 | Bloqueado por Sprint 1 |
571
+ | 3 | 5 | Textual TUI | 6 | Bloqueado por Sprint 2 |
572
+ | 3 | 5 | Progress Renderer | 3 | Bloqueado por Sprint 2 |
573
+ | 3 | 6 | Integration Tests | 4 | Bloqueado por Sprint 2 |
574
+ | 3 | 6 | Real Integration Test | 4 | Bloqueado por Sprint 2 |
575
+ | 4 | 7 | PyInstaller | 4 | Bloqueado por Sprint 3 |
576
+ | 4 | 7 | Documentation | 2 | Bloqueado por Sprint 3 |
577
+ | 4 | 7 | Load Testing | 4 | Bloqueado por Sprint 3 |
578
+ | 4 | 8 | Release Pipeline | 2 | Bloqueado por Sprint 3 |
579
+ | 4 | 8 | Final QA | 3 | Bloqueado por Sprint 3 |
580
+
581
+ **Total: ~84 story points (6-8 semanas com 2-3 eng)**
582
+
583
+ ---
584
+
585
+ ## Velocity + Alocação de Equipe
586
+
587
+ ### Cenário 1: Full Team (2 eng)
588
+ - Sprint de 2 semanas, 2 eng = ~40 pts/sprint
589
+ - Sprint 1-2: 26 pts (semana 1 paralelo, semana 2 paralelo) ✓ cabe
590
+ - Sprint 2 continuation: ~20 pts (cabe)
591
+ - Sprint 3: ~13 pts (cabe)
592
+ - Sprint 4: ~15 pts (cabe)
593
+
594
+ ### Cenário 2: 3 Eng (Backend + Frontend + QA)
595
+ - Paralelizar mais agressivamente
596
+ - Timeline pode reduzir de 8 para 6-7 semanas
597
+
598
+ ---
599
+
600
+ ## Critérios de "Done" por Sprint
601
+
602
+ ### Sprint 1 Done ✓
603
+ - [ ] Todos os módulos base existem (importáveis)
604
+ - [ ] Auth funciona com keyring
605
+ - [ ] Repo pode ser criado no HF
606
+ - [ ] Testes de auth e repo passando
607
+ - [ ] Setup pronto para Sprint 2
608
+
609
+ ### Sprint 2 Done ✓
610
+ - [ ] Scanner varre pasta corretamente
611
+ - [ ] Uploader faz batch com retry
612
+ - [ ] CSV matcher funciona end-to-end
613
+ - [ ] Checkpoint salva e recupera estado
614
+ - [ ] Testes cobrem 80%+ de lógica
615
+
616
+ ### Sprint 3 Done ✓
617
+ - [ ] Fluxo TUI completo (login → config → upload)
618
+ - [ ] Progresso visual atualiza sem flicker
619
+ - [ ] Integração completa mockeada testada
620
+ - [ ] Real integration test com 500MB+ passou
621
+ - [ ] Pronto para build
622
+
623
+ ### Sprint 4 Done ✓
624
+ - [ ] Executável portátil gerado e testado
625
+ - [ ] Release pipeline setup
626
+ - [ ] Documentação completa
627
+ - [ ] Smoke tests passando
628
+ - [ ] Pronto para distribuição
629
+
630
+ ---
631
+
632
+ ## Next Steps
633
+
634
+ 1. **Aprovação**: Confirmar timeline, team size, prioridades
635
+ 2. **Sprint 1 Kickoff**: Semana que vem
636
+ 3. **Daily Standups**: 15min todo dia weekday
637
+ 4. **Sprint Reviews**: Sexta-feira cada semana
638
+ 5. **Retrospectives**: Fim de cada sprint
639
+
640
+ ---
641
+
642
+ ## Notas & Riscos
643
+
644
+ - **Risco 1**: PyInstaller não empacotar corretamente → Mitigar: spike de 1 dia em week 7
645
+ - **Risco 2**: Performance ruim em 50GB+ → Mitigar: load test cedo (week 4)
646
+ - **Risco 3**: Incompatibilidade com validador → Mitigar: validação de manifest early (week 4)
647
+ - **Risco 4**: UX TUI confusa → Mitigar: user testing em week 6
648
+
649
+ Todas as risks têm mitigation strategy atrelada.
app.py CHANGED
@@ -1,9 +1,10 @@
1
  import os
2
 
3
- from src.ui.upload_app_factory import build_upload_app
4
 
5
 
6
  if __name__ == "__main__":
7
- app = build_upload_app()
8
- port = int(os.getenv("PORT") or os.getenv("BIRDNET_UPLOADER_PORT", "7860"))
9
- app.launch(server_name="0.0.0.0", server_port=port)
 
 
1
  import os
2
 
3
+ from src.ui.app_factory import create_app
4
 
5
 
6
  if __name__ == "__main__":
7
+ app = create_app()
8
+ port = int(os.getenv("PORT") or os.getenv("BIRDNET_VALIDATOR_PORT") or "7860")
9
+ host = os.getenv("BIRDNET_VALIDATOR_HOST") or "0.0.0.0"
10
+ app.launch(server_name=host, server_port=port)
build/publish_release_to_hf.py ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import os
5
+ from pathlib import Path
6
+
7
+ from huggingface_hub import HfApi
8
+
9
+
10
+ def main() -> int:
11
+ parser = argparse.ArgumentParser(description="Publish a BirdNET uploader release bundle to Hugging Face")
12
+ parser.add_argument("--repo-id", required=True, help="Target Hugging Face repo id, e.g. user/releases")
13
+ parser.add_argument("--repo-type", default="dataset", choices=["dataset", "model", "space"])
14
+ parser.add_argument("--bundle", required=True, type=Path, help="Path to the release zip bundle")
15
+ parser.add_argument("--checksum", required=True, type=Path, help="Path to the checksum file")
16
+ parser.add_argument("--version", required=True, help="Release version, e.g. 0.1.0")
17
+ args = parser.parse_args()
18
+
19
+ token = os.getenv("HF_TOKEN")
20
+ if not token:
21
+ raise SystemExit("HF_TOKEN is required")
22
+
23
+ if not args.bundle.exists():
24
+ raise SystemExit(f"Bundle not found: {args.bundle}")
25
+ if not args.checksum.exists():
26
+ raise SystemExit(f"Checksum not found: {args.checksum}")
27
+
28
+ api = HfApi(token=token)
29
+ api.create_repo(repo_id=args.repo_id, repo_type=args.repo_type, exist_ok=True)
30
+
31
+ release_prefix = f"releases/v{args.version}"
32
+ uploads = [
33
+ (args.bundle, f"{release_prefix}/{args.bundle.name}"),
34
+ (args.checksum, f"{release_prefix}/{args.checksum.name}"),
35
+ ]
36
+
37
+ for source_path, path_in_repo in uploads:
38
+ api.upload_file(
39
+ path_or_fileobj=str(source_path),
40
+ path_in_repo=path_in_repo,
41
+ repo_id=args.repo_id,
42
+ repo_type=args.repo_type,
43
+ )
44
+
45
+ print(f"Uploaded release artifacts to {args.repo_id}")
46
+ return 0
47
+
48
+
49
+ if __name__ == "__main__":
50
+ raise SystemExit(main())
build/pyinstaller_uploader.spec ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- mode: python ; coding: utf-8 -*-
2
+
3
+ from pathlib import Path
4
+
5
+
6
+ block_cipher = None
7
+ project_root = Path.cwd()
8
+
9
+
10
+ a = Analysis(
11
+ [str(project_root / "src" / "uploader_cli" / "main.py")],
12
+ pathex=[str(project_root)],
13
+ binaries=[],
14
+ datas=[],
15
+ hiddenimports=[
16
+ "click",
17
+ "huggingface_hub",
18
+ "keyring",
19
+ "rich",
20
+ "textual",
21
+ "pydantic",
22
+ "pandas",
23
+ "pyarrow",
24
+ "src.uploader_cli",
25
+ "src.uploader_cli.auth_service",
26
+ "src.uploader_cli.batch_uploader",
27
+ "src.uploader_cli.config",
28
+ "src.uploader_cli.deduplicator",
29
+ "src.uploader_cli.error_handler",
30
+ "src.uploader_cli.exceptions",
31
+ "src.uploader_cli.hash_utils",
32
+ "src.uploader_cli.repo_service",
33
+ "src.uploader_cli.scanner",
34
+ "src.uploader_cli.session_manager",
35
+ "src.uploader_cli.tui",
36
+ ],
37
+ hookspath=[],
38
+ hooksconfig={},
39
+ runtime_hooks=[],
40
+ excludes=[],
41
+ noarchive=False,
42
+ optimize=0,
43
+ )
44
+
45
+ pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
46
+
47
+ exe = EXE(
48
+ pyz,
49
+ a.scripts,
50
+ [],
51
+ exclude_binaries=True,
52
+ name="birdnet-uploader",
53
+ debug=False,
54
+ bootloader_ignore_signals=False,
55
+ strip=False,
56
+ upx=True,
57
+ console=True,
58
+ disable_windowed_traceback=False,
59
+ argv_emulation=False,
60
+ target_arch=None,
61
+ codesign_identity=None,
62
+ entitlements_file=None,
63
+ )
64
+
65
+ coll = COLLECT(
66
+ exe,
67
+ a.binaries,
68
+ a.datas,
69
+ strip=False,
70
+ upx=True,
71
+ upx_exclude=[],
72
+ name="birdnet-uploader",
73
+ )
build/release_pipeline.ps1 ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ param(
2
+ [Parameter(Mandatory = $true)]
3
+ [string]$Version,
4
+
5
+ [string]$RepoId = "",
6
+
7
+ [ValidateSet("dataset", "model", "space")]
8
+ [string]$RepoType = "dataset",
9
+
10
+ [switch]$PublishToHf
11
+ )
12
+
13
+ $ErrorActionPreference = "Stop"
14
+
15
+ $ProjectRoot = Split-Path -Parent $PSScriptRoot
16
+ $Python = "C:/Users/jonat/AppData/Local/Python/pythoncore-3.14-64/python.exe"
17
+ $BuildScript = Join-Path $PSScriptRoot "release_uploader.py"
18
+ $PublishScript = Join-Path $PSScriptRoot "publish_release_to_hf.py"
19
+ $ValidateScript = Join-Path $PSScriptRoot "validate_release.ps1"
20
+
21
+ if (-not (Test-Path $Python)) {
22
+ throw "Python executable not found at $Python"
23
+ }
24
+
25
+ Write-Host "[0/3] Ensuring build dependencies"
26
+ & $Python -m pip install pyinstaller | Out-Null
27
+ if ($LASTEXITCODE -ne 0) {
28
+ throw "Could not install PyInstaller"
29
+ }
30
+
31
+ Write-Host "[1/3] Building release bundle for version $Version"
32
+ & $Python $BuildScript --version $Version
33
+ if ($LASTEXITCODE -ne 0) {
34
+ throw "Release build failed"
35
+ }
36
+
37
+ Write-Host "[2/3] Running isolated smoke validation"
38
+ & $ValidateScript -Version $Version
39
+ if ($LASTEXITCODE -ne 0) {
40
+ throw "Release validation failed"
41
+ }
42
+
43
+ $ReleaseDir = Join-Path $PSScriptRoot "release"
44
+ $Bundle = Get-ChildItem -Path $ReleaseDir -Filter "*.zip" | Sort-Object LastWriteTime -Descending | Select-Object -First 1
45
+ $Checksum = Get-ChildItem -Path $ReleaseDir -Filter "*.sha256" | Sort-Object LastWriteTime -Descending | Select-Object -First 1
46
+
47
+ if (-not $Bundle) {
48
+ throw "Release bundle was not created"
49
+ }
50
+
51
+ if (-not $Checksum) {
52
+ throw "Checksum file was not created"
53
+ }
54
+
55
+ Write-Host "[2/3] Release bundle ready: $($Bundle.FullName)"
56
+ Write-Host "[2/3] Checksum ready: $($Checksum.FullName)"
57
+
58
+ if ($PublishToHf) {
59
+ if ([string]::IsNullOrWhiteSpace($RepoId)) {
60
+ throw "RepoId is required when -PublishToHf is set"
61
+ }
62
+
63
+ if ([string]::IsNullOrWhiteSpace($env:HF_TOKEN)) {
64
+ throw "HF_TOKEN environment variable is required when publishing to Hugging Face"
65
+ }
66
+
67
+ Write-Host "[3/3] Publishing release artifacts to Hugging Face repo $RepoId"
68
+ & $Python $PublishScript `
69
+ --repo-id $RepoId `
70
+ --repo-type $RepoType `
71
+ --bundle $Bundle.FullName `
72
+ --checksum $Checksum.FullName `
73
+ --version $Version
74
+ if ($LASTEXITCODE -ne 0) {
75
+ throw "Publish step failed"
76
+ }
77
+ } else {
78
+ Write-Host "[3/3] Publish step skipped. Use -PublishToHf -RepoId <repo> to upload to Hugging Face."
79
+ }
build/release_uploader.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import hashlib
5
+ import shutil
6
+ import subprocess
7
+ import sys
8
+ import zipfile
9
+ from datetime import UTC, datetime
10
+ from pathlib import Path
11
+
12
+
13
+ PROJECT_ROOT = Path(__file__).resolve().parents[1]
14
+ DIST_DIR = PROJECT_ROOT / "dist"
15
+ BUILD_DIR = PROJECT_ROOT / "build"
16
+ RELEASE_ROOT = BUILD_DIR / "release"
17
+
18
+
19
+ def run(command: list[str]) -> None:
20
+ subprocess.run(command, cwd=PROJECT_ROOT, check=True)
21
+
22
+
23
+ def clean_release_dirs() -> None:
24
+ if DIST_DIR.exists():
25
+ shutil.rmtree(DIST_DIR)
26
+ pycache = PROJECT_ROOT / "build" / "__pycache__"
27
+ if pycache.exists():
28
+ shutil.rmtree(pycache)
29
+ RELEASE_ROOT.mkdir(parents=True, exist_ok=True)
30
+
31
+
32
+ def build_executable() -> Path:
33
+ run([sys.executable, "-m", "PyInstaller", "--noconfirm", "--clean", str(BUILD_DIR / "pyinstaller_uploader.spec")])
34
+ exe_path = DIST_DIR / "birdnet-uploader" / ("birdnet-uploader.exe" if sys.platform.startswith("win") else "birdnet-uploader")
35
+ if not exe_path.exists():
36
+ raise FileNotFoundError(f"Expected executable not found: {exe_path}")
37
+ return exe_path
38
+
39
+
40
+ def make_release_bundle(executable_path: Path, version: str) -> tuple[Path, Path]:
41
+ bundle_name = f"birdnet-uploader-{version}-{'windows' if sys.platform.startswith('win') else 'linux'}.zip"
42
+ bundle_path = RELEASE_ROOT / bundle_name
43
+ sha256_path = RELEASE_ROOT / f"{bundle_name}.sha256"
44
+
45
+ # PyInstaller is configured as one-folder build, so zip the whole folder,
46
+ # not only the executable, otherwise runtime DLLs (_internal) are missing.
47
+ app_dir = executable_path.parent
48
+ dist_root = app_dir.parent
49
+
50
+ with zipfile.ZipFile(bundle_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
51
+ for path in app_dir.rglob("*"):
52
+ if path.is_file():
53
+ archive.write(path, arcname=str(path.relative_to(dist_root)))
54
+
55
+ digest = hashlib.sha256(bundle_path.read_bytes()).hexdigest()
56
+ sha256_path.write_text(f"{digest} {bundle_name}\n", encoding="utf-8")
57
+ return bundle_path, sha256_path
58
+
59
+
60
+ def main() -> int:
61
+ parser = argparse.ArgumentParser(description="Build a release bundle for BirdNET uploader")
62
+ parser.add_argument("--version", required=True, help="Release version, e.g. 0.1.0")
63
+ args = parser.parse_args()
64
+
65
+ clean_release_dirs()
66
+ executable_path = build_executable()
67
+ bundle_path, sha256_path = make_release_bundle(executable_path, args.version)
68
+ print(f"Release bundle: {bundle_path}")
69
+ print(f"Checksum file: {sha256_path}")
70
+ print(f"Built at: {datetime.now(UTC).isoformat()}")
71
+ return 0
72
+
73
+
74
+ if __name__ == "__main__":
75
+ raise SystemExit(main())
build/validate_release.ps1 ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ param(
2
+ [Parameter(Mandatory = $true)]
3
+ [string]$Version,
4
+
5
+ [string]$ReleaseDir = ".\build\release"
6
+ )
7
+
8
+ $ErrorActionPreference = "Stop"
9
+
10
+ function Get-PlatformLabel {
11
+ if ($IsWindows) { return "windows" }
12
+ if ($IsLinux) { return "linux" }
13
+ return "unknown"
14
+ }
15
+
16
+ $Bundle = Get-ChildItem -Path $ReleaseDir -Filter "birdnet-uploader-$Version-*.zip" | Sort-Object LastWriteTime -Descending | Select-Object -First 1
17
+ if (-not $Bundle) {
18
+ throw "Bundle for version $Version not found under $ReleaseDir"
19
+ }
20
+
21
+ $BundlePath = $Bundle.FullName
22
+ $BundleName = $Bundle.Name
23
+ $ChecksumPath = "$BundlePath.sha256"
24
+
25
+ if (-not (Test-Path $BundlePath)) {
26
+ throw "Bundle not found: $BundlePath"
27
+ }
28
+
29
+ if (-not (Test-Path $ChecksumPath)) {
30
+ throw "Checksum file not found: $ChecksumPath"
31
+ }
32
+
33
+ Write-Host "[1/5] Validating checksum file"
34
+ $ChecksumLine = (Get-Content $ChecksumPath -Raw).Trim()
35
+ if ([string]::IsNullOrWhiteSpace($ChecksumLine)) {
36
+ throw "Checksum file is empty: $ChecksumPath"
37
+ }
38
+
39
+ $ExpectedHash = ($ChecksumLine -split "\s+")[0].ToLower()
40
+ $ComputedHash = (Get-FileHash -Path $BundlePath -Algorithm SHA256).Hash.ToLower()
41
+
42
+ if ($ExpectedHash -ne $ComputedHash) {
43
+ throw "Checksum mismatch for $BundleName"
44
+ }
45
+
46
+ Write-Host "[2/5] Extracting bundle into an isolated temp folder"
47
+ $TempRoot = Join-Path $env:TEMP ("birdnet-uploader-smoke-" + [Guid]::NewGuid().ToString("N"))
48
+ New-Item -ItemType Directory -Path $TempRoot | Out-Null
49
+ Expand-Archive -Path $BundlePath -DestinationPath $TempRoot -Force
50
+
51
+ $Exe = Get-ChildItem -Path $TempRoot -Filter "birdnet-uploader.exe" -Recurse | Select-Object -First 1
52
+ if (-not $Exe) {
53
+ throw "Executable not found in extracted bundle"
54
+ }
55
+ $ExePath = $Exe.FullName
56
+ $ExeDir = $Exe.Directory.FullName
57
+
58
+ if (-not (Test-Path (Join-Path $ExeDir "_internal"))) {
59
+ throw "Missing _internal runtime folder next to executable: $ExeDir"
60
+ }
61
+
62
+ Write-Host "[3/5] Smoke test: executable help"
63
+ Push-Location $ExeDir
64
+ & $ExePath --help | Out-Null
65
+ if ($LASTEXITCODE -ne 0) {
66
+ Pop-Location
67
+ throw "Executable help command failed"
68
+ }
69
+
70
+ Write-Host "[4/5] Smoke test: command help"
71
+ & $ExePath scan --help | Out-Null
72
+ if ($LASTEXITCODE -ne 0) {
73
+ Pop-Location
74
+ throw "Scan help command failed"
75
+ }
76
+ Pop-Location
77
+
78
+ Write-Host "[5/5] Success"
79
+ Write-Host "Bundle validated: $BundlePath"
80
+ Write-Host "Extracted test folder: $TempRoot"
cli/__pycache__/__init__.cpython-314.pyc DELETED
Binary file (167 Bytes)
 
cli/__pycache__/hf_dataset_cli.cpython-314.pyc DELETED
Binary file (56 kB)
 
cli/hf_dataset_cli.py DELETED
@@ -1,1314 +0,0 @@
1
- import argparse
2
- import hashlib
3
- import json
4
- import math
5
- import re
6
- import tempfile
7
- import time
8
- from dataclasses import asdict, dataclass
9
- from datetime import UTC, datetime
10
- from io import BytesIO
11
- from pathlib import Path
12
- from typing import Any, Callable
13
-
14
- import pandas as pd
15
- from huggingface_hub import HfApi, hf_hub_download
16
-
17
-
18
- SCHEMA_VERSION = "1.0.0"
19
- REQUIRED_PROJECT_PREFIXES = (
20
- "audio/",
21
- "index/",
22
- "index/shards/",
23
- "validations/",
24
- "audit/",
25
- )
26
- REQUIRED_PLACEHOLDER_FILES = (
27
- "audio/.gitkeep",
28
- "index/.gitkeep",
29
- "index/shards/.gitkeep",
30
- "validations/.gitkeep",
31
- "audit/.gitkeep",
32
- )
33
- REQUIRED_DETECTIONS_COLUMNS = (
34
- "detection_key",
35
- "audio_id",
36
- "scientific_name",
37
- "confidence",
38
- "start_time",
39
- "end_time",
40
- )
41
- SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
42
- REQUIRED_BIRDNET_COLUMNS = (
43
- "source_file",
44
- "scientific_name",
45
- "confidence",
46
- "start_time",
47
- "end_time",
48
- )
49
- SEGMENT_FILENAME_PATTERN = re.compile(
50
- r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
51
- )
52
-
53
-
54
- @dataclass(slots=True)
55
- class ShardMetadata:
56
- path: str
57
- rows: int
58
- sha256: str
59
- size_bytes: int
60
-
61
-
62
- @dataclass(slots=True)
63
- class SegmentFileRecord:
64
- scientific_name: str
65
- source_stem: str
66
- start_time: float
67
- end_time: float
68
- confidence_pct: int
69
- absolute_path: Path
70
- relative_path: str
71
-
72
-
73
- def _utcnow_iso() -> str:
74
- return datetime.now(UTC).replace(microsecond=0).isoformat()
75
-
76
-
77
- def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
78
- api.upload_file(
79
- path_or_fileobj=BytesIO(content.encode("utf-8")),
80
- path_in_repo=path_in_repo,
81
- repo_id=dataset_repo,
82
- repo_type="dataset",
83
- )
84
-
85
-
86
- def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
87
- api.upload_file(
88
- path_or_fileobj=BytesIO(b""),
89
- path_in_repo=path_in_repo,
90
- repo_id=dataset_repo,
91
- repo_type="dataset",
92
- )
93
-
94
-
95
- def ensure_project_dataset_structure(
96
- api: HfApi,
97
- project_slug: str,
98
- dataset_repo: str,
99
- create_private_repo: bool,
100
- max_retries: int = 3,
101
- retry_delay_seconds: float = 2.0,
102
- ) -> dict[str, Any]:
103
- """Create and initialize dataset repository structure with retry logic."""
104
- import time as time_module
105
-
106
- # Create repository with retries
107
- for attempt in range(1, max_retries + 1):
108
- try:
109
- api.create_repo(
110
- repo_id=dataset_repo,
111
- repo_type="dataset",
112
- private=create_private_repo,
113
- exist_ok=True,
114
- )
115
- break # Success
116
- except Exception as exc:
117
- if attempt >= max_retries:
118
- raise RuntimeError(f"Failed to create repository after {max_retries} attempts: {exc}")
119
- time_module.sleep(retry_delay_seconds)
120
-
121
- # List repository files (needed before uploads)
122
- for attempt in range(1, max_retries + 1):
123
- try:
124
- files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
125
- break # Success
126
- except Exception as exc:
127
- if attempt >= max_retries:
128
- raise RuntimeError(f"Failed to access repository files after {max_retries} attempts: {exc}")
129
- time_module.sleep(retry_delay_seconds)
130
-
131
- created_paths: list[str] = []
132
-
133
- # Upload placeholder files with retries
134
- for placeholder in REQUIRED_PLACEHOLDER_FILES:
135
- if placeholder not in files:
136
- for attempt in range(1, max_retries + 1):
137
- try:
138
- _upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
139
- created_paths.append(placeholder)
140
- break # Success
141
- except Exception as exc:
142
- if attempt >= max_retries:
143
- raise RuntimeError(f"Failed to create {placeholder} after {max_retries} attempts: {exc}")
144
- time_module.sleep(retry_delay_seconds)
145
-
146
- # Upload manifest with retries
147
- manifest_path = "manifest.json"
148
- if manifest_path not in files:
149
- manifest = {
150
- "schema_version": SCHEMA_VERSION,
151
- "project_slug": project_slug,
152
- "dataset_repo_id": dataset_repo,
153
- "created_at": _utcnow_iso(),
154
- "updated_at": _utcnow_iso(),
155
- "index": {
156
- "total_detections": 0,
157
- "total_audio_files": 0,
158
- "shard_size": 0,
159
- "shards": [],
160
- },
161
- }
162
- for attempt in range(1, max_retries + 1):
163
- try:
164
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
165
- created_paths.append(manifest_path)
166
- break # Success
167
- except Exception as exc:
168
- if attempt >= max_retries:
169
- raise RuntimeError(f"Failed to create manifest.json after {max_retries} attempts: {exc}")
170
- time_module.sleep(retry_delay_seconds)
171
-
172
- return {
173
- "dataset_repo": dataset_repo,
174
- "project_slug": project_slug,
175
- "created_paths": created_paths,
176
- }
177
-
178
-
179
- def load_detections_table(detections_file: str) -> pd.DataFrame:
180
- input_path = Path(detections_file)
181
- if not input_path.exists():
182
- raise FileNotFoundError(f"Detections file not found: {detections_file}")
183
-
184
- suffix = input_path.suffix.lower()
185
- if suffix == ".csv":
186
- frame = pd.read_csv(input_path)
187
- elif suffix in {".jsonl", ".ndjson"}:
188
- frame = pd.read_json(input_path, lines=True)
189
- elif suffix == ".parquet":
190
- frame = pd.read_parquet(input_path)
191
- else:
192
- raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
193
-
194
- missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
195
- if missing_columns:
196
- raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
197
-
198
- if frame.empty:
199
- return frame
200
-
201
- deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
202
- return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
203
-
204
-
205
- def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
206
- input_path = Path(detections_csv)
207
- if not input_path.exists():
208
- raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
209
-
210
- frame = pd.read_csv(input_path)
211
- missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
212
- if missing_columns:
213
- raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
214
-
215
- if frame.empty:
216
- return frame
217
-
218
- normalized = frame.copy()
219
- normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
220
- normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
221
- normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
222
- normalized["species_key"] = normalized["scientific_name"].str.casefold()
223
- normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
224
- normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
225
- normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
226
-
227
- normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
228
- return normalized
229
-
230
-
231
- def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
232
- payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
233
- return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
234
-
235
-
236
- def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
237
- stem = Path(file_name).stem
238
- match = SEGMENT_FILENAME_PATTERN.match(stem)
239
- if not match:
240
- return None
241
-
242
- source_stem = match.group("source_stem")
243
- start_time = float(match.group("start"))
244
- end_time = float(match.group("end"))
245
- confidence_pct = int(match.group("confidence_pct"))
246
- return source_stem, start_time, end_time, confidence_pct
247
-
248
-
249
- def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
250
- root = Path(segments_root)
251
- if not root.exists() or not root.is_dir():
252
- raise FileNotFoundError(f"Segments root not found: {segments_root}")
253
-
254
- records: list[SegmentFileRecord] = []
255
- for absolute_path in root.rglob("*"):
256
- if not absolute_path.is_file():
257
- continue
258
- if absolute_path.suffix.lower() != ".wav":
259
- continue
260
-
261
- rel = absolute_path.relative_to(root)
262
- rel_parts = list(rel.parts)
263
- if rel_parts and rel_parts[0].strip().lower() in {"segments", "segment", "audio"}:
264
- rel_parts = rel_parts[1:]
265
-
266
- if rel_parts:
267
- scientific_name = rel_parts[0]
268
- normalized_relpath = Path(*rel_parts).as_posix()
269
- else:
270
- scientific_name = absolute_path.parent.name
271
- normalized_relpath = absolute_path.name
272
-
273
- parsed = parse_segment_filename(absolute_path.name)
274
- if not parsed:
275
- continue
276
- source_stem, start_time, end_time, confidence_pct = parsed
277
- records.append(
278
- SegmentFileRecord(
279
- scientific_name=scientific_name,
280
- source_stem=source_stem,
281
- start_time=start_time,
282
- end_time=end_time,
283
- confidence_pct=confidence_pct,
284
- absolute_path=absolute_path,
285
- relative_path=normalized_relpath,
286
- )
287
- )
288
-
289
- return records
290
-
291
-
292
- def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
293
- frame = load_birdnet_detections_csv(detections_csv=detections_csv)
294
- segment_records = discover_segment_records(segments_root=segments_root)
295
-
296
- by_species: dict[str, list[SegmentFileRecord]] = {}
297
- for record in segment_records:
298
- by_species.setdefault(record.scientific_name.casefold(), []).append(record)
299
-
300
- matched_rows: list[dict[str, Any]] = []
301
- unmatched_rows: list[dict[str, Any]] = []
302
- ambiguous_rows: list[dict[str, Any]] = []
303
-
304
- for row in frame.to_dict(orient="records"):
305
- species_key = str(row.get("species_key", "")).casefold()
306
- source_stem = str(row.get("source_stem", "")).strip()
307
- start_time = float(row["start_time"])
308
- end_time = float(row["end_time"])
309
- confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
310
-
311
- species_candidates = by_species.get(species_key, [])
312
- exact_matches = [
313
- record
314
- for record in species_candidates
315
- if record.source_stem == source_stem
316
- and abs(record.start_time - start_time) < 1e-6
317
- and abs(record.end_time - end_time) < 1e-6
318
- ]
319
-
320
- if not exact_matches:
321
- unmatched_rows.append(
322
- {
323
- "source_file": row.get("source_file", ""),
324
- "scientific_name": row.get("scientific_name", ""),
325
- "start_time": start_time,
326
- "end_time": end_time,
327
- }
328
- )
329
- continue
330
-
331
- selected = exact_matches[0]
332
- if len(exact_matches) > 1:
333
- expected_pct = int(round(confidence * 100))
334
- exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
335
- selected = exact_matches[0]
336
- ambiguous_rows.append(
337
- {
338
- "source_file": row.get("source_file", ""),
339
- "scientific_name": row.get("scientific_name", ""),
340
- "start_time": start_time,
341
- "end_time": end_time,
342
- "candidates": len(exact_matches),
343
- }
344
- )
345
-
346
- matched_rows.append(
347
- {
348
- "project_slug": project_slug,
349
- "detection_key": _build_detection_key(
350
- project_slug=project_slug,
351
- source_file=str(row.get("source_file", "")),
352
- scientific_name=str(row.get("scientific_name", "")),
353
- start_time=start_time,
354
- end_time=end_time,
355
- ),
356
- "audio_id": source_stem,
357
- "segment_filename": selected.absolute_path.name,
358
- "segment_relpath": selected.relative_path,
359
- "segment_abs_path": str(selected.absolute_path),
360
- "scientific_name": row.get("scientific_name", ""),
361
- "common_name": row.get("common_name", ""),
362
- "confidence": confidence,
363
- "start_time": start_time,
364
- "end_time": end_time,
365
- "exact_start": row.get("exact_start", start_time),
366
- "exact_end": row.get("exact_end", end_time),
367
- "locality": row.get("locality", ""),
368
- "point": row.get("point", ""),
369
- "date_folder": row.get("date_folder", ""),
370
- "min_freq": row.get("min_freq", None),
371
- "max_freq": row.get("max_freq", None),
372
- "box_source": row.get("box_source", ""),
373
- "label": row.get("label", ""),
374
- "source_file": row.get("source_file", ""),
375
- "schema_version": SCHEMA_VERSION,
376
- "ingested_at": _utcnow_iso(),
377
- }
378
- )
379
-
380
- return {
381
- "frame": frame,
382
- "segment_records_total": len(segment_records),
383
- "matched_rows": matched_rows,
384
- "unmatched_rows": unmatched_rows,
385
- "ambiguous_rows": ambiguous_rows,
386
- }
387
-
388
-
389
- def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
390
- match_result = _match_birdnet_rows(
391
- project_slug=project_slug,
392
- detections_csv=detections_csv,
393
- segments_root=segments_root,
394
- )
395
- frame = match_result["frame"]
396
- matched_rows = match_result["matched_rows"]
397
- unmatched_rows = match_result["unmatched_rows"]
398
- ambiguous_rows = match_result["ambiguous_rows"]
399
-
400
- unique_audio = {row["audio_id"] for row in matched_rows}
401
- result = {
402
- "mode": "dry-run",
403
- "project_slug": project_slug,
404
- "detections_csv": detections_csv,
405
- "segments_root": segments_root,
406
- "csv_rows_total": int(len(frame)),
407
- "segments_found_total": int(match_result["segment_records_total"]),
408
- "matched_rows": len(matched_rows),
409
- "unmatched_rows": len(unmatched_rows),
410
- "ambiguous_rows": len(ambiguous_rows),
411
- "unique_audio_ids": len(unique_audio),
412
- "sample_unmatched": unmatched_rows[:20],
413
- "sample_ambiguous": ambiguous_rows[:20],
414
- }
415
- return result
416
-
417
-
418
- def _file_sha256(file_path: Path) -> str:
419
- hasher = hashlib.sha256()
420
- with file_path.open("rb") as handle:
421
- for chunk in iter(lambda: handle.read(65536), b""):
422
- hasher.update(chunk)
423
- return hasher.hexdigest()
424
-
425
-
426
- def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
427
- if shard_size <= 0:
428
- raise ValueError("shard_size must be greater than zero")
429
-
430
- output_dir.mkdir(parents=True, exist_ok=True)
431
- shards: list[ShardMetadata] = []
432
-
433
- if frame.empty:
434
- return shards
435
-
436
- total_rows = len(frame)
437
- total_shards = math.ceil(total_rows / shard_size)
438
-
439
- for index in range(total_shards):
440
- start = index * shard_size
441
- end = min(start + shard_size, total_rows)
442
- shard_frame = frame.iloc[start:end]
443
- shard_filename = f"shard-{index:05d}.parquet"
444
- local_path = output_dir / shard_filename
445
- shard_frame.to_parquet(local_path, index=False)
446
-
447
- shards.append(
448
- ShardMetadata(
449
- path=f"index/shards/{shard_filename}",
450
- rows=len(shard_frame),
451
- sha256=_file_sha256(local_path),
452
- size_bytes=local_path.stat().st_size,
453
- )
454
- )
455
-
456
- return shards
457
-
458
-
459
- def build_manifest_payload(
460
- project_slug: str,
461
- dataset_repo: str,
462
- frame: pd.DataFrame,
463
- shard_size: int,
464
- shard_metadata: list[ShardMetadata],
465
- ) -> dict[str, Any]:
466
- now = _utcnow_iso()
467
- return {
468
- "schema_version": SCHEMA_VERSION,
469
- "project_slug": project_slug,
470
- "dataset_repo_id": dataset_repo,
471
- "updated_at": now,
472
- "index": {
473
- "total_detections": len(frame),
474
- "total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
475
- "shard_size": shard_size,
476
- "shards": [asdict(item) for item in shard_metadata],
477
- },
478
- }
479
-
480
-
481
- def discover_audio_files(local_audio_dir: str) -> list[Path]:
482
- audio_root = Path(local_audio_dir)
483
- if not audio_root.exists() or not audio_root.is_dir():
484
- raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
485
-
486
- files = [
487
- path
488
- for path in audio_root.rglob("*")
489
- if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
490
- ]
491
- return sorted(files)
492
-
493
-
494
- def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
495
- if chunk_size <= 0:
496
- raise ValueError("chunk_size must be greater than zero")
497
- return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
498
-
499
-
500
- def _load_resume_state(state_file: Path) -> dict[str, Any]:
501
- if not state_file.exists():
502
- return {"uploaded": [], "failed": []}
503
-
504
- payload = json.loads(state_file.read_text(encoding="utf-8"))
505
- uploaded = payload.get("uploaded", [])
506
- failed = payload.get("failed", [])
507
- return {
508
- "uploaded": uploaded if isinstance(uploaded, list) else [],
509
- "failed": failed if isinstance(failed, list) else [],
510
- }
511
-
512
-
513
- def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
514
- state_file.parent.mkdir(parents=True, exist_ok=True)
515
- payload = {
516
- "updated_at": _utcnow_iso(),
517
- "uploaded": sorted(uploaded),
518
- "failed": sorted(failed),
519
- }
520
- state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
521
-
522
-
523
- def _upload_audio_with_retry(
524
- api: HfApi,
525
- dataset_repo: str,
526
- local_path: Path,
527
- path_in_repo: str,
528
- max_retries: int,
529
- retry_backoff_seconds: float,
530
- ) -> None:
531
- """Upload file with exponential backoff retry logic. Raises exception if all retries exhaust."""
532
- attempts = 0
533
- last_exception = None
534
-
535
- while True:
536
- attempts += 1
537
- try:
538
- api.upload_file(
539
- path_or_fileobj=str(local_path),
540
- path_in_repo=path_in_repo,
541
- repo_id=dataset_repo,
542
- repo_type="dataset",
543
- )
544
- return # Success
545
- except Exception as exc:
546
- last_exception = exc
547
- if attempts > max_retries:
548
- # Include first and last error in final exception for debugging
549
- raise RuntimeError(
550
- f"Upload of {path_in_repo} failed after {max_retries} attempts. "
551
- f"Error: {str(exc)[:200]}"
552
- ) from exc
553
- time.sleep(retry_backoff_seconds * attempts)
554
-
555
-
556
- def ingest_segments_to_hf(
557
- api: HfApi,
558
- project_slug: str,
559
- dataset_repo: str,
560
- detections_csv: str,
561
- segments_root: str,
562
- batch_size: int,
563
- shard_size: int,
564
- max_retries: int,
565
- retry_backoff_seconds: float,
566
- resume_state_file: str,
567
- should_pause: Callable[[], bool] | None = None,
568
- should_cancel: Callable[[], bool] | None = None,
569
- progress_callback: Callable[[dict[str, Any]], None] | None = None,
570
- ) -> dict[str, Any]:
571
- # Ensure repository structure with improved error handling
572
- try:
573
- ensure_project_dataset_structure(
574
- api=api,
575
- project_slug=project_slug,
576
- dataset_repo=dataset_repo,
577
- create_private_repo=False,
578
- max_retries=3,
579
- retry_delay_seconds=1.0,
580
- )
581
- except Exception as exc:
582
- # Return error report instead of raising - allows UI to display meaningful message
583
- return {
584
- "mode": "execute",
585
- "project_slug": project_slug,
586
- "dataset_repo": dataset_repo,
587
- "detections_csv": detections_csv,
588
- "segments_root": segments_root,
589
- "error": f"Repository initialization failed: {str(exc)[:200]}",
590
- "csv_rows_total": 0,
591
- "segments_found_total": 0,
592
- "matched_rows": 0,
593
- "unmatched_rows": 0,
594
- "ambiguous_rows": 0,
595
- "unique_audio_ids": 0,
596
- "pending_audio_uploads": 0,
597
- "uploaded_audio_now": 0,
598
- "uploaded_audio_skipped_existing": 0,
599
- "failed_uploads": 0,
600
- "cancelled": False,
601
- "resume_state_file": str(resume_state_file),
602
- "index_rows_written": 0,
603
- "shards_written": 0,
604
- "sample_unmatched": [],
605
- "sample_ambiguous": [],
606
- }
607
-
608
- match_result = _match_birdnet_rows(
609
- project_slug=project_slug,
610
- detections_csv=detections_csv,
611
- segments_root=segments_root,
612
- )
613
- frame = match_result["frame"]
614
- matched_rows = match_result["matched_rows"]
615
- unmatched_rows = match_result["unmatched_rows"]
616
- ambiguous_rows = match_result["ambiguous_rows"]
617
-
618
- # Deduplicate by remote path so retries/resume work on a stable key.
619
- path_to_local: dict[str, Path] = {}
620
- for row in matched_rows:
621
- repo_path = f"audio/segments/{row['segment_relpath']}"
622
- if repo_path in path_to_local:
623
- continue
624
-
625
- # Prefer exact absolute path discovered during matching. This avoids
626
- # reconstruction errors when archives include wrapper folders.
627
- abs_path_value = str(row.get("segment_abs_path") or "").strip()
628
- if abs_path_value:
629
- abs_path = Path(abs_path_value)
630
- if abs_path.exists():
631
- path_to_local[repo_path] = abs_path
632
- continue
633
-
634
- path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
635
-
636
- state_path = Path(resume_state_file)
637
- state_payload = _load_resume_state(state_file=state_path)
638
- uploaded_state = set(str(item) for item in state_payload["uploaded"])
639
- failed_state = set(str(item) for item in state_payload["failed"])
640
-
641
- remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
642
-
643
- pending: list[tuple[Path, str]] = []
644
- skipped_existing = 0
645
- for repo_path, local_path in path_to_local.items():
646
- if repo_path in remote_files:
647
- uploaded_state.add(repo_path)
648
- skipped_existing += 1
649
- continue
650
-
651
- # Source of truth is remote repository contents. If a path is missing in the
652
- # repo, re-upload even when stale resume state says it was uploaded before.
653
- pending.append((local_path, repo_path))
654
-
655
- uploaded_now: set[str] = set()
656
- failed_now: set[str] = set()
657
- cancelled = False
658
-
659
- if progress_callback:
660
- progress_callback(
661
- {
662
- "event": "upload-start",
663
- "total_files": len(path_to_local),
664
- "pending": len(pending),
665
- "skipped_existing": skipped_existing,
666
- }
667
- )
668
-
669
- for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
670
- batch_uploaded = 0
671
- batch_failed = 0
672
- for local_path, repo_path in batch:
673
- if should_cancel and should_cancel():
674
- cancelled = True
675
- break
676
-
677
- while should_pause and should_pause():
678
- time.sleep(0.5)
679
- if should_cancel and should_cancel():
680
- cancelled = True
681
- break
682
-
683
- if cancelled:
684
- break
685
-
686
- try:
687
- _upload_audio_with_retry(
688
- api=api,
689
- dataset_repo=dataset_repo,
690
- local_path=local_path,
691
- path_in_repo=repo_path,
692
- max_retries=max_retries,
693
- retry_backoff_seconds=retry_backoff_seconds,
694
- )
695
- uploaded_state.add(repo_path)
696
- failed_state.discard(repo_path)
697
- uploaded_now.add(repo_path)
698
- batch_uploaded += 1
699
- except Exception:
700
- failed_state.add(repo_path)
701
- failed_now.add(repo_path)
702
- batch_failed += 1
703
-
704
- if progress_callback:
705
- progress_callback(
706
- {
707
- "event": "upload-progress",
708
- "uploaded": len(uploaded_now),
709
- "failed": len(failed_now),
710
- "skipped_existing": skipped_existing,
711
- "processed_pending": len(uploaded_now) + len(failed_now),
712
- "pending_total": len(pending),
713
- }
714
- )
715
-
716
- print(
717
- json.dumps(
718
- {
719
- "event": "ingest-segments-audio-batch",
720
- "batch_index": batch_index,
721
- "batch_size": len(batch),
722
- "uploaded": batch_uploaded,
723
- "failed": batch_failed,
724
- }
725
- )
726
- )
727
- _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
728
-
729
- if cancelled:
730
- break
731
-
732
- if cancelled:
733
- run_report = {
734
- "mode": "execute",
735
- "project_slug": project_slug,
736
- "dataset_repo": dataset_repo,
737
- "detections_csv": detections_csv,
738
- "segments_root": segments_root,
739
- "csv_rows_total": int(len(frame)),
740
- "segments_found_total": int(match_result["segment_records_total"]),
741
- "matched_rows": len(matched_rows),
742
- "unmatched_rows": len(unmatched_rows),
743
- "ambiguous_rows": len(ambiguous_rows),
744
- "unique_audio_ids": 0,
745
- "pending_audio_uploads": len(pending),
746
- "uploaded_audio_now": len(uploaded_now),
747
- "uploaded_audio_skipped_existing": skipped_existing,
748
- "failed_uploads": len(failed_now),
749
- "cancelled": True,
750
- "resume_state_file": str(state_path),
751
- "index_rows_written": 0,
752
- "shards_written": 0,
753
- "sample_unmatched": unmatched_rows[:20],
754
- "sample_ambiguous": ambiguous_rows[:20],
755
- }
756
- if progress_callback:
757
- progress_callback(
758
- {
759
- "event": "upload-complete",
760
- "uploaded": len(uploaded_now),
761
- "failed": len(failed_now),
762
- "skipped_existing": skipped_existing,
763
- "cancelled": True,
764
- }
765
- )
766
- return run_report
767
-
768
- index_frame = pd.DataFrame(matched_rows)
769
- if not index_frame.empty:
770
- index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
771
-
772
- with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
773
- temp_dir = Path(temp_dir_str)
774
- shard_dir = temp_dir / "index" / "shards"
775
- shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
776
-
777
- for item in shard_metadata:
778
- local_path = temp_dir / item.path
779
- api.upload_file(
780
- path_or_fileobj=str(local_path),
781
- path_in_repo=item.path,
782
- repo_id=dataset_repo,
783
- repo_type="dataset",
784
- )
785
-
786
- manifest = build_manifest_payload(
787
- project_slug=project_slug,
788
- dataset_repo=dataset_repo,
789
- frame=index_frame,
790
- shard_size=shard_size,
791
- shard_metadata=shard_metadata,
792
- )
793
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
794
-
795
- run_report = {
796
- "mode": "execute",
797
- "project_slug": project_slug,
798
- "dataset_repo": dataset_repo,
799
- "detections_csv": detections_csv,
800
- "segments_root": segments_root,
801
- "csv_rows_total": int(len(frame)),
802
- "segments_found_total": int(match_result["segment_records_total"]),
803
- "matched_rows": len(matched_rows),
804
- "unmatched_rows": len(unmatched_rows),
805
- "ambiguous_rows": len(ambiguous_rows),
806
- "unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
807
- "pending_audio_uploads": len(pending),
808
- "uploaded_audio_now": len(uploaded_now),
809
- "uploaded_audio_skipped_existing": skipped_existing,
810
- "failed_uploads": len(failed_now),
811
- "cancelled": cancelled,
812
- "resume_state_file": str(state_path),
813
- "index_rows_written": len(index_frame),
814
- "shards_written": len(shard_metadata),
815
- "sample_unmatched": unmatched_rows[:20],
816
- "sample_ambiguous": ambiguous_rows[:20],
817
- }
818
-
819
- audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
820
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
821
-
822
- if progress_callback:
823
- progress_callback(
824
- {
825
- "event": "upload-complete",
826
- "uploaded": len(uploaded_now),
827
- "failed": len(failed_now),
828
- "skipped_existing": skipped_existing,
829
- "cancelled": cancelled,
830
- }
831
- )
832
-
833
- return run_report
834
-
835
-
836
- def sync_audio_batches(
837
- api: HfApi,
838
- project_slug: str,
839
- dataset_repo: str,
840
- local_audio_dir: str,
841
- batch_size: int,
842
- max_retries: int,
843
- retry_backoff_seconds: float,
844
- resume_state_file: str,
845
- ) -> dict[str, Any]:
846
- ensure_project_dataset_structure(
847
- api=api,
848
- project_slug=project_slug,
849
- dataset_repo=dataset_repo,
850
- create_private_repo=False,
851
- )
852
-
853
- all_files = discover_audio_files(local_audio_dir=local_audio_dir)
854
- state_path = Path(resume_state_file)
855
- state_payload = _load_resume_state(state_file=state_path)
856
- uploaded_state = set(str(item) for item in state_payload["uploaded"])
857
-
858
- remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
859
-
860
- pending: list[tuple[Path, str]] = []
861
- for path in all_files:
862
- rel = path.relative_to(Path(local_audio_dir)).as_posix()
863
- repo_path = f"audio/{rel}"
864
-
865
- if repo_path in remote_files:
866
- uploaded_state.add(repo_path)
867
- continue
868
-
869
- if repo_path in uploaded_state:
870
- continue
871
-
872
- pending.append((path, repo_path))
873
-
874
- pending_paths = [item[0] for item in pending]
875
- by_local_to_repo = {local: repo for local, repo in pending}
876
- batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
877
-
878
- uploaded_now: set[str] = set()
879
- failed: set[str] = set()
880
-
881
- for batch_index, batch in enumerate(batches, start=1):
882
- batch_uploaded = 0
883
- batch_failed = 0
884
-
885
- for local_path in batch:
886
- repo_path = by_local_to_repo[local_path]
887
- try:
888
- _upload_audio_with_retry(
889
- api=api,
890
- dataset_repo=dataset_repo,
891
- local_path=local_path,
892
- path_in_repo=repo_path,
893
- max_retries=max_retries,
894
- retry_backoff_seconds=retry_backoff_seconds,
895
- )
896
- uploaded_now.add(repo_path)
897
- uploaded_state.add(repo_path)
898
- batch_uploaded += 1
899
- except Exception:
900
- failed.add(repo_path)
901
- batch_failed += 1
902
-
903
- print(
904
- json.dumps(
905
- {
906
- "event": "sync-audio-batch",
907
- "batch_index": batch_index,
908
- "batch_size": len(batch),
909
- "uploaded": batch_uploaded,
910
- "failed": batch_failed,
911
- }
912
- )
913
- )
914
- _save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
915
-
916
- return {
917
- "project_slug": project_slug,
918
- "dataset_repo": dataset_repo,
919
- "total_local_audio_files": len(all_files),
920
- "pending_uploads": len(pending),
921
- "uploaded_now": len(uploaded_now),
922
- "failed": len(failed),
923
- "resume_state_file": str(state_path),
924
- }
925
-
926
-
927
- def build_and_upload_index(
928
- api: HfApi,
929
- project_slug: str,
930
- dataset_repo: str,
931
- detections_file: str,
932
- shard_size: int,
933
- ) -> dict[str, Any]:
934
- ensure_project_dataset_structure(
935
- api=api,
936
- project_slug=project_slug,
937
- dataset_repo=dataset_repo,
938
- create_private_repo=False,
939
- )
940
-
941
- frame = load_detections_table(detections_file=detections_file)
942
-
943
- with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
944
- temp_dir = Path(temp_dir_str)
945
- shard_dir = temp_dir / "index" / "shards"
946
- metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
947
-
948
- for item in metadata:
949
- local_path = temp_dir / item.path
950
- api.upload_file(
951
- path_or_fileobj=str(local_path),
952
- path_in_repo=item.path,
953
- repo_id=dataset_repo,
954
- repo_type="dataset",
955
- )
956
-
957
- manifest = build_manifest_payload(
958
- project_slug=project_slug,
959
- dataset_repo=dataset_repo,
960
- frame=frame,
961
- shard_size=shard_size,
962
- shard_metadata=metadata,
963
- )
964
- _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
965
-
966
- return {
967
- "project_slug": project_slug,
968
- "dataset_repo": dataset_repo,
969
- "total_detections": manifest["index"]["total_detections"],
970
- "total_audio_files": manifest["index"]["total_audio_files"],
971
- "total_shards": len(metadata),
972
- }
973
-
974
-
975
- def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
976
- errors: list[str] = []
977
-
978
- for prefix in REQUIRED_PROJECT_PREFIXES:
979
- if not any(path.startswith(prefix) for path in repo_files):
980
- errors.append(f"Missing prefix in dataset repo: {prefix}")
981
-
982
- if manifest_payload.get("schema_version") != SCHEMA_VERSION:
983
- errors.append(
984
- f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
985
- )
986
-
987
- if manifest_payload.get("project_slug") != project_slug:
988
- errors.append(
989
- f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
990
- )
991
-
992
- index_section = manifest_payload.get("index", {})
993
- shards = index_section.get("shards", [])
994
- if not isinstance(shards, list):
995
- errors.append("Manifest index.shards must be a list")
996
- return errors
997
-
998
- for shard in shards:
999
- shard_path = shard.get("path")
1000
- if not shard_path or not isinstance(shard_path, str):
1001
- errors.append("Shard entry missing path")
1002
- continue
1003
- if shard_path not in repo_files:
1004
- errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
1005
-
1006
- return errors
1007
-
1008
-
1009
- def upload_segments_to_hf(
1010
- api: HfApi,
1011
- project_slug: str,
1012
- dataset_repo: str,
1013
- segments_root: str,
1014
- batch_size: int = 50,
1015
- max_retries: int = 3,
1016
- retry_backoff_seconds: float = 1.0,
1017
- should_pause: Callable[[], bool] | None = None,
1018
- should_cancel: Callable[[], bool] | None = None,
1019
- progress_callback: Callable[[dict[str, Any]], None] | None = None,
1020
- ) -> dict[str, Any]:
1021
- """Upload local segments folder to dataset repo maintaining directory structure."""
1022
- segments_path = Path(segments_root)
1023
- if not segments_path.exists() or not segments_path.is_dir():
1024
- raise FileNotFoundError(f"Segments root not found: {segments_root}")
1025
-
1026
- if batch_size <= 0:
1027
- raise ValueError("batch_size must be greater than zero")
1028
-
1029
- ensure_project_dataset_structure(
1030
- api=api,
1031
- project_slug=project_slug,
1032
- dataset_repo=dataset_repo,
1033
- create_private_repo=False,
1034
- )
1035
-
1036
- all_audio_files: list[Path] = []
1037
- for ext in SUPPORTED_AUDIO_EXTENSIONS:
1038
- all_audio_files.extend(segments_path.rglob(f"*{ext}"))
1039
- all_audio_files = sorted(all_audio_files)
1040
-
1041
- remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
1042
- pending: list[tuple[Path, str]] = []
1043
- skipped_existing = 0
1044
- for local_file in all_audio_files:
1045
- relative_path = local_file.relative_to(segments_path)
1046
- repo_path = f"audio/{project_slug}/{relative_path.as_posix()}"
1047
- if repo_path in remote_files:
1048
- skipped_existing += 1
1049
- continue
1050
- pending.append((local_file, repo_path))
1051
-
1052
- if not all_audio_files:
1053
- return {
1054
- "project_slug": project_slug,
1055
- "dataset_repo": dataset_repo,
1056
- "mode": "upload-segments-only",
1057
- "total_files": 0,
1058
- "uploaded": 0,
1059
- "skipped_existing": 0,
1060
- "failed": 0,
1061
- "cancelled": False,
1062
- }
1063
-
1064
- uploaded_count = 0
1065
- failed_count = 0
1066
- cancelled = False
1067
- failed_files: list[str] = []
1068
-
1069
- batches = _chunk_items(pending, batch_size) if pending else []
1070
- total_pending = len(pending)
1071
- if progress_callback:
1072
- progress_callback(
1073
- {
1074
- "event": "upload-start",
1075
- "total_files": len(all_audio_files),
1076
- "pending": total_pending,
1077
- "skipped_existing": skipped_existing,
1078
- }
1079
- )
1080
-
1081
- for batch_idx, batch in enumerate(batches, start=1):
1082
- for local_file, repo_path in batch:
1083
- if should_cancel and should_cancel():
1084
- cancelled = True
1085
- break
1086
-
1087
- while should_pause and should_pause():
1088
- time.sleep(0.5)
1089
- if should_cancel and should_cancel():
1090
- cancelled = True
1091
- break
1092
-
1093
- if cancelled:
1094
- break
1095
-
1096
- relative_path = local_file.relative_to(segments_path)
1097
-
1098
- try:
1099
- _upload_audio_with_retry(
1100
- api=api,
1101
- dataset_repo=dataset_repo,
1102
- local_path=local_file,
1103
- path_in_repo=repo_path,
1104
- max_retries=max_retries,
1105
- retry_backoff_seconds=retry_backoff_seconds,
1106
- )
1107
- uploaded_count += 1
1108
- except Exception as exc:
1109
- failed_count += 1
1110
- failed_files.append(f"{relative_path.as_posix()}: {exc}")
1111
-
1112
- if progress_callback:
1113
- progress_callback(
1114
- {
1115
- "event": "upload-progress",
1116
- "uploaded": uploaded_count,
1117
- "failed": failed_count,
1118
- "skipped_existing": skipped_existing,
1119
- "processed_pending": uploaded_count + failed_count,
1120
- "pending_total": total_pending,
1121
- }
1122
- )
1123
-
1124
- if cancelled:
1125
- break
1126
-
1127
- print(
1128
- json.dumps(
1129
- {
1130
- "event": "upload-batch",
1131
- "batch": batch_idx,
1132
- "total_batches": len(batches),
1133
- "uploaded_so_far": uploaded_count,
1134
- "failed_so_far": failed_count,
1135
- "skipped_existing": skipped_existing,
1136
- }
1137
- )
1138
- )
1139
-
1140
- if progress_callback:
1141
- progress_callback(
1142
- {
1143
- "event": "upload-complete",
1144
- "uploaded": uploaded_count,
1145
- "failed": failed_count,
1146
- "skipped_existing": skipped_existing,
1147
- "cancelled": cancelled,
1148
- }
1149
- )
1150
-
1151
- return {
1152
- "project_slug": project_slug,
1153
- "dataset_repo": dataset_repo,
1154
- "mode": "upload-segments-only",
1155
- "total_files": len(all_audio_files),
1156
- "pending_files": total_pending,
1157
- "uploaded": uploaded_count,
1158
- "skipped_existing": skipped_existing,
1159
- "failed": failed_count,
1160
- "cancelled": cancelled,
1161
- "failed_samples": failed_files[:10],
1162
- }
1163
-
1164
-
1165
- def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
1166
- repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
1167
- if "manifest.json" not in repo_files:
1168
- return {
1169
- "ok": False,
1170
- "errors": ["manifest.json not found in dataset repository"],
1171
- }
1172
-
1173
- manifest_local = hf_hub_download(
1174
- repo_id=dataset_repo,
1175
- repo_type="dataset",
1176
- filename="manifest.json",
1177
- )
1178
- manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
1179
- errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
1180
-
1181
- return {
1182
- "ok": len(errors) == 0,
1183
- "errors": errors,
1184
- "total_files": len(repo_files),
1185
- "total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
1186
- }
1187
-
1188
-
1189
- def build_parser() -> argparse.ArgumentParser:
1190
- parser = argparse.ArgumentParser(description="HF dataset project CLI")
1191
- sub = parser.add_subparsers(dest="command", required=True)
1192
-
1193
- create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
1194
- create_project.add_argument("--project-slug", required=True)
1195
- create_project.add_argument("--dataset-repo", required=True)
1196
- create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
1197
-
1198
- build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
1199
- build_index.add_argument("--project-slug", required=True)
1200
- build_index.add_argument("--dataset-repo", required=True)
1201
- build_index.add_argument("--detections-file", required=True)
1202
- build_index.add_argument("--shard-size", type=int, default=10000)
1203
-
1204
- sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
1205
- sync_audio.add_argument("--project-slug", required=True)
1206
- sync_audio.add_argument("--dataset-repo", required=True)
1207
- sync_audio.add_argument("--local-audio-dir", required=True)
1208
- sync_audio.add_argument("--batch-size", type=int, default=100)
1209
- sync_audio.add_argument("--max-retries", type=int, default=3)
1210
- sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
1211
- sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
1212
-
1213
- verify_project = sub.add_parser("verify-project", help="Verify project integrity")
1214
- verify_project.add_argument("--project-slug", required=True)
1215
- verify_project.add_argument("--dataset-repo", required=True)
1216
-
1217
- ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
1218
- ingest_segments.add_argument("--project-slug", required=True)
1219
- ingest_segments.add_argument("--dataset-repo", required=True)
1220
- ingest_segments.add_argument("--detections-csv", required=True)
1221
- ingest_segments.add_argument("--segments-root", required=True)
1222
- ingest_segments.add_argument("--batch-size", type=int, default=200)
1223
- ingest_segments.add_argument("--shard-size", type=int, default=10000)
1224
- ingest_segments.add_argument("--max-retries", type=int, default=3)
1225
- ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
1226
- ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
1227
- ingest_segments.add_argument("--dry-run", action="store_true")
1228
- ingest_segments.add_argument("--report-file", default="")
1229
-
1230
- return parser
1231
-
1232
-
1233
- def main() -> int:
1234
- parser = build_parser()
1235
- args = parser.parse_args()
1236
- api = HfApi()
1237
-
1238
- if args.command == "create-project":
1239
- result = ensure_project_dataset_structure(
1240
- api=api,
1241
- project_slug=args.project_slug,
1242
- dataset_repo=args.dataset_repo,
1243
- create_private_repo=bool(args.private),
1244
- )
1245
- print(json.dumps(result, indent=2))
1246
- return 0
1247
-
1248
- if args.command == "build-index":
1249
- result = build_and_upload_index(
1250
- api=api,
1251
- project_slug=args.project_slug,
1252
- dataset_repo=args.dataset_repo,
1253
- detections_file=args.detections_file,
1254
- shard_size=args.shard_size,
1255
- )
1256
- print(json.dumps(result, indent=2))
1257
- return 0
1258
-
1259
- if args.command == "sync-audio":
1260
- result = sync_audio_batches(
1261
- api=api,
1262
- project_slug=args.project_slug,
1263
- dataset_repo=args.dataset_repo,
1264
- local_audio_dir=args.local_audio_dir,
1265
- batch_size=args.batch_size,
1266
- max_retries=args.max_retries,
1267
- retry_backoff_seconds=args.retry_backoff_seconds,
1268
- resume_state_file=args.resume_state_file,
1269
- )
1270
- print(json.dumps(result, indent=2))
1271
- return 0 if result["failed"] == 0 else 1
1272
-
1273
- if args.command == "verify-project":
1274
- result = verify_project(
1275
- api=api,
1276
- project_slug=args.project_slug,
1277
- dataset_repo=args.dataset_repo,
1278
- )
1279
- print(json.dumps(result, indent=2))
1280
- return 0 if result["ok"] else 1
1281
-
1282
- if args.command == "ingest-segments":
1283
- if args.dry_run:
1284
- result = run_ingest_segments_dry_run(
1285
- project_slug=args.project_slug,
1286
- detections_csv=args.detections_csv,
1287
- segments_root=args.segments_root,
1288
- )
1289
- else:
1290
- result = ingest_segments_to_hf(
1291
- api=api,
1292
- project_slug=args.project_slug,
1293
- dataset_repo=args.dataset_repo,
1294
- detections_csv=args.detections_csv,
1295
- segments_root=args.segments_root,
1296
- batch_size=args.batch_size,
1297
- shard_size=args.shard_size,
1298
- max_retries=args.max_retries,
1299
- retry_backoff_seconds=args.retry_backoff_seconds,
1300
- resume_state_file=args.resume_state_file,
1301
- )
1302
-
1303
- if args.report_file:
1304
- Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
1305
-
1306
- print(json.dumps(result, indent=2))
1307
- return 0 if result.get("failed_uploads", 0) == 0 else 1
1308
-
1309
- parser.error("Unknown command")
1310
- return 2
1311
-
1312
-
1313
- if __name__ == "__main__":
1314
- raise SystemExit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docs/COLLABORATIVE_WORKFLOW_GUIDE.md ADDED
@@ -0,0 +1,424 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # End-to-End Collaborative Workflow Guide
2
+
3
+ ## Overview
4
+
5
+ The BirdNET Validator App supports two project visibility modes:
6
+
7
+ - **Collaborative**: Multiple users can contribute validations. Any admin can invite collaborators via email.
8
+ - **Private**: Single owner only. No invites allowed. Owner retains exclusive access.
9
+
10
+ ## Feature Summary
11
+
12
+ ### ✅ What's Implemented
13
+
14
+ | Feature | Status | Details |
15
+ |---------|--------|---------|
16
+ | **Collaborative Projects** | ✅ | Multiple users, email invites, per-user tokens |
17
+ | **Private Projects** | ✅ | Owner-only, no invites, strict ACL enforcement |
18
+ | **Email Notifications** | ✅ | SMTP + NoOp fallback, transactional delivery |
19
+ | **Per-User Tokens** | ✅ | Each user's HF token stored separately (secure) |
20
+ | **Session Revocation** | ✅ | Immediate when user removed from project |
21
+ | **Durable State** | ✅ | `/data` persistence, atomic writes, crash-safe |
22
+ | **Invite TTL** | ✅ | Expiring invites (72h default, configurable) |
23
+ | **ACL Enforcement** | ✅ | Private vs Collaborative, admin-only operations |
24
+
25
+ ---
26
+
27
+ ## Workflow Example: Creating & Sharing a Collaborative Project
28
+
29
+ ### Phase 1: Admin Creates Project
30
+
31
+ ```bash
32
+ # Admin logs in (first user = automatic admin)
33
+ # → Session created with admin role
34
+
35
+ # Admin creates project via CLI:
36
+ python -m src.cli.project_cli create-project \
37
+ --projects-file /data/bootstrap/projects.json \
38
+ --slug "parrots-2026" \
39
+ --name "Parrots Dataset 2026" \
40
+ --dataset-repo-id "birdnet/parrots-2026" \
41
+ --visibility "collaborative"
42
+ # OK: project 'parrots-2026' created
43
+ ```
44
+
45
+ **What happened:**
46
+ - Project created in `/data/bootstrap/projects.json`
47
+ - No `owner_username` (collaborative mode)
48
+ - Admin automatically has validator role
49
+
50
+ ### Phase 2: Admin Invites Collaborator
51
+
52
+ Using the Admin Panel UI, admin clicks "Invite User":
53
+
54
+ ```
55
+ Actor: alice_admin (project admin)
56
+ Invitee Username: bob_validator
57
+ Invitee Email: bob@example.com
58
+ Role: validator
59
+ ```
60
+
61
+ **What happens:**
62
+ 1. Authorization check: Alice is admin for this project ✓
63
+ 2. Project visibility check: Collaborative (invites allowed) ✓
64
+ 3. Invite created with 72h TTL
65
+ 4. Email sent via SMTP:
66
+ ```
67
+ Subject: You've been invited to validate "Parrots Dataset 2026"
68
+
69
+ Hello Bob,
70
+
71
+ Alice has invited you to join the "Parrots Dataset 2026" project as a validator.
72
+
73
+ Project: parrots-2026
74
+ Role: validator
75
+ Expires: [72 hours from now]
76
+
77
+ To accept this invitation, log in with your Hugging Face token at:
78
+ https://birdnet-validator.example.com/login
79
+
80
+ After logging in, your invite will appear. Click "Accept" to join.
81
+ ```
82
+ 5. Invite stored in `/data/bootstrap/invites.json`:
83
+ ```json
84
+ {
85
+ "bob_validator": {
86
+ "parrots-2026": {
87
+ "role": "validator",
88
+ "invited_by": "alice_admin",
89
+ "created_at": "2026-04-01T12:00:00Z",
90
+ "expires_at": "2026-04-04T12:00:00Z"
91
+ }
92
+ }
93
+ }
94
+ ```
95
+
96
+ ### Phase 3: Collaborator Receives & Accepts Invite
97
+
98
+ **Bob receives email, opens Space app, logs in:**
99
+
100
+ 1. Bob enters his HF token: `hf_user_***`
101
+ 2. AuthService resolves username from HF API (`whoami()`)
102
+ 3. Bob's session created
103
+ 4. Bob's token stored separately: `_hf_tokens_by_username["bob_validator"] = "hf_user_***"`
104
+ 5. **Pending invites auto-appear** in project selector
105
+ 6. Bob clicks "Accept Invite"
106
+
107
+ **What happens:**
108
+ - Invite moved from pending → accepted
109
+ - Bob added to `user_access.json`:
110
+ ```json
111
+ {
112
+ "bob_validator": {
113
+ "parrots-2026": "validator"
114
+ }
115
+ }
116
+ ```
117
+ - Invite removed from `invites.json`
118
+
119
+ ### Phase 4: Both Users Have Access & Own Tokens
120
+
121
+ **Alice (admin):**
122
+ - Token: `hf_alice_***` (stored in `_hf_tokens_by_username["alice_admin"]`)
123
+ - Role: admin for "parrots-2026"
124
+ - Can: create, assign, remove, validate, manage project
125
+
126
+ **Bob (validator):**
127
+ - Token: `hf_bob_***` (stored in `_hf_tokens_by_username["bob_validator"]`)
128
+ - Role: validator for "parrots-2026"
129
+ - Can: validate detections, see project data
130
+
131
+ **Key Security Feature:**
132
+ - Tokens are **NOT** stored in Session objects
133
+ - Token lookup happens only when needed (validation flow)
134
+ - Each user brings own token → no token sharing
135
+ - If Alice's token revoked on HF, only Alice affected
136
+
137
+ ---
138
+
139
+ ## Private Project Workflow
140
+
141
+ ### Create Private Project
142
+
143
+ ```bash
144
+ python -m src.cli.project_cli create-project \
145
+ --projects-file /data/bootstrap/projects.json \
146
+ --user-access-file /data/bootstrap/user_access.json \
147
+ --slug "proprietary-2026" \
148
+ --name "Proprietary Dataset" \
149
+ --dataset-repo-id "owner/proprietary-2026" \
150
+ --visibility "private" \
151
+ --owner "carol_owner"
152
+ ```
153
+
154
+ **Result:**
155
+ - `owner_username = "carol_owner"` is set (required)
156
+ - Carol automatically gets admin role
157
+ - Invites are **rejected** if attempted
158
+
159
+ ### Try to Invite (Fails)
160
+
161
+ If Carol or another admin tries to invite a collaborator:
162
+
163
+ ```
164
+ Actor: carol_owner
165
+ Invitee: dave_validator
166
+ → ERROR: "Private projects do not accept collaborators"
167
+ ```
168
+
169
+ **No invite created.** Private projects are owner-only.
170
+
171
+ ---
172
+
173
+ ## Configuration for HF Spaces Deployment
174
+
175
+ ### Set Environment Variables
176
+
177
+ On your [HF Space Settings](https://huggingface.co/spaces/YOUR_ORG/BirdNET-Validator-App/settings):
178
+
179
+ ```bash
180
+ # Basic config
181
+ BIRDNET_ENABLE_DEMO_BOOTSTRAP=false
182
+ BIRDNET_BOOTSTRAP_DIR=/data/bootstrap
183
+ BIRDNET_VALIDATIONS_DIR=/data/validations
184
+
185
+ # Invite settings
186
+ BIRDNET_INVITE_EMAIL_ENABLED=true
187
+ BIRDNET_INVITE_EMAIL_SENDER=noreply@birdnet-validator.example.com
188
+ BIRDNET_INVITE_EMAIL_LOGIN_URL=https://huggingface.co/spaces/YOUR_ORG/BirdNET-Validator-App
189
+ BIRDNET_INVITE_TTL_HOURS=72
190
+
191
+ # SMTP settings (transactional email service)
192
+ BIRDNET_SMTP_HOST=smtp.gmail.com # or your email provider
193
+ BIRDNET_SMTP_PORT=587
194
+ BIRDNET_SMTP_USERNAME=your-email@gmail.com
195
+ BIRDNET_SMTP_PASSWORD=your-app-password # Use app-specific password for 2FA
196
+ BIRDNET_SMTP_USE_TLS=true
197
+ ```
198
+
199
+ ### Test SMTP Configuration Locally
200
+
201
+ ```python
202
+ from src.config.runtime_config import RuntimeConfig
203
+ from src.services.invite_email_notifier import SmtpInviteEmailNotifier, InviteEmailPayload
204
+ from datetime import datetime, timedelta, UTC
205
+
206
+ config = RuntimeConfig.from_env()
207
+
208
+ if config.invite_email_enabled and config.smtp_host:
209
+ notifier = SmtpInviteEmailNotifier(
210
+ host=config.smtp_host,
211
+ port=config.smtp_port,
212
+ username=config.smtp_username,
213
+ password=config.smtp_password,
214
+ use_tls=config.smtp_use_tls,
215
+ )
216
+
217
+ test_payload = InviteEmailPayload(
218
+ invitee_username="test_user",
219
+ invitee_email="test@example.com",
220
+ project_slug="test-project",
221
+ role="validator",
222
+ invited_by="admin_user",
223
+ expires_at=datetime.now(UTC) + timedelta(hours=72),
224
+ login_url=config.invite_email_login_url,
225
+ )
226
+
227
+ ok, message = notifier.send(test_payload)
228
+ print(f"Email send: {ok=}, {message=}")
229
+ else:
230
+ print("Email invites disabled or SMTP not configured")
231
+ ```
232
+
233
+ ---
234
+
235
+ ## Data Durability & Persistence
236
+
237
+ ### Bootstrap Files Location
238
+
239
+ On HF Spaces, all bootstrap and validation data persists in `/data/`:
240
+
241
+ ```
242
+ /data/
243
+ ├── bootstrap/
244
+ │ ├── projects.json # All projects (collaborative + private)
245
+ │ ├── user_access.json # User → Project → Role mapping
246
+ │ └── invites.json # Pending invites with TTL
247
+ └── validations/
248
+ ├── project-1/
249
+ │ ├── 2026-04-01.jsonl # Append-only validation events
250
+ │ ├── 2026-04-02.jsonl
251
+ │ └── ...
252
+ └── project-2/
253
+ └── ...
254
+ ```
255
+
256
+ ### Atomic Writes (Crash-Safe)
257
+
258
+ All JSON writes use atomic pattern:
259
+
260
+ ```python
261
+ temp_file = target.with_suffix(target.suffix + ".tmp")
262
+ temp_file.write_text(json.dumps(data))
263
+ temp_file.replace(target) # Atomic on modern filesystems
264
+ ```
265
+
266
+ **Benefit:** If Space crashes during write, file is not corrupted.
267
+
268
+ ### Data Loss Prevention
269
+
270
+ Since `/data` is persistent on HF Spaces:
271
+ - Projects, ACL, invites **survive** Space restarts ✓
272
+ - Unfinished validations are **not** lost ✓
273
+ - User data is **durable** across redeploys ✓
274
+
275
+ ---
276
+
277
+ ## Testing the Workflow Locally
278
+
279
+ ### 1. Create Users & Projects
280
+
281
+ ```bash
282
+ # Set environment for local testing
283
+ export BIRDNET_ENABLE_DEMO_BOOTSTRAP=false
284
+ export BIRDNET_PROJECTS_FILE=/tmp/projects.json
285
+ export BIRDNET_USER_ACCESS_FILE=/tmp/user_access.json
286
+ export BIRDNET_INVITES_FILE=/tmp/invites.json
287
+ export BIRDNET_INVITE_EMAIL_ENABLED=true
288
+ export BIRDNET_INVITE_EMAIL_SENDER=test@example.com
289
+ export BIRDNET_INVITE_EMAIL_LOGIN_URL=http://localhost:7860
290
+
291
+ # Create initial files
292
+ echo '[]' > /tmp/projects.json
293
+ echo '{}' > /tmp/user_access.json
294
+ echo '{}' > /tmp/invites.json
295
+
296
+ # Create project
297
+ python -m src.cli.project_cli create-project \
298
+ --projects-file /tmp/projects.json \
299
+ --user-access-file /tmp/user_access.json \
300
+ --slug test-project \
301
+ --name "Test Project" \
302
+ --dataset-repo-id "test/project" \
303
+ --visibility collaborative \
304
+ --owner alice
305
+ ```
306
+
307
+ ### 2. Start App & Test UI
308
+
309
+ ```bash
310
+ # Terminal 1: Start server
311
+ python app.py # Runs on http://localhost:7860
312
+
313
+ # Terminal 2: Test workflow in UI
314
+ # - Log in with first user (becomes admin)
315
+ # - Create collaborative project
316
+ # - Invite second user
317
+ # - Log out, log in as second user
318
+ # - Accept invite from project selector
319
+ # - Both users see project in dropdown
320
+ ```
321
+
322
+ ### 3. Verify Data Files
323
+
324
+ ```bash
325
+ # Check projects file
326
+ cat /tmp/projects.json | python -m json.tool
327
+
328
+ # Check user access
329
+ cat /tmp/user_access.json | python -m json.tool
330
+
331
+ # Check invites
332
+ cat /tmp/invites.json | python -m json.tool
333
+ ```
334
+
335
+ ---
336
+
337
+ ## Troubleshooting
338
+
339
+ ### "Email not sent" but project created
340
+
341
+ **Cause:** SMTP not configured or Email invites disabled
342
+
343
+ **Fix:**
344
+ 1. Check `BIRDNET_INVITE_EMAIL_ENABLED=true`
345
+ 2. Check SMTP credentials are correct
346
+ 3. For Gmail: use [App Password](https://support.google.com/accounts/answer/185833), not account password
347
+ 4. Check firewall allows outbound SMTP (port 587)
348
+
349
+ **Fallback:** System will create invite without email. User must be told invite URL manually.
350
+
351
+ ### User can't accept invite
352
+
353
+ **Cause:** Session TTL expired, or invite expired (>72h)
354
+
355
+ **Fix:**
356
+ 1. Re-login (creates fresh session)
357
+ 2. Resend invite
358
+ 3. Adjust `BIRDNET_INVITE_TTL_HOURS` if needed
359
+
360
+ ### Private project invite fails
361
+
362
+ **Expected behavior:** Private projects reject invites
363
+
364
+ **Workaround:** Assign users directly in `user_access.json` if bootstrap:
365
+ ```json
366
+ {
367
+ "username": {
368
+ "private-project": "validator"
369
+ }
370
+ }
371
+ ```
372
+
373
+ ### Tokens not being used in validation
374
+
375
+ **Cause:** Check project token precedence in `app_factory.py`:
376
+
377
+ ```python
378
+ # Preference order:
379
+ # 1. User's HF token (from session login)
380
+ # 2. Project token (for private project owners only)
381
+ # 3. None (validation fails)
382
+ ```
383
+
384
+ Set users' HF token via login, or set project token via CLI/UI.
385
+
386
+ ---
387
+
388
+ ## Next Steps
389
+
390
+ ### For Production Deployment
391
+
392
+ 1. ✅ **Configure SMTP** on HF Space with real email service
393
+ 2. ✅ **Test end-to-end**: Create project → Invite user → Accept → Validate
394
+ 3. ✅ **Verify durability**: Create data, restart Space, confirm persistence
395
+ 4. 📋 **Monitor invites**: Set up alerts if emails bounce
396
+ 5. 📋 **Audit trail**: Review `/data/bootstrap/invites.json` periodically
397
+
398
+ ### For Future Enhancements
399
+
400
+ - [ ] Invite expiration auto-cleanup background task
401
+ - [ ] Invite revocation by admin
402
+ - [ ] User deactivation cascade (revoke all invites, remove all roles)
403
+ - [ ] Email template customization
404
+ - [ ] Multi-language invite emails
405
+ - [ ] Webhook notifications (Discord, Slack)
406
+
407
+ ---
408
+
409
+ ## Test Results Summary
410
+
411
+ | Test Suite | Pass Rate | Key Tests |
412
+ |-----------|-----------|-----------|
413
+ | **Unit: AuthService** | 37/37 ✅ | Login, session TTL, ACL, token storage, invite lifecycle |
414
+ | **Unit: AdminPanel** | 5/5 ✅ | Private isolation, email dispatch, authorization |
415
+ | **Unit: AppFactory** | 45/45 ✅ | Bootstrap loading, config handling, audio helpers |
416
+ | **Unit: CLI** | 10/10 ✅ | Create-project, init-dataset, build-index, verify |
417
+ | **Integration** | 2/2 ✅ | Validation flow, detection repository |
418
+ | **Total** | **139/139 ✅** | All workflows tested and passing |
419
+
420
+ ---
421
+
422
+ **Last Updated:** 2026-04-01
423
+ **Deployment Status:** Ready for HF Spaces production
424
+ **Latest Commit:** 5e463e3 (feat: security hardening, durable state, email invites)
docs/END_TO_END_QA_CHECKLIST_SPRINT4.md ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # End-to-End QA Checklist - Sprint 4
2
+
3
+ Status legend:
4
+ - PASS = behavior validated
5
+ - FAIL = behavior not validated
6
+ - BLOCKED = could not validate due to environment/tooling limitation
7
+
8
+ ## 5-Minute Smoke Test (Pre-Deploy)
9
+
10
+ Use this quick run before each deploy or before a manual retest cycle.
11
+
12
+ ### A. Start and Health (about 1 min)
13
+ 1. Start app (use a free port if 7860 is busy).
14
+ 2. Open local URL in browser.
15
+ 3. Confirm app loads all tabs.
16
+
17
+ Pass criteria:
18
+ - App boots without traceback.
19
+ - Home screen renders Login, Select Project, Admin, and Validation tabs.
20
+
21
+ ### B. Auth and ACL Core (about 2 min)
22
+ 1. Login as validator_demo.
23
+ 2. Open Select Project tab and verify only allowed projects are shown.
24
+ 3. Open Admin tab and verify access denied with hidden controls.
25
+ 4. Switch user to admin_user.
26
+ 5. Open Admin tab and verify controls are visible.
27
+
28
+ Pass criteria:
29
+ - validator_demo sees only demo-project and kenya-2024.
30
+ - validator_demo cannot use admin actions.
31
+ - admin_user can see admin controls.
32
+
33
+ ### C. Critical Admin Action (about 1 min)
34
+ 1. As admin_user, create a temporary project using slug/name/repo id.
35
+ 2. Click refresh project list.
36
+ 3. Verify new slug appears in project list.
37
+
38
+ Pass criteria:
39
+ - Project creation returns success message.
40
+ - New project appears in list/dropdown immediately.
41
+
42
+ ### D. Validation Readiness (about 1 min)
43
+ 1. Select an authorized project.
44
+ 2. Open Validation tab.
45
+
46
+ Pass criteria:
47
+ - Status changes to ready for selected project.
48
+ - Message "Project not selected" no longer appears after valid selection.
49
+
50
+ ### Smoke Evidence (copy/paste)
51
+ 1. Date/time:
52
+ 2. Tester:
53
+ 3. Build/commit:
54
+ 4. Result: PASS | FAIL
55
+ 5. Notes:
56
+
57
+ ## Scope
58
+ This checklist validates Sprint 4 multi-project security flow:
59
+ - Login and session creation
60
+ - Project authorization per user
61
+ - Admin-only controls
62
+ - Project creation and assignment workflows
63
+ - Validation tab readiness based on selected project
64
+
65
+ ## Test Data (Demo Users)
66
+ - admin_user
67
+ - validator_demo
68
+ - validator_other
69
+
70
+ Expected access matrix:
71
+ - admin_user -> kenya-2024 (admin), nairobi-2023 (admin)
72
+ - validator_demo -> demo-project (validator), kenya-2024 (validator)
73
+ - validator_other -> nairobi-2023 (validator)
74
+
75
+ ## E2E Cases
76
+
77
+ | ID | Area | Precondition | Steps | Expected Result | Status |
78
+ |---|---|---|---|---|---|
79
+ | E2E-01 | App bootstrap | Virtual env active | Start app and open local URL | App starts without runtime exception | PASS |
80
+ | E2E-02 | Login | App running | Login as admin_user | Session is created and user is authenticated | PASS |
81
+ | E2E-03 | Project selector | admin_user logged in | Open Select Project tab | Authorized projects are listed (kenya-2024, nairobi-2023) | PASS |
82
+ | E2E-04 | Validation readiness | admin_user logged in | Open Validation tab | Validation status is ready with selected project | PASS |
83
+ | E2E-05 | Admin visibility | validator_demo logged in | Open Admin tab | Access denied message and hidden admin controls | PASS |
84
+ | E2E-06 | Admin visibility | admin_user logged in | Open Admin tab | Admin controls are visible | PASS |
85
+ | E2E-07 | Project creation | admin_user logged in | Create project with slug/name/repo_id | New project is created and project list is refreshed | PASS |
86
+ | E2E-08 | Project assignment | admin_user logged in | Assign user to a project | Assignment succeeds and access matrix updates | PASS |
87
+ | E2E-09 | Non-admin write protection | validator_demo logged in | Attempt assignment/create actions | Actions are denied by backend callback checks | PASS |
88
+ | E2E-10 | ACL isolation | Test suite available | Run ACL tests | ACL tests pass (project isolation enforced) | PASS |
89
+
90
+ ## Execution Notes
91
+ - This checklist is aligned with fixes applied in `src/ui/app_factory.py`.
92
+ - The app uses Gradio and callback-driven state transitions.
93
+ - ACL behavior is also covered by unit tests in `tests/unit/test_acl_enforcement.py`.
94
+
95
+ ## Evidence Commands
96
+ 1. python -m pytest tests/unit/test_acl_enforcement.py -q
97
+ 2. python -m pytest -q --tb=line
98
+ 3. Start app on a free port and verify local HTTP response
99
+
100
+ ## Sign-off
101
+ - Sprint 4 E2E status: PASS
102
+ - Date: 2026-03-25
docs/spaces/projects.sample.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "project_slug": "kenya-2024",
4
+ "name": "Kenya Survey 2024",
5
+ "dataset_repo_id": "org/kenya-2024-dataset",
6
+ "active": true
7
+ },
8
+ {
9
+ "project_slug": "birds-local",
10
+ "name": "Birds Local Validation",
11
+ "dataset_repo_id": "org/birds-local-dataset",
12
+ "active": true
13
+ }
14
+ ]
docs/spaces/user_access.sample.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "admin_user": {
3
+ "kenya-2024": "admin",
4
+ "birds-local": "admin"
5
+ },
6
+ "validator_a": {
7
+ "kenya-2024": "validator"
8
+ },
9
+ "validator_b": {
10
+ "birds-local": "validator"
11
+ }
12
+ }
pyproject.toml CHANGED
@@ -1,9 +1,12 @@
1
  [project]
2
- name = "BirdNET-validator-App"
3
  version = "0.1.0"
4
- description = "BirdNET detection validator for Hugging Face Spaces"
5
  readme = "README.md"
6
  requires-python = ">=3.11"
7
 
 
 
 
8
  [tool.pytest.ini_options]
9
  testpaths = ["tests"]
 
1
  [project]
2
+ name = "BirdNET-Validator-App"
3
  version = "0.1.0"
4
+ description = "BirdNET-Validator-App for multi-project bird detection review workflows"
5
  readme = "README.md"
6
  requires-python = ">=3.11"
7
 
8
+ [project.scripts]
9
+ birdnet-uploader = "src.uploader_cli.main:cli"
10
+
11
  [tool.pytest.ini_options]
12
  testpaths = ["tests"]
requirements.txt CHANGED
@@ -1,7 +1,12 @@
1
  gradio==5.23.1
2
- huggingface_hub>=0.28.1,<1.0
3
- pydantic==2.10.6
4
- pandas==2.2.3
5
- pyarrow==17.0.0
6
- soundfile==0.12.1
7
- pytest==8.3.4
 
 
 
 
 
 
1
  gradio==5.23.1
2
+ huggingface_hub==0.28.1
3
+ numpy>=2.0.0
4
+ pandas>=2.2.0
5
+ Pillow>=10.4.0
6
+ pyarrow>=16.0.0
7
+ pydantic>=2.8.0
8
+ pytest>=8.3.0
9
+ click>=8.1.7
10
+ rich>=13.7.1
11
+ textual>=0.58.1
12
+ keyring>=25.2.1
src/__pycache__/__init__.cpython-313.pyc CHANGED
Binary files a/src/__pycache__/__init__.cpython-313.pyc and b/src/__pycache__/__init__.cpython-313.pyc differ
 
src/__pycache__/__init__.cpython-314.pyc CHANGED
Binary files a/src/__pycache__/__init__.cpython-314.pyc and b/src/__pycache__/__init__.cpython-314.pyc differ
 
{cli → src/auth}/__init__.py RENAMED
File without changes
src/auth/auth_service.py ADDED
@@ -0,0 +1,684 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AuthService: User authentication, session management, and project ACL."""
2
+
3
+ from dataclasses import dataclass, field
4
+ from datetime import UTC, datetime, timedelta
5
+ from typing import Dict, List, Optional
6
+ import uuid
7
+
8
+ from huggingface_hub import HfApi
9
+
10
+ from src.domain.models import Role, User
11
+
12
+
13
+ @dataclass
14
+ class UserProjectAccess:
15
+ """Maps a user to projects with specific roles."""
16
+
17
+ username: str
18
+ project_slugs: Dict[str, Role] # Maps project_slug -> role (admin or validator)
19
+ is_active: bool = True
20
+
21
+
22
+ @dataclass
23
+ class Session:
24
+ """Represents an authenticated user session."""
25
+
26
+ session_id: str
27
+ username: str
28
+ role: Role
29
+ authorized_projects: List[str] # Projects this user can access
30
+ created_at: datetime
31
+ last_activity: datetime
32
+ expires_at: datetime
33
+
34
+ def is_expired(self) -> bool:
35
+ """Check if session has expired."""
36
+ return datetime.now(UTC) > self.expires_at
37
+
38
+ def update_activity(self, ttl_minutes: int = 120) -> None:
39
+ """Update last activity timestamp and extend expiration."""
40
+ self.last_activity = datetime.now(UTC)
41
+ self.expires_at = datetime.now(UTC) + timedelta(minutes=ttl_minutes)
42
+
43
+
44
+ @dataclass
45
+ class ProjectInvite:
46
+ """Represents an invitation to join a project.
47
+
48
+ Supports 3 scenarios:
49
+ 1. Username only: internal app invite (no email notification)
50
+ 2. Email only: external email-based invite (no prior HF account required)
51
+ 3. Both: dual invite (internal + email notification)
52
+ """
53
+ project_slug: str
54
+ role: Role
55
+ invited_by: str
56
+ created_at: datetime
57
+ expires_at: datetime
58
+ username: str | None = None # HF username (optional)
59
+ invitee_email: str | None = None # Email address (optional)
60
+
61
+ def __post_init__(self) -> None:
62
+ if not self.username and not self.invitee_email:
63
+ raise ValueError("At least one of username or invitee_email must be provided")
64
+
65
+ def is_expired(self) -> bool:
66
+ return datetime.now(UTC) > self.expires_at
67
+
68
+ @property
69
+ def invite_mode(self) -> str:
70
+ """Return the invitation mode: 'username_only', 'email_only', or 'dual'."""
71
+ if self.username and self.invitee_email:
72
+ return "dual"
73
+ elif self.username:
74
+ return "username_only"
75
+ else:
76
+ return "email_only"
77
+
78
+
79
+ @dataclass
80
+ class UserProfile:
81
+ username: str
82
+ hf_email: str | None = None
83
+
84
+
85
+ class AuthService:
86
+ """Manages user authentication, sessions, and project access control."""
87
+
88
+ def __init__(self, session_ttl_minutes: int = 120, invite_ttl_hours: int = 72):
89
+ """Initialize AuthService with session TTL.
90
+
91
+ Args:
92
+ session_ttl_minutes: Session time-to-live in minutes (default: 2 hours)
93
+ """
94
+ self.session_ttl_minutes = session_ttl_minutes
95
+ self.invite_ttl_hours = max(1, int(invite_ttl_hours))
96
+ self._sessions: Dict[str, Session] = {} # session_id -> Session
97
+ self._user_access: Dict[str, UserProjectAccess] = {} # username -> UserProjectAccess
98
+ self._pending_invites: Dict[str, Dict[str, ProjectInvite]] = {}
99
+ self._hf_tokens_by_username: Dict[str, str] = {}
100
+ self._user_profiles: Dict[str, UserProfile] = {}
101
+
102
+ def _refresh_or_revoke_sessions_for_username(self, username: str) -> None:
103
+ access = self._user_access.get(username)
104
+ session_ids = [sid for sid, session in self._sessions.items() if session.username == username]
105
+ if access is None or not access.is_active:
106
+ for session_id in session_ids:
107
+ self._sessions.pop(session_id, None)
108
+ return
109
+
110
+ projects = list(access.project_slugs.keys())
111
+ role = Role.admin if any(project_role == Role.admin for project_role in access.project_slugs.values()) else Role.validator
112
+ for session_id in session_ids:
113
+ session = self._sessions.get(session_id)
114
+ if session is None:
115
+ continue
116
+ session.authorized_projects = projects
117
+ session.role = role
118
+ session.update_activity(self.session_ttl_minutes)
119
+
120
+ def _prune_expired_invites(self) -> None:
121
+ expired_users: list[str] = []
122
+ for username, invites_by_project in self._pending_invites.items():
123
+ expired_projects = [project_slug for project_slug, invite in invites_by_project.items() if invite.is_expired()]
124
+ for project_slug in expired_projects:
125
+ del invites_by_project[project_slug]
126
+ if not invites_by_project:
127
+ expired_users.append(username)
128
+
129
+ for username in expired_users:
130
+ self._pending_invites.pop(username, None)
131
+
132
+ def register_user_project_access(
133
+ self, username: str, project_access: Dict[str, Role]
134
+ ) -> None:
135
+ """Register or update a user's access to projects.
136
+
137
+ Args:
138
+ username: User name
139
+ project_access: Dict mapping project_slug to Role (admin or validator)
140
+ """
141
+ self._user_access[username] = UserProjectAccess(
142
+ username=username,
143
+ project_slugs=project_access,
144
+ is_active=True,
145
+ )
146
+
147
+ def list_usernames(self, include_inactive: bool = False) -> List[str]:
148
+ """List registered usernames.
149
+
150
+ Args:
151
+ include_inactive: Include disabled users when True
152
+
153
+ Returns:
154
+ Sorted list of usernames
155
+ """
156
+ usernames = []
157
+ for username, access in self._user_access.items():
158
+ if include_inactive or access.is_active:
159
+ usernames.append(username)
160
+ return sorted(usernames)
161
+
162
+ def upsert_user_project_role(self, username: str, project_slug: str, role: Role) -> None:
163
+ """Create/update user assignment for a project.
164
+
165
+ Args:
166
+ username: User name
167
+ project_slug: Project slug
168
+ role: Role to apply for this project
169
+ """
170
+ access = self._user_access.get(username)
171
+ if access is None:
172
+ access = UserProjectAccess(username=username, project_slugs={}, is_active=True)
173
+ self._user_access[username] = access
174
+
175
+ access.project_slugs[project_slug] = role
176
+ self._refresh_or_revoke_sessions_for_username(username)
177
+
178
+ def remove_user_project_role(self, username: str, project_slug: str) -> bool:
179
+ """Remove a user assignment from a project.
180
+
181
+ Args:
182
+ username: User name
183
+ project_slug: Project slug
184
+
185
+ Returns:
186
+ True when assignment existed and was removed
187
+ """
188
+ access = self._user_access.get(username)
189
+ if access is None:
190
+ return False
191
+
192
+ if project_slug not in access.project_slugs:
193
+ return False
194
+
195
+ del access.project_slugs[project_slug]
196
+ pending = self._pending_invites.get(username, {})
197
+ if project_slug in pending:
198
+ del pending[project_slug]
199
+ if not pending:
200
+ self._pending_invites.pop(username, None)
201
+ self._refresh_or_revoke_sessions_for_username(username)
202
+ return True
203
+
204
+ def login(self, username: str) -> Optional[Session]:
205
+ """Authenticate a user and create a new session.
206
+
207
+ Args:
208
+ username: User name to authenticate
209
+
210
+ Returns:
211
+ Session if user exists and is active, None otherwise
212
+ """
213
+ return self.login_internal(username=username)
214
+
215
+ def login_internal(
216
+ self,
217
+ username: str,
218
+ auto_promote_to_admin: bool = False,
219
+ ) -> Optional[Session]:
220
+ if username not in self._user_access:
221
+ if not auto_promote_to_admin:
222
+ self._prune_expired_invites()
223
+ pending_for_user = self._pending_invites.get(username, {})
224
+ if not pending_for_user:
225
+ return None
226
+ self._user_access[username] = UserProjectAccess(
227
+ username=username,
228
+ project_slugs={},
229
+ is_active=True,
230
+ )
231
+
232
+ access = self._user_access[username]
233
+ if not access.is_active:
234
+ return None
235
+
236
+ # Determine highest role across all projects (admin > validator)
237
+ role = Role.admin if auto_promote_to_admin else Role.validator
238
+ if role != Role.admin:
239
+ for proj_role in access.project_slugs.values():
240
+ if proj_role == Role.admin:
241
+ role = Role.admin
242
+ break
243
+
244
+ session_id = str(uuid.uuid4())
245
+ now = datetime.now(UTC)
246
+ session = Session(
247
+ session_id=session_id,
248
+ username=username,
249
+ role=role,
250
+ authorized_projects=list(access.project_slugs.keys()),
251
+ created_at=now,
252
+ last_activity=now,
253
+ expires_at=now + timedelta(minutes=self.session_ttl_minutes),
254
+ )
255
+
256
+ self._sessions[session_id] = session
257
+ return session
258
+
259
+ def login_with_hf_token(self, token: str) -> tuple[Optional[Session], str]:
260
+ """Authenticate using Hugging Face personal token and resolve username via whoami."""
261
+ token_value = (token or "").strip()
262
+ if not token_value:
263
+ return None, "❌ Please provide a Hugging Face token"
264
+
265
+ try:
266
+ whoami = HfApi().whoami(token=token_value)
267
+ except Exception:
268
+ return None, "❌ Invalid Hugging Face token or network error"
269
+
270
+ username = str(whoami.get("name") or "").strip()
271
+ if not username:
272
+ return None, "❌ Unable to resolve Hugging Face username from token"
273
+
274
+ email_value = str(whoami.get("email") or "").strip() or None
275
+ self._hf_tokens_by_username[username] = token_value
276
+ self._user_profiles[username] = UserProfile(username=username, hf_email=email_value)
277
+
278
+ is_first_user = len(self._user_access) == 0
279
+ session = self.login_internal(
280
+ username=username,
281
+ auto_promote_to_admin=is_first_user,
282
+ )
283
+ if session is None:
284
+ return None, f"❌ User '{username}' is not invited to any project yet"
285
+
286
+ if is_first_user:
287
+ return session, f"✅ Welcome, {username}! (Admin)"
288
+ if session.role == Role.admin:
289
+ return session, f"✅ Welcome, {username}! (Admin)"
290
+ return session, f"✅ Welcome, {username}! (Validator)"
291
+
292
+ def get_session(self, session_id: str) -> Optional[Session]:
293
+ """Retrieve an active session by ID.
294
+
295
+ Args:
296
+ session_id: Session ID
297
+
298
+ Returns:
299
+ Session if valid and not expired, None otherwise
300
+ """
301
+ session = self._sessions.get(session_id)
302
+ if session is None:
303
+ return None
304
+
305
+ if session.is_expired():
306
+ self._sessions.pop(session_id, None)
307
+ return None
308
+
309
+ session.update_activity(self.session_ttl_minutes)
310
+ return session
311
+
312
+ def logout(self, session_id: str) -> None:
313
+ """Delete a session.
314
+
315
+ Args:
316
+ session_id: Session ID to invalidate
317
+ """
318
+ self._sessions.pop(session_id, None)
319
+
320
+ def is_user_authorized_for_project(self, username: str, project_slug: str) -> bool:
321
+ """Check if a user has access to a specific project.
322
+
323
+ Args:
324
+ username: User name
325
+ project_slug: Project slug
326
+
327
+ Returns:
328
+ True if user has access (admin or validator), False otherwise
329
+ """
330
+ access = self._user_access.get(username)
331
+ if access is None or not access.is_active:
332
+ return False
333
+
334
+ return project_slug in access.project_slugs
335
+
336
+ def get_user_role_for_project(self, username: str, project_slug: str) -> Optional[Role]:
337
+ """Get the specific role a user has for a project.
338
+
339
+ Args:
340
+ username: User name
341
+ project_slug: Project slug
342
+
343
+ Returns:
344
+ Role if user has access, None otherwise
345
+ """
346
+ access = self._user_access.get(username)
347
+ if access is None or not access.is_active:
348
+ return None
349
+
350
+ return access.project_slugs.get(project_slug)
351
+
352
+ def list_user_projects(self, username: str) -> List[str]:
353
+ """List all projects a user has access to.
354
+
355
+ Args:
356
+ username: User name
357
+
358
+ Returns:
359
+ List of project slugs
360
+ """
361
+ access = self._user_access.get(username)
362
+ if access is None or not access.is_active:
363
+ return []
364
+
365
+ return list(access.project_slugs.keys())
366
+
367
+ def cleanup_expired_sessions(self) -> None:
368
+ """Remove all expired sessions (maintenance cleanup)."""
369
+ now = datetime.now(UTC)
370
+ expired = [
371
+ sid for sid, session in self._sessions.items() if session.is_expired()
372
+ ]
373
+ for sid in expired:
374
+ self._sessions.pop(sid)
375
+
376
+ def refresh_session_authorizations(self, session_id: str) -> Optional[Session]:
377
+ """Refresh a session with latest role and project assignments."""
378
+ session = self._sessions.get(session_id)
379
+ if session is None:
380
+ return None
381
+ if session.is_expired():
382
+ self._sessions.pop(session_id, None)
383
+ return None
384
+
385
+ access = self._user_access.get(session.username)
386
+ if access is None or not access.is_active:
387
+ self._sessions.pop(session_id, None)
388
+ return None
389
+
390
+ projects = list(access.project_slugs.keys())
391
+ session.authorized_projects = projects
392
+ session.role = Role.admin if any(role == Role.admin for role in access.project_slugs.values()) else Role.validator
393
+ session.update_activity(self.session_ttl_minutes)
394
+ return session
395
+
396
+ def set_user_active(self, username: str, active: bool) -> None:
397
+ """Enable or disable a user account.
398
+
399
+ Args:
400
+ username: User name
401
+ active: Whether the user should be active
402
+ """
403
+ if username in self._user_access:
404
+ self._user_access[username].is_active = active
405
+ if not active:
406
+ self._hf_tokens_by_username.pop(username, None)
407
+ self._refresh_or_revoke_sessions_for_username(username)
408
+
409
+ def get_hf_token_for_user(self, username: str) -> str | None:
410
+ token = (self._hf_tokens_by_username.get(username) or "").strip()
411
+ return token or None
412
+
413
+ def get_known_email_for_user(self, username: str) -> str | None:
414
+ profile = self._user_profiles.get(username)
415
+ if profile is None:
416
+ return None
417
+ email = (profile.hf_email or "").strip()
418
+ return email or None
419
+
420
+ def _pending_invite_keys_for_username(self, username: str) -> list[str]:
421
+ keys = [username]
422
+ known_email = self.get_known_email_for_user(username)
423
+ if known_email:
424
+ keys.insert(0, f"email:{known_email}")
425
+ return keys
426
+
427
+ def _find_pending_invite_bucket(self, username: str, project_slug: str) -> tuple[str | None, dict[str, ProjectInvite] | None, ProjectInvite | None]:
428
+ for invite_key in self._pending_invite_keys_for_username(username):
429
+ pending = self._pending_invites.get(invite_key, {})
430
+ invite = pending.get(project_slug)
431
+ if invite is not None:
432
+ return invite_key, pending, invite
433
+ return None, None, None
434
+
435
+ def create_project_invite(
436
+ self,
437
+ project_slug: str,
438
+ role: Role,
439
+ invited_by: str,
440
+ username: str | None = None,
441
+ invitee_email: str | None = None,
442
+ ) -> tuple[bool, str]:
443
+ """Create a project invite supporting 3 scenarios:
444
+
445
+ 1. Username only: internal app invite (no email notification)
446
+ 2. Email only: external email-based invite
447
+ 3. Both: dual invite (internal + email notification)
448
+
449
+ Args:
450
+ project_slug: Target project
451
+ role: Role to assign
452
+ invited_by: Username of the inviter
453
+ username: (Optional) HF username of invitee
454
+ invitee_email: (Optional) Email address of invitee
455
+
456
+ Returns:
457
+ (success, message) tuple
458
+ """
459
+ username = (username or "").strip() or None
460
+ invitee_email = (invitee_email or "").strip() or None
461
+
462
+ if not username and not invitee_email:
463
+ return False, "Please provide either a username or an email address"
464
+
465
+ self._prune_expired_invites()
466
+
467
+ # For username-based invites, check if already exists
468
+ if username:
469
+ user_invites = self._pending_invites.get(username, {})
470
+ if project_slug in user_invites:
471
+ return False, f"Invite for {username} to {project_slug} already exists"
472
+
473
+ now = datetime.now(UTC)
474
+ expires_at = now + timedelta(hours=self.invite_ttl_hours)
475
+
476
+ invite = ProjectInvite(
477
+ project_slug=project_slug,
478
+ role=role,
479
+ invited_by=invited_by,
480
+ created_at=now,
481
+ expires_at=expires_at,
482
+ username=username,
483
+ invitee_email=invitee_email,
484
+ )
485
+
486
+ # Store by username if available, otherwise by email (as temporary key)
487
+ if username:
488
+ self._pending_invites.setdefault(username, {})[project_slug] = invite
489
+ else:
490
+ # For email-only invites, use email as the storage key
491
+ # This allows the user to later link their username
492
+ email_key = f"email:{invitee_email}"
493
+ self._pending_invites.setdefault(email_key, {})[project_slug] = invite
494
+
495
+ mode = invite.invite_mode
496
+ summary = f"✅ {mode.replace('_', ' ').title()} invite"
497
+ return True, summary
498
+
499
+ def list_pending_invites(self, username: str) -> List[ProjectInvite]:
500
+ self._prune_expired_invites()
501
+ invites: list[ProjectInvite] = []
502
+ seen_projects: set[str] = set()
503
+ for invite_key in self._pending_invite_keys_for_username(username):
504
+ for invite in self._pending_invites.get(invite_key, {}).values():
505
+ if invite.project_slug in seen_projects:
506
+ continue
507
+ seen_projects.add(invite.project_slug)
508
+ invites.append(invite)
509
+ invites.sort(key=lambda item: item.created_at)
510
+ return invites
511
+
512
+ def list_all_pending_invites(self) -> List[ProjectInvite]:
513
+ self._prune_expired_invites()
514
+ invites: list[ProjectInvite] = []
515
+ for invites_by_project in self._pending_invites.values():
516
+ invites.extend(invites_by_project.values())
517
+ invites.sort(key=lambda item: item.created_at)
518
+ return invites
519
+
520
+ def accept_project_invite(self, username: str, project_slug: str) -> tuple[bool, str]:
521
+ self._prune_expired_invites()
522
+ invite_key, pending, invite = self._find_pending_invite_bucket(username, project_slug)
523
+ if invite_key is None or pending is None or invite is None:
524
+ return False, f"Invite not found for {username} in {project_slug}"
525
+
526
+ self.upsert_user_project_role(username, project_slug, invite.role)
527
+ del pending[project_slug]
528
+ if not pending:
529
+ self._pending_invites.pop(invite_key, None)
530
+ return True, f"✅ Invite accepted for {project_slug} as {invite.role.value}"
531
+
532
+ def reject_project_invite(self, username: str, project_slug: str) -> tuple[bool, str]:
533
+ self._prune_expired_invites()
534
+ invite_key, pending, invite = self._find_pending_invite_bucket(username, project_slug)
535
+ if invite_key is None or pending is None or invite is None:
536
+ return False, f"Invite not found for {username} in {project_slug}"
537
+
538
+ del pending[project_slug]
539
+ if not pending:
540
+ self._pending_invites.pop(invite_key, None)
541
+ return True, f"✅ Invite rejected for {project_slug}"
542
+
543
+ def accept_all_project_invites(self, username: str) -> tuple[int, int, str]:
544
+ self._prune_expired_invites()
545
+ pending_keys = self._pending_invite_keys_for_username(username)
546
+ pending: dict[str, ProjectInvite] = {}
547
+ for invite_key in pending_keys:
548
+ pending.update(self._pending_invites.get(invite_key, {}))
549
+
550
+ if not pending:
551
+ return 0, 0, "No pending invites"
552
+
553
+ accepted = 0
554
+ failed = 0
555
+ for project_slug in list(pending.keys()):
556
+ ok, _ = self.accept_project_invite(username, project_slug)
557
+ if ok:
558
+ accepted += 1
559
+ else:
560
+ failed += 1
561
+ return accepted, failed, f"Accepted {accepted} invites" + (f", {failed} failed" if failed else "")
562
+
563
+ def revoke_project_invite(self, username: str, project_slug: str) -> tuple[bool, str]:
564
+ self._prune_expired_invites()
565
+ invite_key, pending, invite = self._find_pending_invite_bucket(username, project_slug)
566
+ if invite_key is None or pending is None or invite is None:
567
+ return False, f"Invite not found for {username} in {project_slug}"
568
+ del pending[project_slug]
569
+ if not pending:
570
+ self._pending_invites.pop(invite_key, None)
571
+ return True, f"✅ Invite revoked for {username} in {project_slug}"
572
+
573
+ def export_user_access_map(self, include_inactive: bool = False) -> Dict[str, Dict[str, str]]:
574
+ """Export user/project role mapping for persistence."""
575
+ exported: Dict[str, Dict[str, str]] = {}
576
+ for username in self.list_usernames(include_inactive=include_inactive):
577
+ access = self._user_access.get(username)
578
+ if access is None:
579
+ continue
580
+ exported[username] = {project_slug: role.value for project_slug, role in access.project_slugs.items()}
581
+ return exported
582
+
583
+ def export_pending_invites_map(self) -> Dict[str, Dict[str, Dict[str, str]]]:
584
+ self._prune_expired_invites()
585
+ payload: Dict[str, Dict[str, Dict[str, str]]] = {}
586
+ for username, invites_by_project in self._pending_invites.items():
587
+ payload[username] = {}
588
+ for project_slug, invite in invites_by_project.items():
589
+ payload[username][project_slug] = {
590
+ "role": invite.role.value,
591
+ "invited_by": invite.invited_by,
592
+ "created_at": invite.created_at.isoformat(),
593
+ "expires_at": invite.expires_at.isoformat(),
594
+ }
595
+ return payload
596
+
597
+ def load_pending_invites_map(self, payload: Dict[str, Dict[str, Dict[str, str]]]) -> None:
598
+ self._pending_invites = {}
599
+ for username, invites_by_project in (payload or {}).items():
600
+ if not isinstance(invites_by_project, dict):
601
+ continue
602
+ user_invites: Dict[str, ProjectInvite] = {}
603
+ for project_slug, invite_payload in invites_by_project.items():
604
+ if not isinstance(invite_payload, dict):
605
+ continue
606
+ role_text = str(invite_payload.get("role", "")).strip().lower()
607
+ if role_text not in {"admin", "validator"}:
608
+ continue
609
+ invited_by = str(invite_payload.get("invited_by", "")).strip() or "admin"
610
+ created_at_raw = str(invite_payload.get("created_at", "")).strip()
611
+ expires_at_raw = str(invite_payload.get("expires_at", "")).strip()
612
+ try:
613
+ created_at = datetime.fromisoformat(created_at_raw) if created_at_raw else datetime.now(UTC)
614
+ except Exception:
615
+ created_at = datetime.now(UTC)
616
+ try:
617
+ expires_at = datetime.fromisoformat(expires_at_raw) if expires_at_raw else (created_at + timedelta(hours=self.invite_ttl_hours))
618
+ except Exception:
619
+ expires_at = created_at + timedelta(hours=self.invite_ttl_hours)
620
+ if created_at.tzinfo is None:
621
+ created_at = created_at.replace(tzinfo=UTC)
622
+ if expires_at.tzinfo is None:
623
+ expires_at = expires_at.replace(tzinfo=UTC)
624
+ user_invites[str(project_slug)] = ProjectInvite(
625
+ username=str(username),
626
+ project_slug=str(project_slug),
627
+ role=Role(role_text),
628
+ invited_by=invited_by,
629
+ created_at=created_at,
630
+ expires_at=expires_at,
631
+ )
632
+
633
+ if user_invites:
634
+ self._pending_invites[str(username)] = user_invites
635
+
636
+ self._prune_expired_invites()
637
+
638
+ def revoke_all_invites_for_project(self, project_slug: str) -> int:
639
+ """Remove all pending invites for a project.
640
+
641
+ Returns:
642
+ Number of revoked invites
643
+ """
644
+ self._prune_expired_invites()
645
+ revoked = 0
646
+ empty_users: list[str] = []
647
+ for username, invites_by_project in self._pending_invites.items():
648
+ if project_slug in invites_by_project:
649
+ del invites_by_project[project_slug]
650
+ revoked += 1
651
+ if not invites_by_project:
652
+ empty_users.append(username)
653
+
654
+ for username in empty_users:
655
+ self._pending_invites.pop(username, None)
656
+
657
+ return revoked
658
+
659
+ def remove_project_from_all_users(self, project_slug: str) -> int:
660
+ """Remove a project assignment from all users and refresh active sessions.
661
+
662
+ Returns:
663
+ Number of user assignments removed
664
+ """
665
+ removed = 0
666
+ for access in self._user_access.values():
667
+ if project_slug in access.project_slugs:
668
+ del access.project_slugs[project_slug]
669
+ removed += 1
670
+
671
+ for session in self._sessions.values():
672
+ if project_slug in session.authorized_projects:
673
+ session.authorized_projects = [slug for slug in session.authorized_projects if slug != project_slug]
674
+
675
+ access = self._user_access.get(session.username)
676
+ if access is None or not access.is_active:
677
+ continue
678
+ session.role = (
679
+ Role.admin
680
+ if any(role == Role.admin for role in access.project_slugs.values())
681
+ else Role.validator
682
+ )
683
+
684
+ return removed
src/cache/__init__.py ADDED
File without changes
src/cache/ephemeral_cache_manager.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import hashlib
2
+ import tempfile
3
+ import time
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+
7
+
8
+ @dataclass(slots=True)
9
+ class CacheEntry:
10
+ key: str
11
+ path: Path
12
+ created_at: float
13
+ expires_at: float
14
+ size_bytes: int
15
+
16
+
17
+ class EphemeralCacheManager:
18
+ def __init__(
19
+ self,
20
+ cache_dir: str | None = None,
21
+ ttl_seconds: int = 300,
22
+ max_files: int = 256,
23
+ ) -> None:
24
+ self._base_dir = Path(cache_dir) if cache_dir else Path(tempfile.gettempdir()) / "birdnet-validator-cache"
25
+ self._base_dir.mkdir(parents=True, exist_ok=True)
26
+ self._ttl_seconds = ttl_seconds
27
+ self._max_files = max_files
28
+ self._entries: dict[str, CacheEntry] = {}
29
+
30
+ def get(self, key: str) -> Path | None:
31
+ self.cleanup_expired()
32
+ entry = self._entries.get(key)
33
+ if not entry:
34
+ return None
35
+ if not entry.path.exists():
36
+ self._entries.pop(key, None)
37
+ return None
38
+ return entry.path
39
+
40
+ def put_bytes(self, key: str, data: bytes, suffix: str = ".bin") -> Path:
41
+ self.cleanup_expired()
42
+ self._enforce_capacity()
43
+
44
+ digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
45
+ file_path = self._base_dir / f"{digest}{suffix}"
46
+ file_path.write_bytes(data)
47
+
48
+ now = time.time()
49
+ self._entries[key] = CacheEntry(
50
+ key=key,
51
+ path=file_path,
52
+ created_at=now,
53
+ expires_at=now + self._ttl_seconds,
54
+ size_bytes=len(data),
55
+ )
56
+ return file_path
57
+
58
+ def cleanup_key(self, key: str) -> None:
59
+ entry = self._entries.pop(key, None)
60
+ if not entry:
61
+ return
62
+ if entry.path.exists():
63
+ entry.path.unlink()
64
+
65
+ def cleanup_expired(self) -> None:
66
+ now = time.time()
67
+ expired_keys = [key for key, entry in self._entries.items() if entry.expires_at <= now or not entry.path.exists()]
68
+ for key in expired_keys:
69
+ self.cleanup_key(key)
70
+
71
+ def clear(self) -> None:
72
+ for key in list(self._entries.keys()):
73
+ self.cleanup_key(key)
74
+
75
+ def _enforce_capacity(self) -> None:
76
+ if len(self._entries) < self._max_files:
77
+ return
78
+
79
+ oldest_keys = sorted(self._entries.keys(), key=lambda key: self._entries[key].created_at)
80
+ overflow = (len(self._entries) - self._max_files) + 1
81
+ for key in oldest_keys[:overflow]:
82
+ self.cleanup_key(key)
src/cli/__init__.py ADDED
File without changes
src/cli/project_cli.py ADDED
@@ -0,0 +1,339 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+ import re
6
+ from pathlib import Path
7
+ from typing import Any
8
+ from uuid import uuid4
9
+
10
+
11
+ DEFAULT_DATASET_DIRS = [
12
+ "audio",
13
+ "detections",
14
+ "index",
15
+ "exports",
16
+ ]
17
+
18
+ EXIT_OK = 0
19
+ EXIT_VALIDATION_ERROR = 1
20
+ EXIT_IO_ERROR = 2
21
+
22
+ SLUG_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]{1,62}$")
23
+ REPO_ID_PATTERN = re.compile(r"^[^\s/]+/[^\s/]+$")
24
+
25
+
26
+ class CliValidationError(ValueError):
27
+ pass
28
+
29
+
30
+ def _safe_read_text(path: Path) -> str:
31
+ try:
32
+ return path.read_text(encoding="utf-8")
33
+ except FileNotFoundError:
34
+ raise
35
+ except UnicodeDecodeError as exc:
36
+ raise CliValidationError(f"Invalid UTF-8 content in {path}: {exc}") from exc
37
+ except OSError as exc:
38
+ raise OSError(f"Could not read {path}: {exc}") from exc
39
+
40
+
41
+ def _validate_slug(slug: str) -> None:
42
+ if not SLUG_PATTERN.fullmatch(slug):
43
+ raise CliValidationError(
44
+ "Invalid slug. Use lowercase letters, numbers, and hyphens, length 2-63."
45
+ )
46
+
47
+
48
+ def _validate_repo_id(repo_id: str) -> None:
49
+ if not REPO_ID_PATTERN.fullmatch(repo_id):
50
+ raise CliValidationError(
51
+ "Invalid dataset_repo_id. Expected format 'owner/repo' without spaces."
52
+ )
53
+
54
+
55
+ def _load_json(path: Path, default: Any) -> Any:
56
+ if not path.exists():
57
+ return default
58
+ raw = _safe_read_text(path)
59
+ try:
60
+ return json.loads(raw)
61
+ except json.JSONDecodeError as exc:
62
+ raise CliValidationError(f"Invalid JSON in {path}: {exc}") from exc
63
+
64
+
65
+ def _write_json(path: Path, payload: Any) -> None:
66
+ path.parent.mkdir(parents=True, exist_ok=True)
67
+ temp_path = path.with_suffix(path.suffix + ".tmp")
68
+ temp_path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
69
+ temp_path.replace(path)
70
+
71
+
72
+ def _project_exists(projects: list[dict[str, Any]], slug: str) -> bool:
73
+ return any(p.get("project_slug") == slug for p in projects)
74
+
75
+
76
+ def _as_project_list(payload: Any, source: Path) -> list[dict[str, Any]]:
77
+ if not isinstance(payload, list):
78
+ raise CliValidationError(f"Expected list in {source}, got {type(payload).__name__}")
79
+ for idx, item in enumerate(payload):
80
+ if not isinstance(item, dict):
81
+ raise CliValidationError(f"Project entry at index {idx} in {source} is not an object")
82
+ for field in ("project_slug", "name", "dataset_repo_id"):
83
+ if field not in item or not isinstance(item[field], str) or not item[field].strip():
84
+ raise CliValidationError(
85
+ f"Project entry at index {idx} in {source} has invalid '{field}'"
86
+ )
87
+ return payload
88
+
89
+
90
+ def _as_access_map(payload: Any, source: Path) -> dict[str, dict[str, str]]:
91
+ if not isinstance(payload, dict):
92
+ raise CliValidationError(f"Expected object in {source}, got {type(payload).__name__}")
93
+ for username, roles in payload.items():
94
+ if not isinstance(username, str) or not username.strip():
95
+ raise CliValidationError(f"Invalid username key in {source}")
96
+ if not isinstance(roles, dict):
97
+ raise CliValidationError(f"Roles for '{username}' in {source} must be an object")
98
+ for project_slug, role in roles.items():
99
+ if not isinstance(project_slug, str) or not project_slug.strip():
100
+ raise CliValidationError(f"Invalid project slug for user '{username}' in {source}")
101
+ if role not in {"admin", "validator"}:
102
+ raise CliValidationError(
103
+ f"Invalid role '{role}' for user '{username}' and project '{project_slug}' in {source}"
104
+ )
105
+ return payload
106
+
107
+
108
+ def cmd_create_project(args: argparse.Namespace) -> int:
109
+ _validate_slug(args.slug)
110
+ _validate_repo_id(args.dataset_repo_id)
111
+
112
+ if args.owner and not args.owner.strip():
113
+ raise CliValidationError("Owner cannot be blank")
114
+
115
+ projects_file = Path(args.projects_file)
116
+ projects = _as_project_list(_load_json(projects_file, []), projects_file)
117
+
118
+ if _project_exists(projects, args.slug):
119
+ print(f"ERROR: project '{args.slug}' already exists")
120
+ return EXIT_VALIDATION_ERROR
121
+
122
+ project_entry: dict[str, Any] = {
123
+ "project_id": str(uuid4()),
124
+ "project_slug": args.slug,
125
+ "name": args.name,
126
+ "dataset_repo_id": args.dataset_repo_id,
127
+ "visibility": args.visibility,
128
+ "active": not args.inactive,
129
+ }
130
+ if args.owner:
131
+ project_entry["owner_username"] = args.owner
132
+ if args.dataset_token:
133
+ project_entry["dataset_token"] = args.dataset_token
134
+
135
+ projects.append(project_entry)
136
+ projects.sort(key=lambda p: str(p.get("project_slug", "")))
137
+ _write_json(projects_file, projects)
138
+
139
+ if args.user_access_file and args.owner:
140
+ access_file = Path(args.user_access_file)
141
+ access_payload = _as_access_map(_load_json(access_file, {}), access_file)
142
+ owner_roles = access_payload.setdefault(args.owner, {})
143
+ owner_roles[args.slug] = "admin"
144
+ _write_json(access_file, access_payload)
145
+
146
+ print(f"OK: project '{args.slug}' created")
147
+ return EXIT_OK
148
+
149
+
150
+ def cmd_init_dataset(args: argparse.Namespace) -> int:
151
+ _validate_slug(args.slug)
152
+ _validate_repo_id(args.dataset_repo_id)
153
+
154
+ root = Path(args.dataset_root) / args.slug
155
+ for dirname in DEFAULT_DATASET_DIRS:
156
+ (root / dirname).mkdir(parents=True, exist_ok=True)
157
+
158
+ detections_file = root / "detections" / "detections.jsonl"
159
+ if not detections_file.exists():
160
+ detections_file.write_text("", encoding="utf-8")
161
+
162
+ metadata = {
163
+ "project_slug": args.slug,
164
+ "dataset_repo_id": args.dataset_repo_id,
165
+ }
166
+ if args.name:
167
+ metadata["name"] = args.name
168
+
169
+ _write_json(root / "index" / "project_metadata.json", metadata)
170
+ print(f"OK: dataset scaffold initialized at {root}")
171
+ return EXIT_OK
172
+
173
+
174
+ def _read_jsonl(path: Path) -> list[dict[str, Any]]:
175
+ rows: list[dict[str, Any]] = []
176
+ if not path.exists():
177
+ return rows
178
+
179
+ raw_content = _safe_read_text(path)
180
+ for line_no, raw in enumerate(raw_content.splitlines(), start=1):
181
+ line = raw.strip()
182
+ if not line:
183
+ continue
184
+ try:
185
+ parsed = json.loads(line)
186
+ except json.JSONDecodeError as exc:
187
+ raise CliValidationError(
188
+ f"Invalid JSONL in {path} at line {line_no}: {exc.msg} (column {exc.colno})"
189
+ ) from exc
190
+
191
+ if not isinstance(parsed, dict):
192
+ raise CliValidationError(
193
+ f"Invalid JSONL in {path} at line {line_no}: expected object per line"
194
+ )
195
+
196
+ required = ("detection_key", "audio_id", "scientific_name", "confidence")
197
+ for field in required:
198
+ if field not in parsed:
199
+ raise CliValidationError(
200
+ f"Invalid JSONL in {path} at line {line_no}: missing '{field}'"
201
+ )
202
+
203
+ try:
204
+ parsed["confidence"] = float(parsed["confidence"])
205
+ except (TypeError, ValueError) as exc:
206
+ raise CliValidationError(
207
+ f"Invalid JSONL in {path} at line {line_no}: 'confidence' must be numeric"
208
+ ) from exc
209
+
210
+ rows.append(parsed)
211
+ return rows
212
+
213
+
214
+ def cmd_build_index(args: argparse.Namespace) -> int:
215
+ _validate_slug(args.slug)
216
+
217
+ project_root = Path(args.dataset_root) / args.slug
218
+ detections_file = project_root / "detections" / "detections.jsonl"
219
+ rows = _read_jsonl(detections_file)
220
+
221
+ sorted_rows = sorted(rows, key=lambda d: d["confidence"], reverse=True)
222
+ index_payload = {
223
+ "project_slug": args.slug,
224
+ "count": len(sorted_rows),
225
+ "detections": [
226
+ {
227
+ "detection_key": r.get("detection_key"),
228
+ "audio_id": r.get("audio_id"),
229
+ "scientific_name": r.get("scientific_name"),
230
+ "confidence": r["confidence"],
231
+ }
232
+ for r in sorted_rows
233
+ ],
234
+ }
235
+
236
+ _write_json(project_root / "index" / "detections_index.json", index_payload)
237
+ print(f"OK: index generated with {len(sorted_rows)} detections")
238
+ return EXIT_OK
239
+
240
+
241
+ def cmd_verify_project(args: argparse.Namespace) -> int:
242
+ _validate_slug(args.slug)
243
+
244
+ projects_file = Path(args.projects_file)
245
+ projects = _as_project_list(_load_json(projects_file, []), projects_file)
246
+
247
+ findings: list[str] = []
248
+
249
+ if not _project_exists(projects, args.slug):
250
+ findings.append(f"project '{args.slug}' not found in {projects_file}")
251
+
252
+ project_root = Path(args.dataset_root) / args.slug
253
+ for dirname in DEFAULT_DATASET_DIRS:
254
+ expected = project_root / dirname
255
+ if not expected.exists():
256
+ findings.append(f"missing path: {expected}")
257
+
258
+ required_files = [
259
+ project_root / "detections" / "detections.jsonl",
260
+ project_root / "index" / "project_metadata.json",
261
+ project_root / "index" / "detections_index.json",
262
+ ]
263
+ for required in required_files:
264
+ if not required.exists():
265
+ findings.append(f"missing path: {required}")
266
+
267
+ if findings:
268
+ if args.dry_run:
269
+ print("DRY-RUN: project verification found issues:")
270
+ else:
271
+ print("ERROR: project verification failed:")
272
+ for item in findings:
273
+ print(f" - {item}")
274
+ if args.dry_run:
275
+ print("DRY-RUN: completed without failing exit code")
276
+ return EXIT_OK
277
+ return EXIT_VALIDATION_ERROR
278
+
279
+ print(f"OK: project '{args.slug}' verified")
280
+ return EXIT_OK
281
+
282
+
283
+ def build_parser() -> argparse.ArgumentParser:
284
+ parser = argparse.ArgumentParser(prog="birdnet-project")
285
+ sub = parser.add_subparsers(dest="command", required=True)
286
+
287
+ create_project = sub.add_parser("create-project", help="Create project in bootstrap JSON")
288
+ create_project.add_argument("--projects-file", required=True)
289
+ create_project.add_argument("--user-access-file")
290
+ create_project.add_argument("--slug", required=True)
291
+ create_project.add_argument("--name", required=True)
292
+ create_project.add_argument("--dataset-repo-id", required=True)
293
+ create_project.add_argument("--visibility", choices=["private", "collaborative"], default="collaborative")
294
+ create_project.add_argument("--owner")
295
+ create_project.add_argument("--dataset-token")
296
+ create_project.add_argument("--inactive", action="store_true")
297
+ create_project.set_defaults(func=cmd_create_project)
298
+
299
+ init_dataset = sub.add_parser("init-dataset", help="Create initial dataset folder scaffold")
300
+ init_dataset.add_argument("--dataset-root", required=True)
301
+ init_dataset.add_argument("--slug", required=True)
302
+ init_dataset.add_argument("--dataset-repo-id", required=True)
303
+ init_dataset.add_argument("--name")
304
+ init_dataset.set_defaults(func=cmd_init_dataset)
305
+
306
+ build_index = sub.add_parser("build-index", help="Build simple index from detections JSONL")
307
+ build_index.add_argument("--dataset-root", required=True)
308
+ build_index.add_argument("--slug", required=True)
309
+ build_index.set_defaults(func=cmd_build_index)
310
+
311
+ verify_project = sub.add_parser("verify-project", help="Verify project config and local scaffold")
312
+ verify_project.add_argument("--projects-file", required=True)
313
+ verify_project.add_argument("--dataset-root", required=True)
314
+ verify_project.add_argument("--slug", required=True)
315
+ verify_project.add_argument(
316
+ "--dry-run",
317
+ action="store_true",
318
+ help="Print verification findings without returning a failure exit code",
319
+ )
320
+ verify_project.set_defaults(func=cmd_verify_project)
321
+
322
+ return parser
323
+
324
+
325
+ def main(argv: list[str] | None = None) -> int:
326
+ parser = build_parser()
327
+ try:
328
+ args = parser.parse_args(argv)
329
+ return int(args.func(args))
330
+ except CliValidationError as exc:
331
+ print(f"ERROR: {exc}")
332
+ return EXIT_VALIDATION_ERROR
333
+ except OSError as exc:
334
+ print(f"ERROR: I/O failure: {exc}")
335
+ return EXIT_IO_ERROR
336
+
337
+
338
+ if __name__ == "__main__":
339
+ raise SystemExit(main())
src/config/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Runtime configuration helpers."""
src/config/runtime_config.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ import os
3
+ import tempfile
4
+ from pathlib import Path
5
+
6
+
7
+ @dataclass(frozen=True)
8
+ class RuntimeConfig:
9
+ detection_seed_path: str | None
10
+ validation_base_dir: str
11
+ bootstrap_base_dir: str
12
+ page_size: int
13
+ projects_file_path: str | None
14
+ user_access_file_path: str | None
15
+ invites_file_path: str | None
16
+ invite_ttl_hours: int
17
+ enable_demo_bootstrap: bool
18
+ invite_email_enabled: bool
19
+ invite_email_sender: str
20
+ invite_email_login_url: str
21
+ emailjs_enabled: bool = False
22
+ emailjs_service_id: str | None = None
23
+ emailjs_template_id: str | None = None
24
+ emailjs_template_id_username_only: str | None = None
25
+ emailjs_template_id_email_only: str | None = None
26
+ emailjs_template_id_dual: str | None = None
27
+ emailjs_public_key: str | None = None
28
+ emailjs_endpoint: str = "https://api.emailjs.com/api/v1.0/email/send"
29
+ emailjs_timeout_seconds: int = 20
30
+
31
+ @classmethod
32
+ def from_env(cls) -> "RuntimeConfig":
33
+ detection_seed_path = (os.getenv("BIRDNET_DETECTIONS_FILE") or "").strip() or None
34
+
35
+ data_root = Path("/data") if Path("/data").exists() else (Path(tempfile.gettempdir()) / "birdnet-validator-data")
36
+
37
+ bootstrap_base_dir = (
38
+ os.getenv("BIRDNET_BOOTSTRAP_DIR")
39
+ or str(data_root / "bootstrap")
40
+ )
41
+
42
+ validation_base_dir = (
43
+ os.getenv("BIRDNET_VALIDATIONS_DIR")
44
+ or str(data_root / "validations")
45
+ )
46
+
47
+ raw_page_size = (os.getenv("BIRDNET_PAGE_SIZE") or "").strip()
48
+ page_size = 25
49
+ if raw_page_size:
50
+ try:
51
+ parsed = int(raw_page_size)
52
+ if parsed > 0:
53
+ page_size = parsed
54
+ except ValueError:
55
+ page_size = 25
56
+
57
+ projects_file_path = (os.getenv("BIRDNET_PROJECTS_FILE") or "").strip() or None
58
+ user_access_file_path = (os.getenv("BIRDNET_USER_ACCESS_FILE") or "").strip() or None
59
+ invites_file_path = (os.getenv("BIRDNET_INVITES_FILE") or "").strip() or None
60
+ raw_invite_ttl_hours = (os.getenv("BIRDNET_INVITE_TTL_HOURS") or "").strip()
61
+ invite_ttl_hours = 72
62
+ if raw_invite_ttl_hours:
63
+ try:
64
+ parsed = int(raw_invite_ttl_hours)
65
+ if parsed > 0:
66
+ invite_ttl_hours = parsed
67
+ except ValueError:
68
+ invite_ttl_hours = 72
69
+
70
+ raw_enable_demo_bootstrap = (os.getenv("BIRDNET_ENABLE_DEMO_BOOTSTRAP") or "").strip().lower()
71
+ enable_demo_bootstrap = raw_enable_demo_bootstrap in {"1", "true", "yes", "on"}
72
+
73
+ raw_invite_email_enabled = (os.getenv("BIRDNET_INVITE_EMAIL_ENABLED") or "").strip().lower()
74
+ invite_email_enabled = raw_invite_email_enabled in {"1", "true", "yes", "on"}
75
+ invite_email_sender = (os.getenv("BIRDNET_INVITE_EMAIL_SENDER") or "").strip()
76
+ invite_email_login_url = (os.getenv("BIRDNET_INVITE_EMAIL_LOGIN_URL") or "").strip() or ""
77
+
78
+ raw_emailjs_enabled = (os.getenv("BIRDNET_EMAILJS_ENABLED") or "").strip().lower()
79
+ emailjs_enabled = raw_emailjs_enabled in {"1", "true", "yes", "on"}
80
+ emailjs_service_id = (os.getenv("BIRDNET_EMAILJS_SERVICE_ID") or "").strip() or None
81
+ emailjs_template_id = (os.getenv("BIRDNET_EMAILJS_TEMPLATE_ID") or "").strip() or None
82
+ emailjs_template_id_username_only = (os.getenv("BIRDNET_EMAILJS_TEMPLATE_ID_USERNAME_ONLY") or "").strip() or None
83
+ emailjs_template_id_email_only = (os.getenv("BIRDNET_EMAILJS_TEMPLATE_ID_EMAIL_ONLY") or "").strip() or None
84
+ emailjs_template_id_dual = (os.getenv("BIRDNET_EMAILJS_TEMPLATE_ID_DUAL") or "").strip() or None
85
+ emailjs_public_key = (os.getenv("BIRDNET_EMAILJS_PUBLIC_KEY") or os.getenv("EMAILJS_PUBLIC_KEY") or "").strip() or None
86
+ emailjs_endpoint = (os.getenv("BIRDNET_EMAILJS_ENDPOINT") or "https://api.emailjs.com/api/v1.0/email/send").strip() or "https://api.emailjs.com/api/v1.0/email/send"
87
+ raw_emailjs_timeout = (os.getenv("BIRDNET_EMAILJS_TIMEOUT_SECONDS") or "").strip()
88
+ emailjs_timeout_seconds = 20
89
+ if raw_emailjs_timeout:
90
+ try:
91
+ parsed = int(raw_emailjs_timeout)
92
+ if parsed > 0:
93
+ emailjs_timeout_seconds = parsed
94
+ except ValueError:
95
+ emailjs_timeout_seconds = 20
96
+
97
+ return cls(
98
+ detection_seed_path=detection_seed_path,
99
+ validation_base_dir=validation_base_dir,
100
+ bootstrap_base_dir=bootstrap_base_dir,
101
+ page_size=page_size,
102
+ projects_file_path=projects_file_path,
103
+ user_access_file_path=user_access_file_path,
104
+ invites_file_path=invites_file_path,
105
+ invite_ttl_hours=invite_ttl_hours,
106
+ enable_demo_bootstrap=enable_demo_bootstrap,
107
+ invite_email_enabled=invite_email_enabled,
108
+ invite_email_sender=invite_email_sender,
109
+ invite_email_login_url=invite_email_login_url,
110
+ emailjs_enabled=emailjs_enabled,
111
+ emailjs_service_id=emailjs_service_id,
112
+ emailjs_template_id=emailjs_template_id,
113
+ emailjs_template_id_username_only=emailjs_template_id_username_only,
114
+ emailjs_template_id_email_only=emailjs_template_id_email_only,
115
+ emailjs_template_id_dual=emailjs_template_id_dual,
116
+ emailjs_public_key=emailjs_public_key,
117
+ emailjs_endpoint=emailjs_endpoint,
118
+ emailjs_timeout_seconds=emailjs_timeout_seconds,
119
+ )
src/domain/__init__.py ADDED
File without changes
src/domain/models.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from enum import Enum
2
+ from typing import Optional
3
+ from uuid import uuid4
4
+
5
+ from pydantic import BaseModel, Field
6
+
7
+
8
+ class Role(str, Enum):
9
+ admin = "admin"
10
+ validator = "validator"
11
+
12
+
13
+ class User(BaseModel):
14
+ username: str = Field(min_length=1)
15
+ role: Role
16
+ active: bool = True
17
+
18
+
19
+ class Project(BaseModel):
20
+ project_id: str = Field(default_factory=lambda: str(uuid4()), min_length=8)
21
+ project_slug: str = Field(min_length=3)
22
+ name: str = Field(min_length=1)
23
+ dataset_repo_id: str = Field(min_length=3)
24
+ visibility: str = Field(default="collaborative", pattern="^(private|collaborative)$")
25
+ owner_username: Optional[str] = None
26
+ dataset_token: Optional[str] = None
27
+ active: bool = True
28
+
29
+
30
+ class Detection(BaseModel):
31
+ detection_key: str = Field(min_length=16)
32
+ audio_id: str = Field(min_length=1)
33
+ scientific_name: str = Field(min_length=1)
34
+ confidence: float = Field(ge=0.0, le=1.0)
35
+ start_time: float = Field(ge=0.0)
36
+ end_time: float = Field(gt=0.0)
37
+
38
+
39
+ class Validation(BaseModel):
40
+ detection_key: str = Field(min_length=16)
41
+ status: str = Field(min_length=1)
42
+ corrected_species: Optional[str] = None
43
+ notes: str = ""
44
+ validator: str = Field(min_length=1)
45
+
46
+
47
+ class IndexManifest(BaseModel):
48
+ schema_version: str = "1.0.0"
49
+ project_slug: str
50
+ total_detections: int = Field(ge=0)
51
+ total_audio_files: int = Field(ge=0)
src/repositories/__init__.py ADDED
File without changes
src/repositories/append_only_validation_repository.py ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from datetime import UTC, datetime
3
+ from pathlib import Path
4
+ import threading
5
+ from uuid import uuid4
6
+
7
+ from src.domain.models import Validation
8
+
9
+
10
+ class OptimisticLockError(Exception):
11
+ def __init__(self, detection_key: str, expected_version: int, current_version: int) -> None:
12
+ self.detection_key = detection_key
13
+ self.expected_version = expected_version
14
+ self.current_version = current_version
15
+ super().__init__(
16
+ f"Optimistic lock conflict for detection_key={detection_key}: "
17
+ f"expected_version={expected_version}, current_version={current_version}"
18
+ )
19
+
20
+
21
+ class AppendOnlyValidationRepository:
22
+ def __init__(self, base_dir: str) -> None:
23
+ self._base_dir = Path(base_dir)
24
+ self._locks: dict[str, threading.Lock] = {}
25
+ self._locks_guard = threading.Lock()
26
+
27
+ def _lock_for_project(self, project_slug: str) -> threading.Lock:
28
+ with self._locks_guard:
29
+ lock = self._locks.get(project_slug)
30
+ if lock is None:
31
+ lock = threading.Lock()
32
+ self._locks[project_slug] = lock
33
+ return lock
34
+
35
+ @staticmethod
36
+ def _atomic_write_json(path: Path, payload: object) -> None:
37
+ path.parent.mkdir(parents=True, exist_ok=True)
38
+ tmp_path = path.parent / f".{path.name}.tmp.{uuid4().hex}"
39
+ tmp_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
40
+ tmp_path.replace(path)
41
+
42
+ def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
43
+ project_dir = self._base_dir / project_slug / "validations"
44
+ project_dir.mkdir(parents=True, exist_ok=True)
45
+
46
+ with self._lock_for_project(project_slug):
47
+ current_file = project_dir / "current.json"
48
+ if current_file.exists():
49
+ current_payload = json.loads(current_file.read_text(encoding="utf-8"))
50
+ else:
51
+ current_payload = {}
52
+
53
+ current_item = current_payload.get(item.detection_key, {})
54
+ current_version = int(current_item.get("version", 0))
55
+ expected = expected_version if expected_version is not None else current_version
56
+ if expected != current_version:
57
+ raise OptimisticLockError(
58
+ detection_key=item.detection_key,
59
+ expected_version=expected,
60
+ current_version=current_version,
61
+ )
62
+
63
+ new_version = current_version + 1
64
+
65
+ event_date = datetime.now(UTC).strftime("%Y%m%d")
66
+ events_file = project_dir / f"events-{event_date}.jsonl"
67
+ event = {
68
+ "event_id": str(uuid4()),
69
+ "timestamp": datetime.now(UTC).isoformat(),
70
+ "project_slug": project_slug,
71
+ "expected_version": expected,
72
+ "previous_version": current_version,
73
+ "new_version": new_version,
74
+ **item.model_dump(),
75
+ }
76
+ with events_file.open("a", encoding="utf-8") as handle:
77
+ handle.write(json.dumps(event, ensure_ascii=True) + "\n")
78
+
79
+ current_payload[item.detection_key] = {
80
+ "status": item.status,
81
+ "corrected_species": item.corrected_species,
82
+ "notes": item.notes,
83
+ "validator": item.validator,
84
+ "updated_at": event["timestamp"],
85
+ "version": new_version,
86
+ }
87
+ self._atomic_write_json(current_file, current_payload)
88
+ return new_version
89
+
90
+ def list_events(self, project_slug: str) -> list[dict[str, object]]:
91
+ project_dir = self._base_dir / project_slug / "validations"
92
+ if not project_dir.exists():
93
+ return []
94
+
95
+ events: list[dict[str, object]] = []
96
+ for path in sorted(project_dir.glob("events-*.jsonl")):
97
+ with path.open("r", encoding="utf-8") as handle:
98
+ for line in handle:
99
+ payload = line.strip()
100
+ if payload:
101
+ events.append(json.loads(payload))
102
+ return events
103
+
104
+ def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]:
105
+ current_file = self._base_dir / project_slug / "validations" / "current.json"
106
+ if not current_file.exists():
107
+ return {}
108
+ return json.loads(current_file.read_text(encoding="utf-8"))
src/repositories/contracts.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Protocol
2
+
3
+ from src.domain.models import Detection, Project, User, Validation
4
+
5
+
6
+ class ProjectRepository(Protocol):
7
+ def get_project(self, project_slug: str) -> Project: ...
8
+
9
+
10
+ class DetectionRepository(Protocol):
11
+ def list_detections(
12
+ self,
13
+ project_slug: str,
14
+ page: int,
15
+ page_size: int,
16
+ scientific_name: str | None = None,
17
+ min_confidence: float | None = None,
18
+ max_confidence: float | None = None,
19
+ ) -> list[Detection]: ...
20
+
21
+ def count_detections(
22
+ self,
23
+ project_slug: str,
24
+ scientific_name: str | None = None,
25
+ min_confidence: float | None = None,
26
+ max_confidence: float | None = None,
27
+ ) -> int: ...
28
+
29
+
30
+ class ValidationRepository(Protocol):
31
+ def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int: ...
32
+
33
+
34
+ class AuthRepository(Protocol):
35
+ def authenticate(self, username: str) -> User: ...
src/repositories/in_memory_detection_repository.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import defaultdict
2
+ from typing import DefaultDict
3
+
4
+ from src.domain.models import Detection
5
+
6
+
7
+ class InMemoryDetectionRepository:
8
+ def __init__(self) -> None:
9
+ self._project_items: DefaultDict[str, list[Detection]] = defaultdict(list)
10
+
11
+ def seed(self, project_slug: str, detections: list[Detection]) -> None:
12
+ self._project_items[project_slug] = list(detections)
13
+
14
+ def list_detections(
15
+ self,
16
+ project_slug: str,
17
+ page: int,
18
+ page_size: int,
19
+ scientific_name: str | None = None,
20
+ min_confidence: float | None = None,
21
+ max_confidence: float | None = None,
22
+ ) -> list[Detection]:
23
+ if page < 1:
24
+ raise ValueError("page must be >= 1")
25
+ if page_size < 1:
26
+ raise ValueError("page_size must be >= 1")
27
+
28
+ filtered = self._apply_filters(
29
+ project_slug=project_slug,
30
+ scientific_name=scientific_name,
31
+ min_confidence=min_confidence,
32
+ max_confidence=max_confidence,
33
+ )
34
+
35
+ start = (page - 1) * page_size
36
+ end = start + page_size
37
+ return filtered[start:end]
38
+
39
+ def count_detections(
40
+ self,
41
+ project_slug: str,
42
+ scientific_name: str | None = None,
43
+ min_confidence: float | None = None,
44
+ max_confidence: float | None = None,
45
+ ) -> int:
46
+ return len(
47
+ self._apply_filters(
48
+ project_slug=project_slug,
49
+ scientific_name=scientific_name,
50
+ min_confidence=min_confidence,
51
+ max_confidence=max_confidence,
52
+ )
53
+ )
54
+
55
+ def _apply_filters(
56
+ self,
57
+ project_slug: str,
58
+ scientific_name: str | None,
59
+ min_confidence: float | None,
60
+ max_confidence: float | None,
61
+ ) -> list[Detection]:
62
+ items = self._project_items.get(project_slug, [])
63
+
64
+ def match(item: Detection) -> bool:
65
+ if scientific_name and item.scientific_name != scientific_name:
66
+ return False
67
+ if min_confidence is not None and item.confidence < min_confidence:
68
+ return False
69
+ if max_confidence is not None and item.confidence > max_confidence:
70
+ return False
71
+ return True
72
+
73
+ return [item for item in items if match(item)]
src/repositories/in_memory_validation_repository.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import defaultdict
2
+ from typing import DefaultDict
3
+
4
+ from src.domain.models import Validation
5
+ from src.repositories.append_only_validation_repository import OptimisticLockError
6
+
7
+
8
+ class InMemoryValidationRepository:
9
+ def __init__(self) -> None:
10
+ self._by_project: DefaultDict[str, list[Validation]] = defaultdict(list)
11
+ self._versions: DefaultDict[str, dict[str, int]] = defaultdict(dict)
12
+
13
+ def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
14
+ current_version = int(self._versions[project_slug].get(item.detection_key, 0))
15
+ expected = expected_version if expected_version is not None else current_version
16
+ if expected != current_version:
17
+ raise OptimisticLockError(item.detection_key, expected, current_version)
18
+
19
+ self._by_project[project_slug].append(item)
20
+ new_version = current_version + 1
21
+ self._versions[project_slug][item.detection_key] = new_version
22
+ return new_version
23
+
24
+ def list_validations(self, project_slug: str) -> list[Validation]:
25
+ return list(self._by_project.get(project_slug, []))
src/services/__init__.py ADDED
File without changes
src/services/audio_fetch_service.py ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ import io
3
+ import math
4
+ from pathlib import Path
5
+ import struct
6
+ import wave
7
+
8
+ from huggingface_hub import hf_hub_download
9
+
10
+ from src.cache.ephemeral_cache_manager import EphemeralCacheManager
11
+
12
+
13
+ SUPPORTED_AUDIO_EXTENSIONS = (".wav", ".mp3", ".flac", ".ogg", ".m4a")
14
+
15
+
16
+ @dataclass(slots=True)
17
+ class AudioFetchResult:
18
+ cache_key: str
19
+ local_path: str
20
+ source: str
21
+
22
+
23
+ class AudioFetchService:
24
+ def __init__(self, cache_manager: EphemeralCacheManager) -> None:
25
+ self._cache = cache_manager
26
+
27
+ def fetch(
28
+ self,
29
+ dataset_repo: str,
30
+ audio_id: str,
31
+ allow_demo_fallback: bool = False,
32
+ hf_token: str | None = None,
33
+ ) -> AudioFetchResult:
34
+ cache_key = f"{dataset_repo}:{audio_id}"
35
+ cached_path = self._cache.get(cache_key)
36
+ if cached_path:
37
+ return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="cache")
38
+
39
+ if allow_demo_fallback and self._is_seeded_demo_audio_id(audio_id):
40
+ fallback_bytes = self._build_demo_fallback_wav(audio_id=audio_id)
41
+ local_path = self._cache.put_bytes(cache_key, fallback_bytes, suffix=".wav")
42
+ return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="demo-fallback")
43
+
44
+ try:
45
+ target_filename, downloaded_path = self._resolve_remote_filename(
46
+ dataset_repo=dataset_repo,
47
+ audio_id=audio_id,
48
+ hf_token=hf_token,
49
+ )
50
+ except FileNotFoundError:
51
+ if not allow_demo_fallback:
52
+ raise
53
+
54
+ fallback_bytes = self._build_demo_fallback_wav(audio_id=audio_id)
55
+ local_path = self._cache.put_bytes(cache_key, fallback_bytes, suffix=".wav")
56
+ return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="demo-fallback")
57
+
58
+ if downloaded_path is None:
59
+ downloaded = self._download_dataset_file(
60
+ dataset_repo=dataset_repo,
61
+ filename=target_filename,
62
+ hf_token=hf_token,
63
+ )
64
+ downloaded_path = Path(downloaded)
65
+
66
+ suffix = downloaded_path.suffix if downloaded_path.suffix else ".bin"
67
+ local_path = self._cache.put_bytes(cache_key, downloaded_path.read_bytes(), suffix=suffix)
68
+ return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="remote")
69
+
70
+ def fetch_local(self, local_audio_path: str) -> AudioFetchResult:
71
+ """Load audio from local filesystem (useful for testing with local segments).
72
+
73
+ Args:
74
+ local_audio_path: Full path to local audio file
75
+
76
+ Returns:
77
+ AudioFetchResult with cache_key based on file path
78
+
79
+ Raises:
80
+ FileNotFoundError: If local file doesn't exist or is not a supported audio format
81
+ """
82
+ audio_path = Path(local_audio_path)
83
+ if not audio_path.exists():
84
+ raise FileNotFoundError(f"Local audio file not found: {local_audio_path}")
85
+
86
+ if audio_path.suffix.lower() not in SUPPORTED_AUDIO_EXTENSIONS:
87
+ raise ValueError(f"Unsupported audio format: {audio_path.suffix}. Supported: {SUPPORTED_AUDIO_EXTENSIONS}")
88
+
89
+ cache_key = f"local:{audio_path.name}:{audio_path.stat().st_size}"
90
+ cached_path = self._cache.get(cache_key)
91
+ if cached_path:
92
+ return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="cache")
93
+
94
+ # Cache the local file by copying it into the ephemeral cache
95
+ local_bytes = audio_path.read_bytes()
96
+ cached_path = self._cache.put_bytes(cache_key, local_bytes, suffix=audio_path.suffix)
97
+ return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="local")
98
+
99
+ def cleanup_after_validation(self, cache_key: str) -> None:
100
+ self._cache.cleanup_key(cache_key)
101
+
102
+ def _resolve_remote_filename(
103
+ self,
104
+ dataset_repo: str,
105
+ audio_id: str,
106
+ hf_token: str | None = None,
107
+ ) -> tuple[str, Path | None]:
108
+ audio_path = Path(audio_id)
109
+ if audio_path.suffix:
110
+ return f"audio/{audio_id}", None
111
+
112
+ for extension in SUPPORTED_AUDIO_EXTENSIONS:
113
+ candidate = f"audio/{audio_id}{extension}"
114
+ try:
115
+ downloaded = self._download_dataset_file(
116
+ dataset_repo=dataset_repo,
117
+ filename=candidate,
118
+ hf_token=hf_token,
119
+ )
120
+ return candidate, Path(downloaded)
121
+ except Exception:
122
+ continue
123
+
124
+ raise FileNotFoundError(f"Unable to locate audio file for audio_id: {audio_id}")
125
+
126
+ def _build_demo_fallback_wav(self, audio_id: str, duration_seconds: float = 4.0, sample_rate: int = 16000) -> bytes:
127
+ """Generate a short deterministic tone when demo audio files are missing."""
128
+ frame_count = max(1, int(duration_seconds * sample_rate))
129
+
130
+ # Derive deterministic frequencies from audio_id so each fallback sounds distinct.
131
+ seed = sum(ord(ch) for ch in audio_id) % 200
132
+ f1 = 440.0 + float(seed)
133
+ f2 = f1 + 180.0
134
+
135
+ with io.BytesIO() as buffer:
136
+ with wave.open(buffer, "wb") as wav_file:
137
+ wav_file.setnchannels(1)
138
+ wav_file.setsampwidth(2)
139
+ wav_file.setframerate(sample_rate)
140
+
141
+ frames = bytearray()
142
+ for i in range(frame_count):
143
+ t = i / float(sample_rate)
144
+ # Blend two sines and apply light envelope to avoid clicks.
145
+ env = min(1.0, i / 400.0, (frame_count - i) / 400.0)
146
+ sample = env * (0.55 * math.sin(2.0 * math.pi * f1 * t) + 0.45 * math.sin(2.0 * math.pi * f2 * t))
147
+ pcm = int(max(-1.0, min(1.0, sample)) * 32767)
148
+ frames.extend(struct.pack("<h", pcm))
149
+
150
+ wav_file.writeframes(bytes(frames))
151
+
152
+ return buffer.getvalue()
153
+
154
+ def _is_seeded_demo_audio_id(self, audio_id: str) -> bool:
155
+ normalized = audio_id.strip().lower()
156
+ return "_audio_" in normalized and normalized.endswith(("1001", "1002", "1003", "1004"))
157
+
158
+ def _download_dataset_file(self, dataset_repo: str, filename: str, hf_token: str | None = None) -> str:
159
+ if hf_token:
160
+ return hf_hub_download(
161
+ repo_id=dataset_repo,
162
+ repo_type="dataset",
163
+ filename=filename,
164
+ token=hf_token,
165
+ )
166
+ return hf_hub_download(
167
+ repo_id=dataset_repo,
168
+ repo_type="dataset",
169
+ filename=filename,
170
+ )
src/services/detection_queue_service.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ from src.domain.models import Detection
4
+ from src.repositories.contracts import DetectionRepository
5
+
6
+
7
+ @dataclass(slots=True)
8
+ class DetectionPage:
9
+ project_slug: str
10
+ page: int
11
+ page_size: int
12
+ total_items: int
13
+ total_pages: int
14
+ has_next: bool
15
+ has_previous: bool
16
+ items: list[Detection]
17
+
18
+
19
+ class DetectionQueueService:
20
+ def __init__(self, repository: DetectionRepository) -> None:
21
+ self._repository = repository
22
+
23
+ def get_page(
24
+ self,
25
+ project_slug: str,
26
+ page: int,
27
+ page_size: int,
28
+ scientific_name: str | None = None,
29
+ min_confidence: float | None = None,
30
+ max_confidence: float | None = None,
31
+ ) -> DetectionPage:
32
+ if page < 1:
33
+ raise ValueError("page must be >= 1")
34
+ if page_size < 1:
35
+ raise ValueError("page_size must be >= 1")
36
+
37
+ total_items = self._repository.count_detections(
38
+ project_slug=project_slug,
39
+ scientific_name=scientific_name,
40
+ min_confidence=min_confidence,
41
+ max_confidence=max_confidence,
42
+ )
43
+ total_pages = max(1, ((total_items - 1) // page_size) + 1) if total_items else 1
44
+ page = min(page, total_pages)
45
+
46
+ items = self._repository.list_detections(
47
+ project_slug=project_slug,
48
+ page=page,
49
+ page_size=page_size,
50
+ scientific_name=scientific_name,
51
+ min_confidence=min_confidence,
52
+ max_confidence=max_confidence,
53
+ )
54
+
55
+ return DetectionPage(
56
+ project_slug=project_slug,
57
+ page=page,
58
+ page_size=page_size,
59
+ total_items=total_items,
60
+ total_pages=total_pages,
61
+ has_next=page < total_pages,
62
+ has_previous=page > 1,
63
+ items=items,
64
+ )
src/services/invite_email_notifier.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Email delivery via EmailJS for project collaboration invites."""
2
+
3
+ from dataclasses import dataclass
4
+ from datetime import datetime
5
+ import json
6
+ from urllib import request
7
+
8
+
9
+ @dataclass(frozen=True)
10
+ class InviteEmailPayload:
11
+ """Payload for EmailJS invite notification.
12
+
13
+ At least one of invitee_username or invitee_email must be present.
14
+ - Only username: internal app invite only (no email sent).
15
+ - Only email: external email-based invite (no prior HF account required).
16
+ - Both: dual invite (internal + email).
17
+ """
18
+ project_slug: str
19
+ role: str
20
+ invited_by: str
21
+ expires_at: datetime
22
+ login_url: str
23
+ invitee_username: str | None = None
24
+ invitee_email: str | None = None
25
+
26
+
27
+ class InviteEmailNotifier:
28
+ """Abstract notifier interface for invite emails."""
29
+
30
+ def send(self, payload: InviteEmailPayload) -> tuple[bool, str]:
31
+ """Send an invite notification. Returns (success, message)."""
32
+ raise NotImplementedError
33
+
34
+
35
+ class EmailJSInviteEmailNotifier(InviteEmailNotifier):
36
+ """EmailJS-backed notifier for invite emails (only transport available)."""
37
+
38
+ def __init__(
39
+ self,
40
+ sender_email: str,
41
+ service_id: str,
42
+ template_id: str,
43
+ public_key: str,
44
+ template_id_username_only: str | None = None,
45
+ template_id_email_only: str | None = None,
46
+ template_id_dual: str | None = None,
47
+ endpoint: str = "https://api.emailjs.com/api/v1.0/email/send",
48
+ timeout_seconds: int = 20,
49
+ ):
50
+ self._sender_email = sender_email
51
+ self._service_id = service_id
52
+ self._template_id = template_id
53
+ self._public_key = public_key
54
+ self._template_id_username_only = (template_id_username_only or "").strip() or None
55
+ self._template_id_email_only = (template_id_email_only or "").strip() or None
56
+ self._template_id_dual = (template_id_dual or "").strip() or None
57
+ self._endpoint = endpoint
58
+ self._timeout_seconds = max(5, int(timeout_seconds))
59
+
60
+ def _resolve_invite_mode(self, payload: InviteEmailPayload) -> str:
61
+ if payload.invitee_username and payload.invitee_email:
62
+ return "dual"
63
+ if payload.invitee_email:
64
+ return "email_only"
65
+ return "username_only"
66
+
67
+ def _resolve_template_id(self, mode: str) -> str:
68
+ if mode == "username_only" and self._template_id_username_only:
69
+ return self._template_id_username_only
70
+ if mode == "email_only" and self._template_id_email_only:
71
+ return self._template_id_email_only
72
+ if mode == "dual" and self._template_id_dual:
73
+ return self._template_id_dual
74
+ return self._template_id
75
+
76
+ def send(self, payload: InviteEmailPayload) -> tuple[bool, str]:
77
+ """Send invite email via EmailJS.
78
+
79
+ If invitee_email is missing, returns success with message indicating internal-only invite.
80
+ """
81
+ invite_mode = self._resolve_invite_mode(payload)
82
+ if not payload.invitee_email:
83
+ return True, f"✅ Internal invite created for {payload.invitee_username or 'pending'}"
84
+
85
+ template_id = self._resolve_template_id(invite_mode)
86
+ if not template_id:
87
+ return False, "Internal invite created, but email send failed: EmailJS template ID is not configured"
88
+
89
+ template_params = {
90
+ "from_name": self._sender_email,
91
+ "to_email": payload.invitee_email,
92
+ "invitee_username": payload.invitee_username or "(to be claimed)",
93
+ "project_slug": payload.project_slug,
94
+ "role": payload.role,
95
+ "invited_by": payload.invited_by,
96
+ "expires_at": payload.expires_at.isoformat(),
97
+ "login_url": payload.login_url or "",
98
+ "invite_link": payload.login_url or "",
99
+ "invite_mode": invite_mode,
100
+ }
101
+
102
+ req_payload = {
103
+ "service_id": self._service_id,
104
+ "template_id": template_id,
105
+ "user_id": self._public_key,
106
+ "template_params": template_params,
107
+ }
108
+
109
+ try:
110
+ req = request.Request(
111
+ self._endpoint,
112
+ data=json.dumps(req_payload).encode("utf-8"),
113
+ headers={"Content-Type": "application/json"},
114
+ method="POST",
115
+ )
116
+ with request.urlopen(req, timeout=self._timeout_seconds) as response:
117
+ status_code = int(getattr(response, "status", 0) or response.getcode())
118
+ if 200 <= status_code < 300:
119
+ return True, f"✅ Invite sent to {payload.invitee_email}"
120
+ return False, f"Internal invite created, but email send failed (status {status_code})"
121
+ except Exception as exc:
122
+ return False, f"Internal invite created, but email send failed: {exc}"
src/services/validation_service.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ from src.domain.models import Validation
4
+ from src.repositories.contracts import ValidationRepository
5
+
6
+
7
+ @dataclass(slots=True)
8
+ class ValidationWriteResult:
9
+ item: Validation
10
+ new_version: int
11
+
12
+
13
+ class ValidationService:
14
+ def __init__(self, repository: ValidationRepository) -> None:
15
+ self._repository = repository
16
+
17
+ def validate_detection(
18
+ self,
19
+ project_slug: str,
20
+ detection_key: str,
21
+ status: str,
22
+ validator: str,
23
+ notes: str = "",
24
+ corrected_species: str | None = None,
25
+ expected_version: int | None = None,
26
+ ) -> ValidationWriteResult:
27
+ item = Validation(
28
+ detection_key=detection_key,
29
+ status=status,
30
+ corrected_species=corrected_species,
31
+ notes=notes,
32
+ validator=validator,
33
+ )
34
+ new_version = self._repository.save_validation(
35
+ project_slug=project_slug,
36
+ item=item,
37
+ expected_version=expected_version,
38
+ )
39
+ return ValidationWriteResult(item=item, new_version=new_version)
src/telemetry/__init__.py ADDED
File without changes
src/ui/__pycache__/__init__.cpython-313.pyc CHANGED
Binary files a/src/ui/__pycache__/__init__.cpython-313.pyc and b/src/ui/__pycache__/__init__.cpython-313.pyc differ
 
src/ui/__pycache__/__init__.cpython-314.pyc CHANGED
Binary files a/src/ui/__pycache__/__init__.cpython-314.pyc and b/src/ui/__pycache__/__init__.cpython-314.pyc differ
 
src/ui/__pycache__/upload_app_factory.cpython-313.pyc DELETED
Binary file (41.2 kB)