Spaces:
Sleeping
Sleeping
Commit ·
2f1a5fb
1
Parent(s): 3ab4a64
Forçar atualização total do app
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .github/workflows/check-file-size.yml +0 -15
- .github/workflows/ci.yml +29 -0
- .github/workflows/release-uploader.yml +79 -0
- .github/workflows/sync-hf-space.yml +42 -8
- .gitignore +11 -0
- BIRDNET_UPLOADER_REDESIGN.md +665 -0
- README.md +579 -86
- SPRINT1_KICKOFF.md +769 -0
- SPRINT_ROADMAP.md +649 -0
- app.py +5 -4
- build/publish_release_to_hf.py +50 -0
- build/pyinstaller_uploader.spec +73 -0
- build/release_pipeline.ps1 +79 -0
- build/release_uploader.py +75 -0
- build/validate_release.ps1 +80 -0
- cli/__pycache__/__init__.cpython-314.pyc +0 -0
- cli/__pycache__/hf_dataset_cli.cpython-314.pyc +0 -0
- cli/hf_dataset_cli.py +0 -1314
- docs/COLLABORATIVE_WORKFLOW_GUIDE.md +424 -0
- docs/END_TO_END_QA_CHECKLIST_SPRINT4.md +102 -0
- docs/spaces/projects.sample.json +14 -0
- docs/spaces/user_access.sample.json +12 -0
- pyproject.toml +5 -2
- requirements.txt +11 -6
- src/__pycache__/__init__.cpython-313.pyc +0 -0
- src/__pycache__/__init__.cpython-314.pyc +0 -0
- {cli → src/auth}/__init__.py +0 -0
- src/auth/auth_service.py +684 -0
- src/cache/__init__.py +0 -0
- src/cache/ephemeral_cache_manager.py +82 -0
- src/cli/__init__.py +0 -0
- src/cli/project_cli.py +339 -0
- src/config/__init__.py +1 -0
- src/config/runtime_config.py +119 -0
- src/domain/__init__.py +0 -0
- src/domain/models.py +51 -0
- src/repositories/__init__.py +0 -0
- src/repositories/append_only_validation_repository.py +108 -0
- src/repositories/contracts.py +35 -0
- src/repositories/in_memory_detection_repository.py +73 -0
- src/repositories/in_memory_validation_repository.py +25 -0
- src/services/__init__.py +0 -0
- src/services/audio_fetch_service.py +170 -0
- src/services/detection_queue_service.py +64 -0
- src/services/invite_email_notifier.py +122 -0
- src/services/validation_service.py +39 -0
- src/telemetry/__init__.py +0 -0
- src/ui/__pycache__/__init__.cpython-313.pyc +0 -0
- src/ui/__pycache__/__init__.cpython-314.pyc +0 -0
- src/ui/__pycache__/upload_app_factory.cpython-313.pyc +0 -0
.github/workflows/check-file-size.yml
DELETED
|
@@ -1,15 +0,0 @@
|
|
| 1 |
-
name: Check file size
|
| 2 |
-
|
| 3 |
-
on:
|
| 4 |
-
pull_request:
|
| 5 |
-
branches: [main]
|
| 6 |
-
workflow_dispatch:
|
| 7 |
-
|
| 8 |
-
jobs:
|
| 9 |
-
check-large-files:
|
| 10 |
-
runs-on: ubuntu-latest
|
| 11 |
-
steps:
|
| 12 |
-
- name: Check large files
|
| 13 |
-
uses: ActionsDesk/lfs-warning@v2.0
|
| 14 |
-
with:
|
| 15 |
-
filesizelimit: 10485760
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches: [main]
|
| 6 |
+
pull_request:
|
| 7 |
+
branches: [main]
|
| 8 |
+
|
| 9 |
+
jobs:
|
| 10 |
+
test:
|
| 11 |
+
runs-on: ubuntu-latest
|
| 12 |
+
env:
|
| 13 |
+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
| 14 |
+
steps:
|
| 15 |
+
- name: Checkout
|
| 16 |
+
uses: actions/checkout@v4
|
| 17 |
+
|
| 18 |
+
- name: Setup Python
|
| 19 |
+
uses: actions/setup-python@v5
|
| 20 |
+
with:
|
| 21 |
+
python-version: '3.11'
|
| 22 |
+
|
| 23 |
+
- name: Install dependencies
|
| 24 |
+
run: |
|
| 25 |
+
python -m pip install --upgrade pip
|
| 26 |
+
pip install -r requirements.txt
|
| 27 |
+
|
| 28 |
+
- name: Run tests
|
| 29 |
+
run: pytest -q
|
.github/workflows/release-uploader.yml
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Release BirdNET Uploader
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch:
|
| 5 |
+
inputs:
|
| 6 |
+
version:
|
| 7 |
+
description: "Release version (for example 0.1.0)"
|
| 8 |
+
required: true
|
| 9 |
+
type: string
|
| 10 |
+
hf_repo_id:
|
| 11 |
+
description: "Optional Hugging Face repo id to publish artifacts to (for example jrrribeiro/birdnet-uploader-releases)"
|
| 12 |
+
required: false
|
| 13 |
+
type: string
|
| 14 |
+
hf_repo_type:
|
| 15 |
+
description: "Hugging Face repo type for uploads"
|
| 16 |
+
required: false
|
| 17 |
+
default: "dataset"
|
| 18 |
+
type: choice
|
| 19 |
+
options:
|
| 20 |
+
- dataset
|
| 21 |
+
- model
|
| 22 |
+
publish_to_hf:
|
| 23 |
+
description: "Upload release bundle to Hugging Face"
|
| 24 |
+
required: false
|
| 25 |
+
default: false
|
| 26 |
+
type: boolean
|
| 27 |
+
|
| 28 |
+
jobs:
|
| 29 |
+
build-release:
|
| 30 |
+
runs-on: windows-latest
|
| 31 |
+
permissions:
|
| 32 |
+
contents: read
|
| 33 |
+
env:
|
| 34 |
+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
| 35 |
+
steps:
|
| 36 |
+
- name: Checkout
|
| 37 |
+
uses: actions/checkout@v4
|
| 38 |
+
|
| 39 |
+
- name: Setup Python
|
| 40 |
+
uses: actions/setup-python@v5
|
| 41 |
+
with:
|
| 42 |
+
python-version: "3.11"
|
| 43 |
+
|
| 44 |
+
- name: Install dependencies
|
| 45 |
+
shell: pwsh
|
| 46 |
+
run: |
|
| 47 |
+
python -m pip install --upgrade pip
|
| 48 |
+
pip install -r requirements.txt
|
| 49 |
+
pip install pyinstaller
|
| 50 |
+
|
| 51 |
+
- name: Build release bundle
|
| 52 |
+
shell: pwsh
|
| 53 |
+
run: |
|
| 54 |
+
python build/release_uploader.py --version "${{ inputs.version }}"
|
| 55 |
+
|
| 56 |
+
- name: Upload GitHub artifact
|
| 57 |
+
uses: actions/upload-artifact@v4
|
| 58 |
+
with:
|
| 59 |
+
name: birdnet-uploader-${{ inputs.version }}
|
| 60 |
+
path: |
|
| 61 |
+
build/release/*
|
| 62 |
+
if-no-files-found: error
|
| 63 |
+
|
| 64 |
+
- name: Publish to Hugging Face
|
| 65 |
+
if: ${{ inputs.publish_to_hf == true }}
|
| 66 |
+
shell: pwsh
|
| 67 |
+
env:
|
| 68 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
| 69 |
+
HF_REPO_ID: ${{ inputs.hf_repo_id }}
|
| 70 |
+
HF_REPO_TYPE: ${{ inputs.hf_repo_type }}
|
| 71 |
+
RELEASE_VERSION: ${{ inputs.version }}
|
| 72 |
+
run: |
|
| 73 |
+
if ([string]::IsNullOrWhiteSpace($env:HF_TOKEN)) { throw "HF_TOKEN secret is required when publish_to_hf is true" }
|
| 74 |
+
if ([string]::IsNullOrWhiteSpace($env:HF_REPO_ID)) { throw "hf_repo_id input is required when publish_to_hf is true" }
|
| 75 |
+
python -m pip install huggingface_hub
|
| 76 |
+
$bundle = Get-ChildItem build/release/*.zip | Select-Object -First 1
|
| 77 |
+
$checksum = Get-ChildItem build/release/*.sha256 | Select-Object -First 1
|
| 78 |
+
python build/publish_release_to_hf.py --repo-id "$env:HF_REPO_ID" --repo-type "$env:HF_REPO_TYPE" --bundle "$($bundle.FullName)" --checksum "$($checksum.FullName)" --version "$env:RELEASE_VERSION"
|
| 79 |
+
|
.github/workflows/sync-hf-space.yml
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
|
| 2 |
|
| 3 |
on:
|
| 4 |
push:
|
|
@@ -10,6 +10,8 @@ jobs:
|
|
| 10 |
runs-on: ubuntu-latest
|
| 11 |
permissions:
|
| 12 |
contents: read
|
|
|
|
|
|
|
| 13 |
steps:
|
| 14 |
- name: Checkout repository
|
| 15 |
uses: actions/checkout@v4
|
|
@@ -17,16 +19,48 @@ jobs:
|
|
| 17 |
fetch-depth: 0
|
| 18 |
lfs: true
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
- name: Push to Hugging Face Space
|
| 21 |
env:
|
| 22 |
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
| 23 |
-
HF_USERNAME:
|
| 24 |
-
HF_SPACE_NAME:
|
| 25 |
run: |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
git config --global user.name "github-actions"
|
| 27 |
git config --global user.email "actions@github.com"
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
git push "https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_USERNAME}/${HF_SPACE_NAME}" HEAD:main --force
|
|
|
|
| 1 |
+
name: Sync app to Hugging Face Space
|
| 2 |
|
| 3 |
on:
|
| 4 |
push:
|
|
|
|
| 10 |
runs-on: ubuntu-latest
|
| 11 |
permissions:
|
| 12 |
contents: read
|
| 13 |
+
env:
|
| 14 |
+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
| 15 |
steps:
|
| 16 |
- name: Checkout repository
|
| 17 |
uses: actions/checkout@v4
|
|
|
|
| 19 |
fetch-depth: 0
|
| 20 |
lfs: true
|
| 21 |
|
| 22 |
+
|
| 23 |
+
- name: Ensure Hugging Face Space exists
|
| 24 |
+
env:
|
| 25 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
| 26 |
+
HF_USERNAME: ${{ secrets.HF_USERNAME }}
|
| 27 |
+
HF_SPACE_NAME: ${{ secrets.HF_SPACE_NAME }}
|
| 28 |
+
run: |
|
| 29 |
+
if [ -z "$HF_TOKEN" ] || [ -z "$HF_USERNAME" ] || [ -z "$HF_SPACE_NAME" ]; then
|
| 30 |
+
echo "Skipping: missing secrets"
|
| 31 |
+
exit 0
|
| 32 |
+
fi
|
| 33 |
+
python - <<'PY'
|
| 34 |
+
import os
|
| 35 |
+
from huggingface_hub import HfApi
|
| 36 |
+
|
| 37 |
+
token = os.environ["HF_TOKEN"]
|
| 38 |
+
username = os.environ["HF_USERNAME"]
|
| 39 |
+
space_name = os.environ["HF_SPACE_NAME"]
|
| 40 |
+
|
| 41 |
+
api = HfApi(token=token)
|
| 42 |
+
api.create_repo(
|
| 43 |
+
repo_id=f"{username}/{space_name}",
|
| 44 |
+
repo_type="space",
|
| 45 |
+
exist_ok=True,
|
| 46 |
+
space_sdk="gradio",
|
| 47 |
+
)
|
| 48 |
+
PY
|
| 49 |
+
|
| 50 |
+
|
| 51 |
- name: Push to Hugging Face Space
|
| 52 |
env:
|
| 53 |
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
| 54 |
+
HF_USERNAME: ${{ secrets.HF_USERNAME }}
|
| 55 |
+
HF_SPACE_NAME: ${{ secrets.HF_SPACE_NAME }}
|
| 56 |
run: |
|
| 57 |
+
if [ -z "$HF_TOKEN" ] || [ -z "$HF_USERNAME" ] || [ -z "$HF_SPACE_NAME" ]; then
|
| 58 |
+
echo "Skipping: missing secrets"
|
| 59 |
+
exit 0
|
| 60 |
+
fi
|
| 61 |
git config --global user.name "github-actions"
|
| 62 |
git config --global user.email "actions@github.com"
|
| 63 |
+
git push "https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_USERNAME}/${HF_SPACE_NAME}.git" HEAD:main --force
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
# Step de skip removido pois a checagem agora é interna nos steps acima
|
|
|
.gitignore
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
.venv/
|
| 4 |
+
.pytest_cache/
|
| 5 |
+
.env
|
| 6 |
+
.env.*
|
| 7 |
+
.DS_Store
|
| 8 |
+
.vscode/
|
| 9 |
+
dist/
|
| 10 |
+
build/release/
|
| 11 |
+
build/pyinstaller_uploader/
|
BIRDNET_UPLOADER_REDESIGN.md
ADDED
|
@@ -0,0 +1,665 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BirdNET Uploader: Redesign CLI Moderno
|
| 2 |
+
|
| 3 |
+
## Visão Geral
|
| 4 |
+
|
| 5 |
+
Executável local moderno com **um único fluxo integrado**: login → criar repositório → apontar pasta → fazer upload.
|
| 6 |
+
|
| 7 |
+
Interface limpa, baseada em TUI (Textual) ou rich terminal output, compatível com a estrutura esperada pelo BirdNET Validator.
|
| 8 |
+
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
## 1. FLUXO DE UX ÚNICO INTEGRADO
|
| 12 |
+
|
| 13 |
+
### Tela 1: Autenticação
|
| 14 |
+
```
|
| 15 |
+
┌─────────────────────────────────────────────────┐
|
| 16 |
+
│ BirdNET Uploader │
|
| 17 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 18 |
+
│ │
|
| 19 |
+
│ 🐦 Autentique-se no Hugging Face │
|
| 20 |
+
│ │
|
| 21 |
+
│ [i] Cole seu token HF (Settings → Access │
|
| 22 |
+
│ Tokens em huggingface.co) │
|
| 23 |
+
│ │
|
| 24 |
+
│ Token: [●●●●●●●●●●●●●●●●●●●●●●] │
|
| 25 |
+
│ │
|
| 26 |
+
│ [ Continuar ] [ Sair ] │
|
| 27 |
+
│ │
|
| 28 |
+
└─────────────────────────────────────────────────┘
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
### Tela 2: Configuração (Um formulário integrado)
|
| 32 |
+
```
|
| 33 |
+
┌─────────────────────────────────────────────────┐
|
| 34 |
+
│ BirdNET Uploader - Configuração │
|
| 35 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 36 |
+
│ │
|
| 37 |
+
│ Conectado como: alice@example.com │
|
| 38 |
+
│ │
|
| 39 |
+
│ REPOSITÓRIO │
|
| 40 |
+
│ Nome do repo: [my-birds-2026 ] │
|
| 41 |
+
│ Visibilidade: ( ) Público (●) Privado │
|
| 42 |
+
│ │
|
| 43 |
+
│ ARQUIVOS │
|
| 44 |
+
│ Pasta segmentos: [C:\audio\segments] │
|
| 45 |
+
│ Arquivo CSV: [C:\audio\metadata.csv] (opt)
|
| 46 |
+
│ [ Verificar ] │
|
| 47 |
+
│ ✓ 1.247 arquivos encontrados │
|
| 48 |
+
│ ✓ Tamanho total: 53 GB │
|
| 49 |
+
│ ✓ CSV compatível: 1.200 linhas │
|
| 50 |
+
│ │
|
| 51 |
+
│ [ Criar Repo e Fazer Upload ] │
|
| 52 |
+
│ [ Voltar ] │
|
| 53 |
+
│ │
|
| 54 |
+
└─────────────────────────────────────────────────┘
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
### Tela 3: Resumo e Confirmação
|
| 58 |
+
```
|
| 59 |
+
┌─────────────────────────────────────────────────┐
|
| 60 |
+
│ Pronto para Começar │
|
| 61 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 62 |
+
│ │
|
| 63 |
+
│ Repositório: alice/my-birds-2026 │
|
| 64 |
+
│ Visibilidade: Privado │
|
| 65 |
+
│ URL: https://huggingface.co/datasets/alice/.. │
|
| 66 |
+
│ │
|
| 67 |
+
│ Origem: │
|
| 68 |
+
│ Pasta: C:\audio\segments │
|
| 69 |
+
│ Arquivos: 1.247 │
|
| 70 |
+
│ Tamanho: 53 GB │
|
| 71 |
+
│ CSV: metadata.csv (1.200 linhas) │
|
| 72 |
+
│ │
|
| 73 |
+
│ Duração estimada: ~4 horas (com conexão 10MB/s)
|
| 74 |
+
│ │
|
| 75 |
+
│ [ Iniciar Upload ] [ Cancelar ] │
|
| 76 |
+
│ │
|
| 77 |
+
└─────────────────────────────────────────────────┘
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
### Tela 4: Upload em Andamento
|
| 81 |
+
```
|
| 82 |
+
┌─────────────────────────────────────────────────┐
|
| 83 |
+
│ Upload em Andamento │
|
| 84 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 85 |
+
│ │
|
| 86 |
+
│ Progresso Global │
|
| 87 |
+
│ ████████████████░░░░░░░░░░░░░░ 62.3% │
|
| 88 |
+
│ │
|
| 89 |
+
│ Arquivos: 775/1.247 (skipped: 18, erro: 2) │
|
| 90 |
+
│ Dados: 33 GB / 53 GB │
|
| 91 |
+
│ Velocidade: 9.6 MB/s | ETA: 01h 15m │
|
| 92 |
+
│ │
|
| 93 |
+
│ Arquivo Atual: Parrot_Ara_001.wav │
|
| 94 |
+
│ ████████████░░░░░░░░░░░░░░░░░░░ 45.2% │
|
| 95 |
+
│ 3.2 / 7.1 MB │
|
| 96 |
+
│ │
|
| 97 |
+
│ Último evento: ✓ Thrush_042.wav enviado (2s) │
|
| 98 |
+
│ │
|
| 99 |
+
│ [ Pausar ] [ Cancelar ] [ Detalhes ] │
|
| 100 |
+
│ │
|
| 101 |
+
└─────────────────────────────────────────────────┘
|
| 102 |
+
```
|
| 103 |
+
|
| 104 |
+
### Tela 5: Resumo Final
|
| 105 |
+
```
|
| 106 |
+
┌─────────────────────────────────────────────────┐
|
| 107 |
+
│ ✓ Upload Concluído! │
|
| 108 |
+
│ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │
|
| 109 |
+
│ │
|
| 110 |
+
│ Repositório: alice/my-birds-2026 │
|
| 111 |
+
│ URL: https://huggingface.co/datasets/alice/.. │
|
| 112 |
+
│ │
|
| 113 |
+
│ Resumo: │
|
| 114 |
+
│ ✓ Enviados: 1.227 arquivos │
|
| 115 |
+
│ → Pulados: 18 (já existiam) │
|
| 116 |
+
│ ✗ Erros: 2 (revisar detalhes) │
|
| 117 |
+
│ ⏱ Tempo total: 4h 23min │
|
| 118 |
+
│ 📊 Velocidade média: 9.8 MB/s │
|
| 119 |
+
│ │
|
| 120 |
+
│ Próximos passos: │
|
| 121 |
+
│ 1. Validar dados com BirdNET Validator │
|
| 122 |
+
│ 2. Verificar relatório de ingestão │
|
| 123 |
+
│ │
|
| 124 |
+
│ Relatório salvo em: ~/.birdnet-uploader/ │
|
| 125 |
+
│ sessions/upload-xxx/ │
|
| 126 |
+
│ report.json │
|
| 127 |
+
│ │
|
| 128 |
+
│ [ Ver Relatório Completo ] [ Sair ] │
|
| 129 |
+
│ │
|
| 130 |
+
└─────────────────────────────────────────────────┘
|
| 131 |
+
```
|
| 132 |
+
|
| 133 |
+
---
|
| 134 |
+
|
| 135 |
+
## 2. ARQUITETURA LIMPA E MODULAR
|
| 136 |
+
|
| 137 |
+
### Estrutura de Diretórios do Projeto
|
| 138 |
+
|
| 139 |
+
```
|
| 140 |
+
birdnet-uploader/
|
| 141 |
+
├── src/
|
| 142 |
+
│ ├── __init__.py
|
| 143 |
+
│ ├── main.py # Entrypoint principal
|
| 144 |
+
│ ├── config.py # Config app e constantes
|
| 145 |
+
│ ├── auth/
|
| 146 |
+
│ │ ├── __init__.py
|
| 147 |
+
│ │ ├── auth_service.py # Login, token, keyring
|
| 148 |
+
│ │ └── token_manager.py # Gerenciamento de token
|
| 149 |
+
│ ├── repository/
|
| 150 |
+
│ │ ├── __init__.py
|
| 151 |
+
│ │ ├── repo_service.py # Criar/validar dataset HF
|
| 152 |
+
│ │ └── structure_initializer.py # Init estrutura padrão
|
| 153 |
+
│ ├── upload/
|
| 154 |
+
│ │ ├── __init__.py
|
| 155 |
+
│ │ ├── scanner.py # Varrer pasta local
|
| 156 |
+
│ │ ├── uploader.py # Motor de upload por lotes
|
| 157 |
+
│ │ ├── deduplicator.py # Checagem de duplicação
|
| 158 |
+
│ │ └── batch_processor.py # Processamento de lotes
|
| 159 |
+
│ ├── csv/
|
| 160 |
+
│ │ ├── __init__.py
|
| 161 |
+
│ │ ├── csv_matcher.py # Matching CSV + segmentos
|
| 162 |
+
│ │ └── indexer.py # Geração de índice/shards
|
| 163 |
+
│ ├── session/
|
| 164 |
+
│ │ ├── __init__.py
|
| 165 |
+
│ │ ├── session_manager.py # Gerenciamento de sessão
|
| 166 |
+
│ │ ├── checkpoint.py # Persistência de checkpoint
|
| 167 |
+
│ │ └── state_store.py # State file I/O
|
| 168 |
+
│ ├── ui/
|
| 169 |
+
│ │ ├── __init__.py
|
| 170 |
+
│ │ ├── tui.py # Interface Textual (TUI)
|
| 171 |
+
│ │ ├── progress_renderer.py # Barra de progresso rich
|
| 172 |
+
│ │ └── prompts.py # Diálogos e inputs
|
| 173 |
+
│ ├── logger/
|
| 174 |
+
│ │ ├── __init__.py
|
| 175 |
+
│ │ ├── structured_logger.py # Logs estruturados JSONL
|
| 176 |
+
│ │ └── telemetry.py # Eventos de telemetria
|
| 177 |
+
│ └── utils/
|
| 178 |
+
│ ├── __init__.py
|
| 179 |
+
│ ├── file_utils.py # Operações de arquivo
|
| 180 |
+
│ ├── hash_utils.py # Computação de hash
|
| 181 |
+
│ └── error_handler.py # Tratamento de erros
|
| 182 |
+
├── build/
|
| 183 |
+
│ ├── pyinstaller_config.spec # Config do PyInstaller
|
| 184 |
+
│ └── build.py # Script de build
|
| 185 |
+
├── tests/
|
| 186 |
+
│ ├── __init__.py
|
| 187 |
+
│ ├── test_auth.py
|
| 188 |
+
│ ├── test_uploader.py
|
| 189 |
+
│ ├── test_session.py
|
| 190 |
+
│ └── test_csv_matcher.py
|
| 191 |
+
├── README.md
|
| 192 |
+
├── requirements.txt
|
| 193 |
+
└── setup.py
|
| 194 |
+
```
|
| 195 |
+
|
| 196 |
+
### Dependências Principais
|
| 197 |
+
|
| 198 |
+
```
|
| 199 |
+
# requirements.txt
|
| 200 |
+
huggingface_hub>=0.28.0 # HF Hub API
|
| 201 |
+
click>=8.0.0 # CLI framework
|
| 202 |
+
textual>=0.30.0 # TUI moderna
|
| 203 |
+
rich>=13.0.0 # Terminal formatting
|
| 204 |
+
pydantic>=2.0.0 # Validação de dados
|
| 205 |
+
pandas>=2.0.0 # CSV parsing
|
| 206 |
+
pyarrow>=14.0.0 # Parquet para shards
|
| 207 |
+
keyring>=24.0.0 # Armazenamento seguro
|
| 208 |
+
cryptography>=41.0.0 # Criptografia local
|
| 209 |
+
```
|
| 210 |
+
|
| 211 |
+
---
|
| 212 |
+
|
| 213 |
+
## 3. COMPATIBILIDADE COM ESTRUTURA DO VALIDADOR
|
| 214 |
+
|
| 215 |
+
### Estrutura Esperada do Dataset
|
| 216 |
+
|
| 217 |
+
```
|
| 218 |
+
dataset_repo_id = "username/project-dataset"
|
| 219 |
+
|
| 220 |
+
audio/
|
| 221 |
+
├── {project_slug}/
|
| 222 |
+
│ ├── Specie_A/
|
| 223 |
+
│ │ ├── segment_001.wav
|
| 224 |
+
│ │ ├── segment_002.wav
|
| 225 |
+
│ │ └── ...
|
| 226 |
+
│ ├── Specie_B/
|
| 227 |
+
│ │ └── ...
|
| 228 |
+
│ └── ...
|
| 229 |
+
|
| 230 |
+
index/
|
| 231 |
+
├── shards/
|
| 232 |
+
│ ├── shard-00000.parquet
|
| 233 |
+
│ ├── shard-00001.parquet
|
| 234 |
+
│ └── ...
|
| 235 |
+
└── manifest.json
|
| 236 |
+
|
| 237 |
+
validations/
|
| 238 |
+
├── results-20250419.parquet
|
| 239 |
+
|
| 240 |
+
audit/
|
| 241 |
+
├── ingestion-runs/
|
| 242 |
+
│ └── 20250419T100000Z.json
|
| 243 |
+
```
|
| 244 |
+
|
| 245 |
+
### Manifest Format (Mantém compatibilidade)
|
| 246 |
+
|
| 247 |
+
```json
|
| 248 |
+
{
|
| 249 |
+
"schema_version": "1.0.0",
|
| 250 |
+
"project_slug": "my-birds",
|
| 251 |
+
"dataset_repo_id": "alice/my-birds-2026",
|
| 252 |
+
"updated_at": "2025-04-19T10:45:30Z",
|
| 253 |
+
"index": {
|
| 254 |
+
"total_detections": 1227,
|
| 255 |
+
"total_audio_files": 1227,
|
| 256 |
+
"shard_size": 10000,
|
| 257 |
+
"shards": [
|
| 258 |
+
{
|
| 259 |
+
"path": "index/shards/shard-00000.parquet",
|
| 260 |
+
"rows": 10000,
|
| 261 |
+
"sha256": "abc123...",
|
| 262 |
+
"size_bytes": 5242880
|
| 263 |
+
}
|
| 264 |
+
]
|
| 265 |
+
}
|
| 266 |
+
}
|
| 267 |
+
```
|
| 268 |
+
|
| 269 |
+
---
|
| 270 |
+
|
| 271 |
+
## 4. MÓDULO POR MÓDULO
|
| 272 |
+
|
| 273 |
+
### 4.1 Auth Service (`src/auth/auth_service.py`)
|
| 274 |
+
|
| 275 |
+
```python
|
| 276 |
+
from keyring import get_password, set_password
|
| 277 |
+
from huggingface_hub import HfApi, login
|
| 278 |
+
|
| 279 |
+
class AuthService:
|
| 280 |
+
KEYRING_SERVICE = "birdnet-uploader"
|
| 281 |
+
|
| 282 |
+
def authenticate(self, token: str) -> dict:
|
| 283 |
+
"""Valida token e retorna context do usuário."""
|
| 284 |
+
try:
|
| 285 |
+
api = HfApi(token=token)
|
| 286 |
+
user_info = api.whoami()
|
| 287 |
+
|
| 288 |
+
# Salvar em keyring seguro
|
| 289 |
+
set_password(self.KEYRING_SERVICE, "hf_token", token)
|
| 290 |
+
|
| 291 |
+
return {
|
| 292 |
+
"username": user_info["name"],
|
| 293 |
+
"email": user_info.get("email", ""),
|
| 294 |
+
"user_id": user_info["user_id"],
|
| 295 |
+
"token_expiry": self._estimate_expiry()
|
| 296 |
+
}
|
| 297 |
+
except Exception as e:
|
| 298 |
+
raise AuthenticationError(f"Token inválido: {str(e)}")
|
| 299 |
+
|
| 300 |
+
def get_token(self) -> str:
|
| 301 |
+
"""Recupera token do keyring."""
|
| 302 |
+
token = get_password(self.KEYRING_SERVICE, "hf_token")
|
| 303 |
+
if not token:
|
| 304 |
+
raise TokenNotFoundError("Token não encontrado. Execute 'birdnet auth login'")
|
| 305 |
+
return token
|
| 306 |
+
```
|
| 307 |
+
|
| 308 |
+
### 4.2 Repository Service (`src/repository/repo_service.py`)
|
| 309 |
+
|
| 310 |
+
```python
|
| 311 |
+
class RepositoryService:
|
| 312 |
+
def create_dataset(self,
|
| 313 |
+
repo_name: str,
|
| 314 |
+
private: bool = True) -> str:
|
| 315 |
+
"""Cria dataset no HF com estrutura padrão."""
|
| 316 |
+
api = HfApi(token=self.get_token())
|
| 317 |
+
|
| 318 |
+
repo_id = f"{username}/{repo_name}"
|
| 319 |
+
|
| 320 |
+
# Criar repo
|
| 321 |
+
repo = api.create_repo(
|
| 322 |
+
repo_id=repo_id,
|
| 323 |
+
repo_type="dataset",
|
| 324 |
+
private=private,
|
| 325 |
+
exist_ok=True
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
# Inicializar estrutura padrão
|
| 329 |
+
self._init_structure(repo_id)
|
| 330 |
+
|
| 331 |
+
return repo_id
|
| 332 |
+
|
| 333 |
+
def _init_structure(self, repo_id: str):
|
| 334 |
+
"""Inicializa pastas e arquivos padrão."""
|
| 335 |
+
folders = [
|
| 336 |
+
"audio/{project_slug}",
|
| 337 |
+
"index/shards",
|
| 338 |
+
"validations",
|
| 339 |
+
"audit/ingestion-runs"
|
| 340 |
+
]
|
| 341 |
+
|
| 342 |
+
for folder in folders:
|
| 343 |
+
# Criar .gitkeep
|
| 344 |
+
api.upload_file(
|
| 345 |
+
path_or_fileobj=BytesIO(b""),
|
| 346 |
+
path_in_repo=f"{folder}/.gitkeep",
|
| 347 |
+
repo_id=repo_id,
|
| 348 |
+
repo_type="dataset"
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
# Criar manifest.json inicial
|
| 352 |
+
manifest = {
|
| 353 |
+
"schema_version": "1.0.0",
|
| 354 |
+
"project_slug": project_slug,
|
| 355 |
+
"dataset_repo_id": repo_id,
|
| 356 |
+
"index": {
|
| 357 |
+
"total_detections": 0,
|
| 358 |
+
"total_audio_files": 0,
|
| 359 |
+
"shard_size": 10000,
|
| 360 |
+
"shards": []
|
| 361 |
+
}
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
api.upload_file(
|
| 365 |
+
path_or_fileobj=BytesIO(json.dumps(manifest).encode()),
|
| 366 |
+
path_in_repo="index/manifest.json",
|
| 367 |
+
repo_id=repo_id,
|
| 368 |
+
repo_type="dataset"
|
| 369 |
+
)
|
| 370 |
+
```
|
| 371 |
+
|
| 372 |
+
### 4.3 Scanner (`src/upload/scanner.py`)
|
| 373 |
+
|
| 374 |
+
```python
|
| 375 |
+
class LocalScanner:
|
| 376 |
+
def scan_folder(self, folder_path: str) -> dict:
|
| 377 |
+
"""Varre pasta recursiva e retorna estrutura."""
|
| 378 |
+
structure = {}
|
| 379 |
+
total_size = 0
|
| 380 |
+
file_count = 0
|
| 381 |
+
|
| 382 |
+
for root, dirs, files in os.walk(folder_path):
|
| 383 |
+
for file in files:
|
| 384 |
+
if file.lower().endswith(('.wav', '.mp3', '.flac', '.ogg')):
|
| 385 |
+
full_path = os.path.join(root, file)
|
| 386 |
+
|
| 387 |
+
# Inferir espécie do caminho ou nome
|
| 388 |
+
specie = self._infer_specie(root, file)
|
| 389 |
+
|
| 390 |
+
if specie not in structure:
|
| 391 |
+
structure[specie] = []
|
| 392 |
+
|
| 393 |
+
size = os.path.getsize(full_path)
|
| 394 |
+
structure[specie].append({
|
| 395 |
+
"name": file,
|
| 396 |
+
"full_path": full_path,
|
| 397 |
+
"size": size,
|
| 398 |
+
"local_hash": compute_hash(full_path)
|
| 399 |
+
})
|
| 400 |
+
|
| 401 |
+
total_size += size
|
| 402 |
+
file_count += 1
|
| 403 |
+
|
| 404 |
+
return {
|
| 405 |
+
"total_files": file_count,
|
| 406 |
+
"total_size": total_size,
|
| 407 |
+
"by_specie": structure,
|
| 408 |
+
"scan_timestamp": datetime.now()
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
def _infer_specie(self, root: str, file: str) -> str:
|
| 412 |
+
"""Inferir espécie do caminho da pasta."""
|
| 413 |
+
# Estratégia 1: última pasta em root é a espécie
|
| 414 |
+
parts = root.split(os.sep)
|
| 415 |
+
if len(parts) > 0:
|
| 416 |
+
return parts[-1]
|
| 417 |
+
|
| 418 |
+
# Fallback: usar nome do arquivo
|
| 419 |
+
return file.replace(".wav", "").split("_")[0]
|
| 420 |
+
```
|
| 421 |
+
|
| 422 |
+
### 4.4 Uploader (`src/upload/uploader.py`)
|
| 423 |
+
|
| 424 |
+
```python
|
| 425 |
+
class BatchUploader:
|
| 426 |
+
def __init__(self, repo_id: str, project_slug: str, token: str):
|
| 427 |
+
self.repo_id = repo_id
|
| 428 |
+
self.project_slug = project_slug
|
| 429 |
+
self.api = HfApi(token=token)
|
| 430 |
+
self.session = SessionManager(repo_id)
|
| 431 |
+
|
| 432 |
+
async def upload_files(self,
|
| 433 |
+
structure: dict,
|
| 434 |
+
csv_matcher: Optional[CSVMatcher] = None,
|
| 435 |
+
batch_size: int = 10,
|
| 436 |
+
on_progress: Optional[Callable] = None) -> dict:
|
| 437 |
+
"""Upload em lotes com retomada."""
|
| 438 |
+
checkpoint = self.session.load_checkpoint()
|
| 439 |
+
|
| 440 |
+
results = {
|
| 441 |
+
"total": 0,
|
| 442 |
+
"uploaded": 0,
|
| 443 |
+
"skipped": 0,
|
| 444 |
+
"failed": 0,
|
| 445 |
+
"errors": []
|
| 446 |
+
}
|
| 447 |
+
|
| 448 |
+
for specie, files in structure["by_specie"].items():
|
| 449 |
+
# Processar em lotes
|
| 450 |
+
for i in range(0, len(files), batch_size):
|
| 451 |
+
batch = files[i:i + batch_size]
|
| 452 |
+
|
| 453 |
+
# Verificar checkpoint
|
| 454 |
+
if self.session.is_batch_done(specie, i):
|
| 455 |
+
results["skipped"] += len(batch)
|
| 456 |
+
continue
|
| 457 |
+
|
| 458 |
+
# Enviar batch
|
| 459 |
+
for file_info in batch:
|
| 460 |
+
try:
|
| 461 |
+
if self._should_skip(file_info):
|
| 462 |
+
results["skipped"] += 1
|
| 463 |
+
self.session.mark_skipped(file_info)
|
| 464 |
+
continue
|
| 465 |
+
|
| 466 |
+
remote_path = f"audio/{self.project_slug}/{specie}/{file_info['name']}"
|
| 467 |
+
|
| 468 |
+
await self._upload_file_with_retry(
|
| 469 |
+
file_info["full_path"],
|
| 470 |
+
remote_path
|
| 471 |
+
)
|
| 472 |
+
|
| 473 |
+
results["uploaded"] += 1
|
| 474 |
+
self.session.mark_done(file_info)
|
| 475 |
+
|
| 476 |
+
if on_progress:
|
| 477 |
+
on_progress({
|
| 478 |
+
"total": structure["total_files"],
|
| 479 |
+
"uploaded": results["uploaded"],
|
| 480 |
+
"skipped": results["skipped"],
|
| 481 |
+
"failed": results["failed"],
|
| 482 |
+
"current_file": file_info["name"],
|
| 483 |
+
"current_size": file_info["size"]
|
| 484 |
+
})
|
| 485 |
+
|
| 486 |
+
except Exception as e:
|
| 487 |
+
results["failed"] += 1
|
| 488 |
+
results["errors"].append({
|
| 489 |
+
"file": file_info["name"],
|
| 490 |
+
"error": str(e)
|
| 491 |
+
})
|
| 492 |
+
self.session.mark_failed(file_info)
|
| 493 |
+
|
| 494 |
+
# Checkpoint por batch
|
| 495 |
+
self.session.save_checkpoint()
|
| 496 |
+
|
| 497 |
+
return results
|
| 498 |
+
|
| 499 |
+
async def _upload_file_with_retry(self,
|
| 500 |
+
local_path: str,
|
| 501 |
+
remote_path: str,
|
| 502 |
+
max_retries: int = 3):
|
| 503 |
+
"""Upload com retries e backoff exponencial."""
|
| 504 |
+
for attempt in range(max_retries):
|
| 505 |
+
try:
|
| 506 |
+
self.api.upload_file(
|
| 507 |
+
path_or_fileobj=local_path,
|
| 508 |
+
path_in_repo=remote_path,
|
| 509 |
+
repo_id=self.repo_id,
|
| 510 |
+
repo_type="dataset"
|
| 511 |
+
)
|
| 512 |
+
return
|
| 513 |
+
except Exception as e:
|
| 514 |
+
if attempt < max_retries - 1:
|
| 515 |
+
wait_time = 2 ** attempt + random.uniform(0, 1)
|
| 516 |
+
await asyncio.sleep(wait_time)
|
| 517 |
+
else:
|
| 518 |
+
raise
|
| 519 |
+
|
| 520 |
+
def _should_skip(self, file_info: dict) -> bool:
|
| 521 |
+
"""Verifica se arquivo já existe remotamente."""
|
| 522 |
+
try:
|
| 523 |
+
remote_path = f"audio/{self.project_slug}/{specie}/{file_info['name']}"
|
| 524 |
+
file_info_remote = self.api.get_file_info(
|
| 525 |
+
repo_id=self.repo_id,
|
| 526 |
+
filename=remote_path
|
| 527 |
+
)
|
| 528 |
+
return file_info_remote.size == file_info["size"]
|
| 529 |
+
except:
|
| 530 |
+
return False
|
| 531 |
+
```
|
| 532 |
+
|
| 533 |
+
### 4.5 Session Manager (`src/session/session_manager.py`)
|
| 534 |
+
|
| 535 |
+
```python
|
| 536 |
+
class SessionManager:
|
| 537 |
+
def __init__(self, repo_id: str):
|
| 538 |
+
self.session_id = f"upload-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
|
| 539 |
+
self.session_dir = Path(f"~/.birdnet-uploader/sessions/{self.session_id}")
|
| 540 |
+
self.session_dir.mkdir(parents=True, exist_ok=True)
|
| 541 |
+
|
| 542 |
+
def save_checkpoint(self, data: dict):
|
| 543 |
+
"""Salva checkpoint atomicamente."""
|
| 544 |
+
checkpoint_file = self.session_dir / "checkpoint.json"
|
| 545 |
+
temp_file = checkpoint_file.with_suffix(".tmp")
|
| 546 |
+
|
| 547 |
+
temp_file.write_text(json.dumps(data, indent=2))
|
| 548 |
+
temp_file.replace(checkpoint_file)
|
| 549 |
+
|
| 550 |
+
def load_checkpoint(self) -> dict:
|
| 551 |
+
"""Carrega último checkpoint."""
|
| 552 |
+
checkpoint_file = self.session_dir / "checkpoint.json"
|
| 553 |
+
if checkpoint_file.exists():
|
| 554 |
+
return json.loads(checkpoint_file.read_text())
|
| 555 |
+
return {}
|
| 556 |
+
```
|
| 557 |
+
|
| 558 |
+
### 4.6 CSV Matcher (`src/csv/csv_matcher.py`)
|
| 559 |
+
|
| 560 |
+
```python
|
| 561 |
+
class CSVMatcher:
|
| 562 |
+
def match_and_generate_index(self,
|
| 563 |
+
csv_path: str,
|
| 564 |
+
files_structure: dict) -> pd.DataFrame:
|
| 565 |
+
"""Faz matching entre CSV e segmentos, gera índice."""
|
| 566 |
+
df = pd.read_csv(csv_path)
|
| 567 |
+
|
| 568 |
+
# Validar colunas esperadas
|
| 569 |
+
required_cols = ["source_file", "scientific_name", "confidence",
|
| 570 |
+
"start_time", "end_time"]
|
| 571 |
+
if not all(col in df.columns for col in required_cols):
|
| 572 |
+
raise ValueError(f"CSV deve conter colunas: {required_cols}")
|
| 573 |
+
|
| 574 |
+
# Matching lógica
|
| 575 |
+
matched = []
|
| 576 |
+
for _, row in df.iterrows():
|
| 577 |
+
source_stem = Path(row["source_file"]).stem
|
| 578 |
+
|
| 579 |
+
# Procurar arquivo correspondente
|
| 580 |
+
for specie, files in files_structure["by_specie"].items():
|
| 581 |
+
for file_info in files:
|
| 582 |
+
if source_stem in file_info["name"]:
|
| 583 |
+
matched.append({
|
| 584 |
+
**row,
|
| 585 |
+
"segment_path": f"audio/{self.project_slug}/{specie}/{file_info['name']}"
|
| 586 |
+
})
|
| 587 |
+
|
| 588 |
+
return pd.DataFrame(matched)
|
| 589 |
+
|
| 590 |
+
def generate_shards(self,
|
| 591 |
+
index_df: pd.DataFrame,
|
| 592 |
+
output_dir: str,
|
| 593 |
+
shard_size: int = 10000) -> list:
|
| 594 |
+
"""Gera shards parquet para índice."""
|
| 595 |
+
shards = []
|
| 596 |
+
|
| 597 |
+
for i in range(0, len(index_df), shard_size):
|
| 598 |
+
shard_df = index_df.iloc[i:i+shard_size]
|
| 599 |
+
shard_file = f"{output_dir}/shard-{i//shard_size:05d}.parquet"
|
| 600 |
+
|
| 601 |
+
shard_df.to_parquet(shard_file, index=False)
|
| 602 |
+
shards.append(shard_file)
|
| 603 |
+
|
| 604 |
+
return shards
|
| 605 |
+
```
|
| 606 |
+
|
| 607 |
+
---
|
| 608 |
+
|
| 609 |
+
## 5. FLUXO DE COMANDOS CLI
|
| 610 |
+
|
| 611 |
+
```bash
|
| 612 |
+
# Autenticar
|
| 613 |
+
$ birdnet auth login
|
| 614 |
+
> Enter HF token: hf_...
|
| 615 |
+
|
| 616 |
+
# Upload simples (interactive)
|
| 617 |
+
$ birdnet upload
|
| 618 |
+
|
| 619 |
+
# Upload com flags (não-interativo)
|
| 620 |
+
$ birdnet upload \
|
| 621 |
+
--repo alice/my-birds \
|
| 622 |
+
--segments /path/to/segments \
|
| 623 |
+
--csv /path/to/metadata.csv \
|
| 624 |
+
--private
|
| 625 |
+
|
| 626 |
+
# Listar sessões
|
| 627 |
+
$ birdnet sessions list
|
| 628 |
+
|
| 629 |
+
# Retomar sessão
|
| 630 |
+
$ birdnet upload resume <session-id>
|
| 631 |
+
|
| 632 |
+
# Ver status
|
| 633 |
+
$ birdnet upload status <session-id>
|
| 634 |
+
|
| 635 |
+
# Exportar relatório
|
| 636 |
+
$ birdnet upload report <session-id>
|
| 637 |
+
```
|
| 638 |
+
|
| 639 |
+
---
|
| 640 |
+
|
| 641 |
+
## 6. CRITÉRIOS DE ACEITE DO MVP
|
| 642 |
+
|
| 643 |
+
- [ ] Usuário baixa executável portátil, sem Python pré-instalado
|
| 644 |
+
- [ ] Login com token HF funciona e não vaza em logs
|
| 645 |
+
- [ ] Criar dataset público/privado com estrutura padrão
|
| 646 |
+
- [ ] Varrer pasta local corretamente (recursivo, agrupa por espécie)
|
| 647 |
+
- [ ] Upload de 50GB+ com progresso contínuo e ETA coerente
|
| 648 |
+
- [ ] Pausar/fechar app/retomar sem duplicar upload
|
| 649 |
+
- [ ] Queda de rede + 429 recuperam com retries
|
| 650 |
+
- [ ] CSV opcional com matching integrado
|
| 651 |
+
- [ ] Relatório final estruturado (JSONL/JSON)
|
| 652 |
+
- [ ] Estrutura de arquivos compatível com validador
|
| 653 |
+
- [ ] Compatibilidade 100% com manifest/index esperado
|
| 654 |
+
- [ ] Sem regressão do fluxo atual do Space
|
| 655 |
+
|
| 656 |
+
---
|
| 657 |
+
|
| 658 |
+
## 7. PRÓXIMOS PASSOS
|
| 659 |
+
|
| 660 |
+
1. **Implementar módulos base** (Auth, Repo, Scanner, Uploader)
|
| 661 |
+
2. **Construir UI** (Textual TUI ou Rich + Click)
|
| 662 |
+
3. **Integrar Session Manager** e Checkpoint
|
| 663 |
+
4. **Testes de integração** (50GB+, retomada, rede)
|
| 664 |
+
5. **Construir executável** (PyInstaller)
|
| 665 |
+
6. **Validação end-to-end** com usuários reais
|
README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
---
|
| 2 |
-
title: BirdNET
|
| 3 |
emoji: "🐦"
|
| 4 |
colorFrom: green
|
| 5 |
colorTo: blue
|
|
@@ -10,76 +10,430 @@ app_file: app.py
|
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
-
# BirdNET-
|
| 14 |
|
| 15 |
-
BirdNET-
|
| 16 |
|
| 17 |
-
|
|
|
|
| 18 |
|
| 19 |
-
|
| 20 |
-
This app centralizes ingestion for bioacoustics projects by:
|
| 21 |
|
| 22 |
-
|
| 23 |
-
- creating deterministic detection keys
|
| 24 |
-
- uploading data in resumable batches
|
| 25 |
-
- generating index shards and manifest metadata
|
| 26 |
-
- producing audit-friendly ingestion reports
|
| 27 |
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
-
|
| 31 |
|
| 32 |
-
|
| 33 |
-
- initialize or repair the dataset repository structure
|
| 34 |
-
- choose repository visibility (Public by default, Private optional)
|
| 35 |
-
- prefill Dataset Repo (HF) and Project Slug for the next tabs
|
| 36 |
|
| 37 |
-
|
| 38 |
-
- upload
|
| 39 |
-
- supports pause/resume/cancel with progress tracking
|
| 40 |
|
| 41 |
-
|
| 42 |
-
- ingest detections CSV + segments archive in one run
|
| 43 |
-
- updates index and manifest artifacts for downstream consumers
|
| 44 |
|
| 45 |
-
|
|
|
|
|
|
|
|
|
|
| 46 |
|
| 47 |
-
|
| 48 |
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
-
|
| 52 |
-
2. In "Upload de Segmentos" section:
|
| 53 |
-
- **Locally**: Create compressed file of segments:
|
| 54 |
-
```bash
|
| 55 |
-
tar -czf segments.tar.gz /path/to/BirdNET\ Segments/
|
| 56 |
-
```
|
| 57 |
-
- **Upload**: Drag & drop `segments.tar.gz` to the file input
|
| 58 |
-
- Specify dataset repo (e.g., `username/ppbio-dataset`)
|
| 59 |
-
- Click "Enviar Segmentos para Dataset"
|
| 60 |
-
3. Once segments are uploaded, proceed to dry-run and upload detections
|
| 61 |
|
| 62 |
-
|
| 63 |
|
| 64 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
|
| 66 |
```bash
|
| 67 |
-
python
|
| 68 |
```
|
| 69 |
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
```bash
|
| 72 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
```
|
| 74 |
|
| 75 |
-
##
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
-
-
|
| 78 |
-
-
|
| 79 |
-
-
|
| 80 |
-
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
-
##
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
|
| 84 |
1. Create and activate a Python 3.11+ virtual environment.
|
| 85 |
2. Install dependencies:
|
|
@@ -88,65 +442,204 @@ python upload_segments_local.py "/path" "user/repo" --token YOUR_HF_TOKEN --batc
|
|
| 88 |
pip install -r requirements.txt
|
| 89 |
```
|
| 90 |
|
| 91 |
-
3. Run the app:
|
| 92 |
|
| 93 |
```bash
|
| 94 |
python app.py
|
| 95 |
```
|
| 96 |
|
| 97 |
-
|
| 98 |
|
| 99 |
-
|
|
|
|
|
|
|
| 100 |
|
| 101 |
-
|
| 102 |
-
2. Select detections CSV and segments root folder.
|
| 103 |
-
3. Run Dry-run and inspect output JSON.
|
| 104 |
-
4. Fix data issues if unmatched/ambiguous rows are high.
|
| 105 |
-
5. Run Upload mode when quality is acceptable.
|
| 106 |
-
6. Save run report for audit and reproducibility.
|
| 107 |
|
| 108 |
-
|
| 109 |
|
| 110 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
|
| 112 |
```bash
|
| 113 |
-
python -m
|
| 114 |
-
--project-slug ppbio-rabeca \
|
| 115 |
-
--dataset-repo USER/birdnet-ppbio-rabeca-dataset \
|
| 116 |
-
--detections-csv detections.csv \
|
| 117 |
-
--segments-root "C:\\BirdNET Segments" \
|
| 118 |
-
--dry-run \
|
| 119 |
-
--report-file .ingest-dry-run-report.json
|
| 120 |
```
|
| 121 |
|
|
|
|
|
|
|
| 122 |
```bash
|
| 123 |
-
python -m
|
| 124 |
-
--project-slug ppbio-rabeca \
|
| 125 |
-
--dataset-repo USER/birdnet-ppbio-rabeca-dataset \
|
| 126 |
-
--detections-csv detections.csv \
|
| 127 |
-
--segments-root "C:\\BirdNET Segments" \
|
| 128 |
-
--batch-size 200 \
|
| 129 |
-
--shard-size 10000 \
|
| 130 |
-
--max-retries 3 \
|
| 131 |
-
--retry-backoff-seconds 1.0 \
|
| 132 |
-
--resume-state-file .ingest-segments-state.json \
|
| 133 |
-
--report-file .ingest-run-report.json
|
| 134 |
```
|
| 135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
## Repository Structure
|
| 137 |
|
| 138 |
-
- `app.py`:
|
| 139 |
-
- `src/
|
| 140 |
-
- `
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
|
| 142 |
-
|
| 143 |
|
| 144 |
-
|
| 145 |
-
- Keep run reports for traceability.
|
| 146 |
-
- Prefer project-level dataset repos for cleaner access control.
|
| 147 |
-
- For public deployment, use Hugging Face Spaces with repository sync from GitHub.
|
| 148 |
|
| 149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
|
| 151 |
-
|
| 152 |
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: BirdNET-Validator-App
|
| 3 |
emoji: "🐦"
|
| 4 |
colorFrom: green
|
| 5 |
colorTo: blue
|
|
|
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
+
# BirdNET-Validator-App
|
| 14 |
|
| 15 |
+
BirdNET-Validator-App is a multi-purpose BirdNET workflow repository with two complementary entry points:
|
| 16 |
|
| 17 |
+
1. A Gradio-based web application for human validation of BirdNET detections in multi-project workflows.
|
| 18 |
+
2. A local command-line uploader for resumable Hugging Face dataset uploads with checkpoints, deduplication, and a Windows-friendly interactive flow.
|
| 19 |
|
| 20 |
+
The validator app focuses on review, authorization, and auditability. The uploader CLI focuses on reliable dataset ingestion for large audio collections.
|
|
|
|
| 21 |
|
| 22 |
+
## Contents
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
+
- [Overview](#overview)
|
| 25 |
+
- [Validator App](#validator-app)
|
| 26 |
+
- [Local Uploader CLI](#local-uploader-cli)
|
| 27 |
+
- [Quick Start](#quick-start)
|
| 28 |
+
- [Configuration](#configuration)
|
| 29 |
+
- [Typical Flows](#typical-flows)
|
| 30 |
+
- [Repository Structure](#repository-structure)
|
| 31 |
+
- [Troubleshooting](#troubleshooting)
|
| 32 |
+
- [Development Notes](#development-notes)
|
| 33 |
|
| 34 |
+
## Overview
|
| 35 |
|
| 36 |
+
This repository is designed for two high-confidence operations:
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
+
- controlled, multi-project human validation of detections
|
| 39 |
+
- reliable, resumable upload of large audio datasets to Hugging Face
|
|
|
|
| 40 |
|
| 41 |
+
The validator workflow emphasizes:
|
|
|
|
|
|
|
| 42 |
|
| 43 |
+
- controlled access by user and project
|
| 44 |
+
- rapid decision flow for detections
|
| 45 |
+
- auditability through append-only events
|
| 46 |
+
- conflict-aware updates for concurrent validators
|
| 47 |
|
| 48 |
+
The uploader workflow emphasizes:
|
| 49 |
|
| 50 |
+
- single-flow login → repo → scan → upload
|
| 51 |
+
- resumable sessions stored locally
|
| 52 |
+
- remote deduplication with a local cache
|
| 53 |
+
- progress tracking and checkpoint recovery
|
| 54 |
+
- compatibility with the validator app's expected dataset structure
|
| 55 |
|
| 56 |
+
## Validator App
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
|
| 58 |
+
The validator is a Gradio application for human review of BirdNET detections in multi-project workflows.
|
| 59 |
|
| 60 |
+
### Key Usability Features
|
| 61 |
+
|
| 62 |
+
1. Multi-project login with role-based access
|
| 63 |
+
2. Project selection for authorized datasets
|
| 64 |
+
3. Detection queue with filters and pagination
|
| 65 |
+
4. Audio loading on demand with ephemeral cache
|
| 66 |
+
5. Fast validation actions (positive, negative, uncertain, skip)
|
| 67 |
+
6. Concurrency-safe writes with optimistic lock feedback
|
| 68 |
+
7. Conflict resolution support and validation reporting
|
| 69 |
+
|
| 70 |
+
### Quick Start (Local)
|
| 71 |
+
|
| 72 |
+
1. Create and activate a Python 3.11+ virtual environment.
|
| 73 |
+
2. Install dependencies:
|
| 74 |
+
|
| 75 |
+
```bash
|
| 76 |
+
pip install -r requirements.txt
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
3. Run the app:
|
| 80 |
+
|
| 81 |
+
```bash
|
| 82 |
+
python app.py
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
Default port is `7860`.
|
| 86 |
+
|
| 87 |
+
### Runtime Configuration
|
| 88 |
+
|
| 89 |
+
Optional runtime configuration:
|
| 90 |
+
|
| 91 |
+
- `BIRDNET_DETECTIONS_FILE`: path to JSON file with detections grouped by project slug.
|
| 92 |
+
- `BIRDNET_VALIDATIONS_DIR`: custom directory for append-only validation events and snapshot files.
|
| 93 |
+
- `BIRDNET_PAGE_SIZE`: queue page size (default: `25`).
|
| 94 |
+
- `BIRDNET_PROJECTS_FILE`: JSON file with project catalog used at app startup.
|
| 95 |
+
- `BIRDNET_USER_ACCESS_FILE`: JSON file mapping users to project roles (`admin`/`validator`).
|
| 96 |
+
- `BIRDNET_INVITES_FILE`: JSON file storing pending invites.
|
| 97 |
+
- `BIRDNET_INVITE_TTL_HOURS`: invite expiration in hours (default: `72`).
|
| 98 |
+
- `BIRDNET_BOOTSTRAP_DIR`: base directory for bootstrap state files (`projects.json`, `user_access.json`, `invites.json`).
|
| 99 |
+
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP`: set to `true` only for local/demo mode to load built-in sample users/projects.
|
| 100 |
+
|
| 101 |
+
Invite email settings (EmailJS):
|
| 102 |
+
|
| 103 |
+
- `BIRDNET_EMAILJS_ENABLED`: set to `true` to send invite emails via EmailJS.
|
| 104 |
+
- `BIRDNET_INVITE_EMAIL_ENABLED`: set to `true` to enable invite delivery.
|
| 105 |
+
- `BIRDNET_INVITE_EMAIL_SENDER`: sender label shown in the invite email.
|
| 106 |
+
- `BIRDNET_INVITE_EMAIL_LOGIN_URL`: login URL included in invitation instructions.
|
| 107 |
+
- `BIRDNET_EMAILJS_SERVICE_ID`: EmailJS service ID.
|
| 108 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID`: default/fallback EmailJS template ID.
|
| 109 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_USERNAME_ONLY`: optional template for username-only mode.
|
| 110 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_EMAIL_ONLY`: optional template for email-only mode.
|
| 111 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_DUAL`: optional template for dual mode (username + email).
|
| 112 |
+
- `BIRDNET_EMAILJS_PUBLIC_KEY`: EmailJS public key.
|
| 113 |
+
- `BIRDNET_EMAILJS_ENDPOINT`: EmailJS API endpoint (default: `https://api.emailjs.com/api/v1.0/email/send`).
|
| 114 |
+
- `BIRDNET_EMAILJS_TIMEOUT_SECONDS`: request timeout in seconds (default: `20`).
|
| 115 |
+
|
| 116 |
+
### Validation Data Shapes
|
| 117 |
+
|
| 118 |
+
Detection seed file example:
|
| 119 |
+
|
| 120 |
+
```json
|
| 121 |
+
{
|
| 122 |
+
"kenya-2024": [
|
| 123 |
+
{
|
| 124 |
+
"detection_key": "0000000000001001",
|
| 125 |
+
"audio_id": "audio_1001",
|
| 126 |
+
"scientific_name": "Cyanocorax cyanopogon",
|
| 127 |
+
"confidence": 0.91,
|
| 128 |
+
"start_time": 0.0,
|
| 129 |
+
"end_time": 1.0
|
| 130 |
+
}
|
| 131 |
+
]
|
| 132 |
+
}
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
Projects bootstrap file example (`BIRDNET_PROJECTS_FILE`):
|
| 136 |
+
|
| 137 |
+
```json
|
| 138 |
+
[
|
| 139 |
+
{
|
| 140 |
+
"project_slug": "kenya-2024",
|
| 141 |
+
"name": "Kenya Survey 2024",
|
| 142 |
+
"dataset_repo_id": "org/kenya-2024-dataset",
|
| 143 |
+
"active": true
|
| 144 |
+
}
|
| 145 |
+
]
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
User access bootstrap file example (`BIRDNET_USER_ACCESS_FILE`):
|
| 149 |
+
|
| 150 |
+
```json
|
| 151 |
+
{
|
| 152 |
+
"admin_user": {
|
| 153 |
+
"kenya-2024": "admin"
|
| 154 |
+
},
|
| 155 |
+
"validator_a": {
|
| 156 |
+
"kenya-2024": "validator"
|
| 157 |
+
}
|
| 158 |
+
}
|
| 159 |
+
```
|
| 160 |
+
|
| 161 |
+
### Typical Validation Flow
|
| 162 |
+
|
| 163 |
+
1. Login with a valid user.
|
| 164 |
+
2. Select an authorized project.
|
| 165 |
+
3. Load queue items and apply filters when needed.
|
| 166 |
+
4. Listen to selected audio and submit validation status.
|
| 167 |
+
5. Resolve conflicts when concurrent updates occur.
|
| 168 |
+
6. Export or inspect project validation report.
|
| 169 |
+
|
| 170 |
+
### Deploy on Hugging Face Spaces
|
| 171 |
+
|
| 172 |
+
1. Create a new Space with:
|
| 173 |
+
- SDK: `Gradio`
|
| 174 |
+
- Python: `3.11`
|
| 175 |
+
|
| 176 |
+
2. Push this repository to the Space.
|
| 177 |
+
|
| 178 |
+
3. Configure Variables/Secrets in the Space settings.
|
| 179 |
+
|
| 180 |
+
Required for production bootstrap:
|
| 181 |
+
|
| 182 |
+
- `BIRDNET_PROJECTS_FILE` (example: `docs/spaces/projects.sample.json`)
|
| 183 |
+
- `BIRDNET_USER_ACCESS_FILE` (example: `docs/spaces/user_access.sample.json`)
|
| 184 |
+
|
| 185 |
+
Optional runtime settings:
|
| 186 |
+
|
| 187 |
+
- `BIRDNET_DETECTIONS_FILE` (seed detections JSON)
|
| 188 |
+
- `BIRDNET_VALIDATIONS_DIR` (recommended: `/data/validations`)
|
| 189 |
+
- `BIRDNET_BOOTSTRAP_DIR` (recommended: `/data/bootstrap`)
|
| 190 |
+
- `BIRDNET_INVITES_FILE` (recommended: `/data/bootstrap/invites.json`)
|
| 191 |
+
- `BIRDNET_INVITE_TTL_HOURS` (default `72`)
|
| 192 |
+
- `BIRDNET_PAGE_SIZE` (default `25`)
|
| 193 |
+
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP` (`false` in production)
|
| 194 |
+
|
| 195 |
+
Optional invite email settings:
|
| 196 |
+
|
| 197 |
+
- `BIRDNET_INVITE_EMAIL_ENABLED=true`
|
| 198 |
+
- `BIRDNET_INVITE_EMAIL_SENDER`
|
| 199 |
+
- `BIRDNET_INVITE_EMAIL_LOGIN_URL`
|
| 200 |
+
- `BIRDNET_EMAILJS_ENABLED`
|
| 201 |
+
- `BIRDNET_EMAILJS_SERVICE_ID`
|
| 202 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID`
|
| 203 |
+
- `BIRDNET_EMAILJS_PUBLIC_KEY`
|
| 204 |
+
- `BIRDNET_EMAILJS_ENDPOINT`
|
| 205 |
+
- `BIRDNET_EMAILJS_TIMEOUT_SECONDS`
|
| 206 |
+
|
| 207 |
+
4. For first smoke test only, you may temporarily set:
|
| 208 |
+
|
| 209 |
+
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP=true`
|
| 210 |
+
|
| 211 |
+
Then log in with one of the demo users:
|
| 212 |
+
|
| 213 |
+
- `admin_user`
|
| 214 |
+
- `demo_user`
|
| 215 |
+
- `validator_demo`
|
| 216 |
+
|
| 217 |
+
5. After validation, switch to production bootstrap:
|
| 218 |
+
|
| 219 |
+
- Set `BIRDNET_ENABLE_DEMO_BOOTSTRAP=false`
|
| 220 |
+
- Provide real `BIRDNET_PROJECTS_FILE` and `BIRDNET_USER_ACCESS_FILE`
|
| 221 |
+
|
| 222 |
+
Notes:
|
| 223 |
+
|
| 224 |
+
- The app entrypoint reads `PORT` automatically in Spaces.
|
| 225 |
+
- Keep user/project bootstrap files private if they contain sensitive assignments.
|
| 226 |
+
- Use `/data` paths in Spaces to keep projects, invites, ACL, and validations across redeploys.
|
| 227 |
+
- Collaborative access is token-per-user: each collaborator logs in with their own Hugging Face token.
|
| 228 |
+
|
| 229 |
+
## Local Uploader CLI
|
| 230 |
+
|
| 231 |
+
The repository also includes a local uploader CLI designed for large dataset ingest and resume support.
|
| 232 |
+
|
| 233 |
+
### Release Pipeline
|
| 234 |
+
|
| 235 |
+
The repository includes a release pipeline that produces a portable uploader bundle and can publish it to Hugging Face.
|
| 236 |
+
|
| 237 |
+
One-command Windows pipeline:
|
| 238 |
+
|
| 239 |
+
```powershell
|
| 240 |
+
.\build\release_pipeline.ps1 -Version 0.1.0
|
| 241 |
+
```
|
| 242 |
+
|
| 243 |
+
Build and publish in one step:
|
| 244 |
+
|
| 245 |
+
```powershell
|
| 246 |
+
$env:HF_TOKEN = "hf_xxx"
|
| 247 |
+
.\build\release_pipeline.ps1 -Version 0.1.0 -PublishToHf -RepoId jrrribeiro/birdnet-uploader-releases -RepoType dataset
|
| 248 |
+
```
|
| 249 |
+
|
| 250 |
+
Validation-only step (recommended before publish):
|
| 251 |
+
|
| 252 |
+
```powershell
|
| 253 |
+
.\build\validate_release.ps1 -Version 0.1.0
|
| 254 |
+
```
|
| 255 |
+
|
| 256 |
+
Local build steps:
|
| 257 |
+
|
| 258 |
+
```bash
|
| 259 |
+
python -m pip install --upgrade pip
|
| 260 |
+
pip install -r requirements.txt
|
| 261 |
+
pip install pyinstaller
|
| 262 |
+
python build/release_uploader.py --version 0.1.0
|
| 263 |
+
```
|
| 264 |
+
|
| 265 |
+
The build produces:
|
| 266 |
+
|
| 267 |
+
- a zipped release bundle under `build/release/`
|
| 268 |
+
- a `.sha256` checksum file next to the zip
|
| 269 |
+
|
| 270 |
+
To publish a release bundle to Hugging Face from a local machine:
|
| 271 |
+
|
| 272 |
+
```bash
|
| 273 |
+
set HF_TOKEN=hf_xxx
|
| 274 |
+
python build/publish_release_to_hf.py ^
|
| 275 |
+
--repo-id jrrribeiro/birdnet-uploader-releases ^
|
| 276 |
+
--repo-type dataset ^
|
| 277 |
+
--bundle build\release\birdnet-uploader-0.1.0-windows.zip ^
|
| 278 |
+
--checksum build\release\birdnet-uploader-0.1.0-windows.zip.sha256 ^
|
| 279 |
+
--version 0.1.0
|
| 280 |
+
```
|
| 281 |
+
|
| 282 |
+
Recommended Hugging Face release repo layout:
|
| 283 |
+
|
| 284 |
+
```text
|
| 285 |
+
birdnet-uploader-releases/
|
| 286 |
+
releases/
|
| 287 |
+
v0.1.0/
|
| 288 |
+
birdnet-uploader-0.1.0-windows.zip
|
| 289 |
+
birdnet-uploader-0.1.0-windows.zip.sha256
|
| 290 |
+
```
|
| 291 |
+
|
| 292 |
+
Workflow on GitHub Actions:
|
| 293 |
+
|
| 294 |
+
- run the release workflow manually
|
| 295 |
+
- build the Windows executable with PyInstaller
|
| 296 |
+
- archive the bundle as a workflow artifact
|
| 297 |
+
- optionally upload the bundle and checksum to a Hugging Face repo
|
| 298 |
+
|
| 299 |
+
Recommended user-facing validation after publishing:
|
| 300 |
+
|
| 301 |
+
1. Download the zip from Hugging Face.
|
| 302 |
+
2. Unzip into a clean folder.
|
| 303 |
+
3. Run the executable without Python installed.
|
| 304 |
+
4. Authenticate with a Hugging Face token.
|
| 305 |
+
5. Create a session and confirm resume/recovery works.
|
| 306 |
+
|
| 307 |
+
### What It Does
|
| 308 |
+
|
| 309 |
+
- authenticates with a Hugging Face token stored in the OS keyring
|
| 310 |
+
- creates or validates dataset repositories
|
| 311 |
+
- scans a local audio folder recursively
|
| 312 |
+
- computes per-file SHA-256 hashes
|
| 313 |
+
- deduplicates against a cached remote index
|
| 314 |
+
- uploads files with retries and session checkpoints
|
| 315 |
+
- resumes from the last checkpoint after close or crash
|
| 316 |
+
|
| 317 |
+
### Main Commands
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
The CLI entrypoint is:
|
| 321 |
|
| 322 |
```bash
|
| 323 |
+
python -m src.uploader_cli.main --help
|
| 324 |
```
|
| 325 |
|
| 326 |
+
Commands:
|
| 327 |
+
|
| 328 |
+
- `login`: validate and store a Hugging Face token
|
| 329 |
+
- `init-repo`: create and initialize a dataset repository
|
| 330 |
+
- `scan`: scan a local folder and summarize audio files
|
| 331 |
+
- `start`: create a local upload session checkpoint
|
| 332 |
+
- `resume`: print the saved checkpoint for a session
|
| 333 |
+
|
| 334 |
+
### Example Workflow
|
| 335 |
+
|
| 336 |
```bash
|
| 337 |
+
# Authenticate once
|
| 338 |
+
python -m src.uploader_cli.main login
|
| 339 |
+
|
| 340 |
+
# Create or initialize the dataset repository
|
| 341 |
+
python -m src.uploader_cli.main init-repo --repo-id alice/birdnet-2026 --private
|
| 342 |
+
|
| 343 |
+
# Inspect the local folder before upload
|
| 344 |
+
python -m src.uploader_cli.main scan --segments C:\audio\segments
|
| 345 |
+
|
| 346 |
+
# Create a resumable local session checkpoint
|
| 347 |
+
python -m src.uploader_cli.main start --repo-id alice/birdnet-2026 --segments C:\audio\segments
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
# Resume and inspect the checkpoint later
|
| 351 |
+
python -m src.uploader_cli.main resume upload-20260429T120000Z
|
| 352 |
```
|
| 353 |
|
| 354 |
+
### Upload Engine Building Blocks
|
| 355 |
+
|
| 356 |
+
The uploader CLI is built from the following modules:
|
| 357 |
+
|
| 358 |
+
- `src/uploader_cli/auth_service.py`: token validation and keyring storage
|
| 359 |
+
- `src/uploader_cli/repo_service.py`: dataset creation and structure initialization
|
| 360 |
+
- `src/uploader_cli/scanner.py`: recursive audio discovery and metadata collection
|
| 361 |
+
- `src/uploader_cli/hash_utils.py`: streaming SHA-256 helpers
|
| 362 |
+
- `src/uploader_cli/deduplicator.py`: remote index cache and skip/upload decisions
|
| 363 |
+
- `src/uploader_cli/session_manager.py`: checkpoint and session metadata persistence
|
| 364 |
+
- `src/uploader_cli/batch_uploader.py`: retrying upload orchestration
|
| 365 |
+
- `src/uploader_cli/error_handler.py`: user-friendly error formatting
|
| 366 |
+
|
| 367 |
+
### Local Data Layout
|
| 368 |
+
|
| 369 |
+
The uploader stores local state under the user's profile directory by default:
|
| 370 |
+
|
| 371 |
+
```text
|
| 372 |
+
~/.birdnet-uploader/
|
| 373 |
+
sessions/
|
| 374 |
+
upload-YYYYMMDDTHHMMSSZ/
|
| 375 |
+
metadata.json
|
| 376 |
+
checkpoint.json
|
| 377 |
+
cache/
|
| 378 |
+
dedup/
|
| 379 |
+
<repo-hash>.json
|
| 380 |
+
logs/
|
| 381 |
+
```
|
| 382 |
+
|
| 383 |
+
Environment overrides are supported:
|
| 384 |
+
|
| 385 |
+
- `BIRDNET_UPLOADER_SESSION_DIR`
|
| 386 |
+
- `BIRDNET_UPLOADER_CACHE_DIR`
|
| 387 |
+
- `BIRDNET_UPLOADER_LOG_DIR`
|
| 388 |
+
|
| 389 |
+
### Dataset Layout Expected by the Validator
|
| 390 |
+
|
| 391 |
+
The uploader is designed to preserve the validator's expected structure:
|
| 392 |
+
|
| 393 |
+
```text
|
| 394 |
+
audio/{project_slug}/... # uploaded audio files
|
| 395 |
+
index/shards/... # parquet shard files
|
| 396 |
+
index/manifest.json # index manifest
|
| 397 |
+
validations/... # validation outputs
|
| 398 |
+
audit/ingestion-runs/... # ingestion run history
|
| 399 |
+
```
|
| 400 |
+
|
| 401 |
+
### Uploader Configuration
|
| 402 |
+
|
| 403 |
+
Runtime constants are centralized in `src/uploader_cli/config.py`:
|
| 404 |
|
| 405 |
+
- `APP_NAME`
|
| 406 |
+
- `SCHEMA_VERSION`
|
| 407 |
+
- `AUDIO_EXTENSIONS`
|
| 408 |
+
- `INDEX_SHARD_SIZE`
|
| 409 |
+
- `MAX_BATCH_SIZE`
|
| 410 |
+
- `RETRY_MAX_ATTEMPTS`
|
| 411 |
+
- `RETRY_INITIAL_BACKOFF_SECONDS`
|
| 412 |
+
- `RETRY_MAX_BACKOFF_SECONDS`
|
| 413 |
+
- `KEYRING_SERVICE`
|
| 414 |
+
- `KEYRING_ACCOUNT`
|
| 415 |
|
| 416 |
+
### Resumability Guarantees
|
| 417 |
+
|
| 418 |
+
The uploader uses local checkpoints so it can recover from:
|
| 419 |
+
|
| 420 |
+
- process crashes
|
| 421 |
+
- app close
|
| 422 |
+
- transient network failures
|
| 423 |
+
- partial progress during large uploads
|
| 424 |
+
|
| 425 |
+
The checkpoint stores session metadata and cumulative counters such as:
|
| 426 |
+
|
| 427 |
+
- uploaded count
|
| 428 |
+
- skipped count
|
| 429 |
+
- failed count
|
| 430 |
+
- last completed file
|
| 431 |
+
- last failed file
|
| 432 |
+
- total uploaded bytes
|
| 433 |
+
|
| 434 |
+
## Quick Start
|
| 435 |
+
|
| 436 |
+
### Local Development
|
| 437 |
|
| 438 |
1. Create and activate a Python 3.11+ virtual environment.
|
| 439 |
2. Install dependencies:
|
|
|
|
| 442 |
pip install -r requirements.txt
|
| 443 |
```
|
| 444 |
|
| 445 |
+
3. Run the validator app:
|
| 446 |
|
| 447 |
```bash
|
| 448 |
python app.py
|
| 449 |
```
|
| 450 |
|
| 451 |
+
4. Run the uploader CLI help:
|
| 452 |
|
| 453 |
+
```bash
|
| 454 |
+
python -m src.uploader_cli.main --help
|
| 455 |
+
```
|
| 456 |
|
| 457 |
+
### First-Time Uploader Setup
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 458 |
|
| 459 |
+
1. Log in with a Hugging Face token:
|
| 460 |
|
| 461 |
+
```bash
|
| 462 |
+
python -m src.uploader_cli.main login
|
| 463 |
+
```
|
| 464 |
+
|
| 465 |
+
2. Validate your target repository exists or create it:
|
| 466 |
|
| 467 |
```bash
|
| 468 |
+
python -m src.uploader_cli.main init-repo --repo-id alice/birdnet-2026 --private
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 469 |
```
|
| 470 |
|
| 471 |
+
3. Scan your local dataset folder:
|
| 472 |
+
|
| 473 |
```bash
|
| 474 |
+
python -m src.uploader_cli.main scan --segments C:\audio\segments
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 475 |
```
|
| 476 |
|
| 477 |
+
4. Start a resumable session:
|
| 478 |
+
|
| 479 |
+
```bash
|
| 480 |
+
python -m src.uploader_cli.main start --repo-id alice/birdnet-2026 --segments C:\audio\segments
|
| 481 |
+
```
|
| 482 |
+
|
| 483 |
+
## Configuration
|
| 484 |
+
|
| 485 |
+
### Validator App Environment Variables
|
| 486 |
+
|
| 487 |
+
- `BIRDNET_DETECTIONS_FILE`
|
| 488 |
+
- `BIRDNET_VALIDATIONS_DIR`
|
| 489 |
+
- `BIRDNET_PAGE_SIZE`
|
| 490 |
+
- `BIRDNET_PROJECTS_FILE`
|
| 491 |
+
- `BIRDNET_USER_ACCESS_FILE`
|
| 492 |
+
- `BIRDNET_INVITES_FILE`
|
| 493 |
+
- `BIRDNET_INVITE_TTL_HOURS`
|
| 494 |
+
- `BIRDNET_BOOTSTRAP_DIR`
|
| 495 |
+
- `BIRDNET_ENABLE_DEMO_BOOTSTRAP`
|
| 496 |
+
- `BIRDNET_EMAILJS_ENABLED`
|
| 497 |
+
- `BIRDNET_INVITE_EMAIL_ENABLED`
|
| 498 |
+
- `BIRDNET_INVITE_EMAIL_SENDER`
|
| 499 |
+
- `BIRDNET_INVITE_EMAIL_LOGIN_URL`
|
| 500 |
+
- `BIRDNET_EMAILJS_SERVICE_ID`
|
| 501 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID`
|
| 502 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_USERNAME_ONLY`
|
| 503 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_EMAIL_ONLY`
|
| 504 |
+
- `BIRDNET_EMAILJS_TEMPLATE_ID_DUAL`
|
| 505 |
+
- `BIRDNET_EMAILJS_PUBLIC_KEY`
|
| 506 |
+
- `BIRDNET_EMAILJS_ENDPOINT`
|
| 507 |
+
- `BIRDNET_EMAILJS_TIMEOUT_SECONDS`
|
| 508 |
+
|
| 509 |
+
### Uploader CLI Environment Variables
|
| 510 |
+
|
| 511 |
+
- `BIRDNET_UPLOADER_SESSION_DIR`
|
| 512 |
+
- `BIRDNET_UPLOADER_CACHE_DIR`
|
| 513 |
+
- `BIRDNET_UPLOADER_LOG_DIR`
|
| 514 |
+
|
| 515 |
+
## Typical Flows
|
| 516 |
+
|
| 517 |
+
### Validation Flow
|
| 518 |
+
|
| 519 |
+
1. Login with a valid user.
|
| 520 |
+
2. Select an authorized project.
|
| 521 |
+
3. Load queue items and apply filters when needed.
|
| 522 |
+
4. Listen to selected audio and submit validation status.
|
| 523 |
+
5. Resolve conflicts when concurrent updates occur.
|
| 524 |
+
6. Export or inspect project validation report.
|
| 525 |
+
|
| 526 |
+
### Upload Flow
|
| 527 |
+
|
| 528 |
+
1. Authenticate once with a Hugging Face token.
|
| 529 |
+
2. Create or validate a dataset repository.
|
| 530 |
+
3. Scan the local folder recursively.
|
| 531 |
+
4. Deduplicate against remote files.
|
| 532 |
+
5. Upload with retries and checkpoints.
|
| 533 |
+
6. Resume later from the saved session metadata if needed.
|
| 534 |
+
|
| 535 |
## Repository Structure
|
| 536 |
|
| 537 |
+
- `app.py`: validator app entrypoint
|
| 538 |
+
- `src/auth`: authentication/session and ACL logic
|
| 539 |
+
- `src/domain`: domain models
|
| 540 |
+
- `src/repositories`: in-memory and append-only persistence
|
| 541 |
+
- `src/services`: queue, validation, and audio fetch services
|
| 542 |
+
- `src/ui`: Gradio interface composition and callbacks
|
| 543 |
+
- `src/uploader_cli`: local uploader CLI, session persistence, deduplication, and retrying uploads
|
| 544 |
+
- `tests`: unit and integration test suites
|
| 545 |
+
- `docs`: collaborative workflow and QA checklists
|
| 546 |
+
|
| 547 |
+
## Project Status
|
| 548 |
+
|
| 549 |
+
Validator workflow is active and under continuous development, with emphasis on reliability, auditability, and operator productivity.
|
| 550 |
+
|
| 551 |
+
The uploader CLI is being built as a companion local workflow for large ingest tasks, with a focus on resumability and validator compatibility.
|
| 552 |
+
|
| 553 |
+
## FAQ
|
| 554 |
+
|
| 555 |
+
### Is this app intended for production use?
|
| 556 |
+
|
| 557 |
+
Yes, for validator workflows. It is structured for multi-project access control, audit-friendly writes, and collaborative validation.
|
| 558 |
+
|
| 559 |
+
### Is the uploader CLI meant to replace the validator app?
|
| 560 |
+
|
| 561 |
+
No. It is a companion tool for large dataset uploads and repository initialization.
|
| 562 |
+
|
| 563 |
+
### Does the app preload all audio files?
|
| 564 |
+
|
| 565 |
+
No. Audio is fetched on demand for the selected detection, helping keep memory and network usage under control.
|
| 566 |
+
|
| 567 |
+
### Can multiple validators work at the same time?
|
| 568 |
+
|
| 569 |
+
Yes. The validation flow uses optimistic concurrency checks to detect and handle conflicting updates.
|
| 570 |
+
|
| 571 |
+
### Can this run on Hugging Face Spaces?
|
| 572 |
+
|
| 573 |
+
Yes. The validator app is designed for Gradio deployment on Hugging Face Spaces.
|
| 574 |
+
|
| 575 |
+
### Can the uploader resume after closing the app?
|
| 576 |
+
|
| 577 |
+
Yes. It persists checkpoints and session metadata locally and can be resumed later.
|
| 578 |
+
|
| 579 |
+
### Does the uploader support deduplication?
|
| 580 |
+
|
| 581 |
+
Yes. It checks the remote dataset index, caches the remote file list locally, and skips files that are already present.
|
| 582 |
+
|
| 583 |
+
## Troubleshooting
|
| 584 |
+
|
| 585 |
+
### The app does not start locally
|
| 586 |
+
|
| 587 |
+
1. Confirm Python 3.11+ is active.
|
| 588 |
+
2. Reinstall dependencies with `pip install -r requirements.txt`.
|
| 589 |
+
3. Run `python app.py` from the repository root.
|
| 590 |
+
|
| 591 |
+
### Port 7860 is already in use
|
| 592 |
+
|
| 593 |
+
Stop the existing process using that port, or launch after setting a different `GRADIO_SERVER_PORT` environment variable.
|
| 594 |
+
|
| 595 |
+
### Login works but no project appears
|
| 596 |
+
|
| 597 |
+
Your user likely has no project assignment. Verify `BIRDNET_PROJECTS_FILE` and `BIRDNET_USER_ACCESS_FILE`, or add project access through the admin flow.
|
| 598 |
+
|
| 599 |
+
### Seed warning appears in Validation tab
|
| 600 |
+
|
| 601 |
+
If you see a seed warning banner:
|
| 602 |
+
|
| 603 |
+
1. Verify `BIRDNET_DETECTIONS_FILE` points to an existing file.
|
| 604 |
+
2. Confirm the file is valid UTF-8 JSON.
|
| 605 |
+
3. Confirm each project maps to a list of detections, or each list item includes `project_slug`.
|
| 606 |
+
4. If needed, unset `BIRDNET_DETECTIONS_FILE` to fall back to default demo detections.
|
| 607 |
+
|
| 608 |
+
### Audio does not load for a detection
|
| 609 |
+
|
| 610 |
+
Check that:
|
| 611 |
+
|
| 612 |
+
1. `audio_id` exists and is valid for the selected project.
|
| 613 |
+
2. The dataset repository is reachable.
|
| 614 |
+
3. You have permission to read the project dataset.
|
| 615 |
+
|
| 616 |
+
### I get conflict messages while validating
|
| 617 |
+
|
| 618 |
+
This means another validator updated the same detection first. Refresh and reapply your decision on the newest version.
|
| 619 |
+
|
| 620 |
+
### The uploader says the token cannot be saved
|
| 621 |
+
|
| 622 |
+
Make sure your OS keyring backend is available. On Windows, the default credential store should work automatically.
|
| 623 |
+
|
| 624 |
+
### The uploader skips files unexpectedly
|
| 625 |
+
|
| 626 |
+
It may have found them in the remote dataset index cache. Clear the uploader cache directory if you want to force a fresh remote listing.
|
| 627 |
+
|
| 628 |
+
### Resume shows an empty checkpoint
|
| 629 |
|
| 630 |
+
Check that you ran the uploader `start` command at least once and that the session directory still exists.
|
| 631 |
|
| 632 |
+
## Recent Updates
|
|
|
|
|
|
|
|
|
|
| 633 |
|
| 634 |
+
- Integrated Validation tab flow in the multi-project app (no placeholder stage).
|
| 635 |
+
- Added project-scoped queue badge (`Queue: N`) and improved queue context feedback.
|
| 636 |
+
- Added seed-file warning banner with actionable remediation guidance in Validation.
|
| 637 |
+
- Standardized all user-facing UI text and feedback messages in English.
|
| 638 |
+
- Updated authentication/session timestamps to timezone-aware UTC handling.
|
| 639 |
+
- Added a local uploader CLI with login, repository initialization, scan, deduplication, checkpointing, and resumable upload foundations.
|
| 640 |
|
| 641 |
+
## Related Documents
|
| 642 |
|
| 643 |
+
- `BIRDNET_UPLOADER_REDESIGN.md`: uploader UX and architecture notes
|
| 644 |
+
- `SPRINT_ROADMAP.md`: implementation roadmap
|
| 645 |
+
- `SPRINT1_KICKOFF.md`: sprint kickoff details
|
SPRINT1_KICKOFF.md
ADDED
|
@@ -0,0 +1,769 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Sprint 1 Kickoff: Primeiros Passos (HOJE)
|
| 2 |
+
|
| 3 |
+
## Objetivo
|
| 4 |
+
Estrutura base pronta, dependências instaladas, todos os módulos esqueléticos criados, testes rodando.
|
| 5 |
+
|
| 6 |
+
**Tempo estimado**: 4-5 horas para 1-2 pessoas
|
| 7 |
+
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
## Step 1: Criar Branch de Trabalho
|
| 11 |
+
|
| 12 |
+
```bash
|
| 13 |
+
cd c:\Users\jonat\Documents\Python\BirdNET-validator-App
|
| 14 |
+
git checkout -b feature/birdnet-uploader-cli
|
| 15 |
+
git push -u origin feature/birdnet-uploader-cli
|
| 16 |
+
```
|
| 17 |
+
|
| 18 |
+
---
|
| 19 |
+
|
| 20 |
+
## Step 2: Criar Estrutura de Diretórios
|
| 21 |
+
|
| 22 |
+
```bash
|
| 23 |
+
# Criar estrutura completa
|
| 24 |
+
mkdir -p src\auth
|
| 25 |
+
mkdir -p src\repository
|
| 26 |
+
mkdir -p src\upload
|
| 27 |
+
mkdir -p src\csv
|
| 28 |
+
mkdir -p src\session
|
| 29 |
+
mkdir -p src\ui\screens
|
| 30 |
+
mkdir -p src\logger
|
| 31 |
+
mkdir -p src\utils
|
| 32 |
+
mkdir -p tests
|
| 33 |
+
mkdir -p build
|
| 34 |
+
mkdir -p docs
|
| 35 |
+
|
| 36 |
+
# Criar __init__.py em todos
|
| 37 |
+
echo "" > src\__init__.py
|
| 38 |
+
echo "" > src\auth\__init__.py
|
| 39 |
+
echo "" > src\repository\__init__.py
|
| 40 |
+
echo "" > src\upload\__init__.py
|
| 41 |
+
echo "" > src\csv\__init__.py
|
| 42 |
+
echo "" > src\session\__init__.py
|
| 43 |
+
echo "" > src\ui\__init__.py
|
| 44 |
+
echo "" > src\ui\screens\__init__.py
|
| 45 |
+
echo "" > src\logger\__init__.py
|
| 46 |
+
echo "" > src\utils\__init__.py
|
| 47 |
+
echo "" > tests\__init__.py
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
---
|
| 51 |
+
|
| 52 |
+
## Step 3: Criar `requirements.txt`
|
| 53 |
+
|
| 54 |
+
Criar arquivo: `requirements.txt`
|
| 55 |
+
|
| 56 |
+
```
|
| 57 |
+
# Core CLI
|
| 58 |
+
click>=8.0.0
|
| 59 |
+
typer>=0.9.0
|
| 60 |
+
|
| 61 |
+
# UI/TUI
|
| 62 |
+
textual>=0.30.0
|
| 63 |
+
rich>=13.0.0
|
| 64 |
+
|
| 65 |
+
# HF Integration
|
| 66 |
+
huggingface_hub>=0.28.0
|
| 67 |
+
|
| 68 |
+
# Data Processing
|
| 69 |
+
pandas>=2.0.0
|
| 70 |
+
pyarrow>=14.0.0
|
| 71 |
+
|
| 72 |
+
# Security
|
| 73 |
+
keyring>=24.0.0
|
| 74 |
+
cryptography>=41.0.0
|
| 75 |
+
|
| 76 |
+
# Validation
|
| 77 |
+
pydantic>=2.0.0
|
| 78 |
+
|
| 79 |
+
# Development & Testing
|
| 80 |
+
pytest>=7.0.0
|
| 81 |
+
pytest-cov>=4.0.0
|
| 82 |
+
black>=23.0.0
|
| 83 |
+
flake8>=6.0.0
|
| 84 |
+
mypy>=1.0.0
|
| 85 |
+
|
| 86 |
+
# Optional: async
|
| 87 |
+
asyncio>=3.4.3
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
Instalar:
|
| 91 |
+
```bash
|
| 92 |
+
pip install -r requirements.txt
|
| 93 |
+
```
|
| 94 |
+
|
| 95 |
+
---
|
| 96 |
+
|
| 97 |
+
## Step 4: Criar `pyproject.toml`
|
| 98 |
+
|
| 99 |
+
Criar arquivo: `pyproject.toml`
|
| 100 |
+
|
| 101 |
+
```toml
|
| 102 |
+
[build-system]
|
| 103 |
+
requires = ["setuptools>=65.0", "wheel"]
|
| 104 |
+
build-backend = "setuptools.build_meta"
|
| 105 |
+
|
| 106 |
+
[project]
|
| 107 |
+
name = "birdnet-uploader"
|
| 108 |
+
version = "0.1.0"
|
| 109 |
+
description = "Local CLI for resumable HuggingFace dataset uploads (BirdNET)"
|
| 110 |
+
authors = [
|
| 111 |
+
{name = "BirdNET Team", email = "team@example.com"}
|
| 112 |
+
]
|
| 113 |
+
requires-python = ">=3.9"
|
| 114 |
+
dependencies = [
|
| 115 |
+
"click>=8.0.0",
|
| 116 |
+
"textual>=0.30.0",
|
| 117 |
+
"rich>=13.0.0",
|
| 118 |
+
"huggingface_hub>=0.28.0",
|
| 119 |
+
"pandas>=2.0.0",
|
| 120 |
+
"pyarrow>=14.0.0",
|
| 121 |
+
"keyring>=24.0.0",
|
| 122 |
+
"pydantic>=2.0.0",
|
| 123 |
+
]
|
| 124 |
+
|
| 125 |
+
[project.optional-dependencies]
|
| 126 |
+
dev = [
|
| 127 |
+
"pytest>=7.0.0",
|
| 128 |
+
"pytest-cov>=4.0.0",
|
| 129 |
+
"black>=23.0.0",
|
| 130 |
+
"flake8>=6.0.0",
|
| 131 |
+
"mypy>=1.0.0",
|
| 132 |
+
]
|
| 133 |
+
|
| 134 |
+
[project.scripts]
|
| 135 |
+
birdnet = "src.main:app"
|
| 136 |
+
|
| 137 |
+
[tool.black]
|
| 138 |
+
line-length = 100
|
| 139 |
+
target-version = ['py39']
|
| 140 |
+
|
| 141 |
+
[tool.isort]
|
| 142 |
+
profile = "black"
|
| 143 |
+
line_length = 100
|
| 144 |
+
|
| 145 |
+
[tool.mypy]
|
| 146 |
+
python_version = "3.9"
|
| 147 |
+
warn_return_any = true
|
| 148 |
+
warn_unused_configs = true
|
| 149 |
+
|
| 150 |
+
[tool.pytest.ini_options]
|
| 151 |
+
testpaths = ["tests"]
|
| 152 |
+
addopts = "-v --cov=src --cov-report=html --cov-report=term-missing"
|
| 153 |
+
```
|
| 154 |
+
|
| 155 |
+
---
|
| 156 |
+
|
| 157 |
+
## Step 5: Criar Estrutura Base de Módulos (Skeletons)
|
| 158 |
+
|
| 159 |
+
### 5.1: `src/main.py`
|
| 160 |
+
|
| 161 |
+
```python
|
| 162 |
+
"""BirdNET Uploader - Main Entry Point"""
|
| 163 |
+
|
| 164 |
+
import click
|
| 165 |
+
from src.ui.tui import BirdNETApp
|
| 166 |
+
from src.auth.auth_service import AuthService
|
| 167 |
+
from src.config import get_config
|
| 168 |
+
|
| 169 |
+
__version__ = "0.1.0"
|
| 170 |
+
|
| 171 |
+
@click.group()
|
| 172 |
+
@click.version_option(version=__version__)
|
| 173 |
+
def app():
|
| 174 |
+
"""BirdNET Uploader - Upload audio to HuggingFace Datasets"""
|
| 175 |
+
pass
|
| 176 |
+
|
| 177 |
+
@app.command()
|
| 178 |
+
def upload():
|
| 179 |
+
"""Start the upload wizard (interactive TUI)"""
|
| 180 |
+
try:
|
| 181 |
+
app_ui = BirdNETApp()
|
| 182 |
+
app_ui.run()
|
| 183 |
+
except Exception as e:
|
| 184 |
+
click.echo(f"Error: {e}", err=True)
|
| 185 |
+
|
| 186 |
+
@app.command()
|
| 187 |
+
def login():
|
| 188 |
+
"""Authenticate with HuggingFace token"""
|
| 189 |
+
try:
|
| 190 |
+
token = click.prompt("Enter your HF token", hide_input=True)
|
| 191 |
+
auth = AuthService()
|
| 192 |
+
user_info = auth.authenticate(token)
|
| 193 |
+
click.echo(f"✓ Logged in as {user_info['username']}")
|
| 194 |
+
except Exception as e:
|
| 195 |
+
click.echo(f"Error: {e}", err=True)
|
| 196 |
+
|
| 197 |
+
@app.command()
|
| 198 |
+
@click.argument("session_id")
|
| 199 |
+
def resume(session_id: str):
|
| 200 |
+
"""Resume a previous upload session"""
|
| 201 |
+
click.echo(f"Resuming session: {session_id}")
|
| 202 |
+
# TODO: Implement
|
| 203 |
+
|
| 204 |
+
@app.command()
|
| 205 |
+
def list_sessions():
|
| 206 |
+
"""List all upload sessions"""
|
| 207 |
+
click.echo("Sessions:")
|
| 208 |
+
# TODO: Implement
|
| 209 |
+
|
| 210 |
+
if __name__ == "__main__":
|
| 211 |
+
app()
|
| 212 |
+
```
|
| 213 |
+
|
| 214 |
+
### 5.2: `src/config.py`
|
| 215 |
+
|
| 216 |
+
```python
|
| 217 |
+
"""Configuration and Constants"""
|
| 218 |
+
|
| 219 |
+
from pathlib import Path
|
| 220 |
+
import os
|
| 221 |
+
|
| 222 |
+
# App
|
| 223 |
+
APP_NAME = "BirdNET Uploader"
|
| 224 |
+
APP_VERSION = "0.1.0"
|
| 225 |
+
|
| 226 |
+
# Directories
|
| 227 |
+
SESSION_DIR = Path.home() / ".birdnet-uploader" / "sessions"
|
| 228 |
+
CACHE_DIR = Path.home() / ".birdnet-uploader" / "cache"
|
| 229 |
+
LOG_DIR = Path.home() / ".birdnet-uploader" / "logs"
|
| 230 |
+
|
| 231 |
+
# Create directories if not exist
|
| 232 |
+
SESSION_DIR.mkdir(parents=True, exist_ok=True)
|
| 233 |
+
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
| 234 |
+
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
| 235 |
+
|
| 236 |
+
# Audio
|
| 237 |
+
AUDIO_EXTENSIONS = ['.wav', '.mp3', '.flac', '.ogg', '.m4a']
|
| 238 |
+
|
| 239 |
+
# Upload
|
| 240 |
+
MAX_BATCH_SIZE = 10
|
| 241 |
+
MAX_RETRIES = 3
|
| 242 |
+
RETRY_INITIAL_BACKOFF = 1.0 # seconds
|
| 243 |
+
RETRY_MAX_BACKOFF = 30.0 # seconds
|
| 244 |
+
|
| 245 |
+
# Manifest
|
| 246 |
+
MANIFEST_SCHEMA_VERSION = "1.0.0"
|
| 247 |
+
SHARD_SIZE = 10000
|
| 248 |
+
|
| 249 |
+
# Keyring
|
| 250 |
+
KEYRING_SERVICE = "birdnet-uploader"
|
| 251 |
+
KEYRING_ACCOUNT = "hf_token"
|
| 252 |
+
|
| 253 |
+
def get_config():
|
| 254 |
+
"""Get runtime configuration from environment or defaults"""
|
| 255 |
+
return {
|
| 256 |
+
"session_dir": os.getenv("BIRDNET_SESSION_DIR", str(SESSION_DIR)),
|
| 257 |
+
"cache_dir": os.getenv("BIRDNET_CACHE_DIR", str(CACHE_DIR)),
|
| 258 |
+
"log_dir": os.getenv("BIRDNET_LOG_DIR", str(LOG_DIR)),
|
| 259 |
+
}
|
| 260 |
+
```
|
| 261 |
+
|
| 262 |
+
### 5.3: `src/auth/auth_service.py`
|
| 263 |
+
|
| 264 |
+
```python
|
| 265 |
+
"""Authentication Service"""
|
| 266 |
+
|
| 267 |
+
from typing import Optional, Dict
|
| 268 |
+
import keyring
|
| 269 |
+
from huggingface_hub import HfApi, HfFolder
|
| 270 |
+
from src.config import KEYRING_SERVICE, KEYRING_ACCOUNT
|
| 271 |
+
|
| 272 |
+
class AuthenticationError(Exception):
|
| 273 |
+
"""Authentication failed"""
|
| 274 |
+
pass
|
| 275 |
+
|
| 276 |
+
class AuthService:
|
| 277 |
+
"""Manage authentication with HuggingFace"""
|
| 278 |
+
|
| 279 |
+
def __init__(self):
|
| 280 |
+
self.api = None
|
| 281 |
+
|
| 282 |
+
def authenticate(self, token: str) -> Dict[str, str]:
|
| 283 |
+
"""
|
| 284 |
+
Validate token and store it securely.
|
| 285 |
+
|
| 286 |
+
Args:
|
| 287 |
+
token: HF API token
|
| 288 |
+
|
| 289 |
+
Returns:
|
| 290 |
+
User info dict with username, email, user_id
|
| 291 |
+
|
| 292 |
+
Raises:
|
| 293 |
+
AuthenticationError: If token is invalid
|
| 294 |
+
"""
|
| 295 |
+
try:
|
| 296 |
+
api = HfApi(token=token)
|
| 297 |
+
user_info = api.whoami()
|
| 298 |
+
|
| 299 |
+
# Store token in keyring
|
| 300 |
+
keyring.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, token)
|
| 301 |
+
|
| 302 |
+
self.api = api
|
| 303 |
+
return {
|
| 304 |
+
"username": user_info.get("name"),
|
| 305 |
+
"email": user_info.get("email", ""),
|
| 306 |
+
"user_id": user_info.get("user_id"),
|
| 307 |
+
}
|
| 308 |
+
except Exception as e:
|
| 309 |
+
raise AuthenticationError(f"Invalid token: {str(e)}")
|
| 310 |
+
|
| 311 |
+
def get_token(self) -> Optional[str]:
|
| 312 |
+
"""Get stored token from keyring"""
|
| 313 |
+
return keyring.get_password(KEYRING_SERVICE, KEYRING_ACCOUNT)
|
| 314 |
+
|
| 315 |
+
def clear_token(self):
|
| 316 |
+
"""Clear stored token"""
|
| 317 |
+
keyring.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT)
|
| 318 |
+
```
|
| 319 |
+
|
| 320 |
+
### 5.4: `src/repository/repo_service.py`
|
| 321 |
+
|
| 322 |
+
```python
|
| 323 |
+
"""Repository Service"""
|
| 324 |
+
|
| 325 |
+
from typing import Optional
|
| 326 |
+
from huggingface_hub import HfApi
|
| 327 |
+
import json
|
| 328 |
+
from io import BytesIO
|
| 329 |
+
|
| 330 |
+
class RepositoryError(Exception):
|
| 331 |
+
"""Repository operation failed"""
|
| 332 |
+
pass
|
| 333 |
+
|
| 334 |
+
class RepositoryService:
|
| 335 |
+
"""Manage HuggingFace datasets"""
|
| 336 |
+
|
| 337 |
+
def __init__(self, token: str):
|
| 338 |
+
self.api = HfApi(token=token)
|
| 339 |
+
|
| 340 |
+
def create_dataset(self, repo_name: str, private: bool = True) -> str:
|
| 341 |
+
"""
|
| 342 |
+
Create a new dataset on HuggingFace.
|
| 343 |
+
|
| 344 |
+
Args:
|
| 345 |
+
repo_name: Dataset name
|
| 346 |
+
private: Whether dataset should be private
|
| 347 |
+
|
| 348 |
+
Returns:
|
| 349 |
+
Full repo_id (username/repo_name)
|
| 350 |
+
"""
|
| 351 |
+
try:
|
| 352 |
+
# Create repo
|
| 353 |
+
repo = self.api.create_repo(
|
| 354 |
+
repo_id=repo_name,
|
| 355 |
+
repo_type="dataset",
|
| 356 |
+
private=private,
|
| 357 |
+
exist_ok=True
|
| 358 |
+
)
|
| 359 |
+
repo_id = repo.repo_id
|
| 360 |
+
|
| 361 |
+
# Initialize structure
|
| 362 |
+
self._init_structure(repo_id)
|
| 363 |
+
|
| 364 |
+
return repo_id
|
| 365 |
+
except Exception as e:
|
| 366 |
+
raise RepositoryError(f"Failed to create dataset: {str(e)}")
|
| 367 |
+
|
| 368 |
+
def _init_structure(self, repo_id: str):
|
| 369 |
+
"""Initialize default folder structure"""
|
| 370 |
+
folders = [
|
| 371 |
+
"audio/project",
|
| 372 |
+
"index/shards",
|
| 373 |
+
"validations",
|
| 374 |
+
"audit/ingestion-runs"
|
| 375 |
+
]
|
| 376 |
+
|
| 377 |
+
for folder in folders:
|
| 378 |
+
# Upload placeholder file to create folder
|
| 379 |
+
self.api.upload_file(
|
| 380 |
+
path_or_fileobj=BytesIO(b""),
|
| 381 |
+
path_in_repo=f"{folder}/.gitkeep",
|
| 382 |
+
repo_id=repo_id,
|
| 383 |
+
repo_type="dataset"
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
def validate_repo(self, repo_id: str) -> bool:
|
| 387 |
+
"""Validate dataset has expected structure"""
|
| 388 |
+
# TODO: Implement
|
| 389 |
+
return True
|
| 390 |
+
```
|
| 391 |
+
|
| 392 |
+
### 5.5: `src/upload/scanner.py`
|
| 393 |
+
|
| 394 |
+
```python
|
| 395 |
+
"""Local Folder Scanner"""
|
| 396 |
+
|
| 397 |
+
import os
|
| 398 |
+
from pathlib import Path
|
| 399 |
+
from typing import Dict, List
|
| 400 |
+
from src.config import AUDIO_EXTENSIONS
|
| 401 |
+
|
| 402 |
+
class LocalScanner:
|
| 403 |
+
"""Scan local folders for audio files"""
|
| 404 |
+
|
| 405 |
+
def scan_folder(self, folder_path: str) -> Dict:
|
| 406 |
+
"""
|
| 407 |
+
Recursively scan folder for audio files.
|
| 408 |
+
|
| 409 |
+
Args:
|
| 410 |
+
folder_path: Path to scan
|
| 411 |
+
|
| 412 |
+
Returns:
|
| 413 |
+
Structure dict with files grouped by species
|
| 414 |
+
"""
|
| 415 |
+
structure = {}
|
| 416 |
+
total_size = 0
|
| 417 |
+
file_count = 0
|
| 418 |
+
|
| 419 |
+
for root, dirs, files in os.walk(folder_path):
|
| 420 |
+
for file in files:
|
| 421 |
+
if any(file.lower().endswith(ext) for ext in AUDIO_EXTENSIONS):
|
| 422 |
+
full_path = os.path.join(root, file)
|
| 423 |
+
size = os.path.getsize(full_path)
|
| 424 |
+
|
| 425 |
+
# Infer species
|
| 426 |
+
specie = self._infer_species(root, file)
|
| 427 |
+
|
| 428 |
+
if specie not in structure:
|
| 429 |
+
structure[specie] = []
|
| 430 |
+
|
| 431 |
+
structure[specie].append({
|
| 432 |
+
"name": file,
|
| 433 |
+
"full_path": full_path,
|
| 434 |
+
"size": size,
|
| 435 |
+
})
|
| 436 |
+
|
| 437 |
+
total_size += size
|
| 438 |
+
file_count += 1
|
| 439 |
+
|
| 440 |
+
return {
|
| 441 |
+
"total_files": file_count,
|
| 442 |
+
"total_size": total_size,
|
| 443 |
+
"by_species": structure,
|
| 444 |
+
}
|
| 445 |
+
|
| 446 |
+
def _infer_species(self, root: str, filename: str) -> str:
|
| 447 |
+
"""Infer species from path or filename"""
|
| 448 |
+
parts = root.split(os.sep)
|
| 449 |
+
if len(parts) > 0:
|
| 450 |
+
return parts[-1]
|
| 451 |
+
return "unknown"
|
| 452 |
+
```
|
| 453 |
+
|
| 454 |
+
### 5.6: `src/session/session_manager.py`
|
| 455 |
+
|
| 456 |
+
```python
|
| 457 |
+
"""Session Management"""
|
| 458 |
+
|
| 459 |
+
import json
|
| 460 |
+
from pathlib import Path
|
| 461 |
+
from datetime import datetime
|
| 462 |
+
from src.config import SESSION_DIR
|
| 463 |
+
|
| 464 |
+
class SessionManager:
|
| 465 |
+
"""Manage upload sessions"""
|
| 466 |
+
|
| 467 |
+
def __init__(self, session_id: str = None):
|
| 468 |
+
if session_id is None:
|
| 469 |
+
session_id = f"upload-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
|
| 470 |
+
|
| 471 |
+
self.session_id = session_id
|
| 472 |
+
self.session_dir = Path(SESSION_DIR) / session_id
|
| 473 |
+
self.session_dir.mkdir(parents=True, exist_ok=True)
|
| 474 |
+
|
| 475 |
+
def save_checkpoint(self, data: dict):
|
| 476 |
+
"""Save checkpoint atomically"""
|
| 477 |
+
checkpoint_file = self.session_dir / "checkpoint.json"
|
| 478 |
+
temp_file = checkpoint_file.with_suffix(".tmp")
|
| 479 |
+
|
| 480 |
+
with open(temp_file, "w") as f:
|
| 481 |
+
json.dump(data, f, indent=2)
|
| 482 |
+
|
| 483 |
+
temp_file.replace(checkpoint_file)
|
| 484 |
+
|
| 485 |
+
def load_checkpoint(self) -> dict:
|
| 486 |
+
"""Load last checkpoint"""
|
| 487 |
+
checkpoint_file = self.session_dir / "checkpoint.json"
|
| 488 |
+
if checkpoint_file.exists():
|
| 489 |
+
with open(checkpoint_file) as f:
|
| 490 |
+
return json.load(f)
|
| 491 |
+
return {}
|
| 492 |
+
```
|
| 493 |
+
|
| 494 |
+
### 5.7: `src/ui/tui.py`
|
| 495 |
+
|
| 496 |
+
```python
|
| 497 |
+
"""TUI Application (Textual)"""
|
| 498 |
+
|
| 499 |
+
from textual.app import ComposeResult, App
|
| 500 |
+
from textual.containers import Container
|
| 501 |
+
from textual.widgets import Static, Button
|
| 502 |
+
|
| 503 |
+
class BirdNETApp(App):
|
| 504 |
+
"""Main TUI Application"""
|
| 505 |
+
|
| 506 |
+
def compose(self) -> ComposeResult:
|
| 507 |
+
"""Create child widgets for the app."""
|
| 508 |
+
yield Container(
|
| 509 |
+
Static("BirdNET Uploader", id="header"),
|
| 510 |
+
Button("Start Upload", id="btn_upload"),
|
| 511 |
+
id="container"
|
| 512 |
+
)
|
| 513 |
+
|
| 514 |
+
def on_button_pressed(self) -> None:
|
| 515 |
+
"""Handle button press."""
|
| 516 |
+
self.exit()
|
| 517 |
+
|
| 518 |
+
def run():
|
| 519 |
+
"""Run the TUI app"""
|
| 520 |
+
app = BirdNETApp()
|
| 521 |
+
app.run()
|
| 522 |
+
```
|
| 523 |
+
|
| 524 |
+
### 5.8: `src/utils/error_handler.py`
|
| 525 |
+
|
| 526 |
+
```python
|
| 527 |
+
"""Error Handling"""
|
| 528 |
+
|
| 529 |
+
class BirdNETError(Exception):
|
| 530 |
+
"""Base exception"""
|
| 531 |
+
pass
|
| 532 |
+
|
| 533 |
+
class AuthenticationError(BirdNETError):
|
| 534 |
+
"""Authentication failed"""
|
| 535 |
+
pass
|
| 536 |
+
|
| 537 |
+
class RepositoryError(BirdNETError):
|
| 538 |
+
"""Repository operation failed"""
|
| 539 |
+
pass
|
| 540 |
+
|
| 541 |
+
class UploadError(BirdNETError):
|
| 542 |
+
"""Upload operation failed"""
|
| 543 |
+
pass
|
| 544 |
+
|
| 545 |
+
class ValidationError(BirdNETError):
|
| 546 |
+
"""Validation failed"""
|
| 547 |
+
pass
|
| 548 |
+
```
|
| 549 |
+
|
| 550 |
+
---
|
| 551 |
+
|
| 552 |
+
## Step 6: Criar Test Infrastructure
|
| 553 |
+
|
| 554 |
+
### 6.1: `pytest.ini`
|
| 555 |
+
|
| 556 |
+
```ini
|
| 557 |
+
[pytest]
|
| 558 |
+
testpaths = tests
|
| 559 |
+
addopts = -v --tb=short
|
| 560 |
+
python_files = test_*.py
|
| 561 |
+
python_classes = Test*
|
| 562 |
+
python_functions = test_*
|
| 563 |
+
```
|
| 564 |
+
|
| 565 |
+
### 6.2: `tests/conftest.py`
|
| 566 |
+
|
| 567 |
+
```python
|
| 568 |
+
"""Test Configuration and Fixtures"""
|
| 569 |
+
|
| 570 |
+
import pytest
|
| 571 |
+
from pathlib import Path
|
| 572 |
+
import tempfile
|
| 573 |
+
import json
|
| 574 |
+
|
| 575 |
+
@pytest.fixture
|
| 576 |
+
def temp_folder():
|
| 577 |
+
"""Create temporary folder with test audio files"""
|
| 578 |
+
with tempfile.TemporaryDirectory() as tmpdir:
|
| 579 |
+
# Create test structure
|
| 580 |
+
audio_dir = Path(tmpdir) / "audio" / "Parrot"
|
| 581 |
+
audio_dir.mkdir(parents=True)
|
| 582 |
+
|
| 583 |
+
# Create dummy audio files
|
| 584 |
+
for i in range(5):
|
| 585 |
+
(audio_dir / f"segment_{i:03d}.wav").write_bytes(b"fake audio data")
|
| 586 |
+
|
| 587 |
+
yield tmpdir
|
| 588 |
+
|
| 589 |
+
@pytest.fixture
|
| 590 |
+
def mock_hf_token():
|
| 591 |
+
"""Mock HF token"""
|
| 592 |
+
return "hf_test_token_12345"
|
| 593 |
+
|
| 594 |
+
@pytest.fixture
|
| 595 |
+
def session_dir(tmp_path):
|
| 596 |
+
"""Temporary session directory"""
|
| 597 |
+
session_dir = tmp_path / "sessions" / "test-session"
|
| 598 |
+
session_dir.mkdir(parents=True)
|
| 599 |
+
return session_dir
|
| 600 |
+
```
|
| 601 |
+
|
| 602 |
+
### 6.3: `tests/test_auth.py`
|
| 603 |
+
|
| 604 |
+
```python
|
| 605 |
+
"""Tests for Auth Service"""
|
| 606 |
+
|
| 607 |
+
import pytest
|
| 608 |
+
from src.auth.auth_service import AuthService, AuthenticationError
|
| 609 |
+
|
| 610 |
+
def test_auth_invalid_token(mock_hf_token):
|
| 611 |
+
"""Test that invalid token raises error"""
|
| 612 |
+
auth = AuthService()
|
| 613 |
+
with pytest.raises(AuthenticationError):
|
| 614 |
+
auth.authenticate("invalid_token_xyz")
|
| 615 |
+
|
| 616 |
+
# More tests to be implemented
|
| 617 |
+
```
|
| 618 |
+
|
| 619 |
+
### 6.4: `tests/test_scanner.py`
|
| 620 |
+
|
| 621 |
+
```python
|
| 622 |
+
"""Tests for File Scanner"""
|
| 623 |
+
|
| 624 |
+
import pytest
|
| 625 |
+
from src.upload.scanner import LocalScanner
|
| 626 |
+
|
| 627 |
+
def test_scan_folder_empty(tmp_path):
|
| 628 |
+
"""Test scanning empty folder"""
|
| 629 |
+
scanner = LocalScanner()
|
| 630 |
+
result = scanner.scan_folder(str(tmp_path))
|
| 631 |
+
assert result["total_files"] == 0
|
| 632 |
+
assert result["total_size"] == 0
|
| 633 |
+
|
| 634 |
+
def test_scan_folder_with_audio(temp_folder):
|
| 635 |
+
"""Test scanning folder with audio files"""
|
| 636 |
+
scanner = LocalScanner()
|
| 637 |
+
result = scanner.scan_folder(temp_folder)
|
| 638 |
+
assert result["total_files"] == 5
|
| 639 |
+
assert result["total_size"] > 0
|
| 640 |
+
|
| 641 |
+
# More tests to be implemented
|
| 642 |
+
```
|
| 643 |
+
|
| 644 |
+
---
|
| 645 |
+
|
| 646 |
+
## Step 7: Configurar CI/CD (GitHub Actions)
|
| 647 |
+
|
| 648 |
+
Criar arquivo: `.github/workflows/test.yml`
|
| 649 |
+
|
| 650 |
+
```yaml
|
| 651 |
+
name: Tests
|
| 652 |
+
|
| 653 |
+
on: [push, pull_request]
|
| 654 |
+
|
| 655 |
+
jobs:
|
| 656 |
+
test:
|
| 657 |
+
runs-on: ubuntu-latest
|
| 658 |
+
strategy:
|
| 659 |
+
matrix:
|
| 660 |
+
python-version: ["3.9", "3.10", "3.11"]
|
| 661 |
+
|
| 662 |
+
steps:
|
| 663 |
+
- uses: actions/checkout@v3
|
| 664 |
+
|
| 665 |
+
- name: Set up Python
|
| 666 |
+
uses: actions/setup-python@v4
|
| 667 |
+
with:
|
| 668 |
+
python-version: ${{ matrix.python-version }}
|
| 669 |
+
|
| 670 |
+
- name: Install dependencies
|
| 671 |
+
run: |
|
| 672 |
+
pip install -r requirements.txt
|
| 673 |
+
|
| 674 |
+
- name: Lint
|
| 675 |
+
run: |
|
| 676 |
+
flake8 src tests
|
| 677 |
+
|
| 678 |
+
- name: Run tests
|
| 679 |
+
run: |
|
| 680 |
+
pytest
|
| 681 |
+
|
| 682 |
+
- name: Upload coverage
|
| 683 |
+
uses: codecov/codecov-action@v3
|
| 684 |
+
```
|
| 685 |
+
|
| 686 |
+
---
|
| 687 |
+
|
| 688 |
+
## Step 8: Testar Instalação
|
| 689 |
+
|
| 690 |
+
```bash
|
| 691 |
+
# Instalar dependências
|
| 692 |
+
pip install -r requirements.txt
|
| 693 |
+
|
| 694 |
+
# Rodar testes
|
| 695 |
+
pytest -v
|
| 696 |
+
|
| 697 |
+
# Testar import
|
| 698 |
+
python -c "import src; print('✓ Imports OK')"
|
| 699 |
+
|
| 700 |
+
# Rodar CLI help
|
| 701 |
+
python -m src.main --help
|
| 702 |
+
```
|
| 703 |
+
|
| 704 |
+
**Esperado**: Todos os comandos rodarem sem erro.
|
| 705 |
+
|
| 706 |
+
---
|
| 707 |
+
|
| 708 |
+
## Step 9: Commit Initial
|
| 709 |
+
|
| 710 |
+
```bash
|
| 711 |
+
git add -A
|
| 712 |
+
git commit -m "feat: Sprint 1 - Initial project structure and scaffolding"
|
| 713 |
+
git push origin feature/birdnet-uploader-cli
|
| 714 |
+
```
|
| 715 |
+
|
| 716 |
+
---
|
| 717 |
+
|
| 718 |
+
## Checklist: Sprint 1, Day 1 - DONE ✓
|
| 719 |
+
|
| 720 |
+
- [ ] Branch `feature/birdnet-uploader-cli` criada
|
| 721 |
+
- [ ] Estrutura de diretórios completa
|
| 722 |
+
- [ ] `requirements.txt` instalado
|
| 723 |
+
- [ ] `pyproject.toml` criado
|
| 724 |
+
- [ ] Todos os módulos esqueléticos criados (8+ arquivos)
|
| 725 |
+
- [ ] `pytest.ini` e `conftest.py` configurados
|
| 726 |
+
- [ ] Testes básicos criados
|
| 727 |
+
- [ ] CI/CD workflow criado
|
| 728 |
+
- [ ] Primeira execução sem erro: `pytest -v`
|
| 729 |
+
- [ ] Commit feito e pushed
|
| 730 |
+
|
| 731 |
+
**Tempo**: ~2-3 horas
|
| 732 |
+
|
| 733 |
+
---
|
| 734 |
+
|
| 735 |
+
## Próximos Passos (Task 1.2 - Design Review)
|
| 736 |
+
|
| 737 |
+
Quando tudo acima estiver **DONE**, fazer:
|
| 738 |
+
|
| 739 |
+
1. **Meeting de Design Review** com o time (30min)
|
| 740 |
+
- Confirmar Tech Stack (Textual vs Rich+Click)
|
| 741 |
+
- Confirmar persistência (JSON vs SQLite)
|
| 742 |
+
- Validar estrutura de repo HF
|
| 743 |
+
- Assinar decision log
|
| 744 |
+
|
| 745 |
+
2. **Começar Task 2.1** (Auth Service Implementation)
|
| 746 |
+
- Implementar `AuthService.authenticate()` completo
|
| 747 |
+
- Implementar keyring integration
|
| 748 |
+
- Escrever testes com mocks
|
| 749 |
+
|
| 750 |
+
---
|
| 751 |
+
|
| 752 |
+
## Troubleshooting
|
| 753 |
+
|
| 754 |
+
**Problema**: `pip install` falha com erro de dependência
|
| 755 |
+
**Solução**: Upgrade pip: `pip install --upgrade pip`
|
| 756 |
+
|
| 757 |
+
**Problema**: Imports falhando
|
| 758 |
+
**Solução**: Adicionar `export PYTHONPATH="${PYTHONPATH}:$(pwd)"`
|
| 759 |
+
|
| 760 |
+
**Problema**: Pytest não encontra testes
|
| 761 |
+
**Solução**: Verificar que todos os `__init__.py` existem
|
| 762 |
+
|
| 763 |
+
---
|
| 764 |
+
|
| 765 |
+
## Documentação de Referência
|
| 766 |
+
|
| 767 |
+
- BIRDNET_UPLOADER_REDESIGN.md (arquitetura e UX)
|
| 768 |
+
- SPRINT_ROADMAP.md (timeline e tasks)
|
| 769 |
+
- Este arquivo (Sprint 1 Day 1 setup)
|
SPRINT_ROADMAP.md
ADDED
|
@@ -0,0 +1,649 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BirdNET Uploader: Sprint Roadmap (MVP 6-8 semanas)
|
| 2 |
+
|
| 3 |
+
## Overview
|
| 4 |
+
**Objetivo**: Entregar executável local moderno, um fluxo integrado, pausa/retomada, compatibilidade com validador.
|
| 5 |
+
|
| 6 |
+
**Timeline**:
|
| 7 |
+
- **Semanas 1-2**: Arquitetura + Setup (Foundation)
|
| 8 |
+
- **Semanas 3-4**: Módulos Core (Auth, Repo, Upload)
|
| 9 |
+
- **Semanas 5-6**: UI + Session Manager (Integration)
|
| 10 |
+
- **Semanas 7-8**: Build + Testes + Release (Polish)
|
| 11 |
+
|
| 12 |
+
**Team**: 2-3 engineers (1 backend, 1 frontend/UI, 1 QA/DevOps)
|
| 13 |
+
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
## SPRINT 1: Foundation & Architecture (Semanas 1-2)
|
| 17 |
+
|
| 18 |
+
### Semana 1: Setup Inicial + Design Review
|
| 19 |
+
|
| 20 |
+
#### Task 1.1: Setup do Projeto (2 dias, 3 pts)
|
| 21 |
+
**Objetivo**: Estrutura base, dependências, CI/CD
|
| 22 |
+
|
| 23 |
+
**Tarefas**:
|
| 24 |
+
- [ ] Criar branch `feature/birdnet-uploader-cli`
|
| 25 |
+
- [ ] Criar estrutura de diretórios (`src/`, `tests/`, `build/`)
|
| 26 |
+
- [ ] Setup `pyproject.toml` e `requirements.txt`
|
| 27 |
+
- huggingface_hub>=0.28.0
|
| 28 |
+
- click>=8.0.0
|
| 29 |
+
- textual>=0.30.0 (TUI)
|
| 30 |
+
- rich>=13.0.0
|
| 31 |
+
- pydantic>=2.0.0
|
| 32 |
+
- pandas>=2.0.0
|
| 33 |
+
- pyarrow>=14.0.0
|
| 34 |
+
- keyring>=24.0.0
|
| 35 |
+
- [ ] Setup GitHub Actions para lint/test
|
| 36 |
+
- [ ] Criar `src/__init__.py` e `src/main.py` (skeleton)
|
| 37 |
+
|
| 38 |
+
**Critério de Pronto**:
|
| 39 |
+
- `pip install -r requirements.txt` sem erro
|
| 40 |
+
- `python -m src.main --help` roda e exibe help text
|
| 41 |
+
|
| 42 |
+
---
|
| 43 |
+
|
| 44 |
+
#### Task 1.2: Design Review + Arquitetura Finalizada (2 dias, 2 pts)
|
| 45 |
+
**Objetivo**: Alinhamento com stakeholders, decisões arquiteturais bloqueantes
|
| 46 |
+
|
| 47 |
+
**Tarefas**:
|
| 48 |
+
- [ ] Revisar documento REDESIGN com time
|
| 49 |
+
- [ ] Decisão final: Textual vs Rich+Click vs Typer (recomendação: Textual para melhor UX)
|
| 50 |
+
- [ ] Decidir sobre persistência: SQLite vs JSON files (recomendação: JSON + estrutura de diretórios)
|
| 51 |
+
- [ ] Validar estrutura de repositório HF (audio/index/validations/audit)
|
| 52 |
+
- [ ] Documentar todas as decisões em `docs/architecture-decisions.md`
|
| 53 |
+
|
| 54 |
+
**Critério de Pronto**:
|
| 55 |
+
- Documento de decisões assinado
|
| 56 |
+
- Time alinhado em tecnologia stack
|
| 57 |
+
- Nenhuma bloqueante técnica identificada
|
| 58 |
+
|
| 59 |
+
---
|
| 60 |
+
|
| 61 |
+
#### Task 1.3: Estrutura Base de Módulos (2 dias, 3 pts)
|
| 62 |
+
**Objetivo**: Criar interfaces/contracts de todos os módulos
|
| 63 |
+
|
| 64 |
+
**Tarefas**:
|
| 65 |
+
- [ ] Criar `src/auth/__init__.py` + `auth_service.py` (skeleton com docstrings)
|
| 66 |
+
- [ ] Criar `src/repository/__init__.py` + `repo_service.py` (skeleton)
|
| 67 |
+
- [ ] Criar `src/upload/__init__.py` + `scanner.py`, `uploader.py`, `deduplicator.py` (skeleton)
|
| 68 |
+
- [ ] Criar `src/csv/__init__.py` + `csv_matcher.py`, `indexer.py` (skeleton)
|
| 69 |
+
- [ ] Criar `src/session/__init__.py` + `session_manager.py`, `checkpoint.py` (skeleton)
|
| 70 |
+
- [ ] Criar `src/ui/__init__.py` + `tui.py`, `progress_renderer.py` (skeleton)
|
| 71 |
+
- [ ] Criar `src/logger/__init__.py` + `structured_logger.py` (skeleton)
|
| 72 |
+
- [ ] Criar `src/utils/__init__.py` + `file_utils.py`, `hash_utils.py`, `error_handler.py` (skeleton)
|
| 73 |
+
|
| 74 |
+
**Critério de Pronto**:
|
| 75 |
+
- Todos os módulos existem com `pass` ou docstrings
|
| 76 |
+
- Nenhum erro de import circular
|
| 77 |
+
- `python -c "import src; print('OK')"` funciona
|
| 78 |
+
|
| 79 |
+
---
|
| 80 |
+
|
| 81 |
+
#### Task 1.4: Testes Base + Test Fixtures (1 dia, 2 pts)
|
| 82 |
+
**Objetivo**: Infraestrutura de testes pronta
|
| 83 |
+
|
| 84 |
+
**Tarefas**:
|
| 85 |
+
- [ ] Setup `pytest.ini` e `conftest.py`
|
| 86 |
+
- [ ] Criar test fixtures para:
|
| 87 |
+
- Mock HfApi
|
| 88 |
+
- Pasta local temporária com arquivos de teste
|
| 89 |
+
- Session mock
|
| 90 |
+
- Token mock
|
| 91 |
+
- [ ] Criar `tests/test_auth.py` (skeleton)
|
| 92 |
+
- [ ] Criar `tests/test_uploader.py` (skeleton)
|
| 93 |
+
- [ ] Criar `tests/test_session.py` (skeleton)
|
| 94 |
+
|
| 95 |
+
**Critério de Pronto**:
|
| 96 |
+
- `pytest` executa sem erro (mesmo que tests sejam vazios)
|
| 97 |
+
- Fixtures estão disponíveis para todos os testes
|
| 98 |
+
|
| 99 |
+
---
|
| 100 |
+
|
| 101 |
+
### Semana 2: Auth + Repository Services
|
| 102 |
+
|
| 103 |
+
#### Task 2.1: Auth Service - Implementação Completa (2.5 dias, 5 pts)
|
| 104 |
+
**Objetivo**: Login com token, validação, armazenamento seguro
|
| 105 |
+
|
| 106 |
+
**Tarefas**:
|
| 107 |
+
- [ ] Implementar `AuthService.authenticate(token)`:
|
| 108 |
+
- Validar token contra HF API
|
| 109 |
+
- Recuperar info do usuário (username, email, user_id)
|
| 110 |
+
- Salvar em keyring (Windows DPAPI)
|
| 111 |
+
- Retornar user context
|
| 112 |
+
- [ ] Implementar `AuthService.get_stored_token()`:
|
| 113 |
+
- Recuperar token do keyring
|
| 114 |
+
- Lançar erro claro se não encontrado
|
| 115 |
+
- [ ] Implementar `AuthService.clear_token()`:
|
| 116 |
+
- Remover token do keyring
|
| 117 |
+
- [ ] Implementar `TokenManager`:
|
| 118 |
+
- Gerenciar TTL de token
|
| 119 |
+
- Detectar expiração
|
| 120 |
+
- Forçar re-autenticação se necessário
|
| 121 |
+
- [ ] Tests:
|
| 122 |
+
- [ ] `test_auth_valid_token()`
|
| 123 |
+
- [ ] `test_auth_invalid_token()`
|
| 124 |
+
- [ ] `test_auth_token_stored_in_keyring()`
|
| 125 |
+
- [ ] `test_auth_token_retrieval()`
|
| 126 |
+
|
| 127 |
+
**Critério de Pronto**:
|
| 128 |
+
- Todos os testes de auth passando
|
| 129 |
+
- Token nunca é logado/printed
|
| 130 |
+
- Falha de auth retorna mensagem clara
|
| 131 |
+
|
| 132 |
+
---
|
| 133 |
+
|
| 134 |
+
#### Task 2.2: Repository Service - Criar Dataset (2 dias, 4 pts)
|
| 135 |
+
**Objetivo**: Criar dataset HF e inicializar estrutura padrão
|
| 136 |
+
|
| 137 |
+
**Tarefas**:
|
| 138 |
+
- [ ] Implementar `RepositoryService.create_dataset()`:
|
| 139 |
+
- Validar nome do repositório (slug format)
|
| 140 |
+
- Criar dataset no HF (público/privado)
|
| 141 |
+
- Inicializar pastas padrão (audio/, index/shards/, validations/, audit/)
|
| 142 |
+
- Criar manifest.json inicial
|
| 143 |
+
- Retornar repo_id
|
| 144 |
+
- [ ] Implementar `RepositoryService.validate_repo()`:
|
| 145 |
+
- Verificar permissão de escrita
|
| 146 |
+
- Verificar estrutura esperada
|
| 147 |
+
- Retornar validation result
|
| 148 |
+
- [ ] Tests:
|
| 149 |
+
- [ ] `test_create_dataset_public()`
|
| 150 |
+
- [ ] `test_create_dataset_private()`
|
| 151 |
+
- [ ] `test_create_dataset_name_validation()`
|
| 152 |
+
- [ ] `test_validate_repo_structure()`
|
| 153 |
+
|
| 154 |
+
**Critério de Pronto**:
|
| 155 |
+
- Dataset criado no HF com structure correta
|
| 156 |
+
- Manifest.json com schema_version, project_slug, index
|
| 157 |
+
- Testes mockeados passando
|
| 158 |
+
|
| 159 |
+
---
|
| 160 |
+
|
| 161 |
+
#### Task 2.3: Config + Constants Finalizadas (1 dia, 2 pts)
|
| 162 |
+
**Objetivo**: Configuração centralizada do app
|
| 163 |
+
|
| 164 |
+
**Tarefas**:
|
| 165 |
+
- [ ] Criar `src/config.py`:
|
| 166 |
+
- `AUDIO_EXTENSIONS = ['.wav', '.mp3', '.flac', '.ogg']`
|
| 167 |
+
- `SESSION_DIR = ~/.birdnet-uploader/sessions/`
|
| 168 |
+
- `CACHE_DIR = ~/.birdnet-uploader/cache/`
|
| 169 |
+
- `MAX_BATCH_SIZE = 10`
|
| 170 |
+
- `RETRY_MAX_ATTEMPTS = 3`
|
| 171 |
+
- `RETRY_INITIAL_BACKOFF = 1.0`
|
| 172 |
+
- `MANIFEST_SCHEMA_VERSION = "1.0.0"`
|
| 173 |
+
- [ ] Implementar config via `.env` se necessário
|
| 174 |
+
|
| 175 |
+
**Critério de Pronto**:
|
| 176 |
+
- Todas as constantes centralizadas
|
| 177 |
+
- Nenhum hardcoded value em módulos
|
| 178 |
+
|
| 179 |
+
---
|
| 180 |
+
|
| 181 |
+
#### Task 2.4: Error Handling Framework (1.5 dias, 3 pts)
|
| 182 |
+
**Objetivo**: Exceções customizadas e mensagens claras
|
| 183 |
+
|
| 184 |
+
**Tarefas**:
|
| 185 |
+
- [ ] Criar `src/exceptions.py`:
|
| 186 |
+
- `AuthenticationError`
|
| 187 |
+
- `RepositoryError`
|
| 188 |
+
- `UploadError`
|
| 189 |
+
- `SessionError`
|
| 190 |
+
- `ValidationError`
|
| 191 |
+
- [ ] Implementar `src/utils/error_handler.py`:
|
| 192 |
+
- Converter exceções em mensagens user-friendly
|
| 193 |
+
- Log estruturado de erros
|
| 194 |
+
- Sugestões de ação
|
| 195 |
+
|
| 196 |
+
**Critério de Pronto**:
|
| 197 |
+
- Todas as exceções possuem mensagem clara
|
| 198 |
+
- Tests cobrem cenários de erro
|
| 199 |
+
|
| 200 |
+
---
|
| 201 |
+
|
| 202 |
+
## SPRINT 2: Upload & CSV Matching (Semanas 3-4)
|
| 203 |
+
|
| 204 |
+
### Semana 3: Scanner + File Operations
|
| 205 |
+
|
| 206 |
+
#### Task 3.1: Local Scanner - Varrer Pasta (2 dias, 4 pts)
|
| 207 |
+
**Objetivo**: Descobrir arquivos locais, agrupar por espécie
|
| 208 |
+
|
| 209 |
+
**Tarefas**:
|
| 210 |
+
- [ ] Implementar `LocalScanner.scan_folder()`:
|
| 211 |
+
- Varrer recursivamente
|
| 212 |
+
- Filtrar por extensões de áudio
|
| 213 |
+
- Inferir espécie do caminho/nome
|
| 214 |
+
- Calcular tamanho total e contagem
|
| 215 |
+
- Retornar estrutura normalizada
|
| 216 |
+
- [ ] Implementar estratégia de inferência de espécie:
|
| 217 |
+
- Strategy 1: Usar última pasta como specie
|
| 218 |
+
- Strategy 2: Usar prefixo do arquivo
|
| 219 |
+
- Strategy 3: Permitir mapeamento manual
|
| 220 |
+
- [ ] Tests:
|
| 221 |
+
- [ ] `test_scan_folder_empty()`
|
| 222 |
+
- [ ] `test_scan_folder_multiple_species()`
|
| 223 |
+
- [ ] `test_scan_folder_specie_inference()`
|
| 224 |
+
- [ ] `test_scan_folder_with_nested_dirs()`
|
| 225 |
+
|
| 226 |
+
**Critério de Pronto**:
|
| 227 |
+
- Escaneia pasta com 1000+ arquivos em <5 segundos
|
| 228 |
+
- Estrutura normalizada e validada
|
| 229 |
+
|
| 230 |
+
---
|
| 231 |
+
|
| 232 |
+
#### Task 3.2: Hash Utils + File Operations (1.5 dias, 3 pts)
|
| 233 |
+
**Objetivo**: Computação de hash segura, operações de arquivo
|
| 234 |
+
|
| 235 |
+
**Tarefas**:
|
| 236 |
+
- [ ] Implementar `compute_file_hash(path) -> str`:
|
| 237 |
+
- SHA256 eficiente (streaming para arquivos grandes)
|
| 238 |
+
- Cache local de hashes
|
| 239 |
+
- [ ] Implementar `verify_file_integrity(path, hash)`:
|
| 240 |
+
- Validar arquivo não corrompido
|
| 241 |
+
- [ ] Tests:
|
| 242 |
+
- [ ] `test_compute_hash_small_file()`
|
| 243 |
+
- [ ] `test_compute_hash_large_file()`
|
| 244 |
+
- [ ] `test_hash_consistency()`
|
| 245 |
+
|
| 246 |
+
**Critério de Pronto**:
|
| 247 |
+
- Hash de arquivo 1GB < 2 segundos
|
| 248 |
+
- Testes passando
|
| 249 |
+
|
| 250 |
+
---
|
| 251 |
+
|
| 252 |
+
#### Task 3.3: Deduplicator - Verificar Existência (2 dias, 3 pts)
|
| 253 |
+
**Objetivo**: Detectar arquivos já no destino
|
| 254 |
+
|
| 255 |
+
**Tarefas**:
|
| 256 |
+
- [ ] Implementar `Deduplicator.check_remote()`:
|
| 257 |
+
- Query HF API para verificar existência por caminho
|
| 258 |
+
- Comparar tamanho + hash opcional
|
| 259 |
+
- Retornar status (SKIP, UPLOAD, CONFLICT)
|
| 260 |
+
- [ ] Implementar dedup local cache (evitar queries repetidas)
|
| 261 |
+
- [ ] Tests:
|
| 262 |
+
- [ ] `test_check_remote_new_file()`
|
| 263 |
+
- [ ] `test_check_remote_existing_file()`
|
| 264 |
+
- [ ] `test_check_remote_conflict()`
|
| 265 |
+
|
| 266 |
+
**Critério de Pronto**:
|
| 267 |
+
- Dedup funciona sem re-query excessiva
|
| 268 |
+
- Cache valida corretamente
|
| 269 |
+
|
| 270 |
+
---
|
| 271 |
+
|
| 272 |
+
#### Task 3.4: Session + Checkpoint - State Persistence (2.5 dias, 5 pts)
|
| 273 |
+
**Objetivo**: Persistência atomática de estado para retomada
|
| 274 |
+
|
| 275 |
+
**Tarefas**:
|
| 276 |
+
- [ ] Implementar `SessionManager.create_session()`:
|
| 277 |
+
- Gerar session_id
|
| 278 |
+
- Criar diretório de sessão
|
| 279 |
+
- Inicializar metadata.json
|
| 280 |
+
- [ ] Implementar `CheckpointManager.save_checkpoint()`:
|
| 281 |
+
- Escrita atômica (temp + rename)
|
| 282 |
+
- Validação de JSON
|
| 283 |
+
- Flush para disco (fsync)
|
| 284 |
+
- [ ] Implementar `CheckpointManager.load_checkpoint()`:
|
| 285 |
+
- Recuperar último checkpoint
|
| 286 |
+
- Validação de integridade
|
| 287 |
+
- [ ] Implementar `SessionManager.mark_file_done()`:
|
| 288 |
+
- Atualizar manifest por arquivo
|
| 289 |
+
- [ ] Tests:
|
| 290 |
+
- [ ] `test_session_creation()`
|
| 291 |
+
- [ ] `test_checkpoint_atomic_write()`
|
| 292 |
+
- [ ] `test_checkpoint_recovery_after_crash()`
|
| 293 |
+
- [ ] `test_session_resume_same_id()`
|
| 294 |
+
|
| 295 |
+
**Critério de Pronto**:
|
| 296 |
+
- Checkpoint atomático garante consistência
|
| 297 |
+
- Recuperação de crash funciona
|
| 298 |
+
- Testes cobrem falha de escrita
|
| 299 |
+
|
| 300 |
+
---
|
| 301 |
+
|
| 302 |
+
### Semana 4: Upload Engine + CSV Matching
|
| 303 |
+
|
| 304 |
+
#### Task 4.1: Batch Uploader - Motor Principal (3 dias, 6 pts)
|
| 305 |
+
**Objetivo**: Upload resiliente com retry/dedup/pause
|
| 306 |
+
|
| 307 |
+
**Tarefas**:
|
| 308 |
+
- [ ] Implementar `BatchUploader.upload_files()`:
|
| 309 |
+
- Iterar sobre estrutura de arquivos
|
| 310 |
+
- Aplicar dedup check
|
| 311 |
+
- Upload com retry automático
|
| 312 |
+
- Atualizar checkpoint por lote
|
| 313 |
+
- Callback de progresso
|
| 314 |
+
- [ ] Implementar retry com exponential backoff:
|
| 315 |
+
- Backoff: 1s → 2s → 5s → 10s → 30s
|
| 316 |
+
- Tratamento de 429 (rate limit)
|
| 317 |
+
- Max 3 retries por padrão
|
| 318 |
+
- [ ] Implementar pause/resume:
|
| 319 |
+
- Signal SIGTERM/SIGINT
|
| 320 |
+
- Salvar state
|
| 321 |
+
- Resume com checkpoint
|
| 322 |
+
- [ ] Tests:
|
| 323 |
+
- [ ] `test_upload_batch_success()`
|
| 324 |
+
- [ ] `test_upload_with_retry()`
|
| 325 |
+
- [ ] `test_upload_skip_existing()`
|
| 326 |
+
- [ ] `test_upload_pause_resume()`
|
| 327 |
+
|
| 328 |
+
**Critério de Pronto**:
|
| 329 |
+
- Upload funciona end-to-end
|
| 330 |
+
- Retry logic testado
|
| 331 |
+
- Pause/resume funciona
|
| 332 |
+
|
| 333 |
+
---
|
| 334 |
+
|
| 335 |
+
#### Task 4.2: CSV Matcher + Index Generator (2.5 dias, 5 pts)
|
| 336 |
+
**Objetivo**: Matching CSV com segmentos e geração de índice
|
| 337 |
+
|
| 338 |
+
**Tarefas**:
|
| 339 |
+
- [ ] Implementar `CSVMatcher.match_detections()`:
|
| 340 |
+
- Ler CSV com validação de schema
|
| 341 |
+
- Fazer matching com segmentos (source_file → arquivo local)
|
| 342 |
+
- Gerar detection_key determinístico
|
| 343 |
+
- Retornar matched + unmatched + ambiguous
|
| 344 |
+
- [ ] Implementar `IndexGenerator.generate_shards()`:
|
| 345 |
+
- Particionar dados em shards (tamanho configurável)
|
| 346 |
+
- Salvar como parquet
|
| 347 |
+
- Gerar metadata de shards
|
| 348 |
+
- [ ] Implementar `ManifestBuilder.build()`:
|
| 349 |
+
- Construir manifest.json com schema_version
|
| 350 |
+
- Incluir metadados de index
|
| 351 |
+
- Serializar JSON
|
| 352 |
+
- [ ] Tests:
|
| 353 |
+
- [ ] `test_csv_matching_success()`
|
| 354 |
+
- [ ] `test_csv_missing_columns()`
|
| 355 |
+
- [ ] `test_shard_generation()`
|
| 356 |
+
- [ ] `test_manifest_format()`
|
| 357 |
+
|
| 358 |
+
**Critério de Pronto**:
|
| 359 |
+
- CSV+segmentos matchados corretamente
|
| 360 |
+
- Shards gerados com formato correto
|
| 361 |
+
- Manifest válido e legível
|
| 362 |
+
|
| 363 |
+
---
|
| 364 |
+
|
| 365 |
+
## SPRINT 3: UI + Integration (Semanas 5-6)
|
| 366 |
+
|
| 367 |
+
### Semana 5: UI TUI Implementation
|
| 368 |
+
|
| 369 |
+
#### Task 5.1: Textual TUI - Fluxo Principal (3 dias, 6 pts)
|
| 370 |
+
**Objetivo**: Interface TUI moderna para guiar usuário
|
| 371 |
+
|
| 372 |
+
**Tarefas**:
|
| 373 |
+
- [ ] Criar `src/ui/screens/login_screen.py`:
|
| 374 |
+
- Input para token HF
|
| 375 |
+
- Validação em tempo real
|
| 376 |
+
- Mensagens de erro claras
|
| 377 |
+
- [ ] Criar `src/ui/screens/config_screen.py`:
|
| 378 |
+
- Form com campos: repo name, visibility, segments path, csv path
|
| 379 |
+
- Verificação de pasta
|
| 380 |
+
- Validação de formato
|
| 381 |
+
- [ ] Criar `src/ui/screens/summary_screen.py`:
|
| 382 |
+
- Exibir configuração resumida
|
| 383 |
+
- Botão de confirmação
|
| 384 |
+
- [ ] Criar `src/ui/screens/upload_screen.py`:
|
| 385 |
+
- Barra de progresso principal
|
| 386 |
+
- Progresso por arquivo
|
| 387 |
+
- Arquivo atual
|
| 388 |
+
- Último evento
|
| 389 |
+
- Botões pause/cancel
|
| 390 |
+
- [ ] Implementar navegação entre screens
|
| 391 |
+
|
| 392 |
+
**Critério de Pronto**:
|
| 393 |
+
- Fluxo completo funcionando (login → config → summary → upload)
|
| 394 |
+
- Inputs validados
|
| 395 |
+
- Navegação intuitiva
|
| 396 |
+
|
| 397 |
+
---
|
| 398 |
+
|
| 399 |
+
#### Task 5.2: Progress Renderer (1.5 dias, 3 pts)
|
| 400 |
+
**Objetivo**: Visualização rich de progresso
|
| 401 |
+
|
| 402 |
+
**Tarefas**:
|
| 403 |
+
- [ ] Implementar progress bar customizada:
|
| 404 |
+
- Percentual + barra visual
|
| 405 |
+
- ETA com update em tempo real
|
| 406 |
+
- Arquivo atual
|
| 407 |
+
- Contadores (uploaded/pending/failed/skipped)
|
| 408 |
+
- [ ] Implementar event log:
|
| 409 |
+
- Últimos 10 eventos
|
| 410 |
+
- Código de status/cor por tipo
|
| 411 |
+
|
| 412 |
+
**Critério de Pronto**:
|
| 413 |
+
- Progress bar atualiza sem flicker
|
| 414 |
+
- ETA coerente
|
| 415 |
+
- Eventos visíveis
|
| 416 |
+
|
| 417 |
+
---
|
| 418 |
+
|
| 419 |
+
### Semana 6: Integration + Testes End-to-End
|
| 420 |
+
|
| 421 |
+
#### Task 6.1: Integration Testing + Mocks (2 dias, 4 pts)
|
| 422 |
+
**Objetivo**: Teste de fluxo completo (mockeado)
|
| 423 |
+
|
| 424 |
+
**Tarefas**:
|
| 425 |
+
- [ ] Criar `tests/test_integration.py`:
|
| 426 |
+
- Teste login + config + upload (mockeado)
|
| 427 |
+
- Teste pause/resume
|
| 428 |
+
- Teste com CSV
|
| 429 |
+
- [ ] Mockar HfApi completamente
|
| 430 |
+
- [ ] Rodar testes de carga (1000+ arquivos mockeados)
|
| 431 |
+
|
| 432 |
+
**Critério de Pronto**:
|
| 433 |
+
- Testes de integração passando
|
| 434 |
+
- Sem vazamento de recursos
|
| 435 |
+
|
| 436 |
+
---
|
| 437 |
+
|
| 438 |
+
#### Task 6.2: Real Integration Test (pequena escala) (2 dias, 4 pts)
|
| 439 |
+
**Objetivo**: Teste real com dataset HF de verdade
|
| 440 |
+
|
| 441 |
+
**Tarefas**:
|
| 442 |
+
- [ ] Criar dataset de teste no HF (alice/test-uploader-small)
|
| 443 |
+
- [ ] Teste com 100 arquivos reais (~500MB)
|
| 444 |
+
- [ ] Validar estrutura no HF pós-upload
|
| 445 |
+
- [ ] Validar manifest.json e shards
|
| 446 |
+
|
| 447 |
+
**Critério de Pronto**:
|
| 448 |
+
- Arquivos aparecem no HF
|
| 449 |
+
- Estrutura correta
|
| 450 |
+
- Manifest válido
|
| 451 |
+
|
| 452 |
+
---
|
| 453 |
+
|
| 454 |
+
## SPRINT 4: Build + Polish (Semanas 7-8)
|
| 455 |
+
|
| 456 |
+
### Semana 7: Build + Release
|
| 457 |
+
|
| 458 |
+
#### Task 7.1: PyInstaller Setup (2 dias, 4 pts)
|
| 459 |
+
**Objetivo**: Gerar executável portátil Windows
|
| 460 |
+
|
| 461 |
+
**Tarefas**:
|
| 462 |
+
- [ ] Criar `build/pyinstaller_config.spec`:
|
| 463 |
+
- One-file executable
|
| 464 |
+
- Hidden imports para textual, rich, pandas, pyarrow
|
| 465 |
+
- Resource bundling
|
| 466 |
+
- [ ] Criar `build/build.py` script
|
| 467 |
+
- [ ] Testar executável gerado localmente
|
| 468 |
+
- [ ] Validar tamanho (~100-150MB)
|
| 469 |
+
|
| 470 |
+
**Critério de Pronto**:
|
| 471 |
+
- Executável funciona sem Python instalado
|
| 472 |
+
- Tamanho razoável
|
| 473 |
+
- Sem erros de runtime
|
| 474 |
+
|
| 475 |
+
---
|
| 476 |
+
|
| 477 |
+
#### Task 7.2: Documentação + README (1.5 dias, 2 pts)
|
| 478 |
+
**Objetivo**: Docs para usuários finais
|
| 479 |
+
|
| 480 |
+
**Tarefas**:
|
| 481 |
+
- [ ] Criar README.md com:
|
| 482 |
+
- Quick start (download, run, autenticar)
|
| 483 |
+
- Screenshots
|
| 484 |
+
- Troubleshooting
|
| 485 |
+
- FAQ
|
| 486 |
+
- [ ] Criar INSTALLATION.md
|
| 487 |
+
- [ ] Criar USAGE.md (exemplos de comando)
|
| 488 |
+
|
| 489 |
+
**Critério de Pronto**:
|
| 490 |
+
- Usuário não-técnico consegue rodar pelo README
|
| 491 |
+
|
| 492 |
+
---
|
| 493 |
+
|
| 494 |
+
#### Task 7.3: Testes de Carga + Stress (2 dias, 4 pts)
|
| 495 |
+
**Objetivo**: Validar com dados reais maiores
|
| 496 |
+
|
| 497 |
+
**Tarefas**:
|
| 498 |
+
- [ ] Teste com pasta 5GB (1000+ arquivos)
|
| 499 |
+
- [ ] Teste com CSV 5000+ linhas
|
| 500 |
+
- [ ] Teste pause/resume 5x
|
| 501 |
+
- [ ] Monitorar memória e CPU
|
| 502 |
+
- [ ] Documentar performance (tempo/MB/s)
|
| 503 |
+
|
| 504 |
+
**Critério de Pronto**:
|
| 505 |
+
- Memória < 500MB durante upload
|
| 506 |
+
- CPU < 30% média
|
| 507 |
+
- Upload rate > 5 MB/s (com rede OK)
|
| 508 |
+
|
| 509 |
+
---
|
| 510 |
+
|
| 511 |
+
### Semana 8: Release + Smoke Tests
|
| 512 |
+
|
| 513 |
+
#### Task 8.1: Release Pipeline (1.5 dias, 2 pts)
|
| 514 |
+
**Objetivo**: Setup de distribuição
|
| 515 |
+
|
| 516 |
+
**Tarefas**:
|
| 517 |
+
- [ ] Criar release no GitHub com:
|
| 518 |
+
- Executável (`.exe`)
|
| 519 |
+
- SHA256 hash
|
| 520 |
+
- Release notes
|
| 521 |
+
- Link para docs
|
| 522 |
+
- [ ] Testar download + execução do exe
|
| 523 |
+
- [ ] Validar assinatura (opcional nesta fase)
|
| 524 |
+
|
| 525 |
+
**Critério de Pronto**:
|
| 526 |
+
- Release publicado
|
| 527 |
+
- Download funciona
|
| 528 |
+
- Exe roda após download/extração
|
| 529 |
+
|
| 530 |
+
---
|
| 531 |
+
|
| 532 |
+
#### Task 8.2: Smoke Tests + Final QA (1.5 dias, 3 pts)
|
| 533 |
+
**Objetivo**: Validação final antes do release
|
| 534 |
+
|
| 535 |
+
**Tarefas**:
|
| 536 |
+
- [ ] Checklist final:
|
| 537 |
+
- [ ] Login funciona
|
| 538 |
+
- [ ] Criar repo funciona
|
| 539 |
+
- [ ] Upload de 10GB completa
|
| 540 |
+
- [ ] Pause/resume sem perda
|
| 541 |
+
- [ ] CSV matching correto
|
| 542 |
+
- [ ] Manifest válido no destino
|
| 543 |
+
- [ ] Estrutura compatível com validador
|
| 544 |
+
- [ ] Teste com 3+ usuários externos (team)
|
| 545 |
+
- [ ] Coleta de feedback
|
| 546 |
+
|
| 547 |
+
**Critério de Pronto**:
|
| 548 |
+
- Zero bugs críticos
|
| 549 |
+
- Feedback incorporado
|
| 550 |
+
- Pronto para release
|
| 551 |
+
|
| 552 |
+
---
|
| 553 |
+
|
| 554 |
+
## BACKLOG: Priorização de Tasks
|
| 555 |
+
|
| 556 |
+
| Sprint | Semana | Task | Pts | Status |
|
| 557 |
+
|--------|--------|------|-----|--------|
|
| 558 |
+
| 1 | 1 | Setup + Design | 5 | Não iniciado |
|
| 559 |
+
| 1 | 1 | Arquitetura de Módulos | 3 | Não iniciado |
|
| 560 |
+
| 1 | 1 | Test Infrastructure | 2 | Não iniciado |
|
| 561 |
+
| 1 | 2 | Auth Service | 5 | Bloqueado por 1.1 |
|
| 562 |
+
| 1 | 2 | Repository Service | 4 | Bloqueado por 1.1 |
|
| 563 |
+
| 1 | 2 | Config + Constants | 2 | Bloqueado por 1.1 |
|
| 564 |
+
| 1 | 2 | Error Handling | 3 | Bloqueado por 1.1 |
|
| 565 |
+
| 2 | 3 | Local Scanner | 4 | Bloqueado por Sprint 1 |
|
| 566 |
+
| 2 | 3 | Hash Utils | 3 | Bloqueado por Sprint 1 |
|
| 567 |
+
| 2 | 3 | Deduplicator | 3 | Bloqueado por Sprint 1 |
|
| 568 |
+
| 2 | 3 | Session + Checkpoint | 5 | Bloqueado por Sprint 1 |
|
| 569 |
+
| 2 | 4 | Batch Uploader | 6 | Bloqueado por Sprint 1 |
|
| 570 |
+
| 2 | 4 | CSV Matcher | 5 | Bloqueado por Sprint 1 |
|
| 571 |
+
| 3 | 5 | Textual TUI | 6 | Bloqueado por Sprint 2 |
|
| 572 |
+
| 3 | 5 | Progress Renderer | 3 | Bloqueado por Sprint 2 |
|
| 573 |
+
| 3 | 6 | Integration Tests | 4 | Bloqueado por Sprint 2 |
|
| 574 |
+
| 3 | 6 | Real Integration Test | 4 | Bloqueado por Sprint 2 |
|
| 575 |
+
| 4 | 7 | PyInstaller | 4 | Bloqueado por Sprint 3 |
|
| 576 |
+
| 4 | 7 | Documentation | 2 | Bloqueado por Sprint 3 |
|
| 577 |
+
| 4 | 7 | Load Testing | 4 | Bloqueado por Sprint 3 |
|
| 578 |
+
| 4 | 8 | Release Pipeline | 2 | Bloqueado por Sprint 3 |
|
| 579 |
+
| 4 | 8 | Final QA | 3 | Bloqueado por Sprint 3 |
|
| 580 |
+
|
| 581 |
+
**Total: ~84 story points (6-8 semanas com 2-3 eng)**
|
| 582 |
+
|
| 583 |
+
---
|
| 584 |
+
|
| 585 |
+
## Velocity + Alocação de Equipe
|
| 586 |
+
|
| 587 |
+
### Cenário 1: Full Team (2 eng)
|
| 588 |
+
- Sprint de 2 semanas, 2 eng = ~40 pts/sprint
|
| 589 |
+
- Sprint 1-2: 26 pts (semana 1 paralelo, semana 2 paralelo) ✓ cabe
|
| 590 |
+
- Sprint 2 continuation: ~20 pts (cabe)
|
| 591 |
+
- Sprint 3: ~13 pts (cabe)
|
| 592 |
+
- Sprint 4: ~15 pts (cabe)
|
| 593 |
+
|
| 594 |
+
### Cenário 2: 3 Eng (Backend + Frontend + QA)
|
| 595 |
+
- Paralelizar mais agressivamente
|
| 596 |
+
- Timeline pode reduzir de 8 para 6-7 semanas
|
| 597 |
+
|
| 598 |
+
---
|
| 599 |
+
|
| 600 |
+
## Critérios de "Done" por Sprint
|
| 601 |
+
|
| 602 |
+
### Sprint 1 Done ✓
|
| 603 |
+
- [ ] Todos os módulos base existem (importáveis)
|
| 604 |
+
- [ ] Auth funciona com keyring
|
| 605 |
+
- [ ] Repo pode ser criado no HF
|
| 606 |
+
- [ ] Testes de auth e repo passando
|
| 607 |
+
- [ ] Setup pronto para Sprint 2
|
| 608 |
+
|
| 609 |
+
### Sprint 2 Done ✓
|
| 610 |
+
- [ ] Scanner varre pasta corretamente
|
| 611 |
+
- [ ] Uploader faz batch com retry
|
| 612 |
+
- [ ] CSV matcher funciona end-to-end
|
| 613 |
+
- [ ] Checkpoint salva e recupera estado
|
| 614 |
+
- [ ] Testes cobrem 80%+ de lógica
|
| 615 |
+
|
| 616 |
+
### Sprint 3 Done ✓
|
| 617 |
+
- [ ] Fluxo TUI completo (login → config → upload)
|
| 618 |
+
- [ ] Progresso visual atualiza sem flicker
|
| 619 |
+
- [ ] Integração completa mockeada testada
|
| 620 |
+
- [ ] Real integration test com 500MB+ passou
|
| 621 |
+
- [ ] Pronto para build
|
| 622 |
+
|
| 623 |
+
### Sprint 4 Done ✓
|
| 624 |
+
- [ ] Executável portátil gerado e testado
|
| 625 |
+
- [ ] Release pipeline setup
|
| 626 |
+
- [ ] Documentação completa
|
| 627 |
+
- [ ] Smoke tests passando
|
| 628 |
+
- [ ] Pronto para distribuição
|
| 629 |
+
|
| 630 |
+
---
|
| 631 |
+
|
| 632 |
+
## Next Steps
|
| 633 |
+
|
| 634 |
+
1. **Aprovação**: Confirmar timeline, team size, prioridades
|
| 635 |
+
2. **Sprint 1 Kickoff**: Semana que vem
|
| 636 |
+
3. **Daily Standups**: 15min todo dia weekday
|
| 637 |
+
4. **Sprint Reviews**: Sexta-feira cada semana
|
| 638 |
+
5. **Retrospectives**: Fim de cada sprint
|
| 639 |
+
|
| 640 |
+
---
|
| 641 |
+
|
| 642 |
+
## Notas & Riscos
|
| 643 |
+
|
| 644 |
+
- **Risco 1**: PyInstaller não empacotar corretamente → Mitigar: spike de 1 dia em week 7
|
| 645 |
+
- **Risco 2**: Performance ruim em 50GB+ → Mitigar: load test cedo (week 4)
|
| 646 |
+
- **Risco 3**: Incompatibilidade com validador → Mitigar: validação de manifest early (week 4)
|
| 647 |
+
- **Risco 4**: UX TUI confusa → Mitigar: user testing em week 6
|
| 648 |
+
|
| 649 |
+
Todas as risks têm mitigation strategy atrelada.
|
app.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
| 1 |
import os
|
| 2 |
|
| 3 |
-
from src.ui.
|
| 4 |
|
| 5 |
|
| 6 |
if __name__ == "__main__":
|
| 7 |
-
app =
|
| 8 |
-
port = int(os.getenv("PORT") or os.getenv("
|
| 9 |
-
|
|
|
|
|
|
| 1 |
import os
|
| 2 |
|
| 3 |
+
from src.ui.app_factory import create_app
|
| 4 |
|
| 5 |
|
| 6 |
if __name__ == "__main__":
|
| 7 |
+
app = create_app()
|
| 8 |
+
port = int(os.getenv("PORT") or os.getenv("BIRDNET_VALIDATOR_PORT") or "7860")
|
| 9 |
+
host = os.getenv("BIRDNET_VALIDATOR_HOST") or "0.0.0.0"
|
| 10 |
+
app.launch(server_name=host, server_port=port)
|
build/publish_release_to_hf.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import os
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
from huggingface_hub import HfApi
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def main() -> int:
|
| 11 |
+
parser = argparse.ArgumentParser(description="Publish a BirdNET uploader release bundle to Hugging Face")
|
| 12 |
+
parser.add_argument("--repo-id", required=True, help="Target Hugging Face repo id, e.g. user/releases")
|
| 13 |
+
parser.add_argument("--repo-type", default="dataset", choices=["dataset", "model", "space"])
|
| 14 |
+
parser.add_argument("--bundle", required=True, type=Path, help="Path to the release zip bundle")
|
| 15 |
+
parser.add_argument("--checksum", required=True, type=Path, help="Path to the checksum file")
|
| 16 |
+
parser.add_argument("--version", required=True, help="Release version, e.g. 0.1.0")
|
| 17 |
+
args = parser.parse_args()
|
| 18 |
+
|
| 19 |
+
token = os.getenv("HF_TOKEN")
|
| 20 |
+
if not token:
|
| 21 |
+
raise SystemExit("HF_TOKEN is required")
|
| 22 |
+
|
| 23 |
+
if not args.bundle.exists():
|
| 24 |
+
raise SystemExit(f"Bundle not found: {args.bundle}")
|
| 25 |
+
if not args.checksum.exists():
|
| 26 |
+
raise SystemExit(f"Checksum not found: {args.checksum}")
|
| 27 |
+
|
| 28 |
+
api = HfApi(token=token)
|
| 29 |
+
api.create_repo(repo_id=args.repo_id, repo_type=args.repo_type, exist_ok=True)
|
| 30 |
+
|
| 31 |
+
release_prefix = f"releases/v{args.version}"
|
| 32 |
+
uploads = [
|
| 33 |
+
(args.bundle, f"{release_prefix}/{args.bundle.name}"),
|
| 34 |
+
(args.checksum, f"{release_prefix}/{args.checksum.name}"),
|
| 35 |
+
]
|
| 36 |
+
|
| 37 |
+
for source_path, path_in_repo in uploads:
|
| 38 |
+
api.upload_file(
|
| 39 |
+
path_or_fileobj=str(source_path),
|
| 40 |
+
path_in_repo=path_in_repo,
|
| 41 |
+
repo_id=args.repo_id,
|
| 42 |
+
repo_type=args.repo_type,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
print(f"Uploaded release artifacts to {args.repo_id}")
|
| 46 |
+
return 0
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
if __name__ == "__main__":
|
| 50 |
+
raise SystemExit(main())
|
build/pyinstaller_uploader.spec
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- mode: python ; coding: utf-8 -*-
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
block_cipher = None
|
| 7 |
+
project_root = Path.cwd()
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
a = Analysis(
|
| 11 |
+
[str(project_root / "src" / "uploader_cli" / "main.py")],
|
| 12 |
+
pathex=[str(project_root)],
|
| 13 |
+
binaries=[],
|
| 14 |
+
datas=[],
|
| 15 |
+
hiddenimports=[
|
| 16 |
+
"click",
|
| 17 |
+
"huggingface_hub",
|
| 18 |
+
"keyring",
|
| 19 |
+
"rich",
|
| 20 |
+
"textual",
|
| 21 |
+
"pydantic",
|
| 22 |
+
"pandas",
|
| 23 |
+
"pyarrow",
|
| 24 |
+
"src.uploader_cli",
|
| 25 |
+
"src.uploader_cli.auth_service",
|
| 26 |
+
"src.uploader_cli.batch_uploader",
|
| 27 |
+
"src.uploader_cli.config",
|
| 28 |
+
"src.uploader_cli.deduplicator",
|
| 29 |
+
"src.uploader_cli.error_handler",
|
| 30 |
+
"src.uploader_cli.exceptions",
|
| 31 |
+
"src.uploader_cli.hash_utils",
|
| 32 |
+
"src.uploader_cli.repo_service",
|
| 33 |
+
"src.uploader_cli.scanner",
|
| 34 |
+
"src.uploader_cli.session_manager",
|
| 35 |
+
"src.uploader_cli.tui",
|
| 36 |
+
],
|
| 37 |
+
hookspath=[],
|
| 38 |
+
hooksconfig={},
|
| 39 |
+
runtime_hooks=[],
|
| 40 |
+
excludes=[],
|
| 41 |
+
noarchive=False,
|
| 42 |
+
optimize=0,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
|
| 46 |
+
|
| 47 |
+
exe = EXE(
|
| 48 |
+
pyz,
|
| 49 |
+
a.scripts,
|
| 50 |
+
[],
|
| 51 |
+
exclude_binaries=True,
|
| 52 |
+
name="birdnet-uploader",
|
| 53 |
+
debug=False,
|
| 54 |
+
bootloader_ignore_signals=False,
|
| 55 |
+
strip=False,
|
| 56 |
+
upx=True,
|
| 57 |
+
console=True,
|
| 58 |
+
disable_windowed_traceback=False,
|
| 59 |
+
argv_emulation=False,
|
| 60 |
+
target_arch=None,
|
| 61 |
+
codesign_identity=None,
|
| 62 |
+
entitlements_file=None,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
coll = COLLECT(
|
| 66 |
+
exe,
|
| 67 |
+
a.binaries,
|
| 68 |
+
a.datas,
|
| 69 |
+
strip=False,
|
| 70 |
+
upx=True,
|
| 71 |
+
upx_exclude=[],
|
| 72 |
+
name="birdnet-uploader",
|
| 73 |
+
)
|
build/release_pipeline.ps1
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
param(
|
| 2 |
+
[Parameter(Mandatory = $true)]
|
| 3 |
+
[string]$Version,
|
| 4 |
+
|
| 5 |
+
[string]$RepoId = "",
|
| 6 |
+
|
| 7 |
+
[ValidateSet("dataset", "model", "space")]
|
| 8 |
+
[string]$RepoType = "dataset",
|
| 9 |
+
|
| 10 |
+
[switch]$PublishToHf
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
$ErrorActionPreference = "Stop"
|
| 14 |
+
|
| 15 |
+
$ProjectRoot = Split-Path -Parent $PSScriptRoot
|
| 16 |
+
$Python = "C:/Users/jonat/AppData/Local/Python/pythoncore-3.14-64/python.exe"
|
| 17 |
+
$BuildScript = Join-Path $PSScriptRoot "release_uploader.py"
|
| 18 |
+
$PublishScript = Join-Path $PSScriptRoot "publish_release_to_hf.py"
|
| 19 |
+
$ValidateScript = Join-Path $PSScriptRoot "validate_release.ps1"
|
| 20 |
+
|
| 21 |
+
if (-not (Test-Path $Python)) {
|
| 22 |
+
throw "Python executable not found at $Python"
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
Write-Host "[0/3] Ensuring build dependencies"
|
| 26 |
+
& $Python -m pip install pyinstaller | Out-Null
|
| 27 |
+
if ($LASTEXITCODE -ne 0) {
|
| 28 |
+
throw "Could not install PyInstaller"
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
Write-Host "[1/3] Building release bundle for version $Version"
|
| 32 |
+
& $Python $BuildScript --version $Version
|
| 33 |
+
if ($LASTEXITCODE -ne 0) {
|
| 34 |
+
throw "Release build failed"
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
Write-Host "[2/3] Running isolated smoke validation"
|
| 38 |
+
& $ValidateScript -Version $Version
|
| 39 |
+
if ($LASTEXITCODE -ne 0) {
|
| 40 |
+
throw "Release validation failed"
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
$ReleaseDir = Join-Path $PSScriptRoot "release"
|
| 44 |
+
$Bundle = Get-ChildItem -Path $ReleaseDir -Filter "*.zip" | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
| 45 |
+
$Checksum = Get-ChildItem -Path $ReleaseDir -Filter "*.sha256" | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
| 46 |
+
|
| 47 |
+
if (-not $Bundle) {
|
| 48 |
+
throw "Release bundle was not created"
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
if (-not $Checksum) {
|
| 52 |
+
throw "Checksum file was not created"
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
Write-Host "[2/3] Release bundle ready: $($Bundle.FullName)"
|
| 56 |
+
Write-Host "[2/3] Checksum ready: $($Checksum.FullName)"
|
| 57 |
+
|
| 58 |
+
if ($PublishToHf) {
|
| 59 |
+
if ([string]::IsNullOrWhiteSpace($RepoId)) {
|
| 60 |
+
throw "RepoId is required when -PublishToHf is set"
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
if ([string]::IsNullOrWhiteSpace($env:HF_TOKEN)) {
|
| 64 |
+
throw "HF_TOKEN environment variable is required when publishing to Hugging Face"
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
Write-Host "[3/3] Publishing release artifacts to Hugging Face repo $RepoId"
|
| 68 |
+
& $Python $PublishScript `
|
| 69 |
+
--repo-id $RepoId `
|
| 70 |
+
--repo-type $RepoType `
|
| 71 |
+
--bundle $Bundle.FullName `
|
| 72 |
+
--checksum $Checksum.FullName `
|
| 73 |
+
--version $Version
|
| 74 |
+
if ($LASTEXITCODE -ne 0) {
|
| 75 |
+
throw "Publish step failed"
|
| 76 |
+
}
|
| 77 |
+
} else {
|
| 78 |
+
Write-Host "[3/3] Publish step skipped. Use -PublishToHf -RepoId <repo> to upload to Hugging Face."
|
| 79 |
+
}
|
build/release_uploader.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import hashlib
|
| 5 |
+
import shutil
|
| 6 |
+
import subprocess
|
| 7 |
+
import sys
|
| 8 |
+
import zipfile
|
| 9 |
+
from datetime import UTC, datetime
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 14 |
+
DIST_DIR = PROJECT_ROOT / "dist"
|
| 15 |
+
BUILD_DIR = PROJECT_ROOT / "build"
|
| 16 |
+
RELEASE_ROOT = BUILD_DIR / "release"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def run(command: list[str]) -> None:
|
| 20 |
+
subprocess.run(command, cwd=PROJECT_ROOT, check=True)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def clean_release_dirs() -> None:
|
| 24 |
+
if DIST_DIR.exists():
|
| 25 |
+
shutil.rmtree(DIST_DIR)
|
| 26 |
+
pycache = PROJECT_ROOT / "build" / "__pycache__"
|
| 27 |
+
if pycache.exists():
|
| 28 |
+
shutil.rmtree(pycache)
|
| 29 |
+
RELEASE_ROOT.mkdir(parents=True, exist_ok=True)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def build_executable() -> Path:
|
| 33 |
+
run([sys.executable, "-m", "PyInstaller", "--noconfirm", "--clean", str(BUILD_DIR / "pyinstaller_uploader.spec")])
|
| 34 |
+
exe_path = DIST_DIR / "birdnet-uploader" / ("birdnet-uploader.exe" if sys.platform.startswith("win") else "birdnet-uploader")
|
| 35 |
+
if not exe_path.exists():
|
| 36 |
+
raise FileNotFoundError(f"Expected executable not found: {exe_path}")
|
| 37 |
+
return exe_path
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def make_release_bundle(executable_path: Path, version: str) -> tuple[Path, Path]:
|
| 41 |
+
bundle_name = f"birdnet-uploader-{version}-{'windows' if sys.platform.startswith('win') else 'linux'}.zip"
|
| 42 |
+
bundle_path = RELEASE_ROOT / bundle_name
|
| 43 |
+
sha256_path = RELEASE_ROOT / f"{bundle_name}.sha256"
|
| 44 |
+
|
| 45 |
+
# PyInstaller is configured as one-folder build, so zip the whole folder,
|
| 46 |
+
# not only the executable, otherwise runtime DLLs (_internal) are missing.
|
| 47 |
+
app_dir = executable_path.parent
|
| 48 |
+
dist_root = app_dir.parent
|
| 49 |
+
|
| 50 |
+
with zipfile.ZipFile(bundle_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
| 51 |
+
for path in app_dir.rglob("*"):
|
| 52 |
+
if path.is_file():
|
| 53 |
+
archive.write(path, arcname=str(path.relative_to(dist_root)))
|
| 54 |
+
|
| 55 |
+
digest = hashlib.sha256(bundle_path.read_bytes()).hexdigest()
|
| 56 |
+
sha256_path.write_text(f"{digest} {bundle_name}\n", encoding="utf-8")
|
| 57 |
+
return bundle_path, sha256_path
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def main() -> int:
|
| 61 |
+
parser = argparse.ArgumentParser(description="Build a release bundle for BirdNET uploader")
|
| 62 |
+
parser.add_argument("--version", required=True, help="Release version, e.g. 0.1.0")
|
| 63 |
+
args = parser.parse_args()
|
| 64 |
+
|
| 65 |
+
clean_release_dirs()
|
| 66 |
+
executable_path = build_executable()
|
| 67 |
+
bundle_path, sha256_path = make_release_bundle(executable_path, args.version)
|
| 68 |
+
print(f"Release bundle: {bundle_path}")
|
| 69 |
+
print(f"Checksum file: {sha256_path}")
|
| 70 |
+
print(f"Built at: {datetime.now(UTC).isoformat()}")
|
| 71 |
+
return 0
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
if __name__ == "__main__":
|
| 75 |
+
raise SystemExit(main())
|
build/validate_release.ps1
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
param(
|
| 2 |
+
[Parameter(Mandatory = $true)]
|
| 3 |
+
[string]$Version,
|
| 4 |
+
|
| 5 |
+
[string]$ReleaseDir = ".\build\release"
|
| 6 |
+
)
|
| 7 |
+
|
| 8 |
+
$ErrorActionPreference = "Stop"
|
| 9 |
+
|
| 10 |
+
function Get-PlatformLabel {
|
| 11 |
+
if ($IsWindows) { return "windows" }
|
| 12 |
+
if ($IsLinux) { return "linux" }
|
| 13 |
+
return "unknown"
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
$Bundle = Get-ChildItem -Path $ReleaseDir -Filter "birdnet-uploader-$Version-*.zip" | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
| 17 |
+
if (-not $Bundle) {
|
| 18 |
+
throw "Bundle for version $Version not found under $ReleaseDir"
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
$BundlePath = $Bundle.FullName
|
| 22 |
+
$BundleName = $Bundle.Name
|
| 23 |
+
$ChecksumPath = "$BundlePath.sha256"
|
| 24 |
+
|
| 25 |
+
if (-not (Test-Path $BundlePath)) {
|
| 26 |
+
throw "Bundle not found: $BundlePath"
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
if (-not (Test-Path $ChecksumPath)) {
|
| 30 |
+
throw "Checksum file not found: $ChecksumPath"
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
Write-Host "[1/5] Validating checksum file"
|
| 34 |
+
$ChecksumLine = (Get-Content $ChecksumPath -Raw).Trim()
|
| 35 |
+
if ([string]::IsNullOrWhiteSpace($ChecksumLine)) {
|
| 36 |
+
throw "Checksum file is empty: $ChecksumPath"
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
$ExpectedHash = ($ChecksumLine -split "\s+")[0].ToLower()
|
| 40 |
+
$ComputedHash = (Get-FileHash -Path $BundlePath -Algorithm SHA256).Hash.ToLower()
|
| 41 |
+
|
| 42 |
+
if ($ExpectedHash -ne $ComputedHash) {
|
| 43 |
+
throw "Checksum mismatch for $BundleName"
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
Write-Host "[2/5] Extracting bundle into an isolated temp folder"
|
| 47 |
+
$TempRoot = Join-Path $env:TEMP ("birdnet-uploader-smoke-" + [Guid]::NewGuid().ToString("N"))
|
| 48 |
+
New-Item -ItemType Directory -Path $TempRoot | Out-Null
|
| 49 |
+
Expand-Archive -Path $BundlePath -DestinationPath $TempRoot -Force
|
| 50 |
+
|
| 51 |
+
$Exe = Get-ChildItem -Path $TempRoot -Filter "birdnet-uploader.exe" -Recurse | Select-Object -First 1
|
| 52 |
+
if (-not $Exe) {
|
| 53 |
+
throw "Executable not found in extracted bundle"
|
| 54 |
+
}
|
| 55 |
+
$ExePath = $Exe.FullName
|
| 56 |
+
$ExeDir = $Exe.Directory.FullName
|
| 57 |
+
|
| 58 |
+
if (-not (Test-Path (Join-Path $ExeDir "_internal"))) {
|
| 59 |
+
throw "Missing _internal runtime folder next to executable: $ExeDir"
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
Write-Host "[3/5] Smoke test: executable help"
|
| 63 |
+
Push-Location $ExeDir
|
| 64 |
+
& $ExePath --help | Out-Null
|
| 65 |
+
if ($LASTEXITCODE -ne 0) {
|
| 66 |
+
Pop-Location
|
| 67 |
+
throw "Executable help command failed"
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
Write-Host "[4/5] Smoke test: command help"
|
| 71 |
+
& $ExePath scan --help | Out-Null
|
| 72 |
+
if ($LASTEXITCODE -ne 0) {
|
| 73 |
+
Pop-Location
|
| 74 |
+
throw "Scan help command failed"
|
| 75 |
+
}
|
| 76 |
+
Pop-Location
|
| 77 |
+
|
| 78 |
+
Write-Host "[5/5] Success"
|
| 79 |
+
Write-Host "Bundle validated: $BundlePath"
|
| 80 |
+
Write-Host "Extracted test folder: $TempRoot"
|
cli/__pycache__/__init__.cpython-314.pyc
DELETED
|
Binary file (167 Bytes)
|
|
|
cli/__pycache__/hf_dataset_cli.cpython-314.pyc
DELETED
|
Binary file (56 kB)
|
|
|
cli/hf_dataset_cli.py
DELETED
|
@@ -1,1314 +0,0 @@
|
|
| 1 |
-
import argparse
|
| 2 |
-
import hashlib
|
| 3 |
-
import json
|
| 4 |
-
import math
|
| 5 |
-
import re
|
| 6 |
-
import tempfile
|
| 7 |
-
import time
|
| 8 |
-
from dataclasses import asdict, dataclass
|
| 9 |
-
from datetime import UTC, datetime
|
| 10 |
-
from io import BytesIO
|
| 11 |
-
from pathlib import Path
|
| 12 |
-
from typing import Any, Callable
|
| 13 |
-
|
| 14 |
-
import pandas as pd
|
| 15 |
-
from huggingface_hub import HfApi, hf_hub_download
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
SCHEMA_VERSION = "1.0.0"
|
| 19 |
-
REQUIRED_PROJECT_PREFIXES = (
|
| 20 |
-
"audio/",
|
| 21 |
-
"index/",
|
| 22 |
-
"index/shards/",
|
| 23 |
-
"validations/",
|
| 24 |
-
"audit/",
|
| 25 |
-
)
|
| 26 |
-
REQUIRED_PLACEHOLDER_FILES = (
|
| 27 |
-
"audio/.gitkeep",
|
| 28 |
-
"index/.gitkeep",
|
| 29 |
-
"index/shards/.gitkeep",
|
| 30 |
-
"validations/.gitkeep",
|
| 31 |
-
"audit/.gitkeep",
|
| 32 |
-
)
|
| 33 |
-
REQUIRED_DETECTIONS_COLUMNS = (
|
| 34 |
-
"detection_key",
|
| 35 |
-
"audio_id",
|
| 36 |
-
"scientific_name",
|
| 37 |
-
"confidence",
|
| 38 |
-
"start_time",
|
| 39 |
-
"end_time",
|
| 40 |
-
)
|
| 41 |
-
SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
|
| 42 |
-
REQUIRED_BIRDNET_COLUMNS = (
|
| 43 |
-
"source_file",
|
| 44 |
-
"scientific_name",
|
| 45 |
-
"confidence",
|
| 46 |
-
"start_time",
|
| 47 |
-
"end_time",
|
| 48 |
-
)
|
| 49 |
-
SEGMENT_FILENAME_PATTERN = re.compile(
|
| 50 |
-
r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s_(?P<confidence_pct>\d+)%$"
|
| 51 |
-
)
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
@dataclass(slots=True)
|
| 55 |
-
class ShardMetadata:
|
| 56 |
-
path: str
|
| 57 |
-
rows: int
|
| 58 |
-
sha256: str
|
| 59 |
-
size_bytes: int
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
@dataclass(slots=True)
|
| 63 |
-
class SegmentFileRecord:
|
| 64 |
-
scientific_name: str
|
| 65 |
-
source_stem: str
|
| 66 |
-
start_time: float
|
| 67 |
-
end_time: float
|
| 68 |
-
confidence_pct: int
|
| 69 |
-
absolute_path: Path
|
| 70 |
-
relative_path: str
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
def _utcnow_iso() -> str:
|
| 74 |
-
return datetime.now(UTC).replace(microsecond=0).isoformat()
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
|
| 78 |
-
api.upload_file(
|
| 79 |
-
path_or_fileobj=BytesIO(content.encode("utf-8")),
|
| 80 |
-
path_in_repo=path_in_repo,
|
| 81 |
-
repo_id=dataset_repo,
|
| 82 |
-
repo_type="dataset",
|
| 83 |
-
)
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
|
| 87 |
-
api.upload_file(
|
| 88 |
-
path_or_fileobj=BytesIO(b""),
|
| 89 |
-
path_in_repo=path_in_repo,
|
| 90 |
-
repo_id=dataset_repo,
|
| 91 |
-
repo_type="dataset",
|
| 92 |
-
)
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
def ensure_project_dataset_structure(
|
| 96 |
-
api: HfApi,
|
| 97 |
-
project_slug: str,
|
| 98 |
-
dataset_repo: str,
|
| 99 |
-
create_private_repo: bool,
|
| 100 |
-
max_retries: int = 3,
|
| 101 |
-
retry_delay_seconds: float = 2.0,
|
| 102 |
-
) -> dict[str, Any]:
|
| 103 |
-
"""Create and initialize dataset repository structure with retry logic."""
|
| 104 |
-
import time as time_module
|
| 105 |
-
|
| 106 |
-
# Create repository with retries
|
| 107 |
-
for attempt in range(1, max_retries + 1):
|
| 108 |
-
try:
|
| 109 |
-
api.create_repo(
|
| 110 |
-
repo_id=dataset_repo,
|
| 111 |
-
repo_type="dataset",
|
| 112 |
-
private=create_private_repo,
|
| 113 |
-
exist_ok=True,
|
| 114 |
-
)
|
| 115 |
-
break # Success
|
| 116 |
-
except Exception as exc:
|
| 117 |
-
if attempt >= max_retries:
|
| 118 |
-
raise RuntimeError(f"Failed to create repository after {max_retries} attempts: {exc}")
|
| 119 |
-
time_module.sleep(retry_delay_seconds)
|
| 120 |
-
|
| 121 |
-
# List repository files (needed before uploads)
|
| 122 |
-
for attempt in range(1, max_retries + 1):
|
| 123 |
-
try:
|
| 124 |
-
files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 125 |
-
break # Success
|
| 126 |
-
except Exception as exc:
|
| 127 |
-
if attempt >= max_retries:
|
| 128 |
-
raise RuntimeError(f"Failed to access repository files after {max_retries} attempts: {exc}")
|
| 129 |
-
time_module.sleep(retry_delay_seconds)
|
| 130 |
-
|
| 131 |
-
created_paths: list[str] = []
|
| 132 |
-
|
| 133 |
-
# Upload placeholder files with retries
|
| 134 |
-
for placeholder in REQUIRED_PLACEHOLDER_FILES:
|
| 135 |
-
if placeholder not in files:
|
| 136 |
-
for attempt in range(1, max_retries + 1):
|
| 137 |
-
try:
|
| 138 |
-
_upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
|
| 139 |
-
created_paths.append(placeholder)
|
| 140 |
-
break # Success
|
| 141 |
-
except Exception as exc:
|
| 142 |
-
if attempt >= max_retries:
|
| 143 |
-
raise RuntimeError(f"Failed to create {placeholder} after {max_retries} attempts: {exc}")
|
| 144 |
-
time_module.sleep(retry_delay_seconds)
|
| 145 |
-
|
| 146 |
-
# Upload manifest with retries
|
| 147 |
-
manifest_path = "manifest.json"
|
| 148 |
-
if manifest_path not in files:
|
| 149 |
-
manifest = {
|
| 150 |
-
"schema_version": SCHEMA_VERSION,
|
| 151 |
-
"project_slug": project_slug,
|
| 152 |
-
"dataset_repo_id": dataset_repo,
|
| 153 |
-
"created_at": _utcnow_iso(),
|
| 154 |
-
"updated_at": _utcnow_iso(),
|
| 155 |
-
"index": {
|
| 156 |
-
"total_detections": 0,
|
| 157 |
-
"total_audio_files": 0,
|
| 158 |
-
"shard_size": 0,
|
| 159 |
-
"shards": [],
|
| 160 |
-
},
|
| 161 |
-
}
|
| 162 |
-
for attempt in range(1, max_retries + 1):
|
| 163 |
-
try:
|
| 164 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
|
| 165 |
-
created_paths.append(manifest_path)
|
| 166 |
-
break # Success
|
| 167 |
-
except Exception as exc:
|
| 168 |
-
if attempt >= max_retries:
|
| 169 |
-
raise RuntimeError(f"Failed to create manifest.json after {max_retries} attempts: {exc}")
|
| 170 |
-
time_module.sleep(retry_delay_seconds)
|
| 171 |
-
|
| 172 |
-
return {
|
| 173 |
-
"dataset_repo": dataset_repo,
|
| 174 |
-
"project_slug": project_slug,
|
| 175 |
-
"created_paths": created_paths,
|
| 176 |
-
}
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
def load_detections_table(detections_file: str) -> pd.DataFrame:
|
| 180 |
-
input_path = Path(detections_file)
|
| 181 |
-
if not input_path.exists():
|
| 182 |
-
raise FileNotFoundError(f"Detections file not found: {detections_file}")
|
| 183 |
-
|
| 184 |
-
suffix = input_path.suffix.lower()
|
| 185 |
-
if suffix == ".csv":
|
| 186 |
-
frame = pd.read_csv(input_path)
|
| 187 |
-
elif suffix in {".jsonl", ".ndjson"}:
|
| 188 |
-
frame = pd.read_json(input_path, lines=True)
|
| 189 |
-
elif suffix == ".parquet":
|
| 190 |
-
frame = pd.read_parquet(input_path)
|
| 191 |
-
else:
|
| 192 |
-
raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
|
| 193 |
-
|
| 194 |
-
missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
|
| 195 |
-
if missing_columns:
|
| 196 |
-
raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
|
| 197 |
-
|
| 198 |
-
if frame.empty:
|
| 199 |
-
return frame
|
| 200 |
-
|
| 201 |
-
deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
|
| 202 |
-
return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
def load_birdnet_detections_csv(detections_csv: str) -> pd.DataFrame:
|
| 206 |
-
input_path = Path(detections_csv)
|
| 207 |
-
if not input_path.exists():
|
| 208 |
-
raise FileNotFoundError(f"Detections CSV not found: {detections_csv}")
|
| 209 |
-
|
| 210 |
-
frame = pd.read_csv(input_path)
|
| 211 |
-
missing_columns = [col for col in REQUIRED_BIRDNET_COLUMNS if col not in frame.columns]
|
| 212 |
-
if missing_columns:
|
| 213 |
-
raise ValueError(f"Missing required BirdNET columns: {', '.join(missing_columns)}")
|
| 214 |
-
|
| 215 |
-
if frame.empty:
|
| 216 |
-
return frame
|
| 217 |
-
|
| 218 |
-
normalized = frame.copy()
|
| 219 |
-
normalized["source_file"] = normalized["source_file"].astype(str).str.strip()
|
| 220 |
-
normalized["source_stem"] = normalized["source_file"].apply(lambda value: Path(value).stem)
|
| 221 |
-
normalized["scientific_name"] = normalized["scientific_name"].astype(str).str.strip()
|
| 222 |
-
normalized["species_key"] = normalized["scientific_name"].str.casefold()
|
| 223 |
-
normalized["start_time"] = pd.to_numeric(normalized["start_time"], errors="coerce")
|
| 224 |
-
normalized["end_time"] = pd.to_numeric(normalized["end_time"], errors="coerce")
|
| 225 |
-
normalized["confidence"] = pd.to_numeric(normalized["confidence"], errors="coerce")
|
| 226 |
-
|
| 227 |
-
normalized = normalized.dropna(subset=["start_time", "end_time"]).reset_index(drop=True)
|
| 228 |
-
return normalized
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
def _build_detection_key(project_slug: str, source_file: str, scientific_name: str, start_time: float, end_time: float) -> str:
|
| 232 |
-
payload = f"{project_slug}|{source_file}|{scientific_name}|{start_time:.3f}|{end_time:.3f}"
|
| 233 |
-
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
def parse_segment_filename(file_name: str) -> tuple[str, float, float, int] | None:
|
| 237 |
-
stem = Path(file_name).stem
|
| 238 |
-
match = SEGMENT_FILENAME_PATTERN.match(stem)
|
| 239 |
-
if not match:
|
| 240 |
-
return None
|
| 241 |
-
|
| 242 |
-
source_stem = match.group("source_stem")
|
| 243 |
-
start_time = float(match.group("start"))
|
| 244 |
-
end_time = float(match.group("end"))
|
| 245 |
-
confidence_pct = int(match.group("confidence_pct"))
|
| 246 |
-
return source_stem, start_time, end_time, confidence_pct
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
def discover_segment_records(segments_root: str) -> list[SegmentFileRecord]:
|
| 250 |
-
root = Path(segments_root)
|
| 251 |
-
if not root.exists() or not root.is_dir():
|
| 252 |
-
raise FileNotFoundError(f"Segments root not found: {segments_root}")
|
| 253 |
-
|
| 254 |
-
records: list[SegmentFileRecord] = []
|
| 255 |
-
for absolute_path in root.rglob("*"):
|
| 256 |
-
if not absolute_path.is_file():
|
| 257 |
-
continue
|
| 258 |
-
if absolute_path.suffix.lower() != ".wav":
|
| 259 |
-
continue
|
| 260 |
-
|
| 261 |
-
rel = absolute_path.relative_to(root)
|
| 262 |
-
rel_parts = list(rel.parts)
|
| 263 |
-
if rel_parts and rel_parts[0].strip().lower() in {"segments", "segment", "audio"}:
|
| 264 |
-
rel_parts = rel_parts[1:]
|
| 265 |
-
|
| 266 |
-
if rel_parts:
|
| 267 |
-
scientific_name = rel_parts[0]
|
| 268 |
-
normalized_relpath = Path(*rel_parts).as_posix()
|
| 269 |
-
else:
|
| 270 |
-
scientific_name = absolute_path.parent.name
|
| 271 |
-
normalized_relpath = absolute_path.name
|
| 272 |
-
|
| 273 |
-
parsed = parse_segment_filename(absolute_path.name)
|
| 274 |
-
if not parsed:
|
| 275 |
-
continue
|
| 276 |
-
source_stem, start_time, end_time, confidence_pct = parsed
|
| 277 |
-
records.append(
|
| 278 |
-
SegmentFileRecord(
|
| 279 |
-
scientific_name=scientific_name,
|
| 280 |
-
source_stem=source_stem,
|
| 281 |
-
start_time=start_time,
|
| 282 |
-
end_time=end_time,
|
| 283 |
-
confidence_pct=confidence_pct,
|
| 284 |
-
absolute_path=absolute_path,
|
| 285 |
-
relative_path=normalized_relpath,
|
| 286 |
-
)
|
| 287 |
-
)
|
| 288 |
-
|
| 289 |
-
return records
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
def _match_birdnet_rows(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 293 |
-
frame = load_birdnet_detections_csv(detections_csv=detections_csv)
|
| 294 |
-
segment_records = discover_segment_records(segments_root=segments_root)
|
| 295 |
-
|
| 296 |
-
by_species: dict[str, list[SegmentFileRecord]] = {}
|
| 297 |
-
for record in segment_records:
|
| 298 |
-
by_species.setdefault(record.scientific_name.casefold(), []).append(record)
|
| 299 |
-
|
| 300 |
-
matched_rows: list[dict[str, Any]] = []
|
| 301 |
-
unmatched_rows: list[dict[str, Any]] = []
|
| 302 |
-
ambiguous_rows: list[dict[str, Any]] = []
|
| 303 |
-
|
| 304 |
-
for row in frame.to_dict(orient="records"):
|
| 305 |
-
species_key = str(row.get("species_key", "")).casefold()
|
| 306 |
-
source_stem = str(row.get("source_stem", "")).strip()
|
| 307 |
-
start_time = float(row["start_time"])
|
| 308 |
-
end_time = float(row["end_time"])
|
| 309 |
-
confidence = float(row["confidence"]) if pd.notna(row.get("confidence")) else 0.0
|
| 310 |
-
|
| 311 |
-
species_candidates = by_species.get(species_key, [])
|
| 312 |
-
exact_matches = [
|
| 313 |
-
record
|
| 314 |
-
for record in species_candidates
|
| 315 |
-
if record.source_stem == source_stem
|
| 316 |
-
and abs(record.start_time - start_time) < 1e-6
|
| 317 |
-
and abs(record.end_time - end_time) < 1e-6
|
| 318 |
-
]
|
| 319 |
-
|
| 320 |
-
if not exact_matches:
|
| 321 |
-
unmatched_rows.append(
|
| 322 |
-
{
|
| 323 |
-
"source_file": row.get("source_file", ""),
|
| 324 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 325 |
-
"start_time": start_time,
|
| 326 |
-
"end_time": end_time,
|
| 327 |
-
}
|
| 328 |
-
)
|
| 329 |
-
continue
|
| 330 |
-
|
| 331 |
-
selected = exact_matches[0]
|
| 332 |
-
if len(exact_matches) > 1:
|
| 333 |
-
expected_pct = int(round(confidence * 100))
|
| 334 |
-
exact_matches = sorted(exact_matches, key=lambda item: abs(item.confidence_pct - expected_pct))
|
| 335 |
-
selected = exact_matches[0]
|
| 336 |
-
ambiguous_rows.append(
|
| 337 |
-
{
|
| 338 |
-
"source_file": row.get("source_file", ""),
|
| 339 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 340 |
-
"start_time": start_time,
|
| 341 |
-
"end_time": end_time,
|
| 342 |
-
"candidates": len(exact_matches),
|
| 343 |
-
}
|
| 344 |
-
)
|
| 345 |
-
|
| 346 |
-
matched_rows.append(
|
| 347 |
-
{
|
| 348 |
-
"project_slug": project_slug,
|
| 349 |
-
"detection_key": _build_detection_key(
|
| 350 |
-
project_slug=project_slug,
|
| 351 |
-
source_file=str(row.get("source_file", "")),
|
| 352 |
-
scientific_name=str(row.get("scientific_name", "")),
|
| 353 |
-
start_time=start_time,
|
| 354 |
-
end_time=end_time,
|
| 355 |
-
),
|
| 356 |
-
"audio_id": source_stem,
|
| 357 |
-
"segment_filename": selected.absolute_path.name,
|
| 358 |
-
"segment_relpath": selected.relative_path,
|
| 359 |
-
"segment_abs_path": str(selected.absolute_path),
|
| 360 |
-
"scientific_name": row.get("scientific_name", ""),
|
| 361 |
-
"common_name": row.get("common_name", ""),
|
| 362 |
-
"confidence": confidence,
|
| 363 |
-
"start_time": start_time,
|
| 364 |
-
"end_time": end_time,
|
| 365 |
-
"exact_start": row.get("exact_start", start_time),
|
| 366 |
-
"exact_end": row.get("exact_end", end_time),
|
| 367 |
-
"locality": row.get("locality", ""),
|
| 368 |
-
"point": row.get("point", ""),
|
| 369 |
-
"date_folder": row.get("date_folder", ""),
|
| 370 |
-
"min_freq": row.get("min_freq", None),
|
| 371 |
-
"max_freq": row.get("max_freq", None),
|
| 372 |
-
"box_source": row.get("box_source", ""),
|
| 373 |
-
"label": row.get("label", ""),
|
| 374 |
-
"source_file": row.get("source_file", ""),
|
| 375 |
-
"schema_version": SCHEMA_VERSION,
|
| 376 |
-
"ingested_at": _utcnow_iso(),
|
| 377 |
-
}
|
| 378 |
-
)
|
| 379 |
-
|
| 380 |
-
return {
|
| 381 |
-
"frame": frame,
|
| 382 |
-
"segment_records_total": len(segment_records),
|
| 383 |
-
"matched_rows": matched_rows,
|
| 384 |
-
"unmatched_rows": unmatched_rows,
|
| 385 |
-
"ambiguous_rows": ambiguous_rows,
|
| 386 |
-
}
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
def run_ingest_segments_dry_run(project_slug: str, detections_csv: str, segments_root: str) -> dict[str, Any]:
|
| 390 |
-
match_result = _match_birdnet_rows(
|
| 391 |
-
project_slug=project_slug,
|
| 392 |
-
detections_csv=detections_csv,
|
| 393 |
-
segments_root=segments_root,
|
| 394 |
-
)
|
| 395 |
-
frame = match_result["frame"]
|
| 396 |
-
matched_rows = match_result["matched_rows"]
|
| 397 |
-
unmatched_rows = match_result["unmatched_rows"]
|
| 398 |
-
ambiguous_rows = match_result["ambiguous_rows"]
|
| 399 |
-
|
| 400 |
-
unique_audio = {row["audio_id"] for row in matched_rows}
|
| 401 |
-
result = {
|
| 402 |
-
"mode": "dry-run",
|
| 403 |
-
"project_slug": project_slug,
|
| 404 |
-
"detections_csv": detections_csv,
|
| 405 |
-
"segments_root": segments_root,
|
| 406 |
-
"csv_rows_total": int(len(frame)),
|
| 407 |
-
"segments_found_total": int(match_result["segment_records_total"]),
|
| 408 |
-
"matched_rows": len(matched_rows),
|
| 409 |
-
"unmatched_rows": len(unmatched_rows),
|
| 410 |
-
"ambiguous_rows": len(ambiguous_rows),
|
| 411 |
-
"unique_audio_ids": len(unique_audio),
|
| 412 |
-
"sample_unmatched": unmatched_rows[:20],
|
| 413 |
-
"sample_ambiguous": ambiguous_rows[:20],
|
| 414 |
-
}
|
| 415 |
-
return result
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
def _file_sha256(file_path: Path) -> str:
|
| 419 |
-
hasher = hashlib.sha256()
|
| 420 |
-
with file_path.open("rb") as handle:
|
| 421 |
-
for chunk in iter(lambda: handle.read(65536), b""):
|
| 422 |
-
hasher.update(chunk)
|
| 423 |
-
return hasher.hexdigest()
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
|
| 427 |
-
if shard_size <= 0:
|
| 428 |
-
raise ValueError("shard_size must be greater than zero")
|
| 429 |
-
|
| 430 |
-
output_dir.mkdir(parents=True, exist_ok=True)
|
| 431 |
-
shards: list[ShardMetadata] = []
|
| 432 |
-
|
| 433 |
-
if frame.empty:
|
| 434 |
-
return shards
|
| 435 |
-
|
| 436 |
-
total_rows = len(frame)
|
| 437 |
-
total_shards = math.ceil(total_rows / shard_size)
|
| 438 |
-
|
| 439 |
-
for index in range(total_shards):
|
| 440 |
-
start = index * shard_size
|
| 441 |
-
end = min(start + shard_size, total_rows)
|
| 442 |
-
shard_frame = frame.iloc[start:end]
|
| 443 |
-
shard_filename = f"shard-{index:05d}.parquet"
|
| 444 |
-
local_path = output_dir / shard_filename
|
| 445 |
-
shard_frame.to_parquet(local_path, index=False)
|
| 446 |
-
|
| 447 |
-
shards.append(
|
| 448 |
-
ShardMetadata(
|
| 449 |
-
path=f"index/shards/{shard_filename}",
|
| 450 |
-
rows=len(shard_frame),
|
| 451 |
-
sha256=_file_sha256(local_path),
|
| 452 |
-
size_bytes=local_path.stat().st_size,
|
| 453 |
-
)
|
| 454 |
-
)
|
| 455 |
-
|
| 456 |
-
return shards
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
def build_manifest_payload(
|
| 460 |
-
project_slug: str,
|
| 461 |
-
dataset_repo: str,
|
| 462 |
-
frame: pd.DataFrame,
|
| 463 |
-
shard_size: int,
|
| 464 |
-
shard_metadata: list[ShardMetadata],
|
| 465 |
-
) -> dict[str, Any]:
|
| 466 |
-
now = _utcnow_iso()
|
| 467 |
-
return {
|
| 468 |
-
"schema_version": SCHEMA_VERSION,
|
| 469 |
-
"project_slug": project_slug,
|
| 470 |
-
"dataset_repo_id": dataset_repo,
|
| 471 |
-
"updated_at": now,
|
| 472 |
-
"index": {
|
| 473 |
-
"total_detections": len(frame),
|
| 474 |
-
"total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
|
| 475 |
-
"shard_size": shard_size,
|
| 476 |
-
"shards": [asdict(item) for item in shard_metadata],
|
| 477 |
-
},
|
| 478 |
-
}
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
def discover_audio_files(local_audio_dir: str) -> list[Path]:
|
| 482 |
-
audio_root = Path(local_audio_dir)
|
| 483 |
-
if not audio_root.exists() or not audio_root.is_dir():
|
| 484 |
-
raise FileNotFoundError(f"Audio directory not found: {local_audio_dir}")
|
| 485 |
-
|
| 486 |
-
files = [
|
| 487 |
-
path
|
| 488 |
-
for path in audio_root.rglob("*")
|
| 489 |
-
if path.is_file() and path.suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
|
| 490 |
-
]
|
| 491 |
-
return sorted(files)
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
def _chunk_items(items: list[Any], chunk_size: int) -> list[list[Any]]:
|
| 495 |
-
if chunk_size <= 0:
|
| 496 |
-
raise ValueError("chunk_size must be greater than zero")
|
| 497 |
-
return [items[index : index + chunk_size] for index in range(0, len(items), chunk_size)]
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
def _load_resume_state(state_file: Path) -> dict[str, Any]:
|
| 501 |
-
if not state_file.exists():
|
| 502 |
-
return {"uploaded": [], "failed": []}
|
| 503 |
-
|
| 504 |
-
payload = json.loads(state_file.read_text(encoding="utf-8"))
|
| 505 |
-
uploaded = payload.get("uploaded", [])
|
| 506 |
-
failed = payload.get("failed", [])
|
| 507 |
-
return {
|
| 508 |
-
"uploaded": uploaded if isinstance(uploaded, list) else [],
|
| 509 |
-
"failed": failed if isinstance(failed, list) else [],
|
| 510 |
-
}
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
def _save_resume_state(state_file: Path, uploaded: set[str], failed: set[str]) -> None:
|
| 514 |
-
state_file.parent.mkdir(parents=True, exist_ok=True)
|
| 515 |
-
payload = {
|
| 516 |
-
"updated_at": _utcnow_iso(),
|
| 517 |
-
"uploaded": sorted(uploaded),
|
| 518 |
-
"failed": sorted(failed),
|
| 519 |
-
}
|
| 520 |
-
state_file.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
def _upload_audio_with_retry(
|
| 524 |
-
api: HfApi,
|
| 525 |
-
dataset_repo: str,
|
| 526 |
-
local_path: Path,
|
| 527 |
-
path_in_repo: str,
|
| 528 |
-
max_retries: int,
|
| 529 |
-
retry_backoff_seconds: float,
|
| 530 |
-
) -> None:
|
| 531 |
-
"""Upload file with exponential backoff retry logic. Raises exception if all retries exhaust."""
|
| 532 |
-
attempts = 0
|
| 533 |
-
last_exception = None
|
| 534 |
-
|
| 535 |
-
while True:
|
| 536 |
-
attempts += 1
|
| 537 |
-
try:
|
| 538 |
-
api.upload_file(
|
| 539 |
-
path_or_fileobj=str(local_path),
|
| 540 |
-
path_in_repo=path_in_repo,
|
| 541 |
-
repo_id=dataset_repo,
|
| 542 |
-
repo_type="dataset",
|
| 543 |
-
)
|
| 544 |
-
return # Success
|
| 545 |
-
except Exception as exc:
|
| 546 |
-
last_exception = exc
|
| 547 |
-
if attempts > max_retries:
|
| 548 |
-
# Include first and last error in final exception for debugging
|
| 549 |
-
raise RuntimeError(
|
| 550 |
-
f"Upload of {path_in_repo} failed after {max_retries} attempts. "
|
| 551 |
-
f"Error: {str(exc)[:200]}"
|
| 552 |
-
) from exc
|
| 553 |
-
time.sleep(retry_backoff_seconds * attempts)
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
def ingest_segments_to_hf(
|
| 557 |
-
api: HfApi,
|
| 558 |
-
project_slug: str,
|
| 559 |
-
dataset_repo: str,
|
| 560 |
-
detections_csv: str,
|
| 561 |
-
segments_root: str,
|
| 562 |
-
batch_size: int,
|
| 563 |
-
shard_size: int,
|
| 564 |
-
max_retries: int,
|
| 565 |
-
retry_backoff_seconds: float,
|
| 566 |
-
resume_state_file: str,
|
| 567 |
-
should_pause: Callable[[], bool] | None = None,
|
| 568 |
-
should_cancel: Callable[[], bool] | None = None,
|
| 569 |
-
progress_callback: Callable[[dict[str, Any]], None] | None = None,
|
| 570 |
-
) -> dict[str, Any]:
|
| 571 |
-
# Ensure repository structure with improved error handling
|
| 572 |
-
try:
|
| 573 |
-
ensure_project_dataset_structure(
|
| 574 |
-
api=api,
|
| 575 |
-
project_slug=project_slug,
|
| 576 |
-
dataset_repo=dataset_repo,
|
| 577 |
-
create_private_repo=False,
|
| 578 |
-
max_retries=3,
|
| 579 |
-
retry_delay_seconds=1.0,
|
| 580 |
-
)
|
| 581 |
-
except Exception as exc:
|
| 582 |
-
# Return error report instead of raising - allows UI to display meaningful message
|
| 583 |
-
return {
|
| 584 |
-
"mode": "execute",
|
| 585 |
-
"project_slug": project_slug,
|
| 586 |
-
"dataset_repo": dataset_repo,
|
| 587 |
-
"detections_csv": detections_csv,
|
| 588 |
-
"segments_root": segments_root,
|
| 589 |
-
"error": f"Repository initialization failed: {str(exc)[:200]}",
|
| 590 |
-
"csv_rows_total": 0,
|
| 591 |
-
"segments_found_total": 0,
|
| 592 |
-
"matched_rows": 0,
|
| 593 |
-
"unmatched_rows": 0,
|
| 594 |
-
"ambiguous_rows": 0,
|
| 595 |
-
"unique_audio_ids": 0,
|
| 596 |
-
"pending_audio_uploads": 0,
|
| 597 |
-
"uploaded_audio_now": 0,
|
| 598 |
-
"uploaded_audio_skipped_existing": 0,
|
| 599 |
-
"failed_uploads": 0,
|
| 600 |
-
"cancelled": False,
|
| 601 |
-
"resume_state_file": str(resume_state_file),
|
| 602 |
-
"index_rows_written": 0,
|
| 603 |
-
"shards_written": 0,
|
| 604 |
-
"sample_unmatched": [],
|
| 605 |
-
"sample_ambiguous": [],
|
| 606 |
-
}
|
| 607 |
-
|
| 608 |
-
match_result = _match_birdnet_rows(
|
| 609 |
-
project_slug=project_slug,
|
| 610 |
-
detections_csv=detections_csv,
|
| 611 |
-
segments_root=segments_root,
|
| 612 |
-
)
|
| 613 |
-
frame = match_result["frame"]
|
| 614 |
-
matched_rows = match_result["matched_rows"]
|
| 615 |
-
unmatched_rows = match_result["unmatched_rows"]
|
| 616 |
-
ambiguous_rows = match_result["ambiguous_rows"]
|
| 617 |
-
|
| 618 |
-
# Deduplicate by remote path so retries/resume work on a stable key.
|
| 619 |
-
path_to_local: dict[str, Path] = {}
|
| 620 |
-
for row in matched_rows:
|
| 621 |
-
repo_path = f"audio/segments/{row['segment_relpath']}"
|
| 622 |
-
if repo_path in path_to_local:
|
| 623 |
-
continue
|
| 624 |
-
|
| 625 |
-
# Prefer exact absolute path discovered during matching. This avoids
|
| 626 |
-
# reconstruction errors when archives include wrapper folders.
|
| 627 |
-
abs_path_value = str(row.get("segment_abs_path") or "").strip()
|
| 628 |
-
if abs_path_value:
|
| 629 |
-
abs_path = Path(abs_path_value)
|
| 630 |
-
if abs_path.exists():
|
| 631 |
-
path_to_local[repo_path] = abs_path
|
| 632 |
-
continue
|
| 633 |
-
|
| 634 |
-
path_to_local[repo_path] = Path(segments_root) / row["segment_relpath"]
|
| 635 |
-
|
| 636 |
-
state_path = Path(resume_state_file)
|
| 637 |
-
state_payload = _load_resume_state(state_file=state_path)
|
| 638 |
-
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 639 |
-
failed_state = set(str(item) for item in state_payload["failed"])
|
| 640 |
-
|
| 641 |
-
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 642 |
-
|
| 643 |
-
pending: list[tuple[Path, str]] = []
|
| 644 |
-
skipped_existing = 0
|
| 645 |
-
for repo_path, local_path in path_to_local.items():
|
| 646 |
-
if repo_path in remote_files:
|
| 647 |
-
uploaded_state.add(repo_path)
|
| 648 |
-
skipped_existing += 1
|
| 649 |
-
continue
|
| 650 |
-
|
| 651 |
-
# Source of truth is remote repository contents. If a path is missing in the
|
| 652 |
-
# repo, re-upload even when stale resume state says it was uploaded before.
|
| 653 |
-
pending.append((local_path, repo_path))
|
| 654 |
-
|
| 655 |
-
uploaded_now: set[str] = set()
|
| 656 |
-
failed_now: set[str] = set()
|
| 657 |
-
cancelled = False
|
| 658 |
-
|
| 659 |
-
if progress_callback:
|
| 660 |
-
progress_callback(
|
| 661 |
-
{
|
| 662 |
-
"event": "upload-start",
|
| 663 |
-
"total_files": len(path_to_local),
|
| 664 |
-
"pending": len(pending),
|
| 665 |
-
"skipped_existing": skipped_existing,
|
| 666 |
-
}
|
| 667 |
-
)
|
| 668 |
-
|
| 669 |
-
for batch_index, batch in enumerate(_chunk_items(pending, batch_size), start=1):
|
| 670 |
-
batch_uploaded = 0
|
| 671 |
-
batch_failed = 0
|
| 672 |
-
for local_path, repo_path in batch:
|
| 673 |
-
if should_cancel and should_cancel():
|
| 674 |
-
cancelled = True
|
| 675 |
-
break
|
| 676 |
-
|
| 677 |
-
while should_pause and should_pause():
|
| 678 |
-
time.sleep(0.5)
|
| 679 |
-
if should_cancel and should_cancel():
|
| 680 |
-
cancelled = True
|
| 681 |
-
break
|
| 682 |
-
|
| 683 |
-
if cancelled:
|
| 684 |
-
break
|
| 685 |
-
|
| 686 |
-
try:
|
| 687 |
-
_upload_audio_with_retry(
|
| 688 |
-
api=api,
|
| 689 |
-
dataset_repo=dataset_repo,
|
| 690 |
-
local_path=local_path,
|
| 691 |
-
path_in_repo=repo_path,
|
| 692 |
-
max_retries=max_retries,
|
| 693 |
-
retry_backoff_seconds=retry_backoff_seconds,
|
| 694 |
-
)
|
| 695 |
-
uploaded_state.add(repo_path)
|
| 696 |
-
failed_state.discard(repo_path)
|
| 697 |
-
uploaded_now.add(repo_path)
|
| 698 |
-
batch_uploaded += 1
|
| 699 |
-
except Exception:
|
| 700 |
-
failed_state.add(repo_path)
|
| 701 |
-
failed_now.add(repo_path)
|
| 702 |
-
batch_failed += 1
|
| 703 |
-
|
| 704 |
-
if progress_callback:
|
| 705 |
-
progress_callback(
|
| 706 |
-
{
|
| 707 |
-
"event": "upload-progress",
|
| 708 |
-
"uploaded": len(uploaded_now),
|
| 709 |
-
"failed": len(failed_now),
|
| 710 |
-
"skipped_existing": skipped_existing,
|
| 711 |
-
"processed_pending": len(uploaded_now) + len(failed_now),
|
| 712 |
-
"pending_total": len(pending),
|
| 713 |
-
}
|
| 714 |
-
)
|
| 715 |
-
|
| 716 |
-
print(
|
| 717 |
-
json.dumps(
|
| 718 |
-
{
|
| 719 |
-
"event": "ingest-segments-audio-batch",
|
| 720 |
-
"batch_index": batch_index,
|
| 721 |
-
"batch_size": len(batch),
|
| 722 |
-
"uploaded": batch_uploaded,
|
| 723 |
-
"failed": batch_failed,
|
| 724 |
-
}
|
| 725 |
-
)
|
| 726 |
-
)
|
| 727 |
-
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed_state)
|
| 728 |
-
|
| 729 |
-
if cancelled:
|
| 730 |
-
break
|
| 731 |
-
|
| 732 |
-
if cancelled:
|
| 733 |
-
run_report = {
|
| 734 |
-
"mode": "execute",
|
| 735 |
-
"project_slug": project_slug,
|
| 736 |
-
"dataset_repo": dataset_repo,
|
| 737 |
-
"detections_csv": detections_csv,
|
| 738 |
-
"segments_root": segments_root,
|
| 739 |
-
"csv_rows_total": int(len(frame)),
|
| 740 |
-
"segments_found_total": int(match_result["segment_records_total"]),
|
| 741 |
-
"matched_rows": len(matched_rows),
|
| 742 |
-
"unmatched_rows": len(unmatched_rows),
|
| 743 |
-
"ambiguous_rows": len(ambiguous_rows),
|
| 744 |
-
"unique_audio_ids": 0,
|
| 745 |
-
"pending_audio_uploads": len(pending),
|
| 746 |
-
"uploaded_audio_now": len(uploaded_now),
|
| 747 |
-
"uploaded_audio_skipped_existing": skipped_existing,
|
| 748 |
-
"failed_uploads": len(failed_now),
|
| 749 |
-
"cancelled": True,
|
| 750 |
-
"resume_state_file": str(state_path),
|
| 751 |
-
"index_rows_written": 0,
|
| 752 |
-
"shards_written": 0,
|
| 753 |
-
"sample_unmatched": unmatched_rows[:20],
|
| 754 |
-
"sample_ambiguous": ambiguous_rows[:20],
|
| 755 |
-
}
|
| 756 |
-
if progress_callback:
|
| 757 |
-
progress_callback(
|
| 758 |
-
{
|
| 759 |
-
"event": "upload-complete",
|
| 760 |
-
"uploaded": len(uploaded_now),
|
| 761 |
-
"failed": len(failed_now),
|
| 762 |
-
"skipped_existing": skipped_existing,
|
| 763 |
-
"cancelled": True,
|
| 764 |
-
}
|
| 765 |
-
)
|
| 766 |
-
return run_report
|
| 767 |
-
|
| 768 |
-
index_frame = pd.DataFrame(matched_rows)
|
| 769 |
-
if not index_frame.empty:
|
| 770 |
-
index_frame["segment_path_in_repo"] = index_frame["segment_relpath"].apply(lambda value: f"audio/segments/{value}")
|
| 771 |
-
|
| 772 |
-
with tempfile.TemporaryDirectory(prefix="birdnet-ingest-index-") as temp_dir_str:
|
| 773 |
-
temp_dir = Path(temp_dir_str)
|
| 774 |
-
shard_dir = temp_dir / "index" / "shards"
|
| 775 |
-
shard_metadata = build_shards_in_directory(frame=index_frame, output_dir=shard_dir, shard_size=shard_size)
|
| 776 |
-
|
| 777 |
-
for item in shard_metadata:
|
| 778 |
-
local_path = temp_dir / item.path
|
| 779 |
-
api.upload_file(
|
| 780 |
-
path_or_fileobj=str(local_path),
|
| 781 |
-
path_in_repo=item.path,
|
| 782 |
-
repo_id=dataset_repo,
|
| 783 |
-
repo_type="dataset",
|
| 784 |
-
)
|
| 785 |
-
|
| 786 |
-
manifest = build_manifest_payload(
|
| 787 |
-
project_slug=project_slug,
|
| 788 |
-
dataset_repo=dataset_repo,
|
| 789 |
-
frame=index_frame,
|
| 790 |
-
shard_size=shard_size,
|
| 791 |
-
shard_metadata=shard_metadata,
|
| 792 |
-
)
|
| 793 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 794 |
-
|
| 795 |
-
run_report = {
|
| 796 |
-
"mode": "execute",
|
| 797 |
-
"project_slug": project_slug,
|
| 798 |
-
"dataset_repo": dataset_repo,
|
| 799 |
-
"detections_csv": detections_csv,
|
| 800 |
-
"segments_root": segments_root,
|
| 801 |
-
"csv_rows_total": int(len(frame)),
|
| 802 |
-
"segments_found_total": int(match_result["segment_records_total"]),
|
| 803 |
-
"matched_rows": len(matched_rows),
|
| 804 |
-
"unmatched_rows": len(unmatched_rows),
|
| 805 |
-
"ambiguous_rows": len(ambiguous_rows),
|
| 806 |
-
"unique_audio_ids": int(index_frame["audio_id"].nunique()) if not index_frame.empty else 0,
|
| 807 |
-
"pending_audio_uploads": len(pending),
|
| 808 |
-
"uploaded_audio_now": len(uploaded_now),
|
| 809 |
-
"uploaded_audio_skipped_existing": skipped_existing,
|
| 810 |
-
"failed_uploads": len(failed_now),
|
| 811 |
-
"cancelled": cancelled,
|
| 812 |
-
"resume_state_file": str(state_path),
|
| 813 |
-
"index_rows_written": len(index_frame),
|
| 814 |
-
"shards_written": len(shard_metadata),
|
| 815 |
-
"sample_unmatched": unmatched_rows[:20],
|
| 816 |
-
"sample_ambiguous": ambiguous_rows[:20],
|
| 817 |
-
}
|
| 818 |
-
|
| 819 |
-
audit_path = f"audit/ingestion-runs/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.json"
|
| 820 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=audit_path, content=json.dumps(run_report, indent=2))
|
| 821 |
-
|
| 822 |
-
if progress_callback:
|
| 823 |
-
progress_callback(
|
| 824 |
-
{
|
| 825 |
-
"event": "upload-complete",
|
| 826 |
-
"uploaded": len(uploaded_now),
|
| 827 |
-
"failed": len(failed_now),
|
| 828 |
-
"skipped_existing": skipped_existing,
|
| 829 |
-
"cancelled": cancelled,
|
| 830 |
-
}
|
| 831 |
-
)
|
| 832 |
-
|
| 833 |
-
return run_report
|
| 834 |
-
|
| 835 |
-
|
| 836 |
-
def sync_audio_batches(
|
| 837 |
-
api: HfApi,
|
| 838 |
-
project_slug: str,
|
| 839 |
-
dataset_repo: str,
|
| 840 |
-
local_audio_dir: str,
|
| 841 |
-
batch_size: int,
|
| 842 |
-
max_retries: int,
|
| 843 |
-
retry_backoff_seconds: float,
|
| 844 |
-
resume_state_file: str,
|
| 845 |
-
) -> dict[str, Any]:
|
| 846 |
-
ensure_project_dataset_structure(
|
| 847 |
-
api=api,
|
| 848 |
-
project_slug=project_slug,
|
| 849 |
-
dataset_repo=dataset_repo,
|
| 850 |
-
create_private_repo=False,
|
| 851 |
-
)
|
| 852 |
-
|
| 853 |
-
all_files = discover_audio_files(local_audio_dir=local_audio_dir)
|
| 854 |
-
state_path = Path(resume_state_file)
|
| 855 |
-
state_payload = _load_resume_state(state_file=state_path)
|
| 856 |
-
uploaded_state = set(str(item) for item in state_payload["uploaded"])
|
| 857 |
-
|
| 858 |
-
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 859 |
-
|
| 860 |
-
pending: list[tuple[Path, str]] = []
|
| 861 |
-
for path in all_files:
|
| 862 |
-
rel = path.relative_to(Path(local_audio_dir)).as_posix()
|
| 863 |
-
repo_path = f"audio/{rel}"
|
| 864 |
-
|
| 865 |
-
if repo_path in remote_files:
|
| 866 |
-
uploaded_state.add(repo_path)
|
| 867 |
-
continue
|
| 868 |
-
|
| 869 |
-
if repo_path in uploaded_state:
|
| 870 |
-
continue
|
| 871 |
-
|
| 872 |
-
pending.append((path, repo_path))
|
| 873 |
-
|
| 874 |
-
pending_paths = [item[0] for item in pending]
|
| 875 |
-
by_local_to_repo = {local: repo for local, repo in pending}
|
| 876 |
-
batches = _chunk_items(pending_paths, batch_size) if pending_paths else []
|
| 877 |
-
|
| 878 |
-
uploaded_now: set[str] = set()
|
| 879 |
-
failed: set[str] = set()
|
| 880 |
-
|
| 881 |
-
for batch_index, batch in enumerate(batches, start=1):
|
| 882 |
-
batch_uploaded = 0
|
| 883 |
-
batch_failed = 0
|
| 884 |
-
|
| 885 |
-
for local_path in batch:
|
| 886 |
-
repo_path = by_local_to_repo[local_path]
|
| 887 |
-
try:
|
| 888 |
-
_upload_audio_with_retry(
|
| 889 |
-
api=api,
|
| 890 |
-
dataset_repo=dataset_repo,
|
| 891 |
-
local_path=local_path,
|
| 892 |
-
path_in_repo=repo_path,
|
| 893 |
-
max_retries=max_retries,
|
| 894 |
-
retry_backoff_seconds=retry_backoff_seconds,
|
| 895 |
-
)
|
| 896 |
-
uploaded_now.add(repo_path)
|
| 897 |
-
uploaded_state.add(repo_path)
|
| 898 |
-
batch_uploaded += 1
|
| 899 |
-
except Exception:
|
| 900 |
-
failed.add(repo_path)
|
| 901 |
-
batch_failed += 1
|
| 902 |
-
|
| 903 |
-
print(
|
| 904 |
-
json.dumps(
|
| 905 |
-
{
|
| 906 |
-
"event": "sync-audio-batch",
|
| 907 |
-
"batch_index": batch_index,
|
| 908 |
-
"batch_size": len(batch),
|
| 909 |
-
"uploaded": batch_uploaded,
|
| 910 |
-
"failed": batch_failed,
|
| 911 |
-
}
|
| 912 |
-
)
|
| 913 |
-
)
|
| 914 |
-
_save_resume_state(state_file=state_path, uploaded=uploaded_state, failed=failed)
|
| 915 |
-
|
| 916 |
-
return {
|
| 917 |
-
"project_slug": project_slug,
|
| 918 |
-
"dataset_repo": dataset_repo,
|
| 919 |
-
"total_local_audio_files": len(all_files),
|
| 920 |
-
"pending_uploads": len(pending),
|
| 921 |
-
"uploaded_now": len(uploaded_now),
|
| 922 |
-
"failed": len(failed),
|
| 923 |
-
"resume_state_file": str(state_path),
|
| 924 |
-
}
|
| 925 |
-
|
| 926 |
-
|
| 927 |
-
def build_and_upload_index(
|
| 928 |
-
api: HfApi,
|
| 929 |
-
project_slug: str,
|
| 930 |
-
dataset_repo: str,
|
| 931 |
-
detections_file: str,
|
| 932 |
-
shard_size: int,
|
| 933 |
-
) -> dict[str, Any]:
|
| 934 |
-
ensure_project_dataset_structure(
|
| 935 |
-
api=api,
|
| 936 |
-
project_slug=project_slug,
|
| 937 |
-
dataset_repo=dataset_repo,
|
| 938 |
-
create_private_repo=False,
|
| 939 |
-
)
|
| 940 |
-
|
| 941 |
-
frame = load_detections_table(detections_file=detections_file)
|
| 942 |
-
|
| 943 |
-
with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
|
| 944 |
-
temp_dir = Path(temp_dir_str)
|
| 945 |
-
shard_dir = temp_dir / "index" / "shards"
|
| 946 |
-
metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
|
| 947 |
-
|
| 948 |
-
for item in metadata:
|
| 949 |
-
local_path = temp_dir / item.path
|
| 950 |
-
api.upload_file(
|
| 951 |
-
path_or_fileobj=str(local_path),
|
| 952 |
-
path_in_repo=item.path,
|
| 953 |
-
repo_id=dataset_repo,
|
| 954 |
-
repo_type="dataset",
|
| 955 |
-
)
|
| 956 |
-
|
| 957 |
-
manifest = build_manifest_payload(
|
| 958 |
-
project_slug=project_slug,
|
| 959 |
-
dataset_repo=dataset_repo,
|
| 960 |
-
frame=frame,
|
| 961 |
-
shard_size=shard_size,
|
| 962 |
-
shard_metadata=metadata,
|
| 963 |
-
)
|
| 964 |
-
_upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
|
| 965 |
-
|
| 966 |
-
return {
|
| 967 |
-
"project_slug": project_slug,
|
| 968 |
-
"dataset_repo": dataset_repo,
|
| 969 |
-
"total_detections": manifest["index"]["total_detections"],
|
| 970 |
-
"total_audio_files": manifest["index"]["total_audio_files"],
|
| 971 |
-
"total_shards": len(metadata),
|
| 972 |
-
}
|
| 973 |
-
|
| 974 |
-
|
| 975 |
-
def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
|
| 976 |
-
errors: list[str] = []
|
| 977 |
-
|
| 978 |
-
for prefix in REQUIRED_PROJECT_PREFIXES:
|
| 979 |
-
if not any(path.startswith(prefix) for path in repo_files):
|
| 980 |
-
errors.append(f"Missing prefix in dataset repo: {prefix}")
|
| 981 |
-
|
| 982 |
-
if manifest_payload.get("schema_version") != SCHEMA_VERSION:
|
| 983 |
-
errors.append(
|
| 984 |
-
f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
|
| 985 |
-
)
|
| 986 |
-
|
| 987 |
-
if manifest_payload.get("project_slug") != project_slug:
|
| 988 |
-
errors.append(
|
| 989 |
-
f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
|
| 990 |
-
)
|
| 991 |
-
|
| 992 |
-
index_section = manifest_payload.get("index", {})
|
| 993 |
-
shards = index_section.get("shards", [])
|
| 994 |
-
if not isinstance(shards, list):
|
| 995 |
-
errors.append("Manifest index.shards must be a list")
|
| 996 |
-
return errors
|
| 997 |
-
|
| 998 |
-
for shard in shards:
|
| 999 |
-
shard_path = shard.get("path")
|
| 1000 |
-
if not shard_path or not isinstance(shard_path, str):
|
| 1001 |
-
errors.append("Shard entry missing path")
|
| 1002 |
-
continue
|
| 1003 |
-
if shard_path not in repo_files:
|
| 1004 |
-
errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
|
| 1005 |
-
|
| 1006 |
-
return errors
|
| 1007 |
-
|
| 1008 |
-
|
| 1009 |
-
def upload_segments_to_hf(
|
| 1010 |
-
api: HfApi,
|
| 1011 |
-
project_slug: str,
|
| 1012 |
-
dataset_repo: str,
|
| 1013 |
-
segments_root: str,
|
| 1014 |
-
batch_size: int = 50,
|
| 1015 |
-
max_retries: int = 3,
|
| 1016 |
-
retry_backoff_seconds: float = 1.0,
|
| 1017 |
-
should_pause: Callable[[], bool] | None = None,
|
| 1018 |
-
should_cancel: Callable[[], bool] | None = None,
|
| 1019 |
-
progress_callback: Callable[[dict[str, Any]], None] | None = None,
|
| 1020 |
-
) -> dict[str, Any]:
|
| 1021 |
-
"""Upload local segments folder to dataset repo maintaining directory structure."""
|
| 1022 |
-
segments_path = Path(segments_root)
|
| 1023 |
-
if not segments_path.exists() or not segments_path.is_dir():
|
| 1024 |
-
raise FileNotFoundError(f"Segments root not found: {segments_root}")
|
| 1025 |
-
|
| 1026 |
-
if batch_size <= 0:
|
| 1027 |
-
raise ValueError("batch_size must be greater than zero")
|
| 1028 |
-
|
| 1029 |
-
ensure_project_dataset_structure(
|
| 1030 |
-
api=api,
|
| 1031 |
-
project_slug=project_slug,
|
| 1032 |
-
dataset_repo=dataset_repo,
|
| 1033 |
-
create_private_repo=False,
|
| 1034 |
-
)
|
| 1035 |
-
|
| 1036 |
-
all_audio_files: list[Path] = []
|
| 1037 |
-
for ext in SUPPORTED_AUDIO_EXTENSIONS:
|
| 1038 |
-
all_audio_files.extend(segments_path.rglob(f"*{ext}"))
|
| 1039 |
-
all_audio_files = sorted(all_audio_files)
|
| 1040 |
-
|
| 1041 |
-
remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 1042 |
-
pending: list[tuple[Path, str]] = []
|
| 1043 |
-
skipped_existing = 0
|
| 1044 |
-
for local_file in all_audio_files:
|
| 1045 |
-
relative_path = local_file.relative_to(segments_path)
|
| 1046 |
-
repo_path = f"audio/{project_slug}/{relative_path.as_posix()}"
|
| 1047 |
-
if repo_path in remote_files:
|
| 1048 |
-
skipped_existing += 1
|
| 1049 |
-
continue
|
| 1050 |
-
pending.append((local_file, repo_path))
|
| 1051 |
-
|
| 1052 |
-
if not all_audio_files:
|
| 1053 |
-
return {
|
| 1054 |
-
"project_slug": project_slug,
|
| 1055 |
-
"dataset_repo": dataset_repo,
|
| 1056 |
-
"mode": "upload-segments-only",
|
| 1057 |
-
"total_files": 0,
|
| 1058 |
-
"uploaded": 0,
|
| 1059 |
-
"skipped_existing": 0,
|
| 1060 |
-
"failed": 0,
|
| 1061 |
-
"cancelled": False,
|
| 1062 |
-
}
|
| 1063 |
-
|
| 1064 |
-
uploaded_count = 0
|
| 1065 |
-
failed_count = 0
|
| 1066 |
-
cancelled = False
|
| 1067 |
-
failed_files: list[str] = []
|
| 1068 |
-
|
| 1069 |
-
batches = _chunk_items(pending, batch_size) if pending else []
|
| 1070 |
-
total_pending = len(pending)
|
| 1071 |
-
if progress_callback:
|
| 1072 |
-
progress_callback(
|
| 1073 |
-
{
|
| 1074 |
-
"event": "upload-start",
|
| 1075 |
-
"total_files": len(all_audio_files),
|
| 1076 |
-
"pending": total_pending,
|
| 1077 |
-
"skipped_existing": skipped_existing,
|
| 1078 |
-
}
|
| 1079 |
-
)
|
| 1080 |
-
|
| 1081 |
-
for batch_idx, batch in enumerate(batches, start=1):
|
| 1082 |
-
for local_file, repo_path in batch:
|
| 1083 |
-
if should_cancel and should_cancel():
|
| 1084 |
-
cancelled = True
|
| 1085 |
-
break
|
| 1086 |
-
|
| 1087 |
-
while should_pause and should_pause():
|
| 1088 |
-
time.sleep(0.5)
|
| 1089 |
-
if should_cancel and should_cancel():
|
| 1090 |
-
cancelled = True
|
| 1091 |
-
break
|
| 1092 |
-
|
| 1093 |
-
if cancelled:
|
| 1094 |
-
break
|
| 1095 |
-
|
| 1096 |
-
relative_path = local_file.relative_to(segments_path)
|
| 1097 |
-
|
| 1098 |
-
try:
|
| 1099 |
-
_upload_audio_with_retry(
|
| 1100 |
-
api=api,
|
| 1101 |
-
dataset_repo=dataset_repo,
|
| 1102 |
-
local_path=local_file,
|
| 1103 |
-
path_in_repo=repo_path,
|
| 1104 |
-
max_retries=max_retries,
|
| 1105 |
-
retry_backoff_seconds=retry_backoff_seconds,
|
| 1106 |
-
)
|
| 1107 |
-
uploaded_count += 1
|
| 1108 |
-
except Exception as exc:
|
| 1109 |
-
failed_count += 1
|
| 1110 |
-
failed_files.append(f"{relative_path.as_posix()}: {exc}")
|
| 1111 |
-
|
| 1112 |
-
if progress_callback:
|
| 1113 |
-
progress_callback(
|
| 1114 |
-
{
|
| 1115 |
-
"event": "upload-progress",
|
| 1116 |
-
"uploaded": uploaded_count,
|
| 1117 |
-
"failed": failed_count,
|
| 1118 |
-
"skipped_existing": skipped_existing,
|
| 1119 |
-
"processed_pending": uploaded_count + failed_count,
|
| 1120 |
-
"pending_total": total_pending,
|
| 1121 |
-
}
|
| 1122 |
-
)
|
| 1123 |
-
|
| 1124 |
-
if cancelled:
|
| 1125 |
-
break
|
| 1126 |
-
|
| 1127 |
-
print(
|
| 1128 |
-
json.dumps(
|
| 1129 |
-
{
|
| 1130 |
-
"event": "upload-batch",
|
| 1131 |
-
"batch": batch_idx,
|
| 1132 |
-
"total_batches": len(batches),
|
| 1133 |
-
"uploaded_so_far": uploaded_count,
|
| 1134 |
-
"failed_so_far": failed_count,
|
| 1135 |
-
"skipped_existing": skipped_existing,
|
| 1136 |
-
}
|
| 1137 |
-
)
|
| 1138 |
-
)
|
| 1139 |
-
|
| 1140 |
-
if progress_callback:
|
| 1141 |
-
progress_callback(
|
| 1142 |
-
{
|
| 1143 |
-
"event": "upload-complete",
|
| 1144 |
-
"uploaded": uploaded_count,
|
| 1145 |
-
"failed": failed_count,
|
| 1146 |
-
"skipped_existing": skipped_existing,
|
| 1147 |
-
"cancelled": cancelled,
|
| 1148 |
-
}
|
| 1149 |
-
)
|
| 1150 |
-
|
| 1151 |
-
return {
|
| 1152 |
-
"project_slug": project_slug,
|
| 1153 |
-
"dataset_repo": dataset_repo,
|
| 1154 |
-
"mode": "upload-segments-only",
|
| 1155 |
-
"total_files": len(all_audio_files),
|
| 1156 |
-
"pending_files": total_pending,
|
| 1157 |
-
"uploaded": uploaded_count,
|
| 1158 |
-
"skipped_existing": skipped_existing,
|
| 1159 |
-
"failed": failed_count,
|
| 1160 |
-
"cancelled": cancelled,
|
| 1161 |
-
"failed_samples": failed_files[:10],
|
| 1162 |
-
}
|
| 1163 |
-
|
| 1164 |
-
|
| 1165 |
-
def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
|
| 1166 |
-
repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 1167 |
-
if "manifest.json" not in repo_files:
|
| 1168 |
-
return {
|
| 1169 |
-
"ok": False,
|
| 1170 |
-
"errors": ["manifest.json not found in dataset repository"],
|
| 1171 |
-
}
|
| 1172 |
-
|
| 1173 |
-
manifest_local = hf_hub_download(
|
| 1174 |
-
repo_id=dataset_repo,
|
| 1175 |
-
repo_type="dataset",
|
| 1176 |
-
filename="manifest.json",
|
| 1177 |
-
)
|
| 1178 |
-
manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
|
| 1179 |
-
errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
|
| 1180 |
-
|
| 1181 |
-
return {
|
| 1182 |
-
"ok": len(errors) == 0,
|
| 1183 |
-
"errors": errors,
|
| 1184 |
-
"total_files": len(repo_files),
|
| 1185 |
-
"total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
|
| 1186 |
-
}
|
| 1187 |
-
|
| 1188 |
-
|
| 1189 |
-
def build_parser() -> argparse.ArgumentParser:
|
| 1190 |
-
parser = argparse.ArgumentParser(description="HF dataset project CLI")
|
| 1191 |
-
sub = parser.add_subparsers(dest="command", required=True)
|
| 1192 |
-
|
| 1193 |
-
create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
|
| 1194 |
-
create_project.add_argument("--project-slug", required=True)
|
| 1195 |
-
create_project.add_argument("--dataset-repo", required=True)
|
| 1196 |
-
create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
|
| 1197 |
-
|
| 1198 |
-
build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
|
| 1199 |
-
build_index.add_argument("--project-slug", required=True)
|
| 1200 |
-
build_index.add_argument("--dataset-repo", required=True)
|
| 1201 |
-
build_index.add_argument("--detections-file", required=True)
|
| 1202 |
-
build_index.add_argument("--shard-size", type=int, default=10000)
|
| 1203 |
-
|
| 1204 |
-
sync_audio = sub.add_parser("sync-audio", help="Upload local audio files in resumable batches")
|
| 1205 |
-
sync_audio.add_argument("--project-slug", required=True)
|
| 1206 |
-
sync_audio.add_argument("--dataset-repo", required=True)
|
| 1207 |
-
sync_audio.add_argument("--local-audio-dir", required=True)
|
| 1208 |
-
sync_audio.add_argument("--batch-size", type=int, default=100)
|
| 1209 |
-
sync_audio.add_argument("--max-retries", type=int, default=3)
|
| 1210 |
-
sync_audio.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 1211 |
-
sync_audio.add_argument("--resume-state-file", default=".sync-audio-state.json")
|
| 1212 |
-
|
| 1213 |
-
verify_project = sub.add_parser("verify-project", help="Verify project integrity")
|
| 1214 |
-
verify_project.add_argument("--project-slug", required=True)
|
| 1215 |
-
verify_project.add_argument("--dataset-repo", required=True)
|
| 1216 |
-
|
| 1217 |
-
ingest_segments = sub.add_parser("ingest-segments", help="Match BirdNET CSV rows with segment files")
|
| 1218 |
-
ingest_segments.add_argument("--project-slug", required=True)
|
| 1219 |
-
ingest_segments.add_argument("--dataset-repo", required=True)
|
| 1220 |
-
ingest_segments.add_argument("--detections-csv", required=True)
|
| 1221 |
-
ingest_segments.add_argument("--segments-root", required=True)
|
| 1222 |
-
ingest_segments.add_argument("--batch-size", type=int, default=200)
|
| 1223 |
-
ingest_segments.add_argument("--shard-size", type=int, default=10000)
|
| 1224 |
-
ingest_segments.add_argument("--max-retries", type=int, default=3)
|
| 1225 |
-
ingest_segments.add_argument("--retry-backoff-seconds", type=float, default=1.0)
|
| 1226 |
-
ingest_segments.add_argument("--resume-state-file", default=".ingest-segments-state.json")
|
| 1227 |
-
ingest_segments.add_argument("--dry-run", action="store_true")
|
| 1228 |
-
ingest_segments.add_argument("--report-file", default="")
|
| 1229 |
-
|
| 1230 |
-
return parser
|
| 1231 |
-
|
| 1232 |
-
|
| 1233 |
-
def main() -> int:
|
| 1234 |
-
parser = build_parser()
|
| 1235 |
-
args = parser.parse_args()
|
| 1236 |
-
api = HfApi()
|
| 1237 |
-
|
| 1238 |
-
if args.command == "create-project":
|
| 1239 |
-
result = ensure_project_dataset_structure(
|
| 1240 |
-
api=api,
|
| 1241 |
-
project_slug=args.project_slug,
|
| 1242 |
-
dataset_repo=args.dataset_repo,
|
| 1243 |
-
create_private_repo=bool(args.private),
|
| 1244 |
-
)
|
| 1245 |
-
print(json.dumps(result, indent=2))
|
| 1246 |
-
return 0
|
| 1247 |
-
|
| 1248 |
-
if args.command == "build-index":
|
| 1249 |
-
result = build_and_upload_index(
|
| 1250 |
-
api=api,
|
| 1251 |
-
project_slug=args.project_slug,
|
| 1252 |
-
dataset_repo=args.dataset_repo,
|
| 1253 |
-
detections_file=args.detections_file,
|
| 1254 |
-
shard_size=args.shard_size,
|
| 1255 |
-
)
|
| 1256 |
-
print(json.dumps(result, indent=2))
|
| 1257 |
-
return 0
|
| 1258 |
-
|
| 1259 |
-
if args.command == "sync-audio":
|
| 1260 |
-
result = sync_audio_batches(
|
| 1261 |
-
api=api,
|
| 1262 |
-
project_slug=args.project_slug,
|
| 1263 |
-
dataset_repo=args.dataset_repo,
|
| 1264 |
-
local_audio_dir=args.local_audio_dir,
|
| 1265 |
-
batch_size=args.batch_size,
|
| 1266 |
-
max_retries=args.max_retries,
|
| 1267 |
-
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 1268 |
-
resume_state_file=args.resume_state_file,
|
| 1269 |
-
)
|
| 1270 |
-
print(json.dumps(result, indent=2))
|
| 1271 |
-
return 0 if result["failed"] == 0 else 1
|
| 1272 |
-
|
| 1273 |
-
if args.command == "verify-project":
|
| 1274 |
-
result = verify_project(
|
| 1275 |
-
api=api,
|
| 1276 |
-
project_slug=args.project_slug,
|
| 1277 |
-
dataset_repo=args.dataset_repo,
|
| 1278 |
-
)
|
| 1279 |
-
print(json.dumps(result, indent=2))
|
| 1280 |
-
return 0 if result["ok"] else 1
|
| 1281 |
-
|
| 1282 |
-
if args.command == "ingest-segments":
|
| 1283 |
-
if args.dry_run:
|
| 1284 |
-
result = run_ingest_segments_dry_run(
|
| 1285 |
-
project_slug=args.project_slug,
|
| 1286 |
-
detections_csv=args.detections_csv,
|
| 1287 |
-
segments_root=args.segments_root,
|
| 1288 |
-
)
|
| 1289 |
-
else:
|
| 1290 |
-
result = ingest_segments_to_hf(
|
| 1291 |
-
api=api,
|
| 1292 |
-
project_slug=args.project_slug,
|
| 1293 |
-
dataset_repo=args.dataset_repo,
|
| 1294 |
-
detections_csv=args.detections_csv,
|
| 1295 |
-
segments_root=args.segments_root,
|
| 1296 |
-
batch_size=args.batch_size,
|
| 1297 |
-
shard_size=args.shard_size,
|
| 1298 |
-
max_retries=args.max_retries,
|
| 1299 |
-
retry_backoff_seconds=args.retry_backoff_seconds,
|
| 1300 |
-
resume_state_file=args.resume_state_file,
|
| 1301 |
-
)
|
| 1302 |
-
|
| 1303 |
-
if args.report_file:
|
| 1304 |
-
Path(args.report_file).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
| 1305 |
-
|
| 1306 |
-
print(json.dumps(result, indent=2))
|
| 1307 |
-
return 0 if result.get("failed_uploads", 0) == 0 else 1
|
| 1308 |
-
|
| 1309 |
-
parser.error("Unknown command")
|
| 1310 |
-
return 2
|
| 1311 |
-
|
| 1312 |
-
|
| 1313 |
-
if __name__ == "__main__":
|
| 1314 |
-
raise SystemExit(main())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
docs/COLLABORATIVE_WORKFLOW_GUIDE.md
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# End-to-End Collaborative Workflow Guide
|
| 2 |
+
|
| 3 |
+
## Overview
|
| 4 |
+
|
| 5 |
+
The BirdNET Validator App supports two project visibility modes:
|
| 6 |
+
|
| 7 |
+
- **Collaborative**: Multiple users can contribute validations. Any admin can invite collaborators via email.
|
| 8 |
+
- **Private**: Single owner only. No invites allowed. Owner retains exclusive access.
|
| 9 |
+
|
| 10 |
+
## Feature Summary
|
| 11 |
+
|
| 12 |
+
### ✅ What's Implemented
|
| 13 |
+
|
| 14 |
+
| Feature | Status | Details |
|
| 15 |
+
|---------|--------|---------|
|
| 16 |
+
| **Collaborative Projects** | ✅ | Multiple users, email invites, per-user tokens |
|
| 17 |
+
| **Private Projects** | ✅ | Owner-only, no invites, strict ACL enforcement |
|
| 18 |
+
| **Email Notifications** | ✅ | SMTP + NoOp fallback, transactional delivery |
|
| 19 |
+
| **Per-User Tokens** | ✅ | Each user's HF token stored separately (secure) |
|
| 20 |
+
| **Session Revocation** | ✅ | Immediate when user removed from project |
|
| 21 |
+
| **Durable State** | ✅ | `/data` persistence, atomic writes, crash-safe |
|
| 22 |
+
| **Invite TTL** | ✅ | Expiring invites (72h default, configurable) |
|
| 23 |
+
| **ACL Enforcement** | ✅ | Private vs Collaborative, admin-only operations |
|
| 24 |
+
|
| 25 |
+
---
|
| 26 |
+
|
| 27 |
+
## Workflow Example: Creating & Sharing a Collaborative Project
|
| 28 |
+
|
| 29 |
+
### Phase 1: Admin Creates Project
|
| 30 |
+
|
| 31 |
+
```bash
|
| 32 |
+
# Admin logs in (first user = automatic admin)
|
| 33 |
+
# → Session created with admin role
|
| 34 |
+
|
| 35 |
+
# Admin creates project via CLI:
|
| 36 |
+
python -m src.cli.project_cli create-project \
|
| 37 |
+
--projects-file /data/bootstrap/projects.json \
|
| 38 |
+
--slug "parrots-2026" \
|
| 39 |
+
--name "Parrots Dataset 2026" \
|
| 40 |
+
--dataset-repo-id "birdnet/parrots-2026" \
|
| 41 |
+
--visibility "collaborative"
|
| 42 |
+
# OK: project 'parrots-2026' created
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
**What happened:**
|
| 46 |
+
- Project created in `/data/bootstrap/projects.json`
|
| 47 |
+
- No `owner_username` (collaborative mode)
|
| 48 |
+
- Admin automatically has validator role
|
| 49 |
+
|
| 50 |
+
### Phase 2: Admin Invites Collaborator
|
| 51 |
+
|
| 52 |
+
Using the Admin Panel UI, admin clicks "Invite User":
|
| 53 |
+
|
| 54 |
+
```
|
| 55 |
+
Actor: alice_admin (project admin)
|
| 56 |
+
Invitee Username: bob_validator
|
| 57 |
+
Invitee Email: bob@example.com
|
| 58 |
+
Role: validator
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
**What happens:**
|
| 62 |
+
1. Authorization check: Alice is admin for this project ✓
|
| 63 |
+
2. Project visibility check: Collaborative (invites allowed) ✓
|
| 64 |
+
3. Invite created with 72h TTL
|
| 65 |
+
4. Email sent via SMTP:
|
| 66 |
+
```
|
| 67 |
+
Subject: You've been invited to validate "Parrots Dataset 2026"
|
| 68 |
+
|
| 69 |
+
Hello Bob,
|
| 70 |
+
|
| 71 |
+
Alice has invited you to join the "Parrots Dataset 2026" project as a validator.
|
| 72 |
+
|
| 73 |
+
Project: parrots-2026
|
| 74 |
+
Role: validator
|
| 75 |
+
Expires: [72 hours from now]
|
| 76 |
+
|
| 77 |
+
To accept this invitation, log in with your Hugging Face token at:
|
| 78 |
+
https://birdnet-validator.example.com/login
|
| 79 |
+
|
| 80 |
+
After logging in, your invite will appear. Click "Accept" to join.
|
| 81 |
+
```
|
| 82 |
+
5. Invite stored in `/data/bootstrap/invites.json`:
|
| 83 |
+
```json
|
| 84 |
+
{
|
| 85 |
+
"bob_validator": {
|
| 86 |
+
"parrots-2026": {
|
| 87 |
+
"role": "validator",
|
| 88 |
+
"invited_by": "alice_admin",
|
| 89 |
+
"created_at": "2026-04-01T12:00:00Z",
|
| 90 |
+
"expires_at": "2026-04-04T12:00:00Z"
|
| 91 |
+
}
|
| 92 |
+
}
|
| 93 |
+
}
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
### Phase 3: Collaborator Receives & Accepts Invite
|
| 97 |
+
|
| 98 |
+
**Bob receives email, opens Space app, logs in:**
|
| 99 |
+
|
| 100 |
+
1. Bob enters his HF token: `hf_user_***`
|
| 101 |
+
2. AuthService resolves username from HF API (`whoami()`)
|
| 102 |
+
3. Bob's session created
|
| 103 |
+
4. Bob's token stored separately: `_hf_tokens_by_username["bob_validator"] = "hf_user_***"`
|
| 104 |
+
5. **Pending invites auto-appear** in project selector
|
| 105 |
+
6. Bob clicks "Accept Invite"
|
| 106 |
+
|
| 107 |
+
**What happens:**
|
| 108 |
+
- Invite moved from pending → accepted
|
| 109 |
+
- Bob added to `user_access.json`:
|
| 110 |
+
```json
|
| 111 |
+
{
|
| 112 |
+
"bob_validator": {
|
| 113 |
+
"parrots-2026": "validator"
|
| 114 |
+
}
|
| 115 |
+
}
|
| 116 |
+
```
|
| 117 |
+
- Invite removed from `invites.json`
|
| 118 |
+
|
| 119 |
+
### Phase 4: Both Users Have Access & Own Tokens
|
| 120 |
+
|
| 121 |
+
**Alice (admin):**
|
| 122 |
+
- Token: `hf_alice_***` (stored in `_hf_tokens_by_username["alice_admin"]`)
|
| 123 |
+
- Role: admin for "parrots-2026"
|
| 124 |
+
- Can: create, assign, remove, validate, manage project
|
| 125 |
+
|
| 126 |
+
**Bob (validator):**
|
| 127 |
+
- Token: `hf_bob_***` (stored in `_hf_tokens_by_username["bob_validator"]`)
|
| 128 |
+
- Role: validator for "parrots-2026"
|
| 129 |
+
- Can: validate detections, see project data
|
| 130 |
+
|
| 131 |
+
**Key Security Feature:**
|
| 132 |
+
- Tokens are **NOT** stored in Session objects
|
| 133 |
+
- Token lookup happens only when needed (validation flow)
|
| 134 |
+
- Each user brings own token → no token sharing
|
| 135 |
+
- If Alice's token revoked on HF, only Alice affected
|
| 136 |
+
|
| 137 |
+
---
|
| 138 |
+
|
| 139 |
+
## Private Project Workflow
|
| 140 |
+
|
| 141 |
+
### Create Private Project
|
| 142 |
+
|
| 143 |
+
```bash
|
| 144 |
+
python -m src.cli.project_cli create-project \
|
| 145 |
+
--projects-file /data/bootstrap/projects.json \
|
| 146 |
+
--user-access-file /data/bootstrap/user_access.json \
|
| 147 |
+
--slug "proprietary-2026" \
|
| 148 |
+
--name "Proprietary Dataset" \
|
| 149 |
+
--dataset-repo-id "owner/proprietary-2026" \
|
| 150 |
+
--visibility "private" \
|
| 151 |
+
--owner "carol_owner"
|
| 152 |
+
```
|
| 153 |
+
|
| 154 |
+
**Result:**
|
| 155 |
+
- `owner_username = "carol_owner"` is set (required)
|
| 156 |
+
- Carol automatically gets admin role
|
| 157 |
+
- Invites are **rejected** if attempted
|
| 158 |
+
|
| 159 |
+
### Try to Invite (Fails)
|
| 160 |
+
|
| 161 |
+
If Carol or another admin tries to invite a collaborator:
|
| 162 |
+
|
| 163 |
+
```
|
| 164 |
+
Actor: carol_owner
|
| 165 |
+
Invitee: dave_validator
|
| 166 |
+
→ ERROR: "Private projects do not accept collaborators"
|
| 167 |
+
```
|
| 168 |
+
|
| 169 |
+
**No invite created.** Private projects are owner-only.
|
| 170 |
+
|
| 171 |
+
---
|
| 172 |
+
|
| 173 |
+
## Configuration for HF Spaces Deployment
|
| 174 |
+
|
| 175 |
+
### Set Environment Variables
|
| 176 |
+
|
| 177 |
+
On your [HF Space Settings](https://huggingface.co/spaces/YOUR_ORG/BirdNET-Validator-App/settings):
|
| 178 |
+
|
| 179 |
+
```bash
|
| 180 |
+
# Basic config
|
| 181 |
+
BIRDNET_ENABLE_DEMO_BOOTSTRAP=false
|
| 182 |
+
BIRDNET_BOOTSTRAP_DIR=/data/bootstrap
|
| 183 |
+
BIRDNET_VALIDATIONS_DIR=/data/validations
|
| 184 |
+
|
| 185 |
+
# Invite settings
|
| 186 |
+
BIRDNET_INVITE_EMAIL_ENABLED=true
|
| 187 |
+
BIRDNET_INVITE_EMAIL_SENDER=noreply@birdnet-validator.example.com
|
| 188 |
+
BIRDNET_INVITE_EMAIL_LOGIN_URL=https://huggingface.co/spaces/YOUR_ORG/BirdNET-Validator-App
|
| 189 |
+
BIRDNET_INVITE_TTL_HOURS=72
|
| 190 |
+
|
| 191 |
+
# SMTP settings (transactional email service)
|
| 192 |
+
BIRDNET_SMTP_HOST=smtp.gmail.com # or your email provider
|
| 193 |
+
BIRDNET_SMTP_PORT=587
|
| 194 |
+
BIRDNET_SMTP_USERNAME=your-email@gmail.com
|
| 195 |
+
BIRDNET_SMTP_PASSWORD=your-app-password # Use app-specific password for 2FA
|
| 196 |
+
BIRDNET_SMTP_USE_TLS=true
|
| 197 |
+
```
|
| 198 |
+
|
| 199 |
+
### Test SMTP Configuration Locally
|
| 200 |
+
|
| 201 |
+
```python
|
| 202 |
+
from src.config.runtime_config import RuntimeConfig
|
| 203 |
+
from src.services.invite_email_notifier import SmtpInviteEmailNotifier, InviteEmailPayload
|
| 204 |
+
from datetime import datetime, timedelta, UTC
|
| 205 |
+
|
| 206 |
+
config = RuntimeConfig.from_env()
|
| 207 |
+
|
| 208 |
+
if config.invite_email_enabled and config.smtp_host:
|
| 209 |
+
notifier = SmtpInviteEmailNotifier(
|
| 210 |
+
host=config.smtp_host,
|
| 211 |
+
port=config.smtp_port,
|
| 212 |
+
username=config.smtp_username,
|
| 213 |
+
password=config.smtp_password,
|
| 214 |
+
use_tls=config.smtp_use_tls,
|
| 215 |
+
)
|
| 216 |
+
|
| 217 |
+
test_payload = InviteEmailPayload(
|
| 218 |
+
invitee_username="test_user",
|
| 219 |
+
invitee_email="test@example.com",
|
| 220 |
+
project_slug="test-project",
|
| 221 |
+
role="validator",
|
| 222 |
+
invited_by="admin_user",
|
| 223 |
+
expires_at=datetime.now(UTC) + timedelta(hours=72),
|
| 224 |
+
login_url=config.invite_email_login_url,
|
| 225 |
+
)
|
| 226 |
+
|
| 227 |
+
ok, message = notifier.send(test_payload)
|
| 228 |
+
print(f"Email send: {ok=}, {message=}")
|
| 229 |
+
else:
|
| 230 |
+
print("Email invites disabled or SMTP not configured")
|
| 231 |
+
```
|
| 232 |
+
|
| 233 |
+
---
|
| 234 |
+
|
| 235 |
+
## Data Durability & Persistence
|
| 236 |
+
|
| 237 |
+
### Bootstrap Files Location
|
| 238 |
+
|
| 239 |
+
On HF Spaces, all bootstrap and validation data persists in `/data/`:
|
| 240 |
+
|
| 241 |
+
```
|
| 242 |
+
/data/
|
| 243 |
+
├── bootstrap/
|
| 244 |
+
│ ├── projects.json # All projects (collaborative + private)
|
| 245 |
+
│ ├── user_access.json # User → Project → Role mapping
|
| 246 |
+
│ └── invites.json # Pending invites with TTL
|
| 247 |
+
└── validations/
|
| 248 |
+
├── project-1/
|
| 249 |
+
│ ├── 2026-04-01.jsonl # Append-only validation events
|
| 250 |
+
│ ├── 2026-04-02.jsonl
|
| 251 |
+
│ └── ...
|
| 252 |
+
└── project-2/
|
| 253 |
+
└── ...
|
| 254 |
+
```
|
| 255 |
+
|
| 256 |
+
### Atomic Writes (Crash-Safe)
|
| 257 |
+
|
| 258 |
+
All JSON writes use atomic pattern:
|
| 259 |
+
|
| 260 |
+
```python
|
| 261 |
+
temp_file = target.with_suffix(target.suffix + ".tmp")
|
| 262 |
+
temp_file.write_text(json.dumps(data))
|
| 263 |
+
temp_file.replace(target) # Atomic on modern filesystems
|
| 264 |
+
```
|
| 265 |
+
|
| 266 |
+
**Benefit:** If Space crashes during write, file is not corrupted.
|
| 267 |
+
|
| 268 |
+
### Data Loss Prevention
|
| 269 |
+
|
| 270 |
+
Since `/data` is persistent on HF Spaces:
|
| 271 |
+
- Projects, ACL, invites **survive** Space restarts ✓
|
| 272 |
+
- Unfinished validations are **not** lost ✓
|
| 273 |
+
- User data is **durable** across redeploys ✓
|
| 274 |
+
|
| 275 |
+
---
|
| 276 |
+
|
| 277 |
+
## Testing the Workflow Locally
|
| 278 |
+
|
| 279 |
+
### 1. Create Users & Projects
|
| 280 |
+
|
| 281 |
+
```bash
|
| 282 |
+
# Set environment for local testing
|
| 283 |
+
export BIRDNET_ENABLE_DEMO_BOOTSTRAP=false
|
| 284 |
+
export BIRDNET_PROJECTS_FILE=/tmp/projects.json
|
| 285 |
+
export BIRDNET_USER_ACCESS_FILE=/tmp/user_access.json
|
| 286 |
+
export BIRDNET_INVITES_FILE=/tmp/invites.json
|
| 287 |
+
export BIRDNET_INVITE_EMAIL_ENABLED=true
|
| 288 |
+
export BIRDNET_INVITE_EMAIL_SENDER=test@example.com
|
| 289 |
+
export BIRDNET_INVITE_EMAIL_LOGIN_URL=http://localhost:7860
|
| 290 |
+
|
| 291 |
+
# Create initial files
|
| 292 |
+
echo '[]' > /tmp/projects.json
|
| 293 |
+
echo '{}' > /tmp/user_access.json
|
| 294 |
+
echo '{}' > /tmp/invites.json
|
| 295 |
+
|
| 296 |
+
# Create project
|
| 297 |
+
python -m src.cli.project_cli create-project \
|
| 298 |
+
--projects-file /tmp/projects.json \
|
| 299 |
+
--user-access-file /tmp/user_access.json \
|
| 300 |
+
--slug test-project \
|
| 301 |
+
--name "Test Project" \
|
| 302 |
+
--dataset-repo-id "test/project" \
|
| 303 |
+
--visibility collaborative \
|
| 304 |
+
--owner alice
|
| 305 |
+
```
|
| 306 |
+
|
| 307 |
+
### 2. Start App & Test UI
|
| 308 |
+
|
| 309 |
+
```bash
|
| 310 |
+
# Terminal 1: Start server
|
| 311 |
+
python app.py # Runs on http://localhost:7860
|
| 312 |
+
|
| 313 |
+
# Terminal 2: Test workflow in UI
|
| 314 |
+
# - Log in with first user (becomes admin)
|
| 315 |
+
# - Create collaborative project
|
| 316 |
+
# - Invite second user
|
| 317 |
+
# - Log out, log in as second user
|
| 318 |
+
# - Accept invite from project selector
|
| 319 |
+
# - Both users see project in dropdown
|
| 320 |
+
```
|
| 321 |
+
|
| 322 |
+
### 3. Verify Data Files
|
| 323 |
+
|
| 324 |
+
```bash
|
| 325 |
+
# Check projects file
|
| 326 |
+
cat /tmp/projects.json | python -m json.tool
|
| 327 |
+
|
| 328 |
+
# Check user access
|
| 329 |
+
cat /tmp/user_access.json | python -m json.tool
|
| 330 |
+
|
| 331 |
+
# Check invites
|
| 332 |
+
cat /tmp/invites.json | python -m json.tool
|
| 333 |
+
```
|
| 334 |
+
|
| 335 |
+
---
|
| 336 |
+
|
| 337 |
+
## Troubleshooting
|
| 338 |
+
|
| 339 |
+
### "Email not sent" but project created
|
| 340 |
+
|
| 341 |
+
**Cause:** SMTP not configured or Email invites disabled
|
| 342 |
+
|
| 343 |
+
**Fix:**
|
| 344 |
+
1. Check `BIRDNET_INVITE_EMAIL_ENABLED=true`
|
| 345 |
+
2. Check SMTP credentials are correct
|
| 346 |
+
3. For Gmail: use [App Password](https://support.google.com/accounts/answer/185833), not account password
|
| 347 |
+
4. Check firewall allows outbound SMTP (port 587)
|
| 348 |
+
|
| 349 |
+
**Fallback:** System will create invite without email. User must be told invite URL manually.
|
| 350 |
+
|
| 351 |
+
### User can't accept invite
|
| 352 |
+
|
| 353 |
+
**Cause:** Session TTL expired, or invite expired (>72h)
|
| 354 |
+
|
| 355 |
+
**Fix:**
|
| 356 |
+
1. Re-login (creates fresh session)
|
| 357 |
+
2. Resend invite
|
| 358 |
+
3. Adjust `BIRDNET_INVITE_TTL_HOURS` if needed
|
| 359 |
+
|
| 360 |
+
### Private project invite fails
|
| 361 |
+
|
| 362 |
+
**Expected behavior:** Private projects reject invites
|
| 363 |
+
|
| 364 |
+
**Workaround:** Assign users directly in `user_access.json` if bootstrap:
|
| 365 |
+
```json
|
| 366 |
+
{
|
| 367 |
+
"username": {
|
| 368 |
+
"private-project": "validator"
|
| 369 |
+
}
|
| 370 |
+
}
|
| 371 |
+
```
|
| 372 |
+
|
| 373 |
+
### Tokens not being used in validation
|
| 374 |
+
|
| 375 |
+
**Cause:** Check project token precedence in `app_factory.py`:
|
| 376 |
+
|
| 377 |
+
```python
|
| 378 |
+
# Preference order:
|
| 379 |
+
# 1. User's HF token (from session login)
|
| 380 |
+
# 2. Project token (for private project owners only)
|
| 381 |
+
# 3. None (validation fails)
|
| 382 |
+
```
|
| 383 |
+
|
| 384 |
+
Set users' HF token via login, or set project token via CLI/UI.
|
| 385 |
+
|
| 386 |
+
---
|
| 387 |
+
|
| 388 |
+
## Next Steps
|
| 389 |
+
|
| 390 |
+
### For Production Deployment
|
| 391 |
+
|
| 392 |
+
1. ✅ **Configure SMTP** on HF Space with real email service
|
| 393 |
+
2. ✅ **Test end-to-end**: Create project → Invite user → Accept → Validate
|
| 394 |
+
3. ✅ **Verify durability**: Create data, restart Space, confirm persistence
|
| 395 |
+
4. 📋 **Monitor invites**: Set up alerts if emails bounce
|
| 396 |
+
5. 📋 **Audit trail**: Review `/data/bootstrap/invites.json` periodically
|
| 397 |
+
|
| 398 |
+
### For Future Enhancements
|
| 399 |
+
|
| 400 |
+
- [ ] Invite expiration auto-cleanup background task
|
| 401 |
+
- [ ] Invite revocation by admin
|
| 402 |
+
- [ ] User deactivation cascade (revoke all invites, remove all roles)
|
| 403 |
+
- [ ] Email template customization
|
| 404 |
+
- [ ] Multi-language invite emails
|
| 405 |
+
- [ ] Webhook notifications (Discord, Slack)
|
| 406 |
+
|
| 407 |
+
---
|
| 408 |
+
|
| 409 |
+
## Test Results Summary
|
| 410 |
+
|
| 411 |
+
| Test Suite | Pass Rate | Key Tests |
|
| 412 |
+
|-----------|-----------|-----------|
|
| 413 |
+
| **Unit: AuthService** | 37/37 ✅ | Login, session TTL, ACL, token storage, invite lifecycle |
|
| 414 |
+
| **Unit: AdminPanel** | 5/5 ✅ | Private isolation, email dispatch, authorization |
|
| 415 |
+
| **Unit: AppFactory** | 45/45 ✅ | Bootstrap loading, config handling, audio helpers |
|
| 416 |
+
| **Unit: CLI** | 10/10 ✅ | Create-project, init-dataset, build-index, verify |
|
| 417 |
+
| **Integration** | 2/2 ✅ | Validation flow, detection repository |
|
| 418 |
+
| **Total** | **139/139 ✅** | All workflows tested and passing |
|
| 419 |
+
|
| 420 |
+
---
|
| 421 |
+
|
| 422 |
+
**Last Updated:** 2026-04-01
|
| 423 |
+
**Deployment Status:** Ready for HF Spaces production
|
| 424 |
+
**Latest Commit:** 5e463e3 (feat: security hardening, durable state, email invites)
|
docs/END_TO_END_QA_CHECKLIST_SPRINT4.md
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# End-to-End QA Checklist - Sprint 4
|
| 2 |
+
|
| 3 |
+
Status legend:
|
| 4 |
+
- PASS = behavior validated
|
| 5 |
+
- FAIL = behavior not validated
|
| 6 |
+
- BLOCKED = could not validate due to environment/tooling limitation
|
| 7 |
+
|
| 8 |
+
## 5-Minute Smoke Test (Pre-Deploy)
|
| 9 |
+
|
| 10 |
+
Use this quick run before each deploy or before a manual retest cycle.
|
| 11 |
+
|
| 12 |
+
### A. Start and Health (about 1 min)
|
| 13 |
+
1. Start app (use a free port if 7860 is busy).
|
| 14 |
+
2. Open local URL in browser.
|
| 15 |
+
3. Confirm app loads all tabs.
|
| 16 |
+
|
| 17 |
+
Pass criteria:
|
| 18 |
+
- App boots without traceback.
|
| 19 |
+
- Home screen renders Login, Select Project, Admin, and Validation tabs.
|
| 20 |
+
|
| 21 |
+
### B. Auth and ACL Core (about 2 min)
|
| 22 |
+
1. Login as validator_demo.
|
| 23 |
+
2. Open Select Project tab and verify only allowed projects are shown.
|
| 24 |
+
3. Open Admin tab and verify access denied with hidden controls.
|
| 25 |
+
4. Switch user to admin_user.
|
| 26 |
+
5. Open Admin tab and verify controls are visible.
|
| 27 |
+
|
| 28 |
+
Pass criteria:
|
| 29 |
+
- validator_demo sees only demo-project and kenya-2024.
|
| 30 |
+
- validator_demo cannot use admin actions.
|
| 31 |
+
- admin_user can see admin controls.
|
| 32 |
+
|
| 33 |
+
### C. Critical Admin Action (about 1 min)
|
| 34 |
+
1. As admin_user, create a temporary project using slug/name/repo id.
|
| 35 |
+
2. Click refresh project list.
|
| 36 |
+
3. Verify new slug appears in project list.
|
| 37 |
+
|
| 38 |
+
Pass criteria:
|
| 39 |
+
- Project creation returns success message.
|
| 40 |
+
- New project appears in list/dropdown immediately.
|
| 41 |
+
|
| 42 |
+
### D. Validation Readiness (about 1 min)
|
| 43 |
+
1. Select an authorized project.
|
| 44 |
+
2. Open Validation tab.
|
| 45 |
+
|
| 46 |
+
Pass criteria:
|
| 47 |
+
- Status changes to ready for selected project.
|
| 48 |
+
- Message "Project not selected" no longer appears after valid selection.
|
| 49 |
+
|
| 50 |
+
### Smoke Evidence (copy/paste)
|
| 51 |
+
1. Date/time:
|
| 52 |
+
2. Tester:
|
| 53 |
+
3. Build/commit:
|
| 54 |
+
4. Result: PASS | FAIL
|
| 55 |
+
5. Notes:
|
| 56 |
+
|
| 57 |
+
## Scope
|
| 58 |
+
This checklist validates Sprint 4 multi-project security flow:
|
| 59 |
+
- Login and session creation
|
| 60 |
+
- Project authorization per user
|
| 61 |
+
- Admin-only controls
|
| 62 |
+
- Project creation and assignment workflows
|
| 63 |
+
- Validation tab readiness based on selected project
|
| 64 |
+
|
| 65 |
+
## Test Data (Demo Users)
|
| 66 |
+
- admin_user
|
| 67 |
+
- validator_demo
|
| 68 |
+
- validator_other
|
| 69 |
+
|
| 70 |
+
Expected access matrix:
|
| 71 |
+
- admin_user -> kenya-2024 (admin), nairobi-2023 (admin)
|
| 72 |
+
- validator_demo -> demo-project (validator), kenya-2024 (validator)
|
| 73 |
+
- validator_other -> nairobi-2023 (validator)
|
| 74 |
+
|
| 75 |
+
## E2E Cases
|
| 76 |
+
|
| 77 |
+
| ID | Area | Precondition | Steps | Expected Result | Status |
|
| 78 |
+
|---|---|---|---|---|---|
|
| 79 |
+
| E2E-01 | App bootstrap | Virtual env active | Start app and open local URL | App starts without runtime exception | PASS |
|
| 80 |
+
| E2E-02 | Login | App running | Login as admin_user | Session is created and user is authenticated | PASS |
|
| 81 |
+
| E2E-03 | Project selector | admin_user logged in | Open Select Project tab | Authorized projects are listed (kenya-2024, nairobi-2023) | PASS |
|
| 82 |
+
| E2E-04 | Validation readiness | admin_user logged in | Open Validation tab | Validation status is ready with selected project | PASS |
|
| 83 |
+
| E2E-05 | Admin visibility | validator_demo logged in | Open Admin tab | Access denied message and hidden admin controls | PASS |
|
| 84 |
+
| E2E-06 | Admin visibility | admin_user logged in | Open Admin tab | Admin controls are visible | PASS |
|
| 85 |
+
| E2E-07 | Project creation | admin_user logged in | Create project with slug/name/repo_id | New project is created and project list is refreshed | PASS |
|
| 86 |
+
| E2E-08 | Project assignment | admin_user logged in | Assign user to a project | Assignment succeeds and access matrix updates | PASS |
|
| 87 |
+
| E2E-09 | Non-admin write protection | validator_demo logged in | Attempt assignment/create actions | Actions are denied by backend callback checks | PASS |
|
| 88 |
+
| E2E-10 | ACL isolation | Test suite available | Run ACL tests | ACL tests pass (project isolation enforced) | PASS |
|
| 89 |
+
|
| 90 |
+
## Execution Notes
|
| 91 |
+
- This checklist is aligned with fixes applied in `src/ui/app_factory.py`.
|
| 92 |
+
- The app uses Gradio and callback-driven state transitions.
|
| 93 |
+
- ACL behavior is also covered by unit tests in `tests/unit/test_acl_enforcement.py`.
|
| 94 |
+
|
| 95 |
+
## Evidence Commands
|
| 96 |
+
1. python -m pytest tests/unit/test_acl_enforcement.py -q
|
| 97 |
+
2. python -m pytest -q --tb=line
|
| 98 |
+
3. Start app on a free port and verify local HTTP response
|
| 99 |
+
|
| 100 |
+
## Sign-off
|
| 101 |
+
- Sprint 4 E2E status: PASS
|
| 102 |
+
- Date: 2026-03-25
|
docs/spaces/projects.sample.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"project_slug": "kenya-2024",
|
| 4 |
+
"name": "Kenya Survey 2024",
|
| 5 |
+
"dataset_repo_id": "org/kenya-2024-dataset",
|
| 6 |
+
"active": true
|
| 7 |
+
},
|
| 8 |
+
{
|
| 9 |
+
"project_slug": "birds-local",
|
| 10 |
+
"name": "Birds Local Validation",
|
| 11 |
+
"dataset_repo_id": "org/birds-local-dataset",
|
| 12 |
+
"active": true
|
| 13 |
+
}
|
| 14 |
+
]
|
docs/spaces/user_access.sample.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"admin_user": {
|
| 3 |
+
"kenya-2024": "admin",
|
| 4 |
+
"birds-local": "admin"
|
| 5 |
+
},
|
| 6 |
+
"validator_a": {
|
| 7 |
+
"kenya-2024": "validator"
|
| 8 |
+
},
|
| 9 |
+
"validator_b": {
|
| 10 |
+
"birds-local": "validator"
|
| 11 |
+
}
|
| 12 |
+
}
|
pyproject.toml
CHANGED
|
@@ -1,9 +1,12 @@
|
|
| 1 |
[project]
|
| 2 |
-
name = "BirdNET-
|
| 3 |
version = "0.1.0"
|
| 4 |
-
description = "BirdNET
|
| 5 |
readme = "README.md"
|
| 6 |
requires-python = ">=3.11"
|
| 7 |
|
|
|
|
|
|
|
|
|
|
| 8 |
[tool.pytest.ini_options]
|
| 9 |
testpaths = ["tests"]
|
|
|
|
| 1 |
[project]
|
| 2 |
+
name = "BirdNET-Validator-App"
|
| 3 |
version = "0.1.0"
|
| 4 |
+
description = "BirdNET-Validator-App for multi-project bird detection review workflows"
|
| 5 |
readme = "README.md"
|
| 6 |
requires-python = ">=3.11"
|
| 7 |
|
| 8 |
+
[project.scripts]
|
| 9 |
+
birdnet-uploader = "src.uploader_cli.main:cli"
|
| 10 |
+
|
| 11 |
[tool.pytest.ini_options]
|
| 12 |
testpaths = ["tests"]
|
requirements.txt
CHANGED
|
@@ -1,7 +1,12 @@
|
|
| 1 |
gradio==5.23.1
|
| 2 |
-
huggingface_hub
|
| 3 |
-
|
| 4 |
-
pandas=
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
gradio==5.23.1
|
| 2 |
+
huggingface_hub==0.28.1
|
| 3 |
+
numpy>=2.0.0
|
| 4 |
+
pandas>=2.2.0
|
| 5 |
+
Pillow>=10.4.0
|
| 6 |
+
pyarrow>=16.0.0
|
| 7 |
+
pydantic>=2.8.0
|
| 8 |
+
pytest>=8.3.0
|
| 9 |
+
click>=8.1.7
|
| 10 |
+
rich>=13.7.1
|
| 11 |
+
textual>=0.58.1
|
| 12 |
+
keyring>=25.2.1
|
src/__pycache__/__init__.cpython-313.pyc
CHANGED
|
Binary files a/src/__pycache__/__init__.cpython-313.pyc and b/src/__pycache__/__init__.cpython-313.pyc differ
|
|
|
src/__pycache__/__init__.cpython-314.pyc
CHANGED
|
Binary files a/src/__pycache__/__init__.cpython-314.pyc and b/src/__pycache__/__init__.cpython-314.pyc differ
|
|
|
{cli → src/auth}/__init__.py
RENAMED
|
File without changes
|
src/auth/auth_service.py
ADDED
|
@@ -0,0 +1,684 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AuthService: User authentication, session management, and project ACL."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass, field
|
| 4 |
+
from datetime import UTC, datetime, timedelta
|
| 5 |
+
from typing import Dict, List, Optional
|
| 6 |
+
import uuid
|
| 7 |
+
|
| 8 |
+
from huggingface_hub import HfApi
|
| 9 |
+
|
| 10 |
+
from src.domain.models import Role, User
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class UserProjectAccess:
|
| 15 |
+
"""Maps a user to projects with specific roles."""
|
| 16 |
+
|
| 17 |
+
username: str
|
| 18 |
+
project_slugs: Dict[str, Role] # Maps project_slug -> role (admin or validator)
|
| 19 |
+
is_active: bool = True
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class Session:
|
| 24 |
+
"""Represents an authenticated user session."""
|
| 25 |
+
|
| 26 |
+
session_id: str
|
| 27 |
+
username: str
|
| 28 |
+
role: Role
|
| 29 |
+
authorized_projects: List[str] # Projects this user can access
|
| 30 |
+
created_at: datetime
|
| 31 |
+
last_activity: datetime
|
| 32 |
+
expires_at: datetime
|
| 33 |
+
|
| 34 |
+
def is_expired(self) -> bool:
|
| 35 |
+
"""Check if session has expired."""
|
| 36 |
+
return datetime.now(UTC) > self.expires_at
|
| 37 |
+
|
| 38 |
+
def update_activity(self, ttl_minutes: int = 120) -> None:
|
| 39 |
+
"""Update last activity timestamp and extend expiration."""
|
| 40 |
+
self.last_activity = datetime.now(UTC)
|
| 41 |
+
self.expires_at = datetime.now(UTC) + timedelta(minutes=ttl_minutes)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
@dataclass
|
| 45 |
+
class ProjectInvite:
|
| 46 |
+
"""Represents an invitation to join a project.
|
| 47 |
+
|
| 48 |
+
Supports 3 scenarios:
|
| 49 |
+
1. Username only: internal app invite (no email notification)
|
| 50 |
+
2. Email only: external email-based invite (no prior HF account required)
|
| 51 |
+
3. Both: dual invite (internal + email notification)
|
| 52 |
+
"""
|
| 53 |
+
project_slug: str
|
| 54 |
+
role: Role
|
| 55 |
+
invited_by: str
|
| 56 |
+
created_at: datetime
|
| 57 |
+
expires_at: datetime
|
| 58 |
+
username: str | None = None # HF username (optional)
|
| 59 |
+
invitee_email: str | None = None # Email address (optional)
|
| 60 |
+
|
| 61 |
+
def __post_init__(self) -> None:
|
| 62 |
+
if not self.username and not self.invitee_email:
|
| 63 |
+
raise ValueError("At least one of username or invitee_email must be provided")
|
| 64 |
+
|
| 65 |
+
def is_expired(self) -> bool:
|
| 66 |
+
return datetime.now(UTC) > self.expires_at
|
| 67 |
+
|
| 68 |
+
@property
|
| 69 |
+
def invite_mode(self) -> str:
|
| 70 |
+
"""Return the invitation mode: 'username_only', 'email_only', or 'dual'."""
|
| 71 |
+
if self.username and self.invitee_email:
|
| 72 |
+
return "dual"
|
| 73 |
+
elif self.username:
|
| 74 |
+
return "username_only"
|
| 75 |
+
else:
|
| 76 |
+
return "email_only"
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@dataclass
|
| 80 |
+
class UserProfile:
|
| 81 |
+
username: str
|
| 82 |
+
hf_email: str | None = None
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class AuthService:
|
| 86 |
+
"""Manages user authentication, sessions, and project access control."""
|
| 87 |
+
|
| 88 |
+
def __init__(self, session_ttl_minutes: int = 120, invite_ttl_hours: int = 72):
|
| 89 |
+
"""Initialize AuthService with session TTL.
|
| 90 |
+
|
| 91 |
+
Args:
|
| 92 |
+
session_ttl_minutes: Session time-to-live in minutes (default: 2 hours)
|
| 93 |
+
"""
|
| 94 |
+
self.session_ttl_minutes = session_ttl_minutes
|
| 95 |
+
self.invite_ttl_hours = max(1, int(invite_ttl_hours))
|
| 96 |
+
self._sessions: Dict[str, Session] = {} # session_id -> Session
|
| 97 |
+
self._user_access: Dict[str, UserProjectAccess] = {} # username -> UserProjectAccess
|
| 98 |
+
self._pending_invites: Dict[str, Dict[str, ProjectInvite]] = {}
|
| 99 |
+
self._hf_tokens_by_username: Dict[str, str] = {}
|
| 100 |
+
self._user_profiles: Dict[str, UserProfile] = {}
|
| 101 |
+
|
| 102 |
+
def _refresh_or_revoke_sessions_for_username(self, username: str) -> None:
|
| 103 |
+
access = self._user_access.get(username)
|
| 104 |
+
session_ids = [sid for sid, session in self._sessions.items() if session.username == username]
|
| 105 |
+
if access is None or not access.is_active:
|
| 106 |
+
for session_id in session_ids:
|
| 107 |
+
self._sessions.pop(session_id, None)
|
| 108 |
+
return
|
| 109 |
+
|
| 110 |
+
projects = list(access.project_slugs.keys())
|
| 111 |
+
role = Role.admin if any(project_role == Role.admin for project_role in access.project_slugs.values()) else Role.validator
|
| 112 |
+
for session_id in session_ids:
|
| 113 |
+
session = self._sessions.get(session_id)
|
| 114 |
+
if session is None:
|
| 115 |
+
continue
|
| 116 |
+
session.authorized_projects = projects
|
| 117 |
+
session.role = role
|
| 118 |
+
session.update_activity(self.session_ttl_minutes)
|
| 119 |
+
|
| 120 |
+
def _prune_expired_invites(self) -> None:
|
| 121 |
+
expired_users: list[str] = []
|
| 122 |
+
for username, invites_by_project in self._pending_invites.items():
|
| 123 |
+
expired_projects = [project_slug for project_slug, invite in invites_by_project.items() if invite.is_expired()]
|
| 124 |
+
for project_slug in expired_projects:
|
| 125 |
+
del invites_by_project[project_slug]
|
| 126 |
+
if not invites_by_project:
|
| 127 |
+
expired_users.append(username)
|
| 128 |
+
|
| 129 |
+
for username in expired_users:
|
| 130 |
+
self._pending_invites.pop(username, None)
|
| 131 |
+
|
| 132 |
+
def register_user_project_access(
|
| 133 |
+
self, username: str, project_access: Dict[str, Role]
|
| 134 |
+
) -> None:
|
| 135 |
+
"""Register or update a user's access to projects.
|
| 136 |
+
|
| 137 |
+
Args:
|
| 138 |
+
username: User name
|
| 139 |
+
project_access: Dict mapping project_slug to Role (admin or validator)
|
| 140 |
+
"""
|
| 141 |
+
self._user_access[username] = UserProjectAccess(
|
| 142 |
+
username=username,
|
| 143 |
+
project_slugs=project_access,
|
| 144 |
+
is_active=True,
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
def list_usernames(self, include_inactive: bool = False) -> List[str]:
|
| 148 |
+
"""List registered usernames.
|
| 149 |
+
|
| 150 |
+
Args:
|
| 151 |
+
include_inactive: Include disabled users when True
|
| 152 |
+
|
| 153 |
+
Returns:
|
| 154 |
+
Sorted list of usernames
|
| 155 |
+
"""
|
| 156 |
+
usernames = []
|
| 157 |
+
for username, access in self._user_access.items():
|
| 158 |
+
if include_inactive or access.is_active:
|
| 159 |
+
usernames.append(username)
|
| 160 |
+
return sorted(usernames)
|
| 161 |
+
|
| 162 |
+
def upsert_user_project_role(self, username: str, project_slug: str, role: Role) -> None:
|
| 163 |
+
"""Create/update user assignment for a project.
|
| 164 |
+
|
| 165 |
+
Args:
|
| 166 |
+
username: User name
|
| 167 |
+
project_slug: Project slug
|
| 168 |
+
role: Role to apply for this project
|
| 169 |
+
"""
|
| 170 |
+
access = self._user_access.get(username)
|
| 171 |
+
if access is None:
|
| 172 |
+
access = UserProjectAccess(username=username, project_slugs={}, is_active=True)
|
| 173 |
+
self._user_access[username] = access
|
| 174 |
+
|
| 175 |
+
access.project_slugs[project_slug] = role
|
| 176 |
+
self._refresh_or_revoke_sessions_for_username(username)
|
| 177 |
+
|
| 178 |
+
def remove_user_project_role(self, username: str, project_slug: str) -> bool:
|
| 179 |
+
"""Remove a user assignment from a project.
|
| 180 |
+
|
| 181 |
+
Args:
|
| 182 |
+
username: User name
|
| 183 |
+
project_slug: Project slug
|
| 184 |
+
|
| 185 |
+
Returns:
|
| 186 |
+
True when assignment existed and was removed
|
| 187 |
+
"""
|
| 188 |
+
access = self._user_access.get(username)
|
| 189 |
+
if access is None:
|
| 190 |
+
return False
|
| 191 |
+
|
| 192 |
+
if project_slug not in access.project_slugs:
|
| 193 |
+
return False
|
| 194 |
+
|
| 195 |
+
del access.project_slugs[project_slug]
|
| 196 |
+
pending = self._pending_invites.get(username, {})
|
| 197 |
+
if project_slug in pending:
|
| 198 |
+
del pending[project_slug]
|
| 199 |
+
if not pending:
|
| 200 |
+
self._pending_invites.pop(username, None)
|
| 201 |
+
self._refresh_or_revoke_sessions_for_username(username)
|
| 202 |
+
return True
|
| 203 |
+
|
| 204 |
+
def login(self, username: str) -> Optional[Session]:
|
| 205 |
+
"""Authenticate a user and create a new session.
|
| 206 |
+
|
| 207 |
+
Args:
|
| 208 |
+
username: User name to authenticate
|
| 209 |
+
|
| 210 |
+
Returns:
|
| 211 |
+
Session if user exists and is active, None otherwise
|
| 212 |
+
"""
|
| 213 |
+
return self.login_internal(username=username)
|
| 214 |
+
|
| 215 |
+
def login_internal(
|
| 216 |
+
self,
|
| 217 |
+
username: str,
|
| 218 |
+
auto_promote_to_admin: bool = False,
|
| 219 |
+
) -> Optional[Session]:
|
| 220 |
+
if username not in self._user_access:
|
| 221 |
+
if not auto_promote_to_admin:
|
| 222 |
+
self._prune_expired_invites()
|
| 223 |
+
pending_for_user = self._pending_invites.get(username, {})
|
| 224 |
+
if not pending_for_user:
|
| 225 |
+
return None
|
| 226 |
+
self._user_access[username] = UserProjectAccess(
|
| 227 |
+
username=username,
|
| 228 |
+
project_slugs={},
|
| 229 |
+
is_active=True,
|
| 230 |
+
)
|
| 231 |
+
|
| 232 |
+
access = self._user_access[username]
|
| 233 |
+
if not access.is_active:
|
| 234 |
+
return None
|
| 235 |
+
|
| 236 |
+
# Determine highest role across all projects (admin > validator)
|
| 237 |
+
role = Role.admin if auto_promote_to_admin else Role.validator
|
| 238 |
+
if role != Role.admin:
|
| 239 |
+
for proj_role in access.project_slugs.values():
|
| 240 |
+
if proj_role == Role.admin:
|
| 241 |
+
role = Role.admin
|
| 242 |
+
break
|
| 243 |
+
|
| 244 |
+
session_id = str(uuid.uuid4())
|
| 245 |
+
now = datetime.now(UTC)
|
| 246 |
+
session = Session(
|
| 247 |
+
session_id=session_id,
|
| 248 |
+
username=username,
|
| 249 |
+
role=role,
|
| 250 |
+
authorized_projects=list(access.project_slugs.keys()),
|
| 251 |
+
created_at=now,
|
| 252 |
+
last_activity=now,
|
| 253 |
+
expires_at=now + timedelta(minutes=self.session_ttl_minutes),
|
| 254 |
+
)
|
| 255 |
+
|
| 256 |
+
self._sessions[session_id] = session
|
| 257 |
+
return session
|
| 258 |
+
|
| 259 |
+
def login_with_hf_token(self, token: str) -> tuple[Optional[Session], str]:
|
| 260 |
+
"""Authenticate using Hugging Face personal token and resolve username via whoami."""
|
| 261 |
+
token_value = (token or "").strip()
|
| 262 |
+
if not token_value:
|
| 263 |
+
return None, "❌ Please provide a Hugging Face token"
|
| 264 |
+
|
| 265 |
+
try:
|
| 266 |
+
whoami = HfApi().whoami(token=token_value)
|
| 267 |
+
except Exception:
|
| 268 |
+
return None, "❌ Invalid Hugging Face token or network error"
|
| 269 |
+
|
| 270 |
+
username = str(whoami.get("name") or "").strip()
|
| 271 |
+
if not username:
|
| 272 |
+
return None, "❌ Unable to resolve Hugging Face username from token"
|
| 273 |
+
|
| 274 |
+
email_value = str(whoami.get("email") or "").strip() or None
|
| 275 |
+
self._hf_tokens_by_username[username] = token_value
|
| 276 |
+
self._user_profiles[username] = UserProfile(username=username, hf_email=email_value)
|
| 277 |
+
|
| 278 |
+
is_first_user = len(self._user_access) == 0
|
| 279 |
+
session = self.login_internal(
|
| 280 |
+
username=username,
|
| 281 |
+
auto_promote_to_admin=is_first_user,
|
| 282 |
+
)
|
| 283 |
+
if session is None:
|
| 284 |
+
return None, f"❌ User '{username}' is not invited to any project yet"
|
| 285 |
+
|
| 286 |
+
if is_first_user:
|
| 287 |
+
return session, f"✅ Welcome, {username}! (Admin)"
|
| 288 |
+
if session.role == Role.admin:
|
| 289 |
+
return session, f"✅ Welcome, {username}! (Admin)"
|
| 290 |
+
return session, f"✅ Welcome, {username}! (Validator)"
|
| 291 |
+
|
| 292 |
+
def get_session(self, session_id: str) -> Optional[Session]:
|
| 293 |
+
"""Retrieve an active session by ID.
|
| 294 |
+
|
| 295 |
+
Args:
|
| 296 |
+
session_id: Session ID
|
| 297 |
+
|
| 298 |
+
Returns:
|
| 299 |
+
Session if valid and not expired, None otherwise
|
| 300 |
+
"""
|
| 301 |
+
session = self._sessions.get(session_id)
|
| 302 |
+
if session is None:
|
| 303 |
+
return None
|
| 304 |
+
|
| 305 |
+
if session.is_expired():
|
| 306 |
+
self._sessions.pop(session_id, None)
|
| 307 |
+
return None
|
| 308 |
+
|
| 309 |
+
session.update_activity(self.session_ttl_minutes)
|
| 310 |
+
return session
|
| 311 |
+
|
| 312 |
+
def logout(self, session_id: str) -> None:
|
| 313 |
+
"""Delete a session.
|
| 314 |
+
|
| 315 |
+
Args:
|
| 316 |
+
session_id: Session ID to invalidate
|
| 317 |
+
"""
|
| 318 |
+
self._sessions.pop(session_id, None)
|
| 319 |
+
|
| 320 |
+
def is_user_authorized_for_project(self, username: str, project_slug: str) -> bool:
|
| 321 |
+
"""Check if a user has access to a specific project.
|
| 322 |
+
|
| 323 |
+
Args:
|
| 324 |
+
username: User name
|
| 325 |
+
project_slug: Project slug
|
| 326 |
+
|
| 327 |
+
Returns:
|
| 328 |
+
True if user has access (admin or validator), False otherwise
|
| 329 |
+
"""
|
| 330 |
+
access = self._user_access.get(username)
|
| 331 |
+
if access is None or not access.is_active:
|
| 332 |
+
return False
|
| 333 |
+
|
| 334 |
+
return project_slug in access.project_slugs
|
| 335 |
+
|
| 336 |
+
def get_user_role_for_project(self, username: str, project_slug: str) -> Optional[Role]:
|
| 337 |
+
"""Get the specific role a user has for a project.
|
| 338 |
+
|
| 339 |
+
Args:
|
| 340 |
+
username: User name
|
| 341 |
+
project_slug: Project slug
|
| 342 |
+
|
| 343 |
+
Returns:
|
| 344 |
+
Role if user has access, None otherwise
|
| 345 |
+
"""
|
| 346 |
+
access = self._user_access.get(username)
|
| 347 |
+
if access is None or not access.is_active:
|
| 348 |
+
return None
|
| 349 |
+
|
| 350 |
+
return access.project_slugs.get(project_slug)
|
| 351 |
+
|
| 352 |
+
def list_user_projects(self, username: str) -> List[str]:
|
| 353 |
+
"""List all projects a user has access to.
|
| 354 |
+
|
| 355 |
+
Args:
|
| 356 |
+
username: User name
|
| 357 |
+
|
| 358 |
+
Returns:
|
| 359 |
+
List of project slugs
|
| 360 |
+
"""
|
| 361 |
+
access = self._user_access.get(username)
|
| 362 |
+
if access is None or not access.is_active:
|
| 363 |
+
return []
|
| 364 |
+
|
| 365 |
+
return list(access.project_slugs.keys())
|
| 366 |
+
|
| 367 |
+
def cleanup_expired_sessions(self) -> None:
|
| 368 |
+
"""Remove all expired sessions (maintenance cleanup)."""
|
| 369 |
+
now = datetime.now(UTC)
|
| 370 |
+
expired = [
|
| 371 |
+
sid for sid, session in self._sessions.items() if session.is_expired()
|
| 372 |
+
]
|
| 373 |
+
for sid in expired:
|
| 374 |
+
self._sessions.pop(sid)
|
| 375 |
+
|
| 376 |
+
def refresh_session_authorizations(self, session_id: str) -> Optional[Session]:
|
| 377 |
+
"""Refresh a session with latest role and project assignments."""
|
| 378 |
+
session = self._sessions.get(session_id)
|
| 379 |
+
if session is None:
|
| 380 |
+
return None
|
| 381 |
+
if session.is_expired():
|
| 382 |
+
self._sessions.pop(session_id, None)
|
| 383 |
+
return None
|
| 384 |
+
|
| 385 |
+
access = self._user_access.get(session.username)
|
| 386 |
+
if access is None or not access.is_active:
|
| 387 |
+
self._sessions.pop(session_id, None)
|
| 388 |
+
return None
|
| 389 |
+
|
| 390 |
+
projects = list(access.project_slugs.keys())
|
| 391 |
+
session.authorized_projects = projects
|
| 392 |
+
session.role = Role.admin if any(role == Role.admin for role in access.project_slugs.values()) else Role.validator
|
| 393 |
+
session.update_activity(self.session_ttl_minutes)
|
| 394 |
+
return session
|
| 395 |
+
|
| 396 |
+
def set_user_active(self, username: str, active: bool) -> None:
|
| 397 |
+
"""Enable or disable a user account.
|
| 398 |
+
|
| 399 |
+
Args:
|
| 400 |
+
username: User name
|
| 401 |
+
active: Whether the user should be active
|
| 402 |
+
"""
|
| 403 |
+
if username in self._user_access:
|
| 404 |
+
self._user_access[username].is_active = active
|
| 405 |
+
if not active:
|
| 406 |
+
self._hf_tokens_by_username.pop(username, None)
|
| 407 |
+
self._refresh_or_revoke_sessions_for_username(username)
|
| 408 |
+
|
| 409 |
+
def get_hf_token_for_user(self, username: str) -> str | None:
|
| 410 |
+
token = (self._hf_tokens_by_username.get(username) or "").strip()
|
| 411 |
+
return token or None
|
| 412 |
+
|
| 413 |
+
def get_known_email_for_user(self, username: str) -> str | None:
|
| 414 |
+
profile = self._user_profiles.get(username)
|
| 415 |
+
if profile is None:
|
| 416 |
+
return None
|
| 417 |
+
email = (profile.hf_email or "").strip()
|
| 418 |
+
return email or None
|
| 419 |
+
|
| 420 |
+
def _pending_invite_keys_for_username(self, username: str) -> list[str]:
|
| 421 |
+
keys = [username]
|
| 422 |
+
known_email = self.get_known_email_for_user(username)
|
| 423 |
+
if known_email:
|
| 424 |
+
keys.insert(0, f"email:{known_email}")
|
| 425 |
+
return keys
|
| 426 |
+
|
| 427 |
+
def _find_pending_invite_bucket(self, username: str, project_slug: str) -> tuple[str | None, dict[str, ProjectInvite] | None, ProjectInvite | None]:
|
| 428 |
+
for invite_key in self._pending_invite_keys_for_username(username):
|
| 429 |
+
pending = self._pending_invites.get(invite_key, {})
|
| 430 |
+
invite = pending.get(project_slug)
|
| 431 |
+
if invite is not None:
|
| 432 |
+
return invite_key, pending, invite
|
| 433 |
+
return None, None, None
|
| 434 |
+
|
| 435 |
+
def create_project_invite(
|
| 436 |
+
self,
|
| 437 |
+
project_slug: str,
|
| 438 |
+
role: Role,
|
| 439 |
+
invited_by: str,
|
| 440 |
+
username: str | None = None,
|
| 441 |
+
invitee_email: str | None = None,
|
| 442 |
+
) -> tuple[bool, str]:
|
| 443 |
+
"""Create a project invite supporting 3 scenarios:
|
| 444 |
+
|
| 445 |
+
1. Username only: internal app invite (no email notification)
|
| 446 |
+
2. Email only: external email-based invite
|
| 447 |
+
3. Both: dual invite (internal + email notification)
|
| 448 |
+
|
| 449 |
+
Args:
|
| 450 |
+
project_slug: Target project
|
| 451 |
+
role: Role to assign
|
| 452 |
+
invited_by: Username of the inviter
|
| 453 |
+
username: (Optional) HF username of invitee
|
| 454 |
+
invitee_email: (Optional) Email address of invitee
|
| 455 |
+
|
| 456 |
+
Returns:
|
| 457 |
+
(success, message) tuple
|
| 458 |
+
"""
|
| 459 |
+
username = (username or "").strip() or None
|
| 460 |
+
invitee_email = (invitee_email or "").strip() or None
|
| 461 |
+
|
| 462 |
+
if not username and not invitee_email:
|
| 463 |
+
return False, "Please provide either a username or an email address"
|
| 464 |
+
|
| 465 |
+
self._prune_expired_invites()
|
| 466 |
+
|
| 467 |
+
# For username-based invites, check if already exists
|
| 468 |
+
if username:
|
| 469 |
+
user_invites = self._pending_invites.get(username, {})
|
| 470 |
+
if project_slug in user_invites:
|
| 471 |
+
return False, f"Invite for {username} to {project_slug} already exists"
|
| 472 |
+
|
| 473 |
+
now = datetime.now(UTC)
|
| 474 |
+
expires_at = now + timedelta(hours=self.invite_ttl_hours)
|
| 475 |
+
|
| 476 |
+
invite = ProjectInvite(
|
| 477 |
+
project_slug=project_slug,
|
| 478 |
+
role=role,
|
| 479 |
+
invited_by=invited_by,
|
| 480 |
+
created_at=now,
|
| 481 |
+
expires_at=expires_at,
|
| 482 |
+
username=username,
|
| 483 |
+
invitee_email=invitee_email,
|
| 484 |
+
)
|
| 485 |
+
|
| 486 |
+
# Store by username if available, otherwise by email (as temporary key)
|
| 487 |
+
if username:
|
| 488 |
+
self._pending_invites.setdefault(username, {})[project_slug] = invite
|
| 489 |
+
else:
|
| 490 |
+
# For email-only invites, use email as the storage key
|
| 491 |
+
# This allows the user to later link their username
|
| 492 |
+
email_key = f"email:{invitee_email}"
|
| 493 |
+
self._pending_invites.setdefault(email_key, {})[project_slug] = invite
|
| 494 |
+
|
| 495 |
+
mode = invite.invite_mode
|
| 496 |
+
summary = f"✅ {mode.replace('_', ' ').title()} invite"
|
| 497 |
+
return True, summary
|
| 498 |
+
|
| 499 |
+
def list_pending_invites(self, username: str) -> List[ProjectInvite]:
|
| 500 |
+
self._prune_expired_invites()
|
| 501 |
+
invites: list[ProjectInvite] = []
|
| 502 |
+
seen_projects: set[str] = set()
|
| 503 |
+
for invite_key in self._pending_invite_keys_for_username(username):
|
| 504 |
+
for invite in self._pending_invites.get(invite_key, {}).values():
|
| 505 |
+
if invite.project_slug in seen_projects:
|
| 506 |
+
continue
|
| 507 |
+
seen_projects.add(invite.project_slug)
|
| 508 |
+
invites.append(invite)
|
| 509 |
+
invites.sort(key=lambda item: item.created_at)
|
| 510 |
+
return invites
|
| 511 |
+
|
| 512 |
+
def list_all_pending_invites(self) -> List[ProjectInvite]:
|
| 513 |
+
self._prune_expired_invites()
|
| 514 |
+
invites: list[ProjectInvite] = []
|
| 515 |
+
for invites_by_project in self._pending_invites.values():
|
| 516 |
+
invites.extend(invites_by_project.values())
|
| 517 |
+
invites.sort(key=lambda item: item.created_at)
|
| 518 |
+
return invites
|
| 519 |
+
|
| 520 |
+
def accept_project_invite(self, username: str, project_slug: str) -> tuple[bool, str]:
|
| 521 |
+
self._prune_expired_invites()
|
| 522 |
+
invite_key, pending, invite = self._find_pending_invite_bucket(username, project_slug)
|
| 523 |
+
if invite_key is None or pending is None or invite is None:
|
| 524 |
+
return False, f"Invite not found for {username} in {project_slug}"
|
| 525 |
+
|
| 526 |
+
self.upsert_user_project_role(username, project_slug, invite.role)
|
| 527 |
+
del pending[project_slug]
|
| 528 |
+
if not pending:
|
| 529 |
+
self._pending_invites.pop(invite_key, None)
|
| 530 |
+
return True, f"✅ Invite accepted for {project_slug} as {invite.role.value}"
|
| 531 |
+
|
| 532 |
+
def reject_project_invite(self, username: str, project_slug: str) -> tuple[bool, str]:
|
| 533 |
+
self._prune_expired_invites()
|
| 534 |
+
invite_key, pending, invite = self._find_pending_invite_bucket(username, project_slug)
|
| 535 |
+
if invite_key is None or pending is None or invite is None:
|
| 536 |
+
return False, f"Invite not found for {username} in {project_slug}"
|
| 537 |
+
|
| 538 |
+
del pending[project_slug]
|
| 539 |
+
if not pending:
|
| 540 |
+
self._pending_invites.pop(invite_key, None)
|
| 541 |
+
return True, f"✅ Invite rejected for {project_slug}"
|
| 542 |
+
|
| 543 |
+
def accept_all_project_invites(self, username: str) -> tuple[int, int, str]:
|
| 544 |
+
self._prune_expired_invites()
|
| 545 |
+
pending_keys = self._pending_invite_keys_for_username(username)
|
| 546 |
+
pending: dict[str, ProjectInvite] = {}
|
| 547 |
+
for invite_key in pending_keys:
|
| 548 |
+
pending.update(self._pending_invites.get(invite_key, {}))
|
| 549 |
+
|
| 550 |
+
if not pending:
|
| 551 |
+
return 0, 0, "No pending invites"
|
| 552 |
+
|
| 553 |
+
accepted = 0
|
| 554 |
+
failed = 0
|
| 555 |
+
for project_slug in list(pending.keys()):
|
| 556 |
+
ok, _ = self.accept_project_invite(username, project_slug)
|
| 557 |
+
if ok:
|
| 558 |
+
accepted += 1
|
| 559 |
+
else:
|
| 560 |
+
failed += 1
|
| 561 |
+
return accepted, failed, f"Accepted {accepted} invites" + (f", {failed} failed" if failed else "")
|
| 562 |
+
|
| 563 |
+
def revoke_project_invite(self, username: str, project_slug: str) -> tuple[bool, str]:
|
| 564 |
+
self._prune_expired_invites()
|
| 565 |
+
invite_key, pending, invite = self._find_pending_invite_bucket(username, project_slug)
|
| 566 |
+
if invite_key is None or pending is None or invite is None:
|
| 567 |
+
return False, f"Invite not found for {username} in {project_slug}"
|
| 568 |
+
del pending[project_slug]
|
| 569 |
+
if not pending:
|
| 570 |
+
self._pending_invites.pop(invite_key, None)
|
| 571 |
+
return True, f"✅ Invite revoked for {username} in {project_slug}"
|
| 572 |
+
|
| 573 |
+
def export_user_access_map(self, include_inactive: bool = False) -> Dict[str, Dict[str, str]]:
|
| 574 |
+
"""Export user/project role mapping for persistence."""
|
| 575 |
+
exported: Dict[str, Dict[str, str]] = {}
|
| 576 |
+
for username in self.list_usernames(include_inactive=include_inactive):
|
| 577 |
+
access = self._user_access.get(username)
|
| 578 |
+
if access is None:
|
| 579 |
+
continue
|
| 580 |
+
exported[username] = {project_slug: role.value for project_slug, role in access.project_slugs.items()}
|
| 581 |
+
return exported
|
| 582 |
+
|
| 583 |
+
def export_pending_invites_map(self) -> Dict[str, Dict[str, Dict[str, str]]]:
|
| 584 |
+
self._prune_expired_invites()
|
| 585 |
+
payload: Dict[str, Dict[str, Dict[str, str]]] = {}
|
| 586 |
+
for username, invites_by_project in self._pending_invites.items():
|
| 587 |
+
payload[username] = {}
|
| 588 |
+
for project_slug, invite in invites_by_project.items():
|
| 589 |
+
payload[username][project_slug] = {
|
| 590 |
+
"role": invite.role.value,
|
| 591 |
+
"invited_by": invite.invited_by,
|
| 592 |
+
"created_at": invite.created_at.isoformat(),
|
| 593 |
+
"expires_at": invite.expires_at.isoformat(),
|
| 594 |
+
}
|
| 595 |
+
return payload
|
| 596 |
+
|
| 597 |
+
def load_pending_invites_map(self, payload: Dict[str, Dict[str, Dict[str, str]]]) -> None:
|
| 598 |
+
self._pending_invites = {}
|
| 599 |
+
for username, invites_by_project in (payload or {}).items():
|
| 600 |
+
if not isinstance(invites_by_project, dict):
|
| 601 |
+
continue
|
| 602 |
+
user_invites: Dict[str, ProjectInvite] = {}
|
| 603 |
+
for project_slug, invite_payload in invites_by_project.items():
|
| 604 |
+
if not isinstance(invite_payload, dict):
|
| 605 |
+
continue
|
| 606 |
+
role_text = str(invite_payload.get("role", "")).strip().lower()
|
| 607 |
+
if role_text not in {"admin", "validator"}:
|
| 608 |
+
continue
|
| 609 |
+
invited_by = str(invite_payload.get("invited_by", "")).strip() or "admin"
|
| 610 |
+
created_at_raw = str(invite_payload.get("created_at", "")).strip()
|
| 611 |
+
expires_at_raw = str(invite_payload.get("expires_at", "")).strip()
|
| 612 |
+
try:
|
| 613 |
+
created_at = datetime.fromisoformat(created_at_raw) if created_at_raw else datetime.now(UTC)
|
| 614 |
+
except Exception:
|
| 615 |
+
created_at = datetime.now(UTC)
|
| 616 |
+
try:
|
| 617 |
+
expires_at = datetime.fromisoformat(expires_at_raw) if expires_at_raw else (created_at + timedelta(hours=self.invite_ttl_hours))
|
| 618 |
+
except Exception:
|
| 619 |
+
expires_at = created_at + timedelta(hours=self.invite_ttl_hours)
|
| 620 |
+
if created_at.tzinfo is None:
|
| 621 |
+
created_at = created_at.replace(tzinfo=UTC)
|
| 622 |
+
if expires_at.tzinfo is None:
|
| 623 |
+
expires_at = expires_at.replace(tzinfo=UTC)
|
| 624 |
+
user_invites[str(project_slug)] = ProjectInvite(
|
| 625 |
+
username=str(username),
|
| 626 |
+
project_slug=str(project_slug),
|
| 627 |
+
role=Role(role_text),
|
| 628 |
+
invited_by=invited_by,
|
| 629 |
+
created_at=created_at,
|
| 630 |
+
expires_at=expires_at,
|
| 631 |
+
)
|
| 632 |
+
|
| 633 |
+
if user_invites:
|
| 634 |
+
self._pending_invites[str(username)] = user_invites
|
| 635 |
+
|
| 636 |
+
self._prune_expired_invites()
|
| 637 |
+
|
| 638 |
+
def revoke_all_invites_for_project(self, project_slug: str) -> int:
|
| 639 |
+
"""Remove all pending invites for a project.
|
| 640 |
+
|
| 641 |
+
Returns:
|
| 642 |
+
Number of revoked invites
|
| 643 |
+
"""
|
| 644 |
+
self._prune_expired_invites()
|
| 645 |
+
revoked = 0
|
| 646 |
+
empty_users: list[str] = []
|
| 647 |
+
for username, invites_by_project in self._pending_invites.items():
|
| 648 |
+
if project_slug in invites_by_project:
|
| 649 |
+
del invites_by_project[project_slug]
|
| 650 |
+
revoked += 1
|
| 651 |
+
if not invites_by_project:
|
| 652 |
+
empty_users.append(username)
|
| 653 |
+
|
| 654 |
+
for username in empty_users:
|
| 655 |
+
self._pending_invites.pop(username, None)
|
| 656 |
+
|
| 657 |
+
return revoked
|
| 658 |
+
|
| 659 |
+
def remove_project_from_all_users(self, project_slug: str) -> int:
|
| 660 |
+
"""Remove a project assignment from all users and refresh active sessions.
|
| 661 |
+
|
| 662 |
+
Returns:
|
| 663 |
+
Number of user assignments removed
|
| 664 |
+
"""
|
| 665 |
+
removed = 0
|
| 666 |
+
for access in self._user_access.values():
|
| 667 |
+
if project_slug in access.project_slugs:
|
| 668 |
+
del access.project_slugs[project_slug]
|
| 669 |
+
removed += 1
|
| 670 |
+
|
| 671 |
+
for session in self._sessions.values():
|
| 672 |
+
if project_slug in session.authorized_projects:
|
| 673 |
+
session.authorized_projects = [slug for slug in session.authorized_projects if slug != project_slug]
|
| 674 |
+
|
| 675 |
+
access = self._user_access.get(session.username)
|
| 676 |
+
if access is None or not access.is_active:
|
| 677 |
+
continue
|
| 678 |
+
session.role = (
|
| 679 |
+
Role.admin
|
| 680 |
+
if any(role == Role.admin for role in access.project_slugs.values())
|
| 681 |
+
else Role.validator
|
| 682 |
+
)
|
| 683 |
+
|
| 684 |
+
return removed
|
src/cache/__init__.py
ADDED
|
File without changes
|
src/cache/ephemeral_cache_manager.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import tempfile
|
| 3 |
+
import time
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
@dataclass(slots=True)
|
| 9 |
+
class CacheEntry:
|
| 10 |
+
key: str
|
| 11 |
+
path: Path
|
| 12 |
+
created_at: float
|
| 13 |
+
expires_at: float
|
| 14 |
+
size_bytes: int
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class EphemeralCacheManager:
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
cache_dir: str | None = None,
|
| 21 |
+
ttl_seconds: int = 300,
|
| 22 |
+
max_files: int = 256,
|
| 23 |
+
) -> None:
|
| 24 |
+
self._base_dir = Path(cache_dir) if cache_dir else Path(tempfile.gettempdir()) / "birdnet-validator-cache"
|
| 25 |
+
self._base_dir.mkdir(parents=True, exist_ok=True)
|
| 26 |
+
self._ttl_seconds = ttl_seconds
|
| 27 |
+
self._max_files = max_files
|
| 28 |
+
self._entries: dict[str, CacheEntry] = {}
|
| 29 |
+
|
| 30 |
+
def get(self, key: str) -> Path | None:
|
| 31 |
+
self.cleanup_expired()
|
| 32 |
+
entry = self._entries.get(key)
|
| 33 |
+
if not entry:
|
| 34 |
+
return None
|
| 35 |
+
if not entry.path.exists():
|
| 36 |
+
self._entries.pop(key, None)
|
| 37 |
+
return None
|
| 38 |
+
return entry.path
|
| 39 |
+
|
| 40 |
+
def put_bytes(self, key: str, data: bytes, suffix: str = ".bin") -> Path:
|
| 41 |
+
self.cleanup_expired()
|
| 42 |
+
self._enforce_capacity()
|
| 43 |
+
|
| 44 |
+
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
|
| 45 |
+
file_path = self._base_dir / f"{digest}{suffix}"
|
| 46 |
+
file_path.write_bytes(data)
|
| 47 |
+
|
| 48 |
+
now = time.time()
|
| 49 |
+
self._entries[key] = CacheEntry(
|
| 50 |
+
key=key,
|
| 51 |
+
path=file_path,
|
| 52 |
+
created_at=now,
|
| 53 |
+
expires_at=now + self._ttl_seconds,
|
| 54 |
+
size_bytes=len(data),
|
| 55 |
+
)
|
| 56 |
+
return file_path
|
| 57 |
+
|
| 58 |
+
def cleanup_key(self, key: str) -> None:
|
| 59 |
+
entry = self._entries.pop(key, None)
|
| 60 |
+
if not entry:
|
| 61 |
+
return
|
| 62 |
+
if entry.path.exists():
|
| 63 |
+
entry.path.unlink()
|
| 64 |
+
|
| 65 |
+
def cleanup_expired(self) -> None:
|
| 66 |
+
now = time.time()
|
| 67 |
+
expired_keys = [key for key, entry in self._entries.items() if entry.expires_at <= now or not entry.path.exists()]
|
| 68 |
+
for key in expired_keys:
|
| 69 |
+
self.cleanup_key(key)
|
| 70 |
+
|
| 71 |
+
def clear(self) -> None:
|
| 72 |
+
for key in list(self._entries.keys()):
|
| 73 |
+
self.cleanup_key(key)
|
| 74 |
+
|
| 75 |
+
def _enforce_capacity(self) -> None:
|
| 76 |
+
if len(self._entries) < self._max_files:
|
| 77 |
+
return
|
| 78 |
+
|
| 79 |
+
oldest_keys = sorted(self._entries.keys(), key=lambda key: self._entries[key].created_at)
|
| 80 |
+
overflow = (len(self._entries) - self._max_files) + 1
|
| 81 |
+
for key in oldest_keys[:overflow]:
|
| 82 |
+
self.cleanup_key(key)
|
src/cli/__init__.py
ADDED
|
File without changes
|
src/cli/project_cli.py
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import json
|
| 5 |
+
import re
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
from uuid import uuid4
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
DEFAULT_DATASET_DIRS = [
|
| 12 |
+
"audio",
|
| 13 |
+
"detections",
|
| 14 |
+
"index",
|
| 15 |
+
"exports",
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
EXIT_OK = 0
|
| 19 |
+
EXIT_VALIDATION_ERROR = 1
|
| 20 |
+
EXIT_IO_ERROR = 2
|
| 21 |
+
|
| 22 |
+
SLUG_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]{1,62}$")
|
| 23 |
+
REPO_ID_PATTERN = re.compile(r"^[^\s/]+/[^\s/]+$")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class CliValidationError(ValueError):
|
| 27 |
+
pass
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _safe_read_text(path: Path) -> str:
|
| 31 |
+
try:
|
| 32 |
+
return path.read_text(encoding="utf-8")
|
| 33 |
+
except FileNotFoundError:
|
| 34 |
+
raise
|
| 35 |
+
except UnicodeDecodeError as exc:
|
| 36 |
+
raise CliValidationError(f"Invalid UTF-8 content in {path}: {exc}") from exc
|
| 37 |
+
except OSError as exc:
|
| 38 |
+
raise OSError(f"Could not read {path}: {exc}") from exc
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _validate_slug(slug: str) -> None:
|
| 42 |
+
if not SLUG_PATTERN.fullmatch(slug):
|
| 43 |
+
raise CliValidationError(
|
| 44 |
+
"Invalid slug. Use lowercase letters, numbers, and hyphens, length 2-63."
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _validate_repo_id(repo_id: str) -> None:
|
| 49 |
+
if not REPO_ID_PATTERN.fullmatch(repo_id):
|
| 50 |
+
raise CliValidationError(
|
| 51 |
+
"Invalid dataset_repo_id. Expected format 'owner/repo' without spaces."
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _load_json(path: Path, default: Any) -> Any:
|
| 56 |
+
if not path.exists():
|
| 57 |
+
return default
|
| 58 |
+
raw = _safe_read_text(path)
|
| 59 |
+
try:
|
| 60 |
+
return json.loads(raw)
|
| 61 |
+
except json.JSONDecodeError as exc:
|
| 62 |
+
raise CliValidationError(f"Invalid JSON in {path}: {exc}") from exc
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _write_json(path: Path, payload: Any) -> None:
|
| 66 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 67 |
+
temp_path = path.with_suffix(path.suffix + ".tmp")
|
| 68 |
+
temp_path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
|
| 69 |
+
temp_path.replace(path)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _project_exists(projects: list[dict[str, Any]], slug: str) -> bool:
|
| 73 |
+
return any(p.get("project_slug") == slug for p in projects)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _as_project_list(payload: Any, source: Path) -> list[dict[str, Any]]:
|
| 77 |
+
if not isinstance(payload, list):
|
| 78 |
+
raise CliValidationError(f"Expected list in {source}, got {type(payload).__name__}")
|
| 79 |
+
for idx, item in enumerate(payload):
|
| 80 |
+
if not isinstance(item, dict):
|
| 81 |
+
raise CliValidationError(f"Project entry at index {idx} in {source} is not an object")
|
| 82 |
+
for field in ("project_slug", "name", "dataset_repo_id"):
|
| 83 |
+
if field not in item or not isinstance(item[field], str) or not item[field].strip():
|
| 84 |
+
raise CliValidationError(
|
| 85 |
+
f"Project entry at index {idx} in {source} has invalid '{field}'"
|
| 86 |
+
)
|
| 87 |
+
return payload
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _as_access_map(payload: Any, source: Path) -> dict[str, dict[str, str]]:
|
| 91 |
+
if not isinstance(payload, dict):
|
| 92 |
+
raise CliValidationError(f"Expected object in {source}, got {type(payload).__name__}")
|
| 93 |
+
for username, roles in payload.items():
|
| 94 |
+
if not isinstance(username, str) or not username.strip():
|
| 95 |
+
raise CliValidationError(f"Invalid username key in {source}")
|
| 96 |
+
if not isinstance(roles, dict):
|
| 97 |
+
raise CliValidationError(f"Roles for '{username}' in {source} must be an object")
|
| 98 |
+
for project_slug, role in roles.items():
|
| 99 |
+
if not isinstance(project_slug, str) or not project_slug.strip():
|
| 100 |
+
raise CliValidationError(f"Invalid project slug for user '{username}' in {source}")
|
| 101 |
+
if role not in {"admin", "validator"}:
|
| 102 |
+
raise CliValidationError(
|
| 103 |
+
f"Invalid role '{role}' for user '{username}' and project '{project_slug}' in {source}"
|
| 104 |
+
)
|
| 105 |
+
return payload
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def cmd_create_project(args: argparse.Namespace) -> int:
|
| 109 |
+
_validate_slug(args.slug)
|
| 110 |
+
_validate_repo_id(args.dataset_repo_id)
|
| 111 |
+
|
| 112 |
+
if args.owner and not args.owner.strip():
|
| 113 |
+
raise CliValidationError("Owner cannot be blank")
|
| 114 |
+
|
| 115 |
+
projects_file = Path(args.projects_file)
|
| 116 |
+
projects = _as_project_list(_load_json(projects_file, []), projects_file)
|
| 117 |
+
|
| 118 |
+
if _project_exists(projects, args.slug):
|
| 119 |
+
print(f"ERROR: project '{args.slug}' already exists")
|
| 120 |
+
return EXIT_VALIDATION_ERROR
|
| 121 |
+
|
| 122 |
+
project_entry: dict[str, Any] = {
|
| 123 |
+
"project_id": str(uuid4()),
|
| 124 |
+
"project_slug": args.slug,
|
| 125 |
+
"name": args.name,
|
| 126 |
+
"dataset_repo_id": args.dataset_repo_id,
|
| 127 |
+
"visibility": args.visibility,
|
| 128 |
+
"active": not args.inactive,
|
| 129 |
+
}
|
| 130 |
+
if args.owner:
|
| 131 |
+
project_entry["owner_username"] = args.owner
|
| 132 |
+
if args.dataset_token:
|
| 133 |
+
project_entry["dataset_token"] = args.dataset_token
|
| 134 |
+
|
| 135 |
+
projects.append(project_entry)
|
| 136 |
+
projects.sort(key=lambda p: str(p.get("project_slug", "")))
|
| 137 |
+
_write_json(projects_file, projects)
|
| 138 |
+
|
| 139 |
+
if args.user_access_file and args.owner:
|
| 140 |
+
access_file = Path(args.user_access_file)
|
| 141 |
+
access_payload = _as_access_map(_load_json(access_file, {}), access_file)
|
| 142 |
+
owner_roles = access_payload.setdefault(args.owner, {})
|
| 143 |
+
owner_roles[args.slug] = "admin"
|
| 144 |
+
_write_json(access_file, access_payload)
|
| 145 |
+
|
| 146 |
+
print(f"OK: project '{args.slug}' created")
|
| 147 |
+
return EXIT_OK
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def cmd_init_dataset(args: argparse.Namespace) -> int:
|
| 151 |
+
_validate_slug(args.slug)
|
| 152 |
+
_validate_repo_id(args.dataset_repo_id)
|
| 153 |
+
|
| 154 |
+
root = Path(args.dataset_root) / args.slug
|
| 155 |
+
for dirname in DEFAULT_DATASET_DIRS:
|
| 156 |
+
(root / dirname).mkdir(parents=True, exist_ok=True)
|
| 157 |
+
|
| 158 |
+
detections_file = root / "detections" / "detections.jsonl"
|
| 159 |
+
if not detections_file.exists():
|
| 160 |
+
detections_file.write_text("", encoding="utf-8")
|
| 161 |
+
|
| 162 |
+
metadata = {
|
| 163 |
+
"project_slug": args.slug,
|
| 164 |
+
"dataset_repo_id": args.dataset_repo_id,
|
| 165 |
+
}
|
| 166 |
+
if args.name:
|
| 167 |
+
metadata["name"] = args.name
|
| 168 |
+
|
| 169 |
+
_write_json(root / "index" / "project_metadata.json", metadata)
|
| 170 |
+
print(f"OK: dataset scaffold initialized at {root}")
|
| 171 |
+
return EXIT_OK
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
| 175 |
+
rows: list[dict[str, Any]] = []
|
| 176 |
+
if not path.exists():
|
| 177 |
+
return rows
|
| 178 |
+
|
| 179 |
+
raw_content = _safe_read_text(path)
|
| 180 |
+
for line_no, raw in enumerate(raw_content.splitlines(), start=1):
|
| 181 |
+
line = raw.strip()
|
| 182 |
+
if not line:
|
| 183 |
+
continue
|
| 184 |
+
try:
|
| 185 |
+
parsed = json.loads(line)
|
| 186 |
+
except json.JSONDecodeError as exc:
|
| 187 |
+
raise CliValidationError(
|
| 188 |
+
f"Invalid JSONL in {path} at line {line_no}: {exc.msg} (column {exc.colno})"
|
| 189 |
+
) from exc
|
| 190 |
+
|
| 191 |
+
if not isinstance(parsed, dict):
|
| 192 |
+
raise CliValidationError(
|
| 193 |
+
f"Invalid JSONL in {path} at line {line_no}: expected object per line"
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
required = ("detection_key", "audio_id", "scientific_name", "confidence")
|
| 197 |
+
for field in required:
|
| 198 |
+
if field not in parsed:
|
| 199 |
+
raise CliValidationError(
|
| 200 |
+
f"Invalid JSONL in {path} at line {line_no}: missing '{field}'"
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
try:
|
| 204 |
+
parsed["confidence"] = float(parsed["confidence"])
|
| 205 |
+
except (TypeError, ValueError) as exc:
|
| 206 |
+
raise CliValidationError(
|
| 207 |
+
f"Invalid JSONL in {path} at line {line_no}: 'confidence' must be numeric"
|
| 208 |
+
) from exc
|
| 209 |
+
|
| 210 |
+
rows.append(parsed)
|
| 211 |
+
return rows
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def cmd_build_index(args: argparse.Namespace) -> int:
|
| 215 |
+
_validate_slug(args.slug)
|
| 216 |
+
|
| 217 |
+
project_root = Path(args.dataset_root) / args.slug
|
| 218 |
+
detections_file = project_root / "detections" / "detections.jsonl"
|
| 219 |
+
rows = _read_jsonl(detections_file)
|
| 220 |
+
|
| 221 |
+
sorted_rows = sorted(rows, key=lambda d: d["confidence"], reverse=True)
|
| 222 |
+
index_payload = {
|
| 223 |
+
"project_slug": args.slug,
|
| 224 |
+
"count": len(sorted_rows),
|
| 225 |
+
"detections": [
|
| 226 |
+
{
|
| 227 |
+
"detection_key": r.get("detection_key"),
|
| 228 |
+
"audio_id": r.get("audio_id"),
|
| 229 |
+
"scientific_name": r.get("scientific_name"),
|
| 230 |
+
"confidence": r["confidence"],
|
| 231 |
+
}
|
| 232 |
+
for r in sorted_rows
|
| 233 |
+
],
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
_write_json(project_root / "index" / "detections_index.json", index_payload)
|
| 237 |
+
print(f"OK: index generated with {len(sorted_rows)} detections")
|
| 238 |
+
return EXIT_OK
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def cmd_verify_project(args: argparse.Namespace) -> int:
|
| 242 |
+
_validate_slug(args.slug)
|
| 243 |
+
|
| 244 |
+
projects_file = Path(args.projects_file)
|
| 245 |
+
projects = _as_project_list(_load_json(projects_file, []), projects_file)
|
| 246 |
+
|
| 247 |
+
findings: list[str] = []
|
| 248 |
+
|
| 249 |
+
if not _project_exists(projects, args.slug):
|
| 250 |
+
findings.append(f"project '{args.slug}' not found in {projects_file}")
|
| 251 |
+
|
| 252 |
+
project_root = Path(args.dataset_root) / args.slug
|
| 253 |
+
for dirname in DEFAULT_DATASET_DIRS:
|
| 254 |
+
expected = project_root / dirname
|
| 255 |
+
if not expected.exists():
|
| 256 |
+
findings.append(f"missing path: {expected}")
|
| 257 |
+
|
| 258 |
+
required_files = [
|
| 259 |
+
project_root / "detections" / "detections.jsonl",
|
| 260 |
+
project_root / "index" / "project_metadata.json",
|
| 261 |
+
project_root / "index" / "detections_index.json",
|
| 262 |
+
]
|
| 263 |
+
for required in required_files:
|
| 264 |
+
if not required.exists():
|
| 265 |
+
findings.append(f"missing path: {required}")
|
| 266 |
+
|
| 267 |
+
if findings:
|
| 268 |
+
if args.dry_run:
|
| 269 |
+
print("DRY-RUN: project verification found issues:")
|
| 270 |
+
else:
|
| 271 |
+
print("ERROR: project verification failed:")
|
| 272 |
+
for item in findings:
|
| 273 |
+
print(f" - {item}")
|
| 274 |
+
if args.dry_run:
|
| 275 |
+
print("DRY-RUN: completed without failing exit code")
|
| 276 |
+
return EXIT_OK
|
| 277 |
+
return EXIT_VALIDATION_ERROR
|
| 278 |
+
|
| 279 |
+
print(f"OK: project '{args.slug}' verified")
|
| 280 |
+
return EXIT_OK
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 284 |
+
parser = argparse.ArgumentParser(prog="birdnet-project")
|
| 285 |
+
sub = parser.add_subparsers(dest="command", required=True)
|
| 286 |
+
|
| 287 |
+
create_project = sub.add_parser("create-project", help="Create project in bootstrap JSON")
|
| 288 |
+
create_project.add_argument("--projects-file", required=True)
|
| 289 |
+
create_project.add_argument("--user-access-file")
|
| 290 |
+
create_project.add_argument("--slug", required=True)
|
| 291 |
+
create_project.add_argument("--name", required=True)
|
| 292 |
+
create_project.add_argument("--dataset-repo-id", required=True)
|
| 293 |
+
create_project.add_argument("--visibility", choices=["private", "collaborative"], default="collaborative")
|
| 294 |
+
create_project.add_argument("--owner")
|
| 295 |
+
create_project.add_argument("--dataset-token")
|
| 296 |
+
create_project.add_argument("--inactive", action="store_true")
|
| 297 |
+
create_project.set_defaults(func=cmd_create_project)
|
| 298 |
+
|
| 299 |
+
init_dataset = sub.add_parser("init-dataset", help="Create initial dataset folder scaffold")
|
| 300 |
+
init_dataset.add_argument("--dataset-root", required=True)
|
| 301 |
+
init_dataset.add_argument("--slug", required=True)
|
| 302 |
+
init_dataset.add_argument("--dataset-repo-id", required=True)
|
| 303 |
+
init_dataset.add_argument("--name")
|
| 304 |
+
init_dataset.set_defaults(func=cmd_init_dataset)
|
| 305 |
+
|
| 306 |
+
build_index = sub.add_parser("build-index", help="Build simple index from detections JSONL")
|
| 307 |
+
build_index.add_argument("--dataset-root", required=True)
|
| 308 |
+
build_index.add_argument("--slug", required=True)
|
| 309 |
+
build_index.set_defaults(func=cmd_build_index)
|
| 310 |
+
|
| 311 |
+
verify_project = sub.add_parser("verify-project", help="Verify project config and local scaffold")
|
| 312 |
+
verify_project.add_argument("--projects-file", required=True)
|
| 313 |
+
verify_project.add_argument("--dataset-root", required=True)
|
| 314 |
+
verify_project.add_argument("--slug", required=True)
|
| 315 |
+
verify_project.add_argument(
|
| 316 |
+
"--dry-run",
|
| 317 |
+
action="store_true",
|
| 318 |
+
help="Print verification findings without returning a failure exit code",
|
| 319 |
+
)
|
| 320 |
+
verify_project.set_defaults(func=cmd_verify_project)
|
| 321 |
+
|
| 322 |
+
return parser
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def main(argv: list[str] | None = None) -> int:
|
| 326 |
+
parser = build_parser()
|
| 327 |
+
try:
|
| 328 |
+
args = parser.parse_args(argv)
|
| 329 |
+
return int(args.func(args))
|
| 330 |
+
except CliValidationError as exc:
|
| 331 |
+
print(f"ERROR: {exc}")
|
| 332 |
+
return EXIT_VALIDATION_ERROR
|
| 333 |
+
except OSError as exc:
|
| 334 |
+
print(f"ERROR: I/O failure: {exc}")
|
| 335 |
+
return EXIT_IO_ERROR
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
if __name__ == "__main__":
|
| 339 |
+
raise SystemExit(main())
|
src/config/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Runtime configuration helpers."""
|
src/config/runtime_config.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
import os
|
| 3 |
+
import tempfile
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass(frozen=True)
|
| 8 |
+
class RuntimeConfig:
|
| 9 |
+
detection_seed_path: str | None
|
| 10 |
+
validation_base_dir: str
|
| 11 |
+
bootstrap_base_dir: str
|
| 12 |
+
page_size: int
|
| 13 |
+
projects_file_path: str | None
|
| 14 |
+
user_access_file_path: str | None
|
| 15 |
+
invites_file_path: str | None
|
| 16 |
+
invite_ttl_hours: int
|
| 17 |
+
enable_demo_bootstrap: bool
|
| 18 |
+
invite_email_enabled: bool
|
| 19 |
+
invite_email_sender: str
|
| 20 |
+
invite_email_login_url: str
|
| 21 |
+
emailjs_enabled: bool = False
|
| 22 |
+
emailjs_service_id: str | None = None
|
| 23 |
+
emailjs_template_id: str | None = None
|
| 24 |
+
emailjs_template_id_username_only: str | None = None
|
| 25 |
+
emailjs_template_id_email_only: str | None = None
|
| 26 |
+
emailjs_template_id_dual: str | None = None
|
| 27 |
+
emailjs_public_key: str | None = None
|
| 28 |
+
emailjs_endpoint: str = "https://api.emailjs.com/api/v1.0/email/send"
|
| 29 |
+
emailjs_timeout_seconds: int = 20
|
| 30 |
+
|
| 31 |
+
@classmethod
|
| 32 |
+
def from_env(cls) -> "RuntimeConfig":
|
| 33 |
+
detection_seed_path = (os.getenv("BIRDNET_DETECTIONS_FILE") or "").strip() or None
|
| 34 |
+
|
| 35 |
+
data_root = Path("/data") if Path("/data").exists() else (Path(tempfile.gettempdir()) / "birdnet-validator-data")
|
| 36 |
+
|
| 37 |
+
bootstrap_base_dir = (
|
| 38 |
+
os.getenv("BIRDNET_BOOTSTRAP_DIR")
|
| 39 |
+
or str(data_root / "bootstrap")
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
validation_base_dir = (
|
| 43 |
+
os.getenv("BIRDNET_VALIDATIONS_DIR")
|
| 44 |
+
or str(data_root / "validations")
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
raw_page_size = (os.getenv("BIRDNET_PAGE_SIZE") or "").strip()
|
| 48 |
+
page_size = 25
|
| 49 |
+
if raw_page_size:
|
| 50 |
+
try:
|
| 51 |
+
parsed = int(raw_page_size)
|
| 52 |
+
if parsed > 0:
|
| 53 |
+
page_size = parsed
|
| 54 |
+
except ValueError:
|
| 55 |
+
page_size = 25
|
| 56 |
+
|
| 57 |
+
projects_file_path = (os.getenv("BIRDNET_PROJECTS_FILE") or "").strip() or None
|
| 58 |
+
user_access_file_path = (os.getenv("BIRDNET_USER_ACCESS_FILE") or "").strip() or None
|
| 59 |
+
invites_file_path = (os.getenv("BIRDNET_INVITES_FILE") or "").strip() or None
|
| 60 |
+
raw_invite_ttl_hours = (os.getenv("BIRDNET_INVITE_TTL_HOURS") or "").strip()
|
| 61 |
+
invite_ttl_hours = 72
|
| 62 |
+
if raw_invite_ttl_hours:
|
| 63 |
+
try:
|
| 64 |
+
parsed = int(raw_invite_ttl_hours)
|
| 65 |
+
if parsed > 0:
|
| 66 |
+
invite_ttl_hours = parsed
|
| 67 |
+
except ValueError:
|
| 68 |
+
invite_ttl_hours = 72
|
| 69 |
+
|
| 70 |
+
raw_enable_demo_bootstrap = (os.getenv("BIRDNET_ENABLE_DEMO_BOOTSTRAP") or "").strip().lower()
|
| 71 |
+
enable_demo_bootstrap = raw_enable_demo_bootstrap in {"1", "true", "yes", "on"}
|
| 72 |
+
|
| 73 |
+
raw_invite_email_enabled = (os.getenv("BIRDNET_INVITE_EMAIL_ENABLED") or "").strip().lower()
|
| 74 |
+
invite_email_enabled = raw_invite_email_enabled in {"1", "true", "yes", "on"}
|
| 75 |
+
invite_email_sender = (os.getenv("BIRDNET_INVITE_EMAIL_SENDER") or "").strip()
|
| 76 |
+
invite_email_login_url = (os.getenv("BIRDNET_INVITE_EMAIL_LOGIN_URL") or "").strip() or ""
|
| 77 |
+
|
| 78 |
+
raw_emailjs_enabled = (os.getenv("BIRDNET_EMAILJS_ENABLED") or "").strip().lower()
|
| 79 |
+
emailjs_enabled = raw_emailjs_enabled in {"1", "true", "yes", "on"}
|
| 80 |
+
emailjs_service_id = (os.getenv("BIRDNET_EMAILJS_SERVICE_ID") or "").strip() or None
|
| 81 |
+
emailjs_template_id = (os.getenv("BIRDNET_EMAILJS_TEMPLATE_ID") or "").strip() or None
|
| 82 |
+
emailjs_template_id_username_only = (os.getenv("BIRDNET_EMAILJS_TEMPLATE_ID_USERNAME_ONLY") or "").strip() or None
|
| 83 |
+
emailjs_template_id_email_only = (os.getenv("BIRDNET_EMAILJS_TEMPLATE_ID_EMAIL_ONLY") or "").strip() or None
|
| 84 |
+
emailjs_template_id_dual = (os.getenv("BIRDNET_EMAILJS_TEMPLATE_ID_DUAL") or "").strip() or None
|
| 85 |
+
emailjs_public_key = (os.getenv("BIRDNET_EMAILJS_PUBLIC_KEY") or os.getenv("EMAILJS_PUBLIC_KEY") or "").strip() or None
|
| 86 |
+
emailjs_endpoint = (os.getenv("BIRDNET_EMAILJS_ENDPOINT") or "https://api.emailjs.com/api/v1.0/email/send").strip() or "https://api.emailjs.com/api/v1.0/email/send"
|
| 87 |
+
raw_emailjs_timeout = (os.getenv("BIRDNET_EMAILJS_TIMEOUT_SECONDS") or "").strip()
|
| 88 |
+
emailjs_timeout_seconds = 20
|
| 89 |
+
if raw_emailjs_timeout:
|
| 90 |
+
try:
|
| 91 |
+
parsed = int(raw_emailjs_timeout)
|
| 92 |
+
if parsed > 0:
|
| 93 |
+
emailjs_timeout_seconds = parsed
|
| 94 |
+
except ValueError:
|
| 95 |
+
emailjs_timeout_seconds = 20
|
| 96 |
+
|
| 97 |
+
return cls(
|
| 98 |
+
detection_seed_path=detection_seed_path,
|
| 99 |
+
validation_base_dir=validation_base_dir,
|
| 100 |
+
bootstrap_base_dir=bootstrap_base_dir,
|
| 101 |
+
page_size=page_size,
|
| 102 |
+
projects_file_path=projects_file_path,
|
| 103 |
+
user_access_file_path=user_access_file_path,
|
| 104 |
+
invites_file_path=invites_file_path,
|
| 105 |
+
invite_ttl_hours=invite_ttl_hours,
|
| 106 |
+
enable_demo_bootstrap=enable_demo_bootstrap,
|
| 107 |
+
invite_email_enabled=invite_email_enabled,
|
| 108 |
+
invite_email_sender=invite_email_sender,
|
| 109 |
+
invite_email_login_url=invite_email_login_url,
|
| 110 |
+
emailjs_enabled=emailjs_enabled,
|
| 111 |
+
emailjs_service_id=emailjs_service_id,
|
| 112 |
+
emailjs_template_id=emailjs_template_id,
|
| 113 |
+
emailjs_template_id_username_only=emailjs_template_id_username_only,
|
| 114 |
+
emailjs_template_id_email_only=emailjs_template_id_email_only,
|
| 115 |
+
emailjs_template_id_dual=emailjs_template_id_dual,
|
| 116 |
+
emailjs_public_key=emailjs_public_key,
|
| 117 |
+
emailjs_endpoint=emailjs_endpoint,
|
| 118 |
+
emailjs_timeout_seconds=emailjs_timeout_seconds,
|
| 119 |
+
)
|
src/domain/__init__.py
ADDED
|
File without changes
|
src/domain/models.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from enum import Enum
|
| 2 |
+
from typing import Optional
|
| 3 |
+
from uuid import uuid4
|
| 4 |
+
|
| 5 |
+
from pydantic import BaseModel, Field
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class Role(str, Enum):
|
| 9 |
+
admin = "admin"
|
| 10 |
+
validator = "validator"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class User(BaseModel):
|
| 14 |
+
username: str = Field(min_length=1)
|
| 15 |
+
role: Role
|
| 16 |
+
active: bool = True
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class Project(BaseModel):
|
| 20 |
+
project_id: str = Field(default_factory=lambda: str(uuid4()), min_length=8)
|
| 21 |
+
project_slug: str = Field(min_length=3)
|
| 22 |
+
name: str = Field(min_length=1)
|
| 23 |
+
dataset_repo_id: str = Field(min_length=3)
|
| 24 |
+
visibility: str = Field(default="collaborative", pattern="^(private|collaborative)$")
|
| 25 |
+
owner_username: Optional[str] = None
|
| 26 |
+
dataset_token: Optional[str] = None
|
| 27 |
+
active: bool = True
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class Detection(BaseModel):
|
| 31 |
+
detection_key: str = Field(min_length=16)
|
| 32 |
+
audio_id: str = Field(min_length=1)
|
| 33 |
+
scientific_name: str = Field(min_length=1)
|
| 34 |
+
confidence: float = Field(ge=0.0, le=1.0)
|
| 35 |
+
start_time: float = Field(ge=0.0)
|
| 36 |
+
end_time: float = Field(gt=0.0)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class Validation(BaseModel):
|
| 40 |
+
detection_key: str = Field(min_length=16)
|
| 41 |
+
status: str = Field(min_length=1)
|
| 42 |
+
corrected_species: Optional[str] = None
|
| 43 |
+
notes: str = ""
|
| 44 |
+
validator: str = Field(min_length=1)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class IndexManifest(BaseModel):
|
| 48 |
+
schema_version: str = "1.0.0"
|
| 49 |
+
project_slug: str
|
| 50 |
+
total_detections: int = Field(ge=0)
|
| 51 |
+
total_audio_files: int = Field(ge=0)
|
src/repositories/__init__.py
ADDED
|
File without changes
|
src/repositories/append_only_validation_repository.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from datetime import UTC, datetime
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
import threading
|
| 5 |
+
from uuid import uuid4
|
| 6 |
+
|
| 7 |
+
from src.domain.models import Validation
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class OptimisticLockError(Exception):
|
| 11 |
+
def __init__(self, detection_key: str, expected_version: int, current_version: int) -> None:
|
| 12 |
+
self.detection_key = detection_key
|
| 13 |
+
self.expected_version = expected_version
|
| 14 |
+
self.current_version = current_version
|
| 15 |
+
super().__init__(
|
| 16 |
+
f"Optimistic lock conflict for detection_key={detection_key}: "
|
| 17 |
+
f"expected_version={expected_version}, current_version={current_version}"
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class AppendOnlyValidationRepository:
|
| 22 |
+
def __init__(self, base_dir: str) -> None:
|
| 23 |
+
self._base_dir = Path(base_dir)
|
| 24 |
+
self._locks: dict[str, threading.Lock] = {}
|
| 25 |
+
self._locks_guard = threading.Lock()
|
| 26 |
+
|
| 27 |
+
def _lock_for_project(self, project_slug: str) -> threading.Lock:
|
| 28 |
+
with self._locks_guard:
|
| 29 |
+
lock = self._locks.get(project_slug)
|
| 30 |
+
if lock is None:
|
| 31 |
+
lock = threading.Lock()
|
| 32 |
+
self._locks[project_slug] = lock
|
| 33 |
+
return lock
|
| 34 |
+
|
| 35 |
+
@staticmethod
|
| 36 |
+
def _atomic_write_json(path: Path, payload: object) -> None:
|
| 37 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 38 |
+
tmp_path = path.parent / f".{path.name}.tmp.{uuid4().hex}"
|
| 39 |
+
tmp_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
| 40 |
+
tmp_path.replace(path)
|
| 41 |
+
|
| 42 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 43 |
+
project_dir = self._base_dir / project_slug / "validations"
|
| 44 |
+
project_dir.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
|
| 46 |
+
with self._lock_for_project(project_slug):
|
| 47 |
+
current_file = project_dir / "current.json"
|
| 48 |
+
if current_file.exists():
|
| 49 |
+
current_payload = json.loads(current_file.read_text(encoding="utf-8"))
|
| 50 |
+
else:
|
| 51 |
+
current_payload = {}
|
| 52 |
+
|
| 53 |
+
current_item = current_payload.get(item.detection_key, {})
|
| 54 |
+
current_version = int(current_item.get("version", 0))
|
| 55 |
+
expected = expected_version if expected_version is not None else current_version
|
| 56 |
+
if expected != current_version:
|
| 57 |
+
raise OptimisticLockError(
|
| 58 |
+
detection_key=item.detection_key,
|
| 59 |
+
expected_version=expected,
|
| 60 |
+
current_version=current_version,
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
new_version = current_version + 1
|
| 64 |
+
|
| 65 |
+
event_date = datetime.now(UTC).strftime("%Y%m%d")
|
| 66 |
+
events_file = project_dir / f"events-{event_date}.jsonl"
|
| 67 |
+
event = {
|
| 68 |
+
"event_id": str(uuid4()),
|
| 69 |
+
"timestamp": datetime.now(UTC).isoformat(),
|
| 70 |
+
"project_slug": project_slug,
|
| 71 |
+
"expected_version": expected,
|
| 72 |
+
"previous_version": current_version,
|
| 73 |
+
"new_version": new_version,
|
| 74 |
+
**item.model_dump(),
|
| 75 |
+
}
|
| 76 |
+
with events_file.open("a", encoding="utf-8") as handle:
|
| 77 |
+
handle.write(json.dumps(event, ensure_ascii=True) + "\n")
|
| 78 |
+
|
| 79 |
+
current_payload[item.detection_key] = {
|
| 80 |
+
"status": item.status,
|
| 81 |
+
"corrected_species": item.corrected_species,
|
| 82 |
+
"notes": item.notes,
|
| 83 |
+
"validator": item.validator,
|
| 84 |
+
"updated_at": event["timestamp"],
|
| 85 |
+
"version": new_version,
|
| 86 |
+
}
|
| 87 |
+
self._atomic_write_json(current_file, current_payload)
|
| 88 |
+
return new_version
|
| 89 |
+
|
| 90 |
+
def list_events(self, project_slug: str) -> list[dict[str, object]]:
|
| 91 |
+
project_dir = self._base_dir / project_slug / "validations"
|
| 92 |
+
if not project_dir.exists():
|
| 93 |
+
return []
|
| 94 |
+
|
| 95 |
+
events: list[dict[str, object]] = []
|
| 96 |
+
for path in sorted(project_dir.glob("events-*.jsonl")):
|
| 97 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 98 |
+
for line in handle:
|
| 99 |
+
payload = line.strip()
|
| 100 |
+
if payload:
|
| 101 |
+
events.append(json.loads(payload))
|
| 102 |
+
return events
|
| 103 |
+
|
| 104 |
+
def load_current_snapshot(self, project_slug: str) -> dict[str, dict[str, object]]:
|
| 105 |
+
current_file = self._base_dir / project_slug / "validations" / "current.json"
|
| 106 |
+
if not current_file.exists():
|
| 107 |
+
return {}
|
| 108 |
+
return json.loads(current_file.read_text(encoding="utf-8"))
|
src/repositories/contracts.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Protocol
|
| 2 |
+
|
| 3 |
+
from src.domain.models import Detection, Project, User, Validation
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class ProjectRepository(Protocol):
|
| 7 |
+
def get_project(self, project_slug: str) -> Project: ...
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class DetectionRepository(Protocol):
|
| 11 |
+
def list_detections(
|
| 12 |
+
self,
|
| 13 |
+
project_slug: str,
|
| 14 |
+
page: int,
|
| 15 |
+
page_size: int,
|
| 16 |
+
scientific_name: str | None = None,
|
| 17 |
+
min_confidence: float | None = None,
|
| 18 |
+
max_confidence: float | None = None,
|
| 19 |
+
) -> list[Detection]: ...
|
| 20 |
+
|
| 21 |
+
def count_detections(
|
| 22 |
+
self,
|
| 23 |
+
project_slug: str,
|
| 24 |
+
scientific_name: str | None = None,
|
| 25 |
+
min_confidence: float | None = None,
|
| 26 |
+
max_confidence: float | None = None,
|
| 27 |
+
) -> int: ...
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class ValidationRepository(Protocol):
|
| 31 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int: ...
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class AuthRepository(Protocol):
|
| 35 |
+
def authenticate(self, username: str) -> User: ...
|
src/repositories/in_memory_detection_repository.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import defaultdict
|
| 2 |
+
from typing import DefaultDict
|
| 3 |
+
|
| 4 |
+
from src.domain.models import Detection
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class InMemoryDetectionRepository:
|
| 8 |
+
def __init__(self) -> None:
|
| 9 |
+
self._project_items: DefaultDict[str, list[Detection]] = defaultdict(list)
|
| 10 |
+
|
| 11 |
+
def seed(self, project_slug: str, detections: list[Detection]) -> None:
|
| 12 |
+
self._project_items[project_slug] = list(detections)
|
| 13 |
+
|
| 14 |
+
def list_detections(
|
| 15 |
+
self,
|
| 16 |
+
project_slug: str,
|
| 17 |
+
page: int,
|
| 18 |
+
page_size: int,
|
| 19 |
+
scientific_name: str | None = None,
|
| 20 |
+
min_confidence: float | None = None,
|
| 21 |
+
max_confidence: float | None = None,
|
| 22 |
+
) -> list[Detection]:
|
| 23 |
+
if page < 1:
|
| 24 |
+
raise ValueError("page must be >= 1")
|
| 25 |
+
if page_size < 1:
|
| 26 |
+
raise ValueError("page_size must be >= 1")
|
| 27 |
+
|
| 28 |
+
filtered = self._apply_filters(
|
| 29 |
+
project_slug=project_slug,
|
| 30 |
+
scientific_name=scientific_name,
|
| 31 |
+
min_confidence=min_confidence,
|
| 32 |
+
max_confidence=max_confidence,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
start = (page - 1) * page_size
|
| 36 |
+
end = start + page_size
|
| 37 |
+
return filtered[start:end]
|
| 38 |
+
|
| 39 |
+
def count_detections(
|
| 40 |
+
self,
|
| 41 |
+
project_slug: str,
|
| 42 |
+
scientific_name: str | None = None,
|
| 43 |
+
min_confidence: float | None = None,
|
| 44 |
+
max_confidence: float | None = None,
|
| 45 |
+
) -> int:
|
| 46 |
+
return len(
|
| 47 |
+
self._apply_filters(
|
| 48 |
+
project_slug=project_slug,
|
| 49 |
+
scientific_name=scientific_name,
|
| 50 |
+
min_confidence=min_confidence,
|
| 51 |
+
max_confidence=max_confidence,
|
| 52 |
+
)
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
def _apply_filters(
|
| 56 |
+
self,
|
| 57 |
+
project_slug: str,
|
| 58 |
+
scientific_name: str | None,
|
| 59 |
+
min_confidence: float | None,
|
| 60 |
+
max_confidence: float | None,
|
| 61 |
+
) -> list[Detection]:
|
| 62 |
+
items = self._project_items.get(project_slug, [])
|
| 63 |
+
|
| 64 |
+
def match(item: Detection) -> bool:
|
| 65 |
+
if scientific_name and item.scientific_name != scientific_name:
|
| 66 |
+
return False
|
| 67 |
+
if min_confidence is not None and item.confidence < min_confidence:
|
| 68 |
+
return False
|
| 69 |
+
if max_confidence is not None and item.confidence > max_confidence:
|
| 70 |
+
return False
|
| 71 |
+
return True
|
| 72 |
+
|
| 73 |
+
return [item for item in items if match(item)]
|
src/repositories/in_memory_validation_repository.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import defaultdict
|
| 2 |
+
from typing import DefaultDict
|
| 3 |
+
|
| 4 |
+
from src.domain.models import Validation
|
| 5 |
+
from src.repositories.append_only_validation_repository import OptimisticLockError
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class InMemoryValidationRepository:
|
| 9 |
+
def __init__(self) -> None:
|
| 10 |
+
self._by_project: DefaultDict[str, list[Validation]] = defaultdict(list)
|
| 11 |
+
self._versions: DefaultDict[str, dict[str, int]] = defaultdict(dict)
|
| 12 |
+
|
| 13 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 14 |
+
current_version = int(self._versions[project_slug].get(item.detection_key, 0))
|
| 15 |
+
expected = expected_version if expected_version is not None else current_version
|
| 16 |
+
if expected != current_version:
|
| 17 |
+
raise OptimisticLockError(item.detection_key, expected, current_version)
|
| 18 |
+
|
| 19 |
+
self._by_project[project_slug].append(item)
|
| 20 |
+
new_version = current_version + 1
|
| 21 |
+
self._versions[project_slug][item.detection_key] = new_version
|
| 22 |
+
return new_version
|
| 23 |
+
|
| 24 |
+
def list_validations(self, project_slug: str) -> list[Validation]:
|
| 25 |
+
return list(self._by_project.get(project_slug, []))
|
src/services/__init__.py
ADDED
|
File without changes
|
src/services/audio_fetch_service.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
import io
|
| 3 |
+
import math
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import struct
|
| 6 |
+
import wave
|
| 7 |
+
|
| 8 |
+
from huggingface_hub import hf_hub_download
|
| 9 |
+
|
| 10 |
+
from src.cache.ephemeral_cache_manager import EphemeralCacheManager
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
SUPPORTED_AUDIO_EXTENSIONS = (".wav", ".mp3", ".flac", ".ogg", ".m4a")
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass(slots=True)
|
| 17 |
+
class AudioFetchResult:
|
| 18 |
+
cache_key: str
|
| 19 |
+
local_path: str
|
| 20 |
+
source: str
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class AudioFetchService:
|
| 24 |
+
def __init__(self, cache_manager: EphemeralCacheManager) -> None:
|
| 25 |
+
self._cache = cache_manager
|
| 26 |
+
|
| 27 |
+
def fetch(
|
| 28 |
+
self,
|
| 29 |
+
dataset_repo: str,
|
| 30 |
+
audio_id: str,
|
| 31 |
+
allow_demo_fallback: bool = False,
|
| 32 |
+
hf_token: str | None = None,
|
| 33 |
+
) -> AudioFetchResult:
|
| 34 |
+
cache_key = f"{dataset_repo}:{audio_id}"
|
| 35 |
+
cached_path = self._cache.get(cache_key)
|
| 36 |
+
if cached_path:
|
| 37 |
+
return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="cache")
|
| 38 |
+
|
| 39 |
+
if allow_demo_fallback and self._is_seeded_demo_audio_id(audio_id):
|
| 40 |
+
fallback_bytes = self._build_demo_fallback_wav(audio_id=audio_id)
|
| 41 |
+
local_path = self._cache.put_bytes(cache_key, fallback_bytes, suffix=".wav")
|
| 42 |
+
return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="demo-fallback")
|
| 43 |
+
|
| 44 |
+
try:
|
| 45 |
+
target_filename, downloaded_path = self._resolve_remote_filename(
|
| 46 |
+
dataset_repo=dataset_repo,
|
| 47 |
+
audio_id=audio_id,
|
| 48 |
+
hf_token=hf_token,
|
| 49 |
+
)
|
| 50 |
+
except FileNotFoundError:
|
| 51 |
+
if not allow_demo_fallback:
|
| 52 |
+
raise
|
| 53 |
+
|
| 54 |
+
fallback_bytes = self._build_demo_fallback_wav(audio_id=audio_id)
|
| 55 |
+
local_path = self._cache.put_bytes(cache_key, fallback_bytes, suffix=".wav")
|
| 56 |
+
return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="demo-fallback")
|
| 57 |
+
|
| 58 |
+
if downloaded_path is None:
|
| 59 |
+
downloaded = self._download_dataset_file(
|
| 60 |
+
dataset_repo=dataset_repo,
|
| 61 |
+
filename=target_filename,
|
| 62 |
+
hf_token=hf_token,
|
| 63 |
+
)
|
| 64 |
+
downloaded_path = Path(downloaded)
|
| 65 |
+
|
| 66 |
+
suffix = downloaded_path.suffix if downloaded_path.suffix else ".bin"
|
| 67 |
+
local_path = self._cache.put_bytes(cache_key, downloaded_path.read_bytes(), suffix=suffix)
|
| 68 |
+
return AudioFetchResult(cache_key=cache_key, local_path=str(local_path), source="remote")
|
| 69 |
+
|
| 70 |
+
def fetch_local(self, local_audio_path: str) -> AudioFetchResult:
|
| 71 |
+
"""Load audio from local filesystem (useful for testing with local segments).
|
| 72 |
+
|
| 73 |
+
Args:
|
| 74 |
+
local_audio_path: Full path to local audio file
|
| 75 |
+
|
| 76 |
+
Returns:
|
| 77 |
+
AudioFetchResult with cache_key based on file path
|
| 78 |
+
|
| 79 |
+
Raises:
|
| 80 |
+
FileNotFoundError: If local file doesn't exist or is not a supported audio format
|
| 81 |
+
"""
|
| 82 |
+
audio_path = Path(local_audio_path)
|
| 83 |
+
if not audio_path.exists():
|
| 84 |
+
raise FileNotFoundError(f"Local audio file not found: {local_audio_path}")
|
| 85 |
+
|
| 86 |
+
if audio_path.suffix.lower() not in SUPPORTED_AUDIO_EXTENSIONS:
|
| 87 |
+
raise ValueError(f"Unsupported audio format: {audio_path.suffix}. Supported: {SUPPORTED_AUDIO_EXTENSIONS}")
|
| 88 |
+
|
| 89 |
+
cache_key = f"local:{audio_path.name}:{audio_path.stat().st_size}"
|
| 90 |
+
cached_path = self._cache.get(cache_key)
|
| 91 |
+
if cached_path:
|
| 92 |
+
return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="cache")
|
| 93 |
+
|
| 94 |
+
# Cache the local file by copying it into the ephemeral cache
|
| 95 |
+
local_bytes = audio_path.read_bytes()
|
| 96 |
+
cached_path = self._cache.put_bytes(cache_key, local_bytes, suffix=audio_path.suffix)
|
| 97 |
+
return AudioFetchResult(cache_key=cache_key, local_path=str(cached_path), source="local")
|
| 98 |
+
|
| 99 |
+
def cleanup_after_validation(self, cache_key: str) -> None:
|
| 100 |
+
self._cache.cleanup_key(cache_key)
|
| 101 |
+
|
| 102 |
+
def _resolve_remote_filename(
|
| 103 |
+
self,
|
| 104 |
+
dataset_repo: str,
|
| 105 |
+
audio_id: str,
|
| 106 |
+
hf_token: str | None = None,
|
| 107 |
+
) -> tuple[str, Path | None]:
|
| 108 |
+
audio_path = Path(audio_id)
|
| 109 |
+
if audio_path.suffix:
|
| 110 |
+
return f"audio/{audio_id}", None
|
| 111 |
+
|
| 112 |
+
for extension in SUPPORTED_AUDIO_EXTENSIONS:
|
| 113 |
+
candidate = f"audio/{audio_id}{extension}"
|
| 114 |
+
try:
|
| 115 |
+
downloaded = self._download_dataset_file(
|
| 116 |
+
dataset_repo=dataset_repo,
|
| 117 |
+
filename=candidate,
|
| 118 |
+
hf_token=hf_token,
|
| 119 |
+
)
|
| 120 |
+
return candidate, Path(downloaded)
|
| 121 |
+
except Exception:
|
| 122 |
+
continue
|
| 123 |
+
|
| 124 |
+
raise FileNotFoundError(f"Unable to locate audio file for audio_id: {audio_id}")
|
| 125 |
+
|
| 126 |
+
def _build_demo_fallback_wav(self, audio_id: str, duration_seconds: float = 4.0, sample_rate: int = 16000) -> bytes:
|
| 127 |
+
"""Generate a short deterministic tone when demo audio files are missing."""
|
| 128 |
+
frame_count = max(1, int(duration_seconds * sample_rate))
|
| 129 |
+
|
| 130 |
+
# Derive deterministic frequencies from audio_id so each fallback sounds distinct.
|
| 131 |
+
seed = sum(ord(ch) for ch in audio_id) % 200
|
| 132 |
+
f1 = 440.0 + float(seed)
|
| 133 |
+
f2 = f1 + 180.0
|
| 134 |
+
|
| 135 |
+
with io.BytesIO() as buffer:
|
| 136 |
+
with wave.open(buffer, "wb") as wav_file:
|
| 137 |
+
wav_file.setnchannels(1)
|
| 138 |
+
wav_file.setsampwidth(2)
|
| 139 |
+
wav_file.setframerate(sample_rate)
|
| 140 |
+
|
| 141 |
+
frames = bytearray()
|
| 142 |
+
for i in range(frame_count):
|
| 143 |
+
t = i / float(sample_rate)
|
| 144 |
+
# Blend two sines and apply light envelope to avoid clicks.
|
| 145 |
+
env = min(1.0, i / 400.0, (frame_count - i) / 400.0)
|
| 146 |
+
sample = env * (0.55 * math.sin(2.0 * math.pi * f1 * t) + 0.45 * math.sin(2.0 * math.pi * f2 * t))
|
| 147 |
+
pcm = int(max(-1.0, min(1.0, sample)) * 32767)
|
| 148 |
+
frames.extend(struct.pack("<h", pcm))
|
| 149 |
+
|
| 150 |
+
wav_file.writeframes(bytes(frames))
|
| 151 |
+
|
| 152 |
+
return buffer.getvalue()
|
| 153 |
+
|
| 154 |
+
def _is_seeded_demo_audio_id(self, audio_id: str) -> bool:
|
| 155 |
+
normalized = audio_id.strip().lower()
|
| 156 |
+
return "_audio_" in normalized and normalized.endswith(("1001", "1002", "1003", "1004"))
|
| 157 |
+
|
| 158 |
+
def _download_dataset_file(self, dataset_repo: str, filename: str, hf_token: str | None = None) -> str:
|
| 159 |
+
if hf_token:
|
| 160 |
+
return hf_hub_download(
|
| 161 |
+
repo_id=dataset_repo,
|
| 162 |
+
repo_type="dataset",
|
| 163 |
+
filename=filename,
|
| 164 |
+
token=hf_token,
|
| 165 |
+
)
|
| 166 |
+
return hf_hub_download(
|
| 167 |
+
repo_id=dataset_repo,
|
| 168 |
+
repo_type="dataset",
|
| 169 |
+
filename=filename,
|
| 170 |
+
)
|
src/services/detection_queue_service.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
|
| 3 |
+
from src.domain.models import Detection
|
| 4 |
+
from src.repositories.contracts import DetectionRepository
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass(slots=True)
|
| 8 |
+
class DetectionPage:
|
| 9 |
+
project_slug: str
|
| 10 |
+
page: int
|
| 11 |
+
page_size: int
|
| 12 |
+
total_items: int
|
| 13 |
+
total_pages: int
|
| 14 |
+
has_next: bool
|
| 15 |
+
has_previous: bool
|
| 16 |
+
items: list[Detection]
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class DetectionQueueService:
|
| 20 |
+
def __init__(self, repository: DetectionRepository) -> None:
|
| 21 |
+
self._repository = repository
|
| 22 |
+
|
| 23 |
+
def get_page(
|
| 24 |
+
self,
|
| 25 |
+
project_slug: str,
|
| 26 |
+
page: int,
|
| 27 |
+
page_size: int,
|
| 28 |
+
scientific_name: str | None = None,
|
| 29 |
+
min_confidence: float | None = None,
|
| 30 |
+
max_confidence: float | None = None,
|
| 31 |
+
) -> DetectionPage:
|
| 32 |
+
if page < 1:
|
| 33 |
+
raise ValueError("page must be >= 1")
|
| 34 |
+
if page_size < 1:
|
| 35 |
+
raise ValueError("page_size must be >= 1")
|
| 36 |
+
|
| 37 |
+
total_items = self._repository.count_detections(
|
| 38 |
+
project_slug=project_slug,
|
| 39 |
+
scientific_name=scientific_name,
|
| 40 |
+
min_confidence=min_confidence,
|
| 41 |
+
max_confidence=max_confidence,
|
| 42 |
+
)
|
| 43 |
+
total_pages = max(1, ((total_items - 1) // page_size) + 1) if total_items else 1
|
| 44 |
+
page = min(page, total_pages)
|
| 45 |
+
|
| 46 |
+
items = self._repository.list_detections(
|
| 47 |
+
project_slug=project_slug,
|
| 48 |
+
page=page,
|
| 49 |
+
page_size=page_size,
|
| 50 |
+
scientific_name=scientific_name,
|
| 51 |
+
min_confidence=min_confidence,
|
| 52 |
+
max_confidence=max_confidence,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
return DetectionPage(
|
| 56 |
+
project_slug=project_slug,
|
| 57 |
+
page=page,
|
| 58 |
+
page_size=page_size,
|
| 59 |
+
total_items=total_items,
|
| 60 |
+
total_pages=total_pages,
|
| 61 |
+
has_next=page < total_pages,
|
| 62 |
+
has_previous=page > 1,
|
| 63 |
+
items=items,
|
| 64 |
+
)
|
src/services/invite_email_notifier.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Email delivery via EmailJS for project collaboration invites."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
import json
|
| 6 |
+
from urllib import request
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@dataclass(frozen=True)
|
| 10 |
+
class InviteEmailPayload:
|
| 11 |
+
"""Payload for EmailJS invite notification.
|
| 12 |
+
|
| 13 |
+
At least one of invitee_username or invitee_email must be present.
|
| 14 |
+
- Only username: internal app invite only (no email sent).
|
| 15 |
+
- Only email: external email-based invite (no prior HF account required).
|
| 16 |
+
- Both: dual invite (internal + email).
|
| 17 |
+
"""
|
| 18 |
+
project_slug: str
|
| 19 |
+
role: str
|
| 20 |
+
invited_by: str
|
| 21 |
+
expires_at: datetime
|
| 22 |
+
login_url: str
|
| 23 |
+
invitee_username: str | None = None
|
| 24 |
+
invitee_email: str | None = None
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class InviteEmailNotifier:
|
| 28 |
+
"""Abstract notifier interface for invite emails."""
|
| 29 |
+
|
| 30 |
+
def send(self, payload: InviteEmailPayload) -> tuple[bool, str]:
|
| 31 |
+
"""Send an invite notification. Returns (success, message)."""
|
| 32 |
+
raise NotImplementedError
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class EmailJSInviteEmailNotifier(InviteEmailNotifier):
|
| 36 |
+
"""EmailJS-backed notifier for invite emails (only transport available)."""
|
| 37 |
+
|
| 38 |
+
def __init__(
|
| 39 |
+
self,
|
| 40 |
+
sender_email: str,
|
| 41 |
+
service_id: str,
|
| 42 |
+
template_id: str,
|
| 43 |
+
public_key: str,
|
| 44 |
+
template_id_username_only: str | None = None,
|
| 45 |
+
template_id_email_only: str | None = None,
|
| 46 |
+
template_id_dual: str | None = None,
|
| 47 |
+
endpoint: str = "https://api.emailjs.com/api/v1.0/email/send",
|
| 48 |
+
timeout_seconds: int = 20,
|
| 49 |
+
):
|
| 50 |
+
self._sender_email = sender_email
|
| 51 |
+
self._service_id = service_id
|
| 52 |
+
self._template_id = template_id
|
| 53 |
+
self._public_key = public_key
|
| 54 |
+
self._template_id_username_only = (template_id_username_only or "").strip() or None
|
| 55 |
+
self._template_id_email_only = (template_id_email_only or "").strip() or None
|
| 56 |
+
self._template_id_dual = (template_id_dual or "").strip() or None
|
| 57 |
+
self._endpoint = endpoint
|
| 58 |
+
self._timeout_seconds = max(5, int(timeout_seconds))
|
| 59 |
+
|
| 60 |
+
def _resolve_invite_mode(self, payload: InviteEmailPayload) -> str:
|
| 61 |
+
if payload.invitee_username and payload.invitee_email:
|
| 62 |
+
return "dual"
|
| 63 |
+
if payload.invitee_email:
|
| 64 |
+
return "email_only"
|
| 65 |
+
return "username_only"
|
| 66 |
+
|
| 67 |
+
def _resolve_template_id(self, mode: str) -> str:
|
| 68 |
+
if mode == "username_only" and self._template_id_username_only:
|
| 69 |
+
return self._template_id_username_only
|
| 70 |
+
if mode == "email_only" and self._template_id_email_only:
|
| 71 |
+
return self._template_id_email_only
|
| 72 |
+
if mode == "dual" and self._template_id_dual:
|
| 73 |
+
return self._template_id_dual
|
| 74 |
+
return self._template_id
|
| 75 |
+
|
| 76 |
+
def send(self, payload: InviteEmailPayload) -> tuple[bool, str]:
|
| 77 |
+
"""Send invite email via EmailJS.
|
| 78 |
+
|
| 79 |
+
If invitee_email is missing, returns success with message indicating internal-only invite.
|
| 80 |
+
"""
|
| 81 |
+
invite_mode = self._resolve_invite_mode(payload)
|
| 82 |
+
if not payload.invitee_email:
|
| 83 |
+
return True, f"✅ Internal invite created for {payload.invitee_username or 'pending'}"
|
| 84 |
+
|
| 85 |
+
template_id = self._resolve_template_id(invite_mode)
|
| 86 |
+
if not template_id:
|
| 87 |
+
return False, "Internal invite created, but email send failed: EmailJS template ID is not configured"
|
| 88 |
+
|
| 89 |
+
template_params = {
|
| 90 |
+
"from_name": self._sender_email,
|
| 91 |
+
"to_email": payload.invitee_email,
|
| 92 |
+
"invitee_username": payload.invitee_username or "(to be claimed)",
|
| 93 |
+
"project_slug": payload.project_slug,
|
| 94 |
+
"role": payload.role,
|
| 95 |
+
"invited_by": payload.invited_by,
|
| 96 |
+
"expires_at": payload.expires_at.isoformat(),
|
| 97 |
+
"login_url": payload.login_url or "",
|
| 98 |
+
"invite_link": payload.login_url or "",
|
| 99 |
+
"invite_mode": invite_mode,
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
req_payload = {
|
| 103 |
+
"service_id": self._service_id,
|
| 104 |
+
"template_id": template_id,
|
| 105 |
+
"user_id": self._public_key,
|
| 106 |
+
"template_params": template_params,
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
try:
|
| 110 |
+
req = request.Request(
|
| 111 |
+
self._endpoint,
|
| 112 |
+
data=json.dumps(req_payload).encode("utf-8"),
|
| 113 |
+
headers={"Content-Type": "application/json"},
|
| 114 |
+
method="POST",
|
| 115 |
+
)
|
| 116 |
+
with request.urlopen(req, timeout=self._timeout_seconds) as response:
|
| 117 |
+
status_code = int(getattr(response, "status", 0) or response.getcode())
|
| 118 |
+
if 200 <= status_code < 300:
|
| 119 |
+
return True, f"✅ Invite sent to {payload.invitee_email}"
|
| 120 |
+
return False, f"Internal invite created, but email send failed (status {status_code})"
|
| 121 |
+
except Exception as exc:
|
| 122 |
+
return False, f"Internal invite created, but email send failed: {exc}"
|
src/services/validation_service.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
|
| 3 |
+
from src.domain.models import Validation
|
| 4 |
+
from src.repositories.contracts import ValidationRepository
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass(slots=True)
|
| 8 |
+
class ValidationWriteResult:
|
| 9 |
+
item: Validation
|
| 10 |
+
new_version: int
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ValidationService:
|
| 14 |
+
def __init__(self, repository: ValidationRepository) -> None:
|
| 15 |
+
self._repository = repository
|
| 16 |
+
|
| 17 |
+
def validate_detection(
|
| 18 |
+
self,
|
| 19 |
+
project_slug: str,
|
| 20 |
+
detection_key: str,
|
| 21 |
+
status: str,
|
| 22 |
+
validator: str,
|
| 23 |
+
notes: str = "",
|
| 24 |
+
corrected_species: str | None = None,
|
| 25 |
+
expected_version: int | None = None,
|
| 26 |
+
) -> ValidationWriteResult:
|
| 27 |
+
item = Validation(
|
| 28 |
+
detection_key=detection_key,
|
| 29 |
+
status=status,
|
| 30 |
+
corrected_species=corrected_species,
|
| 31 |
+
notes=notes,
|
| 32 |
+
validator=validator,
|
| 33 |
+
)
|
| 34 |
+
new_version = self._repository.save_validation(
|
| 35 |
+
project_slug=project_slug,
|
| 36 |
+
item=item,
|
| 37 |
+
expected_version=expected_version,
|
| 38 |
+
)
|
| 39 |
+
return ValidationWriteResult(item=item, new_version=new_version)
|
src/telemetry/__init__.py
ADDED
|
File without changes
|
src/ui/__pycache__/__init__.cpython-313.pyc
CHANGED
|
Binary files a/src/ui/__pycache__/__init__.cpython-313.pyc and b/src/ui/__pycache__/__init__.cpython-313.pyc differ
|
|
|
src/ui/__pycache__/__init__.cpython-314.pyc
CHANGED
|
Binary files a/src/ui/__pycache__/__init__.cpython-314.pyc and b/src/ui/__pycache__/__init__.cpython-314.pyc differ
|
|
|
src/ui/__pycache__/upload_app_factory.cpython-313.pyc
DELETED
|
Binary file (41.2 kB)
|
|
|