Spaces:
Sleeping
Sleeping
Commit ·
4e31d1f
1
Parent(s): f961d5d
Import uploader files from validator workspace
Browse files- README.md +6 -6
- app.py +3 -2
- birdnet-uploader.spec +38 -0
- build/build_exe.py +34 -0
- pyproject.toml +3 -3
- requirements.txt +1 -0
- src/uploader/__init__.py +17 -0
- src/{uploader_cli → uploader}/auth_service.py +4 -6
- src/{uploader_cli → uploader}/batch_uploader.py +0 -11
- src/{uploader_cli → uploader}/config.py +0 -1
- src/{uploader_cli → uploader}/deduplicator.py +0 -0
- src/{uploader_cli → uploader}/error_handler.py +0 -0
- src/{uploader_cli → uploader}/exceptions.py +0 -0
- src/{uploader_cli → uploader}/hash_utils.py +0 -0
- src/{uploader_cli → uploader}/main.py +11 -21
- src/{uploader_cli → uploader}/manifest.py +0 -11
- src/{uploader_cli → uploader}/repo_service.py +0 -0
- src/{uploader_cli → uploader}/scanner.py +1 -2
- src/{uploader_cli → uploader}/session_manager.py +0 -0
- src/{uploader_cli → uploader}/tui.py +0 -0
- src/uploader/uploader/__init__.py +17 -0
- src/uploader/uploader/auth_service.py +60 -0
- src/uploader/uploader/batch_uploader.py +102 -0
- src/uploader/uploader/config.py +61 -0
- src/uploader/uploader/deduplicator.py +115 -0
- src/uploader/uploader/error_handler.py +29 -0
- src/uploader/uploader/exceptions.py +26 -0
- src/uploader/uploader/hash_utils.py +21 -0
- src/uploader/uploader/main.py +107 -0
- src/uploader/uploader/manifest.py +58 -0
- src/uploader/uploader/repo_service.py +110 -0
- src/uploader/uploader/scanner.py +62 -0
- src/uploader/uploader/session_manager.py +98 -0
- src/uploader/uploader/tui.py +22 -0
- src/uploader/web_ui.py +193 -0
- src/uploader_cli/__init__.py +0 -7
README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
---
|
| 2 |
-
title: BirdNET-
|
| 3 |
emoji: "🐦"
|
| 4 |
colorFrom: green
|
| 5 |
colorTo: blue
|
|
@@ -10,14 +10,14 @@ app_file: app.py
|
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
-
# BirdNET-
|
| 14 |
|
| 15 |
-
BirdNET-
|
| 16 |
|
| 17 |
-
|
| 18 |
-
|
| 19 |
|
| 20 |
-
|
| 21 |
|
| 22 |
## Contents
|
| 23 |
|
|
|
|
| 1 |
---
|
| 2 |
+
title: BirdNET-Uploader-App
|
| 3 |
emoji: "🐦"
|
| 4 |
colorFrom: green
|
| 5 |
colorTo: blue
|
|
|
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
+
# BirdNET-Uploader-App
|
| 14 |
|
| 15 |
+
BirdNET-Uploader-App is a focused tool for reliably uploading large audio collections and optional detection CSVs to Hugging Face datasets. It provides:
|
| 16 |
|
| 17 |
+
- A unified Gradio web interface for small-to-medium uploads (single flow; CSV optional and auto-detected)
|
| 18 |
+
- A Windows standalone container (downloadable) for large uploads without installing Python
|
| 19 |
|
| 20 |
+
This repository contains a resumable uploader with deduplication, manifest/shard generation, and a simple web UI for convenience.
|
| 21 |
|
| 22 |
## Contents
|
| 23 |
|
app.py
CHANGED
|
@@ -1,10 +1,11 @@
|
|
| 1 |
import os
|
| 2 |
|
| 3 |
-
|
|
|
|
| 4 |
|
| 5 |
|
| 6 |
if __name__ == "__main__":
|
| 7 |
-
app =
|
| 8 |
port = int(os.getenv("PORT") or os.getenv("BIRDNET_VALIDATOR_PORT") or "7860")
|
| 9 |
host = os.getenv("BIRDNET_VALIDATOR_HOST") or "0.0.0.0"
|
| 10 |
app.launch(server_name=host, server_port=port)
|
|
|
|
| 1 |
import os
|
| 2 |
|
| 3 |
+
# New uploader web UI entry
|
| 4 |
+
from src.uploader.web_ui import create_uploader_app
|
| 5 |
|
| 6 |
|
| 7 |
if __name__ == "__main__":
|
| 8 |
+
app = create_uploader_app()
|
| 9 |
port = int(os.getenv("PORT") or os.getenv("BIRDNET_VALIDATOR_PORT") or "7860")
|
| 10 |
host = os.getenv("BIRDNET_VALIDATOR_HOST") or "0.0.0.0"
|
| 11 |
app.launch(server_name=host, server_port=port)
|
birdnet-uploader.spec
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- mode: python ; coding: utf-8 -*-
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
a = Analysis(
|
| 5 |
+
['C:\\Users\\jonat\\Documents\\Python\\BirdNET-validator-App\\app.py'],
|
| 6 |
+
pathex=[],
|
| 7 |
+
binaries=[],
|
| 8 |
+
datas=[],
|
| 9 |
+
hiddenimports=[],
|
| 10 |
+
hookspath=[],
|
| 11 |
+
hooksconfig={},
|
| 12 |
+
runtime_hooks=[],
|
| 13 |
+
excludes=[],
|
| 14 |
+
noarchive=False,
|
| 15 |
+
optimize=0,
|
| 16 |
+
)
|
| 17 |
+
pyz = PYZ(a.pure)
|
| 18 |
+
|
| 19 |
+
exe = EXE(
|
| 20 |
+
pyz,
|
| 21 |
+
a.scripts,
|
| 22 |
+
a.binaries,
|
| 23 |
+
a.datas,
|
| 24 |
+
[],
|
| 25 |
+
name='birdnet-uploader',
|
| 26 |
+
debug=False,
|
| 27 |
+
bootloader_ignore_signals=False,
|
| 28 |
+
strip=False,
|
| 29 |
+
upx=True,
|
| 30 |
+
upx_exclude=[],
|
| 31 |
+
runtime_tmpdir=None,
|
| 32 |
+
console=True,
|
| 33 |
+
disable_windowed_traceback=False,
|
| 34 |
+
argv_emulation=False,
|
| 35 |
+
target_arch=None,
|
| 36 |
+
codesign_identity=None,
|
| 37 |
+
entitlements_file=None,
|
| 38 |
+
)
|
build/build_exe.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import sys
|
| 4 |
+
import subprocess
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 8 |
+
ENTRY = ROOT / "app.py"
|
| 9 |
+
DIST = ROOT / "dist"
|
| 10 |
+
|
| 11 |
+
def main():
|
| 12 |
+
if not ENTRY.exists():
|
| 13 |
+
print("Entrypoint app.py not found; aborting.")
|
| 14 |
+
raise SystemExit(1)
|
| 15 |
+
|
| 16 |
+
cmd = [
|
| 17 |
+
sys.executable,
|
| 18 |
+
"-m",
|
| 19 |
+
"PyInstaller",
|
| 20 |
+
"--clean",
|
| 21 |
+
"--onefile",
|
| 22 |
+
"--noconfirm",
|
| 23 |
+
"--name",
|
| 24 |
+
"birdnet-uploader",
|
| 25 |
+
str(ENTRY),
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
print("Running:", " ".join(cmd))
|
| 29 |
+
subprocess.check_call(cmd)
|
| 30 |
+
print("Build finished. See dist/ for output.")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
if __name__ == "__main__":
|
| 34 |
+
main()
|
pyproject.toml
CHANGED
|
@@ -1,12 +1,12 @@
|
|
| 1 |
[project]
|
| 2 |
-
name = "BirdNET-
|
| 3 |
version = "0.1.0"
|
| 4 |
-
description = "BirdNET-
|
| 5 |
readme = "README.md"
|
| 6 |
requires-python = ">=3.11"
|
| 7 |
|
| 8 |
[project.scripts]
|
| 9 |
-
birdnet-uploader = "src.
|
| 10 |
|
| 11 |
[tool.pytest.ini_options]
|
| 12 |
testpaths = ["tests"]
|
|
|
|
| 1 |
[project]
|
| 2 |
+
name = "BirdNET-Uploader-App"
|
| 3 |
version = "0.1.0"
|
| 4 |
+
description = "BirdNET-Uploader-App: resumable uploader for large audio datasets to Hugging Face"
|
| 5 |
readme = "README.md"
|
| 6 |
requires-python = ">=3.11"
|
| 7 |
|
| 8 |
[project.scripts]
|
| 9 |
+
birdnet-uploader = "src.uploader.main:cli"
|
| 10 |
|
| 11 |
[tool.pytest.ini_options]
|
| 12 |
testpaths = ["tests"]
|
requirements.txt
CHANGED
|
@@ -10,3 +10,4 @@ click>=8.1.7
|
|
| 10 |
rich>=13.7.1
|
| 11 |
textual>=0.58.1
|
| 12 |
keyring>=25.2.1
|
|
|
|
|
|
| 10 |
rich>=13.7.1
|
| 11 |
textual>=0.58.1
|
| 12 |
keyring>=25.2.1
|
| 13 |
+
pyinstaller>=6.20.0
|
src/uploader/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""BirdNET Uploader package (refactored from uploader_cli)."""
|
| 2 |
+
|
| 3 |
+
__all__ = [
|
| 4 |
+
"auth_service",
|
| 5 |
+
"batch_uploader",
|
| 6 |
+
"config",
|
| 7 |
+
"deduplicator",
|
| 8 |
+
"error_handler",
|
| 9 |
+
"exceptions",
|
| 10 |
+
"hash_utils",
|
| 11 |
+
"main",
|
| 12 |
+
"manifest",
|
| 13 |
+
"repo_service",
|
| 14 |
+
"scanner",
|
| 15 |
+
"session_manager",
|
| 16 |
+
"tui",
|
| 17 |
+
]
|
src/{uploader_cli → uploader}/auth_service.py
RENAMED
|
@@ -12,10 +12,10 @@ from .exceptions import AuthenticationError
|
|
| 12 |
|
| 13 |
class AuthService:
|
| 14 |
"""Authenticate and securely store Hugging Face tokens.
|
| 15 |
-
|
| 16 |
Token resolution order (highest to lowest priority):
|
| 17 |
1. HF_TOKEN environment variable (useful for container/CI)
|
| 18 |
-
2. keyring (OS-specific secure storage
|
| 19 |
3. Manual --token CLI option
|
| 20 |
"""
|
| 21 |
|
|
@@ -46,15 +46,13 @@ class AuthService:
|
|
| 46 |
|
| 47 |
def get_token(self) -> str | None:
|
| 48 |
"""Retrieve token from environment or secure storage.
|
| 49 |
-
|
| 50 |
Priority: HF_TOKEN env var > keyring storage.
|
| 51 |
"""
|
| 52 |
-
# First check environment variable (useful for Docker/CI)
|
| 53 |
env_token = os.getenv(TOKEN_ENV_VAR)
|
| 54 |
if env_token:
|
| 55 |
return env_token.strip()
|
| 56 |
-
|
| 57 |
-
# Fall back to keyring (OS-specific secure storage)
|
| 58 |
try:
|
| 59 |
return keyring.get_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
|
| 60 |
except Exception as exc: # pragma: no cover - backend-specific behavior
|
|
|
|
| 12 |
|
| 13 |
class AuthService:
|
| 14 |
"""Authenticate and securely store Hugging Face tokens.
|
| 15 |
+
|
| 16 |
Token resolution order (highest to lowest priority):
|
| 17 |
1. HF_TOKEN environment variable (useful for container/CI)
|
| 18 |
+
2. keyring (OS-specific secure storage)
|
| 19 |
3. Manual --token CLI option
|
| 20 |
"""
|
| 21 |
|
|
|
|
| 46 |
|
| 47 |
def get_token(self) -> str | None:
|
| 48 |
"""Retrieve token from environment or secure storage.
|
| 49 |
+
|
| 50 |
Priority: HF_TOKEN env var > keyring storage.
|
| 51 |
"""
|
|
|
|
| 52 |
env_token = os.getenv(TOKEN_ENV_VAR)
|
| 53 |
if env_token:
|
| 54 |
return env_token.strip()
|
| 55 |
+
|
|
|
|
| 56 |
try:
|
| 57 |
return keyring.get_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
|
| 58 |
except Exception as exc: # pragma: no cover - backend-specific behavior
|
src/{uploader_cli → uploader}/batch_uploader.py
RENAMED
|
@@ -40,8 +40,6 @@ class BatchUploader:
|
|
| 40 |
last_exc: Exception | None = None
|
| 41 |
while attempts <= self.max_retries:
|
| 42 |
try:
|
| 43 |
-
# The HfApi.upload_file signature varies across versions; callers/tests
|
| 44 |
-
# should accept the positional usage shown here and monkeypatch in tests.
|
| 45 |
self._api.upload_file(path_or_file=local_path, path_in_repo=remote_path, repo_id=self.repo_id)
|
| 46 |
return
|
| 47 |
except Exception as exc: # pragma: no cover - network behavior
|
|
@@ -60,16 +58,11 @@ class BatchUploader:
|
|
| 60 |
batch_size: int | None = None,
|
| 61 |
on_progress: Callable[[dict[str, Any]], None] | None = None,
|
| 62 |
) -> dict[str, Any]:
|
| 63 |
-
"""Upload an iterable of file info dictionaries.
|
| 64 |
-
|
| 65 |
-
Each file_info must include `full_path` and `relative_path`. `size` is optional.
|
| 66 |
-
"""
|
| 67 |
batch_size = batch_size or MAX_BATCH_SIZE
|
| 68 |
uploaded = 0
|
| 69 |
skipped = 0
|
| 70 |
failed = 0
|
| 71 |
|
| 72 |
-
# First pass: decide which files to skip and which to upload
|
| 73 |
to_upload: List[Tuple[str, str, int]] = []
|
| 74 |
for info in file_infos:
|
| 75 |
full_path = str(info["full_path"])
|
|
@@ -87,7 +80,6 @@ class BatchUploader:
|
|
| 87 |
|
| 88 |
to_upload.append((full_path, remote_path, int(info.get("size", 0))))
|
| 89 |
|
| 90 |
-
# If no parallelism requested, perform sequential upload to preserve behavior
|
| 91 |
if not self.max_workers or self.max_workers <= 1:
|
| 92 |
for full_path, remote_path, size in to_upload:
|
| 93 |
try:
|
|
@@ -107,7 +99,6 @@ class BatchUploader:
|
|
| 107 |
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 108 |
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
| 109 |
|
| 110 |
-
# Parallel upload path
|
| 111 |
def _worker(task: Tuple[str, str, int]) -> Tuple[str, str]:
|
| 112 |
full_path, remote_path, size = task
|
| 113 |
try:
|
|
@@ -134,5 +125,3 @@ class BatchUploader:
|
|
| 134 |
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 135 |
|
| 136 |
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
| 137 |
-
|
| 138 |
-
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
|
|
|
| 40 |
last_exc: Exception | None = None
|
| 41 |
while attempts <= self.max_retries:
|
| 42 |
try:
|
|
|
|
|
|
|
| 43 |
self._api.upload_file(path_or_file=local_path, path_in_repo=remote_path, repo_id=self.repo_id)
|
| 44 |
return
|
| 45 |
except Exception as exc: # pragma: no cover - network behavior
|
|
|
|
| 58 |
batch_size: int | None = None,
|
| 59 |
on_progress: Callable[[dict[str, Any]], None] | None = None,
|
| 60 |
) -> dict[str, Any]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
batch_size = batch_size or MAX_BATCH_SIZE
|
| 62 |
uploaded = 0
|
| 63 |
skipped = 0
|
| 64 |
failed = 0
|
| 65 |
|
|
|
|
| 66 |
to_upload: List[Tuple[str, str, int]] = []
|
| 67 |
for info in file_infos:
|
| 68 |
full_path = str(info["full_path"])
|
|
|
|
| 80 |
|
| 81 |
to_upload.append((full_path, remote_path, int(info.get("size", 0))))
|
| 82 |
|
|
|
|
| 83 |
if not self.max_workers or self.max_workers <= 1:
|
| 84 |
for full_path, remote_path, size in to_upload:
|
| 85 |
try:
|
|
|
|
| 99 |
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 100 |
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
| 101 |
|
|
|
|
| 102 |
def _worker(task: Tuple[str, str, int]) -> Tuple[str, str]:
|
| 103 |
full_path, remote_path, size = task
|
| 104 |
try:
|
|
|
|
| 125 |
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 126 |
|
| 127 |
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
|
|
|
|
|
src/{uploader_cli → uploader}/config.py
RENAMED
|
@@ -28,7 +28,6 @@ def _resolve_path_from_env(env_name: str, default: Path) -> Path:
|
|
| 28 |
|
| 29 |
|
| 30 |
def get_session_root() -> Path:
|
| 31 |
-
"""Resolve and create the session storage root directory."""
|
| 32 |
data_root = os.getenv("BIRDNET_UPLOADER_DATA_DIR")
|
| 33 |
if data_root:
|
| 34 |
return _resolve_path_from_env("BIRDNET_UPLOADER_SESSION_DIR", Path(data_root).expanduser() / "sessions")
|
|
|
|
| 28 |
|
| 29 |
|
| 30 |
def get_session_root() -> Path:
|
|
|
|
| 31 |
data_root = os.getenv("BIRDNET_UPLOADER_DATA_DIR")
|
| 32 |
if data_root:
|
| 33 |
return _resolve_path_from_env("BIRDNET_UPLOADER_SESSION_DIR", Path(data_root).expanduser() / "sessions")
|
src/{uploader_cli → uploader}/deduplicator.py
RENAMED
|
File without changes
|
src/{uploader_cli → uploader}/error_handler.py
RENAMED
|
File without changes
|
src/{uploader_cli → uploader}/exceptions.py
RENAMED
|
File without changes
|
src/{uploader_cli → uploader}/hash_utils.py
RENAMED
|
File without changes
|
src/{uploader_cli → uploader}/main.py
RENAMED
|
@@ -7,11 +7,11 @@ from typing import Any, Callable
|
|
| 7 |
|
| 8 |
import click
|
| 9 |
|
| 10 |
-
from
|
| 11 |
-
from
|
| 12 |
-
from
|
| 13 |
-
from
|
| 14 |
-
from
|
| 15 |
|
| 16 |
|
| 17 |
def handle_cli_errors(fn: Callable[..., Any]) -> Callable[..., Any]:
|
|
@@ -124,7 +124,7 @@ def upload_cmd(
|
|
| 124 |
batch_size: int | None,
|
| 125 |
workers: int | None,
|
| 126 |
dry_run: bool,
|
| 127 |
-
)
|
| 128 |
"""Upload local segments (and optional CSV) into the HF dataset.
|
| 129 |
|
| 130 |
Token resolution (in order of priority):
|
|
@@ -134,13 +134,11 @@ def upload_cmd(
|
|
| 134 |
|
| 135 |
The command creates/uses a session checkpoint for resumable uploads.
|
| 136 |
"""
|
| 137 |
-
# Resolve token from CLI, env var, or secure storage
|
| 138 |
if token:
|
| 139 |
hf_token = token
|
| 140 |
else:
|
| 141 |
hf_token = AuthService().require_token()
|
| 142 |
|
| 143 |
-
# Scan local segments
|
| 144 |
scanner = LocalScanner()
|
| 145 |
summary = scanner.scan_folder(segments_dir)
|
| 146 |
total_files = summary["total_files"]
|
|
@@ -163,7 +161,6 @@ def upload_cmd(
|
|
| 163 |
click.echo(f"Would also upload CSV: {csv_file} -> index/detections.csv")
|
| 164 |
return
|
| 165 |
|
| 166 |
-
# Prepare API and session
|
| 167 |
api = None
|
| 168 |
try:
|
| 169 |
from huggingface_hub import HfApi
|
|
@@ -172,28 +169,23 @@ def upload_cmd(
|
|
| 172 |
except Exception as exc: # pragma: no cover - external
|
| 173 |
raise click.ClickException(build_error_message(exc)) from exc
|
| 174 |
|
| 175 |
-
# Validate dataset exists
|
| 176 |
repo_service = RepositoryService(hf_token)
|
| 177 |
validation = repo_service.validate_repo(repo_id)
|
| 178 |
if not validation.get("is_valid"):
|
| 179 |
click.echo(f"Warning: dataset {repo_id} may be missing structure: {validation.get('missing_prefixes')}")
|
| 180 |
|
| 181 |
-
|
| 182 |
-
from
|
| 183 |
-
from src.uploader_cli.batch_uploader import BatchUploader
|
| 184 |
|
| 185 |
dedup = Deduplicator(api=api, repo_id=repo_id)
|
| 186 |
session = SessionManager(session_id=session_id) if session_id else SessionManager()
|
| 187 |
uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_retries=None, initial_backoff=None, max_workers=workers)
|
| 188 |
|
| 189 |
-
# Flatten file infos
|
| 190 |
file_infos: list[dict] = []
|
| 191 |
for species, items in summary["by_species"].items():
|
| 192 |
for it in items:
|
| 193 |
-
# keep the relative path as scanned (preserves species directories)
|
| 194 |
file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
|
| 195 |
|
| 196 |
-
# Upload CSV first if present
|
| 197 |
if csv_file:
|
| 198 |
click.echo(f"Uploading CSV to index/detections.csv...")
|
| 199 |
try:
|
|
@@ -202,9 +194,8 @@ def upload_cmd(
|
|
| 202 |
except Exception as exc:
|
| 203 |
raise click.ClickException(f"CSV upload failed: {exc}") from exc
|
| 204 |
|
| 205 |
-
# Build and upload manifest.json based on scan and optional CSV
|
| 206 |
try:
|
| 207 |
-
from
|
| 208 |
|
| 209 |
csv_rows = None
|
| 210 |
if csv_file:
|
|
@@ -220,13 +211,12 @@ def upload_cmd(
|
|
| 220 |
except Exception as exc: # pragma: no cover - external API
|
| 221 |
raise click.ClickException(f"Manifest upload failed: {exc}") from exc
|
| 222 |
|
| 223 |
-
# Generate and upload shards based on CSV rows (if any)
|
| 224 |
try:
|
| 225 |
-
from
|
| 226 |
|
| 227 |
if csv_file and csv_rows:
|
| 228 |
click.echo("Generating index shards from CSV detections...")
|
| 229 |
-
shards = write_shards_from_csv_rows(csv_rows, shard_size=int(__import__("src.
|
| 230 |
for shard_path in shards:
|
| 231 |
shard_name = shard_path.name
|
| 232 |
click.echo(f"Uploading shard: index/shards/{shard_name}")
|
|
|
|
| 7 |
|
| 8 |
import click
|
| 9 |
|
| 10 |
+
from .auth_service import AuthService
|
| 11 |
+
from .error_handler import build_error_message
|
| 12 |
+
from .repo_service import RepositoryService
|
| 13 |
+
from .scanner import LocalScanner
|
| 14 |
+
from .session_manager import SessionManager
|
| 15 |
|
| 16 |
|
| 17 |
def handle_cli_errors(fn: Callable[..., Any]) -> Callable[..., Any]:
|
|
|
|
| 124 |
batch_size: int | None,
|
| 125 |
workers: int | None,
|
| 126 |
dry_run: bool,
|
| 127 |
+
):
|
| 128 |
"""Upload local segments (and optional CSV) into the HF dataset.
|
| 129 |
|
| 130 |
Token resolution (in order of priority):
|
|
|
|
| 134 |
|
| 135 |
The command creates/uses a session checkpoint for resumable uploads.
|
| 136 |
"""
|
|
|
|
| 137 |
if token:
|
| 138 |
hf_token = token
|
| 139 |
else:
|
| 140 |
hf_token = AuthService().require_token()
|
| 141 |
|
|
|
|
| 142 |
scanner = LocalScanner()
|
| 143 |
summary = scanner.scan_folder(segments_dir)
|
| 144 |
total_files = summary["total_files"]
|
|
|
|
| 161 |
click.echo(f"Would also upload CSV: {csv_file} -> index/detections.csv")
|
| 162 |
return
|
| 163 |
|
|
|
|
| 164 |
api = None
|
| 165 |
try:
|
| 166 |
from huggingface_hub import HfApi
|
|
|
|
| 169 |
except Exception as exc: # pragma: no cover - external
|
| 170 |
raise click.ClickException(build_error_message(exc)) from exc
|
| 171 |
|
|
|
|
| 172 |
repo_service = RepositoryService(hf_token)
|
| 173 |
validation = repo_service.validate_repo(repo_id)
|
| 174 |
if not validation.get("is_valid"):
|
| 175 |
click.echo(f"Warning: dataset {repo_id} may be missing structure: {validation.get('missing_prefixes')}")
|
| 176 |
|
| 177 |
+
from .deduplicator import Deduplicator
|
| 178 |
+
from .batch_uploader import BatchUploader
|
|
|
|
| 179 |
|
| 180 |
dedup = Deduplicator(api=api, repo_id=repo_id)
|
| 181 |
session = SessionManager(session_id=session_id) if session_id else SessionManager()
|
| 182 |
uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_retries=None, initial_backoff=None, max_workers=workers)
|
| 183 |
|
|
|
|
| 184 |
file_infos: list[dict] = []
|
| 185 |
for species, items in summary["by_species"].items():
|
| 186 |
for it in items:
|
|
|
|
| 187 |
file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
|
| 188 |
|
|
|
|
| 189 |
if csv_file:
|
| 190 |
click.echo(f"Uploading CSV to index/detections.csv...")
|
| 191 |
try:
|
|
|
|
| 194 |
except Exception as exc:
|
| 195 |
raise click.ClickException(f"CSV upload failed: {exc}") from exc
|
| 196 |
|
|
|
|
| 197 |
try:
|
| 198 |
+
from .manifest import build_manifest_from_scan, manifest_to_bytes
|
| 199 |
|
| 200 |
csv_rows = None
|
| 201 |
if csv_file:
|
|
|
|
| 211 |
except Exception as exc: # pragma: no cover - external API
|
| 212 |
raise click.ClickException(f"Manifest upload failed: {exc}") from exc
|
| 213 |
|
|
|
|
| 214 |
try:
|
| 215 |
+
from .manifest import write_shards_from_csv_rows
|
| 216 |
|
| 217 |
if csv_file and csv_rows:
|
| 218 |
click.echo("Generating index shards from CSV detections...")
|
| 219 |
+
shards = write_shards_from_csv_rows(csv_rows, shard_size=int(__import__("src.uploader.config", fromlist=["INDEX_SHARD_SIZE"]).INDEX_SHARD_SIZE))
|
| 220 |
for shard_path in shards:
|
| 221 |
shard_name = shard_path.name
|
| 222 |
click.echo(f"Uploading shard: index/shards/{shard_name}")
|
src/{uploader_cli → uploader}/manifest.py
RENAMED
|
@@ -32,10 +32,8 @@ def build_manifest_from_scan(repo_id: str, scan_summary: dict[str, Any], *, csv_
|
|
| 32 |
shards: list[str] = []
|
| 33 |
|
| 34 |
if csv_rows is not None:
|
| 35 |
-
# csv_rows expected to be an iterable of detection dicts with at least 'audio_file' key
|
| 36 |
rows = list(csv_rows)
|
| 37 |
total_detections = len(rows)
|
| 38 |
-
# derive unique audio files referenced by CSV
|
| 39 |
unique_files = {str(r.get("audio_file") or r.get("file") or "") for r in rows}
|
| 40 |
unique_files = {p for p in unique_files if p}
|
| 41 |
if unique_files:
|
|
@@ -61,13 +59,6 @@ def manifest_to_bytes(manifest: dict[str, Any]) -> bytes:
|
|
| 61 |
|
| 62 |
|
| 63 |
def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: int = INDEX_SHARD_SIZE) -> List[Path]:
|
| 64 |
-
"""Write shards from CSV rows.
|
| 65 |
-
|
| 66 |
-
Prefer Parquet (pandas/pyarrow). If pandas (and its parquet engine) is
|
| 67 |
-
unavailable, fall back to newline-delimited JSON (.jsonl).
|
| 68 |
-
|
| 69 |
-
Returns list of Path objects for shard files created.
|
| 70 |
-
"""
|
| 71 |
rows_list = list(rows)
|
| 72 |
if not rows_list:
|
| 73 |
return []
|
|
@@ -83,7 +74,6 @@ def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: in
|
|
| 83 |
df = pd.DataFrame.from_records(chunk)
|
| 84 |
shard_name = f"shard-{shard_index:06d}.parquet"
|
| 85 |
shard_path = tmpdir / shard_name
|
| 86 |
-
# pandas will choose pyarrow or fastparquet if available
|
| 87 |
df.to_parquet(shard_path, index=False)
|
| 88 |
else:
|
| 89 |
shard_name = f"shard-{shard_index:06d}.jsonl"
|
|
@@ -96,7 +86,6 @@ def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: in
|
|
| 96 |
|
| 97 |
return out_paths
|
| 98 |
except Exception:
|
| 99 |
-
# If anything fails, attempt to cleanup created files then re-raise
|
| 100 |
try:
|
| 101 |
for p in tmpdir.iterdir():
|
| 102 |
p.unlink()
|
|
|
|
| 32 |
shards: list[str] = []
|
| 33 |
|
| 34 |
if csv_rows is not None:
|
|
|
|
| 35 |
rows = list(csv_rows)
|
| 36 |
total_detections = len(rows)
|
|
|
|
| 37 |
unique_files = {str(r.get("audio_file") or r.get("file") or "") for r in rows}
|
| 38 |
unique_files = {p for p in unique_files if p}
|
| 39 |
if unique_files:
|
|
|
|
| 59 |
|
| 60 |
|
| 61 |
def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: int = INDEX_SHARD_SIZE) -> List[Path]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
rows_list = list(rows)
|
| 63 |
if not rows_list:
|
| 64 |
return []
|
|
|
|
| 74 |
df = pd.DataFrame.from_records(chunk)
|
| 75 |
shard_name = f"shard-{shard_index:06d}.parquet"
|
| 76 |
shard_path = tmpdir / shard_name
|
|
|
|
| 77 |
df.to_parquet(shard_path, index=False)
|
| 78 |
else:
|
| 79 |
shard_name = f"shard-{shard_index:06d}.jsonl"
|
|
|
|
| 86 |
|
| 87 |
return out_paths
|
| 88 |
except Exception:
|
|
|
|
| 89 |
try:
|
| 90 |
for p in tmpdir.iterdir():
|
| 91 |
p.unlink()
|
src/{uploader_cli → uploader}/repo_service.py
RENAMED
|
File without changes
|
src/{uploader_cli → uploader}/scanner.py
RENAMED
|
@@ -7,7 +7,6 @@ from typing import Any
|
|
| 7 |
from .config import AUDIO_EXTENSIONS
|
| 8 |
from .hash_utils import compute_file_hash
|
| 9 |
|
| 10 |
-
|
| 11 |
class LocalScanner:
|
| 12 |
"""Scan local directories and group files by inferred species."""
|
| 13 |
|
|
@@ -59,4 +58,4 @@ class LocalScanner:
|
|
| 59 |
return Path(relative_parts[0]).stem.split("_")[0] or "unknown"
|
| 60 |
|
| 61 |
stem = file_path.stem
|
| 62 |
-
return stem.split("_")[0] if "_" in stem else "unknown"
|
|
|
|
| 7 |
from .config import AUDIO_EXTENSIONS
|
| 8 |
from .hash_utils import compute_file_hash
|
| 9 |
|
|
|
|
| 10 |
class LocalScanner:
|
| 11 |
"""Scan local directories and group files by inferred species."""
|
| 12 |
|
|
|
|
| 58 |
return Path(relative_parts[0]).stem.split("_")[0] or "unknown"
|
| 59 |
|
| 60 |
stem = file_path.stem
|
| 61 |
+
return stem.split("_")[0] if "_" in stem else "unknown"
|
src/{uploader_cli → uploader}/session_manager.py
RENAMED
|
File without changes
|
src/{uploader_cli → uploader}/tui.py
RENAMED
|
File without changes
|
src/uploader/uploader/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compatibility shim package exporting uploader_cli modules under src.uploader."""
|
| 2 |
+
|
| 3 |
+
__all__ = [
|
| 4 |
+
"auth_service",
|
| 5 |
+
"batch_uploader",
|
| 6 |
+
"config",
|
| 7 |
+
"deduplicator",
|
| 8 |
+
"error_handler",
|
| 9 |
+
"exceptions",
|
| 10 |
+
"hash_utils",
|
| 11 |
+
"main",
|
| 12 |
+
"manifest",
|
| 13 |
+
"repo_service",
|
| 14 |
+
"scanner",
|
| 15 |
+
"session_manager",
|
| 16 |
+
"tui",
|
| 17 |
+
]
|
src/uploader/uploader/auth_service.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
import keyring
|
| 6 |
+
from huggingface_hub import HfApi
|
| 7 |
+
|
| 8 |
+
from .config import KEYRING_ACCOUNT, KEYRING_SERVICE
|
| 9 |
+
from .exceptions import AuthenticationError
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AuthService:
|
| 13 |
+
"""Authenticate and securely store Hugging Face tokens."""
|
| 14 |
+
|
| 15 |
+
KEYRING_SERVICE = KEYRING_SERVICE
|
| 16 |
+
KEYRING_ACCOUNT = KEYRING_ACCOUNT
|
| 17 |
+
|
| 18 |
+
def authenticate(self, token: str) -> dict[str, Any]:
|
| 19 |
+
token = token.strip()
|
| 20 |
+
if not token:
|
| 21 |
+
raise AuthenticationError("Token is required")
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
api = HfApi(token=token)
|
| 25 |
+
whoami = api.whoami()
|
| 26 |
+
except Exception as exc: # pragma: no cover - external API behavior
|
| 27 |
+
raise AuthenticationError(f"Token validation failed: {exc}") from exc
|
| 28 |
+
|
| 29 |
+
try:
|
| 30 |
+
keyring.set_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT, token)
|
| 31 |
+
except Exception as exc: # pragma: no cover - backend-specific behavior
|
| 32 |
+
raise AuthenticationError(f"Token validated, but could not be saved securely: {exc}") from exc
|
| 33 |
+
|
| 34 |
+
return {
|
| 35 |
+
"username": whoami.get("name") or "unknown",
|
| 36 |
+
"email": whoami.get("email", ""),
|
| 37 |
+
"user_id": whoami.get("id") or whoami.get("user_id", ""),
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
def get_token(self) -> str | None:
|
| 41 |
+
try:
|
| 42 |
+
return keyring.get_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
|
| 43 |
+
except Exception as exc: # pragma: no cover - backend-specific behavior
|
| 44 |
+
raise AuthenticationError(f"Could not read stored token: {exc}") from exc
|
| 45 |
+
|
| 46 |
+
def require_token(self) -> str:
|
| 47 |
+
token = self.get_token()
|
| 48 |
+
if not token:
|
| 49 |
+
raise AuthenticationError("No stored token found. Run 'birdnet-uploader login' first.")
|
| 50 |
+
return token
|
| 51 |
+
|
| 52 |
+
def clear_token(self) -> None:
|
| 53 |
+
try:
|
| 54 |
+
keyring.delete_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
|
| 55 |
+
except keyring.errors.PasswordDeleteError:
|
| 56 |
+
return
|
| 57 |
+
except Exception as exc: # pragma: no cover - backend-specific behavior
|
| 58 |
+
raise AuthenticationError(f"Could not clear stored token: {exc}") from exc
|
| 59 |
+
else:
|
| 60 |
+
return
|
src/uploader/uploader/batch_uploader.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import time
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any, Callable, Iterable
|
| 6 |
+
|
| 7 |
+
from huggingface_hub import HfApi
|
| 8 |
+
|
| 9 |
+
from .config import MAX_BATCH_SIZE, RETRY_MAX_ATTEMPTS, RETRY_INITIAL_BACKOFF_SECONDS
|
| 10 |
+
from .deduplicator import Deduplicator
|
| 11 |
+
from .session_manager import SessionManager
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class BatchUploader:
|
| 15 |
+
"""Orchestrate uploads with deduplication, retries and session checkpoints."""
|
| 16 |
+
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
api: HfApi,
|
| 20 |
+
repo_id: str,
|
| 21 |
+
deduplicator: Deduplicator,
|
| 22 |
+
session: SessionManager | None = None,
|
| 23 |
+
*,
|
| 24 |
+
max_retries: int | None = None,
|
| 25 |
+
initial_backoff: float | None = None,
|
| 26 |
+
) -> None:
|
| 27 |
+
self._api = api
|
| 28 |
+
self.repo_id = repo_id
|
| 29 |
+
self.deduplicator = deduplicator
|
| 30 |
+
self.session = session or SessionManager.create_session()
|
| 31 |
+
self.max_retries = max_retries or RETRY_MAX_ATTEMPTS
|
| 32 |
+
self.initial_backoff = initial_backoff or RETRY_INITIAL_BACKOFF_SECONDS
|
| 33 |
+
|
| 34 |
+
def _upload_file_with_retry(self, local_path: str | Path, remote_path: str) -> None:
|
| 35 |
+
attempts = 0
|
| 36 |
+
last_exc: Exception | None = None
|
| 37 |
+
while attempts <= self.max_retries:
|
| 38 |
+
try:
|
| 39 |
+
# The HfApi.upload_file signature varies across versions; callers/tests
|
| 40 |
+
# should accept the positional usage shown here and monkeypatch in tests.
|
| 41 |
+
self._api.upload_file(path_or_file=local_path, path_in_repo=remote_path, repo_id=self.repo_id)
|
| 42 |
+
return
|
| 43 |
+
except Exception as exc: # pragma: no cover - network behavior
|
| 44 |
+
last_exc = exc
|
| 45 |
+
attempts += 1
|
| 46 |
+
if attempts > self.max_retries:
|
| 47 |
+
raise
|
| 48 |
+
backoff = self.initial_backoff * (2 ** (attempts - 1))
|
| 49 |
+
time.sleep(backoff)
|
| 50 |
+
|
| 51 |
+
def upload_files(
|
| 52 |
+
self,
|
| 53 |
+
file_infos: Iterable[dict[str, Any]],
|
| 54 |
+
*,
|
| 55 |
+
remote_base: str = "",
|
| 56 |
+
batch_size: int | None = None,
|
| 57 |
+
on_progress: Callable[[dict[str, Any]], None] | None = None,
|
| 58 |
+
) -> dict[str, Any]:
|
| 59 |
+
"""Upload an iterable of file info dictionaries.
|
| 60 |
+
|
| 61 |
+
Each file_info must include `full_path` and `relative_path`. `size` is optional.
|
| 62 |
+
"""
|
| 63 |
+
batch_size = batch_size or MAX_BATCH_SIZE
|
| 64 |
+
uploaded = 0
|
| 65 |
+
skipped = 0
|
| 66 |
+
failed = 0
|
| 67 |
+
|
| 68 |
+
for info in file_infos:
|
| 69 |
+
full_path = str(info["full_path"])
|
| 70 |
+
relative = info["relative_path"].lstrip("/")
|
| 71 |
+
remote_path = f"{remote_base.rstrip('/')}/{relative}" if remote_base else relative
|
| 72 |
+
remote_path = remote_path.lstrip("/")
|
| 73 |
+
|
| 74 |
+
decision = self.deduplicator.check_remote(remote_path, file_path=full_path)
|
| 75 |
+
if decision.get("status") == "skip":
|
| 76 |
+
skipped += 1
|
| 77 |
+
self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=0, status="skipped")
|
| 78 |
+
if on_progress:
|
| 79 |
+
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 80 |
+
continue
|
| 81 |
+
|
| 82 |
+
try:
|
| 83 |
+
self._upload_file_with_retry(full_path, remote_path)
|
| 84 |
+
# mark remote as present in local dedup cache
|
| 85 |
+
try:
|
| 86 |
+
self.deduplicator.mark_uploaded(remote_path)
|
| 87 |
+
except Exception:
|
| 88 |
+
# non-fatal cache update failure should not break overall flow
|
| 89 |
+
pass
|
| 90 |
+
|
| 91 |
+
uploaded += 1
|
| 92 |
+
bytes_uploaded = int(info.get("size", 0))
|
| 93 |
+
self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=bytes_uploaded)
|
| 94 |
+
if on_progress:
|
| 95 |
+
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 96 |
+
except Exception as exc:
|
| 97 |
+
failed += 1
|
| 98 |
+
self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
|
| 99 |
+
if on_progress:
|
| 100 |
+
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 101 |
+
|
| 102 |
+
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
src/uploader/uploader/config.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
APP_NAME = "BirdNET Uploader"
|
| 9 |
+
SCHEMA_VERSION = "1.0.0"
|
| 10 |
+
AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
|
| 11 |
+
INDEX_SHARD_SIZE = 10_000
|
| 12 |
+
|
| 13 |
+
MAX_BATCH_SIZE = 10
|
| 14 |
+
RETRY_MAX_ATTEMPTS = 3
|
| 15 |
+
RETRY_INITIAL_BACKOFF_SECONDS = 1.0
|
| 16 |
+
RETRY_MAX_BACKOFF_SECONDS = 30.0
|
| 17 |
+
|
| 18 |
+
KEYRING_SERVICE = "birdnet-uploader"
|
| 19 |
+
KEYRING_ACCOUNT = "hf_token"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _resolve_path_from_env(env_name: str, default: Path) -> Path:
|
| 23 |
+
raw = os.getenv(env_name)
|
| 24 |
+
value = Path(raw).expanduser() if raw else default
|
| 25 |
+
value.mkdir(parents=True, exist_ok=True)
|
| 26 |
+
return value
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def get_session_root() -> Path:
|
| 30 |
+
"""Resolve and create the session storage root directory."""
|
| 31 |
+
return _resolve_path_from_env(
|
| 32 |
+
"BIRDNET_UPLOADER_SESSION_DIR",
|
| 33 |
+
Path.home() / ".birdnet-uploader" / "sessions",
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def get_cache_root() -> Path:
|
| 38 |
+
return _resolve_path_from_env(
|
| 39 |
+
"BIRDNET_UPLOADER_CACHE_DIR",
|
| 40 |
+
Path.home() / ".birdnet-uploader" / "cache",
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def get_log_root() -> Path:
|
| 45 |
+
return _resolve_path_from_env(
|
| 46 |
+
"BIRDNET_UPLOADER_LOG_DIR",
|
| 47 |
+
Path.home() / ".birdnet-uploader" / "logs",
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def get_runtime_config() -> dict[str, Any]:
|
| 52 |
+
return {
|
| 53 |
+
"session_root": str(get_session_root()),
|
| 54 |
+
"cache_root": str(get_cache_root()),
|
| 55 |
+
"log_root": str(get_log_root()),
|
| 56 |
+
"max_batch_size": MAX_BATCH_SIZE,
|
| 57 |
+
"retry_max_attempts": RETRY_MAX_ATTEMPTS,
|
| 58 |
+
"retry_initial_backoff_seconds": RETRY_INITIAL_BACKOFF_SECONDS,
|
| 59 |
+
"retry_max_backoff_seconds": RETRY_MAX_BACKOFF_SECONDS,
|
| 60 |
+
"index_shard_size": INDEX_SHARD_SIZE,
|
| 61 |
+
}
|
src/uploader/uploader/deduplicator.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import hashlib
|
| 5 |
+
from datetime import UTC, datetime
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from huggingface_hub import HfApi
|
| 10 |
+
|
| 11 |
+
from .config import get_cache_root
|
| 12 |
+
from .hash_utils import compute_file_hash
|
| 13 |
+
from .exceptions import ValidationError
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Deduplicator:
|
| 17 |
+
"""Check remote file presence and cache the remote path index locally."""
|
| 18 |
+
|
| 19 |
+
def __init__(self, api: HfApi, repo_id: str, *, repo_type: str = "dataset") -> None:
|
| 20 |
+
self._api = api
|
| 21 |
+
self.repo_id = repo_id
|
| 22 |
+
self.repo_type = repo_type
|
| 23 |
+
self._remote_paths: set[str] | None = None
|
| 24 |
+
self._decision_cache: dict[str, dict[str, Any]] = {}
|
| 25 |
+
self.cache_path = self._build_cache_path()
|
| 26 |
+
|
| 27 |
+
def _build_cache_path(self) -> Path:
|
| 28 |
+
cache_root = get_cache_root() / "dedup"
|
| 29 |
+
cache_root.mkdir(parents=True, exist_ok=True)
|
| 30 |
+
digest = hashlib.sha256(f"{self.repo_type}:{self.repo_id}".encode("utf-8")).hexdigest()[:16]
|
| 31 |
+
return cache_root / f"{digest}.json"
|
| 32 |
+
|
| 33 |
+
def load_cached_index(self) -> set[str]:
|
| 34 |
+
if self._remote_paths is not None:
|
| 35 |
+
return self._remote_paths
|
| 36 |
+
|
| 37 |
+
if self.cache_path.exists():
|
| 38 |
+
raw = self.cache_path.read_text(encoding="utf-8")
|
| 39 |
+
try:
|
| 40 |
+
payload = json.loads(raw)
|
| 41 |
+
except json.JSONDecodeError as exc:
|
| 42 |
+
raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc
|
| 43 |
+
|
| 44 |
+
remote_paths = payload.get("remote_paths", [])
|
| 45 |
+
if not isinstance(remote_paths, list):
|
| 46 |
+
raise ValidationError(f"Invalid dedup cache format at {self.cache_path}")
|
| 47 |
+
self._remote_paths = {str(path) for path in remote_paths}
|
| 48 |
+
return self._remote_paths
|
| 49 |
+
|
| 50 |
+
self._remote_paths = self.refresh_remote_index()
|
| 51 |
+
return self._remote_paths
|
| 52 |
+
|
| 53 |
+
def refresh_remote_index(self) -> set[str]:
|
| 54 |
+
try:
|
| 55 |
+
repo_files = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
|
| 56 |
+
except Exception as exc: # pragma: no cover - external API behavior
|
| 57 |
+
raise ValidationError(f"Could not read remote file listing for {self.repo_id}: {exc}") from exc
|
| 58 |
+
|
| 59 |
+
remote_paths = {str(path) for path in repo_files}
|
| 60 |
+
self._remote_paths = remote_paths
|
| 61 |
+
self._write_cache(remote_paths)
|
| 62 |
+
return remote_paths
|
| 63 |
+
|
| 64 |
+
def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
|
| 65 |
+
cache_key = self._build_decision_key(remote_path, file_path=file_path)
|
| 66 |
+
cached = self._decision_cache.get(cache_key)
|
| 67 |
+
if cached is not None:
|
| 68 |
+
return dict(cached)
|
| 69 |
+
|
| 70 |
+
remote_paths = self.load_cached_index()
|
| 71 |
+
exists = remote_path in remote_paths
|
| 72 |
+
|
| 73 |
+
payload = {
|
| 74 |
+
"remote_path": remote_path,
|
| 75 |
+
"status": "skip" if exists else "upload",
|
| 76 |
+
"cached_remote_index": True,
|
| 77 |
+
}
|
| 78 |
+
if file_path is not None:
|
| 79 |
+
file_path_obj = Path(file_path)
|
| 80 |
+
payload["local_size"] = file_path_obj.stat().st_size
|
| 81 |
+
payload["local_sha256"] = compute_file_hash(file_path_obj)
|
| 82 |
+
|
| 83 |
+
self._decision_cache[cache_key] = dict(payload)
|
| 84 |
+
return payload
|
| 85 |
+
|
| 86 |
+
def mark_uploaded(self, remote_path: str) -> None:
|
| 87 |
+
remote_paths = self.load_cached_index()
|
| 88 |
+
if remote_path not in remote_paths:
|
| 89 |
+
remote_paths.add(remote_path)
|
| 90 |
+
self._write_cache(remote_paths)
|
| 91 |
+
|
| 92 |
+
keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path]
|
| 93 |
+
for key in keys_to_update:
|
| 94 |
+
self._decision_cache[key]["status"] = "skip"
|
| 95 |
+
|
| 96 |
+
def _write_cache(self, remote_paths: set[str]) -> None:
|
| 97 |
+
payload = {
|
| 98 |
+
"repo_id": self.repo_id,
|
| 99 |
+
"repo_type": self.repo_type,
|
| 100 |
+
"updated_at": datetime.now(UTC).isoformat(),
|
| 101 |
+
"remote_paths": sorted(remote_paths),
|
| 102 |
+
}
|
| 103 |
+
temp_path = self.cache_path.with_suffix(".json.tmp")
|
| 104 |
+
temp_path.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
|
| 105 |
+
temp_path.replace(self.cache_path)
|
| 106 |
+
|
| 107 |
+
def _build_decision_key(self, remote_path: str, *, file_path: str | Path | None) -> str:
|
| 108 |
+
if file_path is None:
|
| 109 |
+
return remote_path
|
| 110 |
+
file_path_obj = Path(file_path)
|
| 111 |
+
try:
|
| 112 |
+
stat = file_path_obj.stat()
|
| 113 |
+
except FileNotFoundError as exc:
|
| 114 |
+
raise ValidationError(f"Local file not found: {file_path_obj}") from exc
|
| 115 |
+
return f"{remote_path}:{file_path_obj.resolve()}:{stat.st_size}:{stat.st_mtime_ns}"
|
src/uploader/uploader/error_handler.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from .exceptions import (
|
| 4 |
+
AuthenticationError,
|
| 5 |
+
RepositoryError,
|
| 6 |
+
SessionError,
|
| 7 |
+
UploaderCliError,
|
| 8 |
+
UploadError,
|
| 9 |
+
ValidationError,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
ERROR_HINTS: dict[type[UploaderCliError], str] = {
|
| 14 |
+
AuthenticationError: "Run 'birdnet-uploader login' and verify your Hugging Face token.",
|
| 15 |
+
RepositoryError: "Check repository id format (owner/name) and write permissions on Hugging Face.",
|
| 16 |
+
SessionError: "Verify session id and local checkpoint files under .birdnet-uploader/sessions.",
|
| 17 |
+
ValidationError: "Fix invalid paths or inputs and retry the command.",
|
| 18 |
+
UploadError: "Check your network and run the command again to resume from checkpoint.",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def build_error_message(error: Exception) -> str:
|
| 23 |
+
if isinstance(error, UploaderCliError):
|
| 24 |
+
hint = error.suggestion or ERROR_HINTS.get(type(error), "")
|
| 25 |
+
if hint:
|
| 26 |
+
return f"{error} Hint: {hint}"
|
| 27 |
+
return str(error)
|
| 28 |
+
|
| 29 |
+
return f"Unexpected error: {error}"
|
src/uploader/uploader/exceptions.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class UploaderCliError(Exception):
|
| 2 |
+
"""Base exception for uploader CLI."""
|
| 3 |
+
|
| 4 |
+
def __init__(self, message: str, *, suggestion: str = ""):
|
| 5 |
+
super().__init__(message)
|
| 6 |
+
self.suggestion = suggestion
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class AuthenticationError(UploaderCliError):
|
| 10 |
+
"""Raised when Hugging Face authentication fails."""
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class RepositoryError(UploaderCliError):
|
| 14 |
+
"""Raised for dataset repository failures."""
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class SessionError(UploaderCliError):
|
| 18 |
+
"""Raised when session state cannot be read or written."""
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ValidationError(UploaderCliError):
|
| 22 |
+
"""Raised when user input or local resources are invalid."""
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class UploadError(UploaderCliError):
|
| 26 |
+
"""Raised when upload operations fail after retries."""
|
src/uploader/uploader/hash_utils.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def compute_file_hash(path: str | Path, *, chunk_size: int = 1024 * 1024) -> str:
|
| 8 |
+
"""Compute a streaming SHA-256 hash for a file."""
|
| 9 |
+
digest = hashlib.sha256()
|
| 10 |
+
file_path = Path(path)
|
| 11 |
+
|
| 12 |
+
with file_path.open("rb") as handle:
|
| 13 |
+
for chunk in iter(lambda: handle.read(chunk_size), b""):
|
| 14 |
+
digest.update(chunk)
|
| 15 |
+
|
| 16 |
+
return digest.hexdigest()
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def verify_file_integrity(path: str | Path, expected_hash: str, *, chunk_size: int = 1024 * 1024) -> bool:
|
| 20 |
+
"""Return True when a file's SHA-256 matches the expected hash."""
|
| 21 |
+
return compute_file_hash(path, chunk_size=chunk_size) == expected_hash
|
src/uploader/uploader/main.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from functools import wraps
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Any, Callable
|
| 7 |
+
|
| 8 |
+
import click
|
| 9 |
+
|
| 10 |
+
from src.uploader.auth_service import AuthService
|
| 11 |
+
from src.uploader.error_handler import build_error_message
|
| 12 |
+
from src.uploader.repo_service import RepositoryService
|
| 13 |
+
from src.uploader.scanner import LocalScanner
|
| 14 |
+
from src.uploader.session_manager import SessionManager
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def handle_cli_errors(fn: Callable[..., Any]) -> Callable[..., Any]:
|
| 18 |
+
@wraps(fn)
|
| 19 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 20 |
+
try:
|
| 21 |
+
return fn(*args, **kwargs)
|
| 22 |
+
except click.ClickException:
|
| 23 |
+
raise
|
| 24 |
+
except Exception as exc:
|
| 25 |
+
raise click.ClickException(build_error_message(exc)) from exc
|
| 26 |
+
|
| 27 |
+
return wrapper
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@click.group(help="BirdNET local uploader CLI")
|
| 31 |
+
def cli() -> None:
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@cli.command("login")
|
| 36 |
+
@click.option("--token", prompt=True, hide_input=True, help="Hugging Face token")
|
| 37 |
+
@handle_cli_errors
|
| 38 |
+
def login_cmd(token: str) -> None:
|
| 39 |
+
service = AuthService()
|
| 40 |
+
user = service.authenticate(token)
|
| 41 |
+
click.echo(f"OK: authenticated as {user['username']}")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
@cli.command("init-repo")
|
| 45 |
+
@click.option("--repo-id", required=True, help="Dataset repo id in owner/name format")
|
| 46 |
+
@click.option("--private/--public", "private_repo", default=True, show_default=True)
|
| 47 |
+
@handle_cli_errors
|
| 48 |
+
def init_repo_cmd(repo_id: str, private_repo: bool) -> None:
|
| 49 |
+
token = AuthService().require_token()
|
| 50 |
+
|
| 51 |
+
created = RepositoryService(token).create_dataset(repo_id, private=private_repo)
|
| 52 |
+
click.echo(f"OK: dataset ready at {created}")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@cli.command("scan")
|
| 56 |
+
@click.option("--segments", "segments_dir", required=True, type=click.Path(exists=True, file_okay=False))
|
| 57 |
+
@handle_cli_errors
|
| 58 |
+
def scan_cmd(segments_dir: str) -> None:
|
| 59 |
+
summary = LocalScanner().scan_folder(segments_dir)
|
| 60 |
+
click.echo(
|
| 61 |
+
json.dumps(
|
| 62 |
+
{
|
| 63 |
+
"total_files": summary["total_files"],
|
| 64 |
+
"total_size": summary["total_size"],
|
| 65 |
+
"species_count": len(summary["by_species"]),
|
| 66 |
+
},
|
| 67 |
+
ensure_ascii=True,
|
| 68 |
+
indent=2,
|
| 69 |
+
)
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@cli.command("start")
|
| 74 |
+
@click.option("--repo-id", required=True)
|
| 75 |
+
@click.option("--segments", "segments_dir", required=True, type=click.Path(exists=True, file_okay=False))
|
| 76 |
+
@handle_cli_errors
|
| 77 |
+
def start_cmd(repo_id: str, segments_dir: str) -> None:
|
| 78 |
+
scanner = LocalScanner()
|
| 79 |
+
summary = scanner.scan_folder(segments_dir)
|
| 80 |
+
session = SessionManager()
|
| 81 |
+
payload = {
|
| 82 |
+
"repo_id": repo_id,
|
| 83 |
+
"segments_dir": str(Path(segments_dir).resolve()),
|
| 84 |
+
"total_files": summary["total_files"],
|
| 85 |
+
"total_size": summary["total_size"],
|
| 86 |
+
"uploaded": 0,
|
| 87 |
+
"failed": 0,
|
| 88 |
+
"status": "ready",
|
| 89 |
+
}
|
| 90 |
+
session.save_checkpoint(payload)
|
| 91 |
+
click.echo(f"OK: session created: {session.session_id}")
|
| 92 |
+
click.echo(f"Checkpoint: {session.checkpoint_path}")
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
@cli.command("resume")
|
| 96 |
+
@click.argument("session_id")
|
| 97 |
+
@handle_cli_errors
|
| 98 |
+
def resume_cmd(session_id: str) -> None:
|
| 99 |
+
session = SessionManager(session_id=session_id)
|
| 100 |
+
payload = session.load_checkpoint()
|
| 101 |
+
if not payload:
|
| 102 |
+
raise click.ClickException(f"Session has no checkpoint: {session_id}")
|
| 103 |
+
click.echo(json.dumps(payload, ensure_ascii=True, indent=2))
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
if __name__ == "__main__":
|
| 107 |
+
cli()
|
src/uploader/uploader/manifest.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from io import BytesIO
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Any, Iterable, List
|
| 7 |
+
|
| 8 |
+
from .config import SCHEMA_VERSION, INDEX_SHARD_SIZE
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def build_manifest_from_scan(scan: dict[str, Any], repo_id: str | None = None) -> dict[str, Any]:
|
| 12 |
+
project_slug = None
|
| 13 |
+
if repo_id and "/" in repo_id:
|
| 14 |
+
project_slug = repo_id.split("/", 1)[1]
|
| 15 |
+
|
| 16 |
+
return {
|
| 17 |
+
"schema_version": SCHEMA_VERSION,
|
| 18 |
+
"project_slug": project_slug or "unknown",
|
| 19 |
+
"dataset_repo_id": repo_id,
|
| 20 |
+
"index": {
|
| 21 |
+
"total_detections": 0,
|
| 22 |
+
"total_audio_files": scan.get("total_files", 0),
|
| 23 |
+
"shard_size": INDEX_SHARD_SIZE,
|
| 24 |
+
"shards": [],
|
| 25 |
+
},
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def manifest_to_bytes(manifest: dict[str, Any]) -> bytes:
|
| 30 |
+
return json.dumps(manifest, ensure_ascii=True, indent=2).encode("utf-8")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], out_dir: Path) -> List[Path]:
|
| 34 |
+
"""Write simple JSONL shards from CSV rows. Returns list of shard paths."""
|
| 35 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 36 |
+
shard_paths: List[Path] = []
|
| 37 |
+
shard_index = 0
|
| 38 |
+
batch: List[dict[str, Any]] = []
|
| 39 |
+
for row in rows:
|
| 40 |
+
batch.append(row)
|
| 41 |
+
if len(batch) >= INDEX_SHARD_SIZE:
|
| 42 |
+
path = out_dir / f"shard-{shard_index:04d}.jsonl"
|
| 43 |
+
with path.open("w", encoding="utf-8") as fh:
|
| 44 |
+
for item in batch:
|
| 45 |
+
fh.write(json.dumps(item, ensure_ascii=True) + "\n")
|
| 46 |
+
shard_paths.append(path)
|
| 47 |
+
shard_index += 1
|
| 48 |
+
batch = []
|
| 49 |
+
|
| 50 |
+
if batch:
|
| 51 |
+
path = out_dir / f"shard-{shard_index:04d}.jsonl"
|
| 52 |
+
with path.open("w", encoding="utf-8") as fh:
|
| 53 |
+
for item in batch:
|
| 54 |
+
fh.write(json.dumps(item, ensure_ascii=True) + "\n")
|
| 55 |
+
shard_paths.append(path)
|
| 56 |
+
|
| 57 |
+
return shard_paths
|
| 58 |
+
from src.uploader_cli.manifest import *
|
src/uploader/uploader/repo_service.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from io import BytesIO
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from huggingface_hub import HfApi
|
| 8 |
+
|
| 9 |
+
from .config import INDEX_SHARD_SIZE, SCHEMA_VERSION
|
| 10 |
+
from .exceptions import RepositoryError
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class RepositoryService:
|
| 14 |
+
"""Create and initialize dataset repositories for uploader sessions."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, token: str):
|
| 17 |
+
self._api = HfApi(token=token)
|
| 18 |
+
|
| 19 |
+
def create_dataset(self, repo_id: str, *, private: bool = True) -> str:
|
| 20 |
+
if not self._is_valid_repo_id(repo_id):
|
| 21 |
+
raise RepositoryError("Repository id must be in the form 'owner/name'")
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
self._api.create_repo(repo_id=repo_id, repo_type="dataset", private=private, exist_ok=True)
|
| 25 |
+
self._init_structure(repo_id)
|
| 26 |
+
except Exception as exc: # pragma: no cover - external API behavior
|
| 27 |
+
raise RepositoryError(f"Could not create dataset: {exc}") from exc
|
| 28 |
+
|
| 29 |
+
return repo_id
|
| 30 |
+
|
| 31 |
+
def _init_structure(self, repo_id: str) -> None:
|
| 32 |
+
project_slug = self._project_slug_from_repo_id(repo_id)
|
| 33 |
+
placeholder_dirs = [
|
| 34 |
+
f"audio/{project_slug}/.gitkeep",
|
| 35 |
+
"index/shards/.gitkeep",
|
| 36 |
+
"validations/.gitkeep",
|
| 37 |
+
"audit/ingestion-runs/.gitkeep",
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
for path_in_repo in placeholder_dirs:
|
| 41 |
+
self._api.upload_file(
|
| 42 |
+
path_or_fileobj=BytesIO(b""),
|
| 43 |
+
path_in_repo=path_in_repo,
|
| 44 |
+
repo_id=repo_id,
|
| 45 |
+
repo_type="dataset",
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
manifest = {
|
| 49 |
+
"schema_version": SCHEMA_VERSION,
|
| 50 |
+
"project_slug": project_slug,
|
| 51 |
+
"dataset_repo_id": repo_id,
|
| 52 |
+
"index": {
|
| 53 |
+
"total_detections": 0,
|
| 54 |
+
"total_audio_files": 0,
|
| 55 |
+
"shard_size": INDEX_SHARD_SIZE,
|
| 56 |
+
"shards": [],
|
| 57 |
+
},
|
| 58 |
+
}
|
| 59 |
+
self._api.upload_file(
|
| 60 |
+
path_or_fileobj=BytesIO(json.dumps(manifest, ensure_ascii=True, indent=2).encode("utf-8")),
|
| 61 |
+
path_in_repo="index/manifest.json",
|
| 62 |
+
repo_id=repo_id,
|
| 63 |
+
repo_type="dataset",
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
def validate_repo(self, repo_id: str) -> dict[str, Any]:
|
| 67 |
+
if not self._is_valid_repo_id(repo_id):
|
| 68 |
+
raise RepositoryError("Repository id must be in the form 'owner/name'")
|
| 69 |
+
|
| 70 |
+
try:
|
| 71 |
+
repo_files = self._api.list_repo_files(repo_id=repo_id, repo_type="dataset")
|
| 72 |
+
except Exception as exc: # pragma: no cover - external API behavior
|
| 73 |
+
raise RepositoryError(f"Could not list dataset files: {exc}") from exc
|
| 74 |
+
|
| 75 |
+
project_slug = self._project_slug_from_repo_id(repo_id)
|
| 76 |
+
required_prefixes = [
|
| 77 |
+
f"audio/{project_slug}/",
|
| 78 |
+
"index/shards/",
|
| 79 |
+
"validations/",
|
| 80 |
+
"audit/ingestion-runs/",
|
| 81 |
+
]
|
| 82 |
+
missing_prefixes = [
|
| 83 |
+
prefix for prefix in required_prefixes if not any(path.startswith(prefix) for path in repo_files)
|
| 84 |
+
]
|
| 85 |
+
|
| 86 |
+
has_manifest = "index/manifest.json" in repo_files
|
| 87 |
+
manifest_ok = has_manifest
|
| 88 |
+
manifest_error = ""
|
| 89 |
+
|
| 90 |
+
is_valid = not missing_prefixes and has_manifest
|
| 91 |
+
return {
|
| 92 |
+
"repo_id": repo_id,
|
| 93 |
+
"is_valid": is_valid,
|
| 94 |
+
"project_slug": project_slug,
|
| 95 |
+
"missing_prefixes": missing_prefixes,
|
| 96 |
+
"has_manifest": has_manifest,
|
| 97 |
+
"manifest_ok": manifest_ok if has_manifest else False,
|
| 98 |
+
"manifest_error": manifest_error,
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
@staticmethod
|
| 102 |
+
def _is_valid_repo_id(repo_id: str) -> bool:
|
| 103 |
+
if not repo_id or repo_id.count("/") != 1:
|
| 104 |
+
return False
|
| 105 |
+
owner, name = repo_id.split("/", maxsplit=1)
|
| 106 |
+
return bool(owner.strip()) and bool(name.strip()) and " " not in repo_id
|
| 107 |
+
|
| 108 |
+
@staticmethod
|
| 109 |
+
def _project_slug_from_repo_id(repo_id: str) -> str:
|
| 110 |
+
return repo_id.split("/", maxsplit=1)[1]
|
src/uploader/uploader/scanner.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from .config import AUDIO_EXTENSIONS
|
| 8 |
+
from .hash_utils import compute_file_hash
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class LocalScanner:
|
| 12 |
+
"""Scan local directories and group files by inferred species."""
|
| 13 |
+
|
| 14 |
+
def scan_folder(self, folder_path: str) -> dict[str, Any]:
|
| 15 |
+
root_path = Path(folder_path).resolve()
|
| 16 |
+
by_species: dict[str, list[dict[str, Any]]] = {}
|
| 17 |
+
total_files = 0
|
| 18 |
+
total_size = 0
|
| 19 |
+
|
| 20 |
+
for root, _, files in os.walk(root_path):
|
| 21 |
+
current_root = Path(root)
|
| 22 |
+
for filename in files:
|
| 23 |
+
ext = Path(filename).suffix.lower()
|
| 24 |
+
if ext not in AUDIO_EXTENSIONS:
|
| 25 |
+
continue
|
| 26 |
+
|
| 27 |
+
full_path = current_root / filename
|
| 28 |
+
relative_path = full_path.relative_to(root_path)
|
| 29 |
+
species = self._infer_species(root_path, full_path)
|
| 30 |
+
item = {
|
| 31 |
+
"name": filename,
|
| 32 |
+
"full_path": str(full_path),
|
| 33 |
+
"relative_path": relative_path.as_posix(),
|
| 34 |
+
"species": species,
|
| 35 |
+
"size": full_path.stat().st_size,
|
| 36 |
+
"sha256": compute_file_hash(full_path),
|
| 37 |
+
}
|
| 38 |
+
by_species.setdefault(species, []).append(item)
|
| 39 |
+
total_files += 1
|
| 40 |
+
total_size += item["size"]
|
| 41 |
+
|
| 42 |
+
return {
|
| 43 |
+
"total_files": total_files,
|
| 44 |
+
"total_size": total_size,
|
| 45 |
+
"by_species": by_species,
|
| 46 |
+
"root_path": str(root_path),
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
def _infer_species(self, root_path: Path, file_path: Path) -> str:
|
| 50 |
+
try:
|
| 51 |
+
relative_parts = file_path.relative_to(root_path).parts
|
| 52 |
+
except ValueError:
|
| 53 |
+
relative_parts = file_path.parts
|
| 54 |
+
|
| 55 |
+
if len(relative_parts) >= 2:
|
| 56 |
+
return relative_parts[0]
|
| 57 |
+
|
| 58 |
+
if len(relative_parts) == 1:
|
| 59 |
+
return Path(relative_parts[0]).stem.split("_")[0] or "unknown"
|
| 60 |
+
|
| 61 |
+
stem = file_path.stem
|
| 62 |
+
return stem.split("_")[0] if "_" in stem else "unknown"
|
src/uploader/uploader/session_manager.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import json
|
| 5 |
+
from datetime import UTC, datetime
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from .config import get_session_root
|
| 10 |
+
from .exceptions import SessionError
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class SessionManager:
|
| 14 |
+
"""Persist and recover upload checkpoints."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, session_id: str | None = None):
|
| 17 |
+
if session_id is None:
|
| 18 |
+
now = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
| 19 |
+
session_id = f"upload-{now}"
|
| 20 |
+
self.session_id = session_id
|
| 21 |
+
self.session_dir = get_session_root() / session_id
|
| 22 |
+
self.session_dir.mkdir(parents=True, exist_ok=True)
|
| 23 |
+
|
| 24 |
+
@classmethod
|
| 25 |
+
def create_session(cls, metadata: dict[str, Any] | None = None) -> "SessionManager":
|
| 26 |
+
session = cls()
|
| 27 |
+
if metadata:
|
| 28 |
+
session.save_metadata(metadata)
|
| 29 |
+
return session
|
| 30 |
+
|
| 31 |
+
@property
|
| 32 |
+
def checkpoint_path(self) -> Path:
|
| 33 |
+
return self.session_dir / "checkpoint.json"
|
| 34 |
+
|
| 35 |
+
@property
|
| 36 |
+
def metadata_path(self) -> Path:
|
| 37 |
+
return self.session_dir / "metadata.json"
|
| 38 |
+
|
| 39 |
+
def _atomic_write_json(self, path: Path, payload: dict[str, Any]) -> None:
|
| 40 |
+
temp_path = path.with_suffix(path.suffix + ".tmp")
|
| 41 |
+
serialized = json.dumps(payload, ensure_ascii=True, indent=2) + "\n"
|
| 42 |
+
temp_path.write_text(serialized, encoding="utf-8")
|
| 43 |
+
with temp_path.open("a", encoding="utf-8") as handle:
|
| 44 |
+
handle.flush()
|
| 45 |
+
os.fsync(handle.fileno())
|
| 46 |
+
temp_path.replace(path)
|
| 47 |
+
|
| 48 |
+
def save_metadata(self, payload: dict[str, Any]) -> None:
|
| 49 |
+
self._atomic_write_json(self.metadata_path, payload)
|
| 50 |
+
|
| 51 |
+
def load_metadata(self) -> dict[str, Any]:
|
| 52 |
+
if not self.metadata_path.exists():
|
| 53 |
+
return {}
|
| 54 |
+
raw = self.metadata_path.read_text(encoding="utf-8")
|
| 55 |
+
try:
|
| 56 |
+
return json.loads(raw)
|
| 57 |
+
except json.JSONDecodeError as exc:
|
| 58 |
+
raise SessionError(f"Invalid metadata JSON for session '{self.session_id}'") from exc
|
| 59 |
+
|
| 60 |
+
def save_checkpoint(self, payload: dict[str, Any]) -> None:
|
| 61 |
+
self._atomic_write_json(self.checkpoint_path, payload)
|
| 62 |
+
|
| 63 |
+
def load_checkpoint(self) -> dict[str, Any]:
|
| 64 |
+
if not self.checkpoint_path.exists():
|
| 65 |
+
return {}
|
| 66 |
+
raw = self.checkpoint_path.read_text(encoding="utf-8")
|
| 67 |
+
try:
|
| 68 |
+
return json.loads(raw)
|
| 69 |
+
except json.JSONDecodeError as exc:
|
| 70 |
+
raise SessionError(f"Invalid checkpoint JSON for session '{self.session_id}'") from exc
|
| 71 |
+
|
| 72 |
+
def mark_file_done(self, *, remote_path: str, bytes_uploaded: int, status: str = "uploaded") -> dict[str, Any]:
|
| 73 |
+
checkpoint = self.load_checkpoint()
|
| 74 |
+
checkpoint.setdefault("uploaded_files", [])
|
| 75 |
+
checkpoint.setdefault("uploaded", 0)
|
| 76 |
+
checkpoint.setdefault("failed", 0)
|
| 77 |
+
checkpoint["uploaded"] = int(checkpoint["uploaded"]) + 1
|
| 78 |
+
checkpoint["status"] = status
|
| 79 |
+
checkpoint["last_completed_file"] = remote_path
|
| 80 |
+
checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
|
| 81 |
+
checkpoint["bytes_uploaded"] = int(checkpoint.get("bytes_uploaded", 0)) + int(bytes_uploaded)
|
| 82 |
+
checkpoint["uploaded_files"].append(remote_path)
|
| 83 |
+
self.save_checkpoint(checkpoint)
|
| 84 |
+
return checkpoint
|
| 85 |
+
|
| 86 |
+
def mark_file_failed(self, *, remote_path: str, error: str) -> dict[str, Any]:
|
| 87 |
+
checkpoint = self.load_checkpoint()
|
| 88 |
+
checkpoint.setdefault("failed_files", [])
|
| 89 |
+
checkpoint.setdefault("uploaded", 0)
|
| 90 |
+
checkpoint.setdefault("failed", 0)
|
| 91 |
+
checkpoint["failed"] = int(checkpoint["failed"]) + 1
|
| 92 |
+
checkpoint["status"] = "failed"
|
| 93 |
+
checkpoint["last_failed_file"] = remote_path
|
| 94 |
+
checkpoint["last_error"] = error
|
| 95 |
+
checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
|
| 96 |
+
checkpoint["failed_files"].append({"remote_path": remote_path, "error": error})
|
| 97 |
+
self.save_checkpoint(checkpoint)
|
| 98 |
+
return checkpoint
|
src/uploader/uploader/tui.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from textual.app import App, ComposeResult
|
| 4 |
+
from textual.containers import Container
|
| 5 |
+
from textual.widgets import Button, Static
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class BirdNetUploaderTui(App[None]):
|
| 9 |
+
"""Minimal shell used during sprint scaffolding."""
|
| 10 |
+
|
| 11 |
+
TITLE = "BirdNET Uploader"
|
| 12 |
+
|
| 13 |
+
def compose(self) -> ComposeResult:
|
| 14 |
+
yield Container(
|
| 15 |
+
Static("BirdNET Uploader\nLogin -> Repo -> Upload", id="title"),
|
| 16 |
+
Button("Exit", id="exit"),
|
| 17 |
+
id="main",
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
| 21 |
+
if event.button.id == "exit":
|
| 22 |
+
self.exit()
|
src/uploader/web_ui.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import shutil
|
| 5 |
+
import tempfile
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import List
|
| 8 |
+
|
| 9 |
+
import gradio as gr
|
| 10 |
+
|
| 11 |
+
from huggingface_hub import HfApi
|
| 12 |
+
|
| 13 |
+
from .auth_service import AuthService
|
| 14 |
+
from .repo_service import RepositoryService
|
| 15 |
+
from .scanner import LocalScanner
|
| 16 |
+
from .deduplicator import Deduplicator
|
| 17 |
+
from .batch_uploader import BatchUploader
|
| 18 |
+
from .manifest import build_manifest_from_scan, manifest_to_bytes, write_shards_from_csv_rows
|
| 19 |
+
from .session_manager import SessionManager
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _store_uploaded_files(files: List, dst: Path) -> None:
|
| 23 |
+
dst.mkdir(parents=True, exist_ok=True)
|
| 24 |
+
if not files:
|
| 25 |
+
return
|
| 26 |
+
for f in files:
|
| 27 |
+
# Gradio may pass either a path string, a file-like object, or a dict
|
| 28 |
+
src_path = None
|
| 29 |
+
if isinstance(f, str) and os.path.exists(f):
|
| 30 |
+
src_path = f
|
| 31 |
+
elif hasattr(f, "name") and isinstance(getattr(f, "name"), str) and os.path.exists(getattr(f, "name")):
|
| 32 |
+
src_path = getattr(f, "name")
|
| 33 |
+
elif isinstance(f, dict) and f.get("name") and os.path.exists(f.get("name")):
|
| 34 |
+
src_path = f.get("name")
|
| 35 |
+
|
| 36 |
+
if src_path:
|
| 37 |
+
shutil.copy(src_path, dst / Path(src_path).name)
|
| 38 |
+
else:
|
| 39 |
+
# fallback: try reading bytes and write
|
| 40 |
+
try:
|
| 41 |
+
data = f.read()
|
| 42 |
+
out_path = dst / (getattr(f, "name", "uploaded_file"))
|
| 43 |
+
with open(out_path, "wb") as fh:
|
| 44 |
+
fh.write(data)
|
| 45 |
+
except Exception:
|
| 46 |
+
# ignore silently for best-effort behavior
|
| 47 |
+
continue
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _handle_upload(token: str, repo_id: str, files, csv_file, remote_base: str, workers: int | None):
|
| 51 |
+
# Basic validation
|
| 52 |
+
if not token or not repo_id:
|
| 53 |
+
return "Error: token and repo_id are required." , None
|
| 54 |
+
|
| 55 |
+
# Store uploaded files to a temp directory
|
| 56 |
+
tmpdir = Path(tempfile.mkdtemp(prefix="birdnet-upload-"))
|
| 57 |
+
try:
|
| 58 |
+
_store_uploaded_files(files or [], tmpdir)
|
| 59 |
+
|
| 60 |
+
csv_path = None
|
| 61 |
+
if csv_file:
|
| 62 |
+
csv_tmp = tmpdir / Path(csv_file.name).name if hasattr(csv_file, "name") else tmpdir / "detections.csv"
|
| 63 |
+
try:
|
| 64 |
+
# csv_file may be a dict or file-like
|
| 65 |
+
if isinstance(csv_file, str) and os.path.exists(csv_file):
|
| 66 |
+
shutil.copy(csv_file, csv_tmp)
|
| 67 |
+
elif hasattr(csv_file, "name") and os.path.exists(getattr(csv_file, "name")):
|
| 68 |
+
shutil.copy(getattr(csv_file, "name"), csv_tmp)
|
| 69 |
+
elif isinstance(csv_file, dict) and csv_file.get("name") and os.path.exists(csv_file.get("name")):
|
| 70 |
+
shutil.copy(csv_file.get("name"), csv_tmp)
|
| 71 |
+
else:
|
| 72 |
+
# try read() then write
|
| 73 |
+
data = csv_file.read()
|
| 74 |
+
with open(csv_tmp, "wb") as fh:
|
| 75 |
+
fh.write(data)
|
| 76 |
+
csv_path = str(csv_tmp)
|
| 77 |
+
except Exception:
|
| 78 |
+
csv_path = None
|
| 79 |
+
|
| 80 |
+
# Validate repo / create if necessary
|
| 81 |
+
api = HfApi(token=token)
|
| 82 |
+
repo_service = RepositoryService(token)
|
| 83 |
+
validation = repo_service.validate_repo(repo_id)
|
| 84 |
+
if not validation.get("is_valid"):
|
| 85 |
+
# attempt to create required structure
|
| 86 |
+
try:
|
| 87 |
+
repo_service.create_dataset(repo_id, private=True)
|
| 88 |
+
except Exception:
|
| 89 |
+
# fall back to warning
|
| 90 |
+
pass
|
| 91 |
+
|
| 92 |
+
# Build scan summary from files on disk
|
| 93 |
+
scanner = LocalScanner()
|
| 94 |
+
# LocalScanner expects a directory of segments; use tmpdir
|
| 95 |
+
summary = scanner.scan_folder(str(tmpdir))
|
| 96 |
+
|
| 97 |
+
# Upload CSV if present
|
| 98 |
+
try:
|
| 99 |
+
if csv_path:
|
| 100 |
+
api.upload_file(path_or_file=str(csv_path), path_in_repo="index/detections.csv", repo_id=repo_id, repo_type="dataset")
|
| 101 |
+
except Exception as exc:
|
| 102 |
+
return f"CSV upload failed: {exc}", None
|
| 103 |
+
|
| 104 |
+
# Build and upload manifest
|
| 105 |
+
try:
|
| 106 |
+
csv_rows = None
|
| 107 |
+
if csv_path:
|
| 108 |
+
import csv
|
| 109 |
+
|
| 110 |
+
with open(csv_path, newline="", encoding="utf-8") as fh:
|
| 111 |
+
reader = csv.DictReader(fh)
|
| 112 |
+
csv_rows = list(reader)
|
| 113 |
+
|
| 114 |
+
manifest = build_manifest_from_scan(repo_id, summary, csv_rows=csv_rows)
|
| 115 |
+
api.upload_file(path_or_fileobj=manifest_to_bytes(manifest), path_in_repo="index/manifest.json", repo_id=repo_id, repo_type="dataset")
|
| 116 |
+
except Exception as exc:
|
| 117 |
+
return f"Manifest upload failed: {exc}", None
|
| 118 |
+
|
| 119 |
+
# Generate and upload shards if CSV provided
|
| 120 |
+
try:
|
| 121 |
+
if csv_path and csv_rows:
|
| 122 |
+
shards = write_shards_from_csv_rows(csv_rows)
|
| 123 |
+
for shard_path in shards:
|
| 124 |
+
try:
|
| 125 |
+
api.upload_file(path_or_file=str(shard_path), path_in_repo=f"index/shards/{shard_path.name}", repo_id=repo_id, repo_type="dataset")
|
| 126 |
+
except Exception:
|
| 127 |
+
# continue with other shards
|
| 128 |
+
continue
|
| 129 |
+
except Exception:
|
| 130 |
+
pass
|
| 131 |
+
|
| 132 |
+
# Dedup + batch upload
|
| 133 |
+
dedup = Deduplicator(api=api, repo_id=repo_id)
|
| 134 |
+
session = SessionManager()
|
| 135 |
+
uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_workers=workers)
|
| 136 |
+
|
| 137 |
+
# Build file_infos from scanned summary
|
| 138 |
+
file_infos = []
|
| 139 |
+
for species, items in summary.get("by_species", {}).items():
|
| 140 |
+
for it in items:
|
| 141 |
+
file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
|
| 142 |
+
|
| 143 |
+
def on_progress(state: dict):
|
| 144 |
+
# for now, we don't stream progress; could integrate via websocket
|
| 145 |
+
pass
|
| 146 |
+
|
| 147 |
+
result = uploader.upload_files(file_infos, remote_base=remote_base or "audio", on_progress=on_progress)
|
| 148 |
+
return f"Upload finished: {result}", result
|
| 149 |
+
finally:
|
| 150 |
+
# cleanup tempdir
|
| 151 |
+
try:
|
| 152 |
+
shutil.rmtree(tmpdir)
|
| 153 |
+
except Exception:
|
| 154 |
+
pass
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def create_uploader_app():
|
| 158 |
+
with gr.Blocks(title="BirdNET Uploader") as app:
|
| 159 |
+
gr.Markdown("""
|
| 160 |
+
# 🐦 BirdNET Uploader
|
| 161 |
+
Upload audio segments (multiple files) and optional CSV of detections to a Hugging Face dataset.
|
| 162 |
+
|
| 163 |
+
If you need to upload many files, download the Windows container and run the standalone uploader locally for best performance.
|
| 164 |
+
""")
|
| 165 |
+
|
| 166 |
+
with gr.Group("Authentication"):
|
| 167 |
+
token = gr.Textbox(label="Hugging Face Token", type="password", placeholder="hf_xxx...", interactive=True)
|
| 168 |
+
|
| 169 |
+
with gr.Group("Dataset"):
|
| 170 |
+
repo_id = gr.Textbox(label="Dataset repo id (owner/name)", placeholder="username/dataset-name")
|
| 171 |
+
remote_base = gr.Textbox(label="Remote base path (default: audio)", value="audio")
|
| 172 |
+
|
| 173 |
+
with gr.Group("Files"):
|
| 174 |
+
files = gr.File(label="Audio files (multiple)", file_count="multiple")
|
| 175 |
+
csv_file = gr.File(label="Optional detections CSV", file_count="single", file_types=[".csv"])
|
| 176 |
+
|
| 177 |
+
with gr.Row():
|
| 178 |
+
workers = gr.Number(label="Parallel workers (for large uploads)", value=4)
|
| 179 |
+
upload_btn = gr.Button("Start Upload")
|
| 180 |
+
|
| 181 |
+
with gr.Group("Progress"):
|
| 182 |
+
status = gr.Textbox(label="Status", interactive=False)
|
| 183 |
+
|
| 184 |
+
with gr.Group("Download"):
|
| 185 |
+
gr.Markdown("[Download Windows standalone uploader (zip)](https://github.com/jrrribeiro/BirdNET-Uploader-App/releases/latest/download/birdnet-uploader-windows.zip)")
|
| 186 |
+
|
| 187 |
+
def _start(token_val, repo_val, files_val, csv_val, remote_base_val, workers_val):
|
| 188 |
+
status_text, result = _handle_upload(token_val, repo_val, files_val, csv_val, remote_base_val, int(workers_val) if workers_val else None)
|
| 189 |
+
return status_text
|
| 190 |
+
|
| 191 |
+
upload_btn.click(_start, inputs=[token, repo_id, files, csv_file, remote_base, workers], outputs=[status])
|
| 192 |
+
|
| 193 |
+
return app
|
src/uploader_cli/__init__.py
DELETED
|
@@ -1,7 +0,0 @@
|
|
| 1 |
-
"""BirdNET local uploader CLI package."""
|
| 2 |
-
|
| 3 |
-
__all__ = [
|
| 4 |
-
"__version__",
|
| 5 |
-
]
|
| 6 |
-
|
| 7 |
-
__version__ = "0.1.0"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|