jrrribeiro commited on
Commit
4e31d1f
·
1 Parent(s): f961d5d

Import uploader files from validator workspace

Browse files
Files changed (36) hide show
  1. README.md +6 -6
  2. app.py +3 -2
  3. birdnet-uploader.spec +38 -0
  4. build/build_exe.py +34 -0
  5. pyproject.toml +3 -3
  6. requirements.txt +1 -0
  7. src/uploader/__init__.py +17 -0
  8. src/{uploader_cli → uploader}/auth_service.py +4 -6
  9. src/{uploader_cli → uploader}/batch_uploader.py +0 -11
  10. src/{uploader_cli → uploader}/config.py +0 -1
  11. src/{uploader_cli → uploader}/deduplicator.py +0 -0
  12. src/{uploader_cli → uploader}/error_handler.py +0 -0
  13. src/{uploader_cli → uploader}/exceptions.py +0 -0
  14. src/{uploader_cli → uploader}/hash_utils.py +0 -0
  15. src/{uploader_cli → uploader}/main.py +11 -21
  16. src/{uploader_cli → uploader}/manifest.py +0 -11
  17. src/{uploader_cli → uploader}/repo_service.py +0 -0
  18. src/{uploader_cli → uploader}/scanner.py +1 -2
  19. src/{uploader_cli → uploader}/session_manager.py +0 -0
  20. src/{uploader_cli → uploader}/tui.py +0 -0
  21. src/uploader/uploader/__init__.py +17 -0
  22. src/uploader/uploader/auth_service.py +60 -0
  23. src/uploader/uploader/batch_uploader.py +102 -0
  24. src/uploader/uploader/config.py +61 -0
  25. src/uploader/uploader/deduplicator.py +115 -0
  26. src/uploader/uploader/error_handler.py +29 -0
  27. src/uploader/uploader/exceptions.py +26 -0
  28. src/uploader/uploader/hash_utils.py +21 -0
  29. src/uploader/uploader/main.py +107 -0
  30. src/uploader/uploader/manifest.py +58 -0
  31. src/uploader/uploader/repo_service.py +110 -0
  32. src/uploader/uploader/scanner.py +62 -0
  33. src/uploader/uploader/session_manager.py +98 -0
  34. src/uploader/uploader/tui.py +22 -0
  35. src/uploader/web_ui.py +193 -0
  36. src/uploader_cli/__init__.py +0 -7
README.md CHANGED
@@ -1,5 +1,5 @@
1
  ---
2
- title: BirdNET-Validator-App
3
  emoji: "🐦"
4
  colorFrom: green
5
  colorTo: blue
@@ -10,14 +10,14 @@ app_file: app.py
10
  pinned: false
11
  ---
12
 
13
- # BirdNET-Validator-App
14
 
15
- BirdNET-Validator-App is a multi-purpose BirdNET workflow repository with two complementary entry points:
16
 
17
- 1. A Gradio-based web application for human validation of BirdNET detections in multi-project workflows.
18
- 2. A local command-line uploader for resumable Hugging Face dataset uploads with checkpoints, deduplication, and a Windows-friendly interactive flow.
19
 
20
- The validator app focuses on review, authorization, and auditability. The uploader CLI focuses on reliable dataset ingestion for large audio collections.
21
 
22
  ## Contents
23
 
 
1
  ---
2
+ title: BirdNET-Uploader-App
3
  emoji: "🐦"
4
  colorFrom: green
5
  colorTo: blue
 
10
  pinned: false
11
  ---
12
 
13
+ # BirdNET-Uploader-App
14
 
15
+ BirdNET-Uploader-App is a focused tool for reliably uploading large audio collections and optional detection CSVs to Hugging Face datasets. It provides:
16
 
17
+ - A unified Gradio web interface for small-to-medium uploads (single flow; CSV optional and auto-detected)
18
+ - A Windows standalone container (downloadable) for large uploads without installing Python
19
 
20
+ This repository contains a resumable uploader with deduplication, manifest/shard generation, and a simple web UI for convenience.
21
 
22
  ## Contents
23
 
app.py CHANGED
@@ -1,10 +1,11 @@
1
  import os
2
 
3
- from src.ui.app_factory import create_app
 
4
 
5
 
6
  if __name__ == "__main__":
7
- app = create_app()
8
  port = int(os.getenv("PORT") or os.getenv("BIRDNET_VALIDATOR_PORT") or "7860")
9
  host = os.getenv("BIRDNET_VALIDATOR_HOST") or "0.0.0.0"
10
  app.launch(server_name=host, server_port=port)
 
1
  import os
2
 
3
+ # New uploader web UI entry
4
+ from src.uploader.web_ui import create_uploader_app
5
 
6
 
7
  if __name__ == "__main__":
8
+ app = create_uploader_app()
9
  port = int(os.getenv("PORT") or os.getenv("BIRDNET_VALIDATOR_PORT") or "7860")
10
  host = os.getenv("BIRDNET_VALIDATOR_HOST") or "0.0.0.0"
11
  app.launch(server_name=host, server_port=port)
birdnet-uploader.spec ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- mode: python ; coding: utf-8 -*-
2
+
3
+
4
+ a = Analysis(
5
+ ['C:\\Users\\jonat\\Documents\\Python\\BirdNET-validator-App\\app.py'],
6
+ pathex=[],
7
+ binaries=[],
8
+ datas=[],
9
+ hiddenimports=[],
10
+ hookspath=[],
11
+ hooksconfig={},
12
+ runtime_hooks=[],
13
+ excludes=[],
14
+ noarchive=False,
15
+ optimize=0,
16
+ )
17
+ pyz = PYZ(a.pure)
18
+
19
+ exe = EXE(
20
+ pyz,
21
+ a.scripts,
22
+ a.binaries,
23
+ a.datas,
24
+ [],
25
+ name='birdnet-uploader',
26
+ debug=False,
27
+ bootloader_ignore_signals=False,
28
+ strip=False,
29
+ upx=True,
30
+ upx_exclude=[],
31
+ runtime_tmpdir=None,
32
+ console=True,
33
+ disable_windowed_traceback=False,
34
+ argv_emulation=False,
35
+ target_arch=None,
36
+ codesign_identity=None,
37
+ entitlements_file=None,
38
+ )
build/build_exe.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import sys
4
+ import subprocess
5
+ from pathlib import Path
6
+
7
+ ROOT = Path(__file__).resolve().parents[1]
8
+ ENTRY = ROOT / "app.py"
9
+ DIST = ROOT / "dist"
10
+
11
+ def main():
12
+ if not ENTRY.exists():
13
+ print("Entrypoint app.py not found; aborting.")
14
+ raise SystemExit(1)
15
+
16
+ cmd = [
17
+ sys.executable,
18
+ "-m",
19
+ "PyInstaller",
20
+ "--clean",
21
+ "--onefile",
22
+ "--noconfirm",
23
+ "--name",
24
+ "birdnet-uploader",
25
+ str(ENTRY),
26
+ ]
27
+
28
+ print("Running:", " ".join(cmd))
29
+ subprocess.check_call(cmd)
30
+ print("Build finished. See dist/ for output.")
31
+
32
+
33
+ if __name__ == "__main__":
34
+ main()
pyproject.toml CHANGED
@@ -1,12 +1,12 @@
1
  [project]
2
- name = "BirdNET-Validator-App"
3
  version = "0.1.0"
4
- description = "BirdNET-Validator-App for multi-project bird detection review workflows"
5
  readme = "README.md"
6
  requires-python = ">=3.11"
7
 
8
  [project.scripts]
9
- birdnet-uploader = "src.uploader_cli.main:cli"
10
 
11
  [tool.pytest.ini_options]
12
  testpaths = ["tests"]
 
1
  [project]
2
+ name = "BirdNET-Uploader-App"
3
  version = "0.1.0"
4
+ description = "BirdNET-Uploader-App: resumable uploader for large audio datasets to Hugging Face"
5
  readme = "README.md"
6
  requires-python = ">=3.11"
7
 
8
  [project.scripts]
9
+ birdnet-uploader = "src.uploader.main:cli"
10
 
11
  [tool.pytest.ini_options]
12
  testpaths = ["tests"]
requirements.txt CHANGED
@@ -10,3 +10,4 @@ click>=8.1.7
10
  rich>=13.7.1
11
  textual>=0.58.1
12
  keyring>=25.2.1
 
 
10
  rich>=13.7.1
11
  textual>=0.58.1
12
  keyring>=25.2.1
13
+ pyinstaller>=6.20.0
src/uploader/__init__.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """BirdNET Uploader package (refactored from uploader_cli)."""
2
+
3
+ __all__ = [
4
+ "auth_service",
5
+ "batch_uploader",
6
+ "config",
7
+ "deduplicator",
8
+ "error_handler",
9
+ "exceptions",
10
+ "hash_utils",
11
+ "main",
12
+ "manifest",
13
+ "repo_service",
14
+ "scanner",
15
+ "session_manager",
16
+ "tui",
17
+ ]
src/{uploader_cli → uploader}/auth_service.py RENAMED
@@ -12,10 +12,10 @@ from .exceptions import AuthenticationError
12
 
13
  class AuthService:
14
  """Authenticate and securely store Hugging Face tokens.
15
-
16
  Token resolution order (highest to lowest priority):
17
  1. HF_TOKEN environment variable (useful for container/CI)
18
- 2. keyring (OS-specific secure storage: Credential Manager on Windows, Keychain on macOS, Secret Service on Linux)
19
  3. Manual --token CLI option
20
  """
21
 
@@ -46,15 +46,13 @@ class AuthService:
46
 
47
  def get_token(self) -> str | None:
48
  """Retrieve token from environment or secure storage.
49
-
50
  Priority: HF_TOKEN env var > keyring storage.
51
  """
52
- # First check environment variable (useful for Docker/CI)
53
  env_token = os.getenv(TOKEN_ENV_VAR)
54
  if env_token:
55
  return env_token.strip()
56
-
57
- # Fall back to keyring (OS-specific secure storage)
58
  try:
59
  return keyring.get_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
60
  except Exception as exc: # pragma: no cover - backend-specific behavior
 
12
 
13
  class AuthService:
14
  """Authenticate and securely store Hugging Face tokens.
15
+
16
  Token resolution order (highest to lowest priority):
17
  1. HF_TOKEN environment variable (useful for container/CI)
18
+ 2. keyring (OS-specific secure storage)
19
  3. Manual --token CLI option
20
  """
21
 
 
46
 
47
  def get_token(self) -> str | None:
48
  """Retrieve token from environment or secure storage.
49
+
50
  Priority: HF_TOKEN env var > keyring storage.
51
  """
 
52
  env_token = os.getenv(TOKEN_ENV_VAR)
53
  if env_token:
54
  return env_token.strip()
55
+
 
56
  try:
57
  return keyring.get_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
58
  except Exception as exc: # pragma: no cover - backend-specific behavior
src/{uploader_cli → uploader}/batch_uploader.py RENAMED
@@ -40,8 +40,6 @@ class BatchUploader:
40
  last_exc: Exception | None = None
41
  while attempts <= self.max_retries:
42
  try:
43
- # The HfApi.upload_file signature varies across versions; callers/tests
44
- # should accept the positional usage shown here and monkeypatch in tests.
45
  self._api.upload_file(path_or_file=local_path, path_in_repo=remote_path, repo_id=self.repo_id)
46
  return
47
  except Exception as exc: # pragma: no cover - network behavior
@@ -60,16 +58,11 @@ class BatchUploader:
60
  batch_size: int | None = None,
61
  on_progress: Callable[[dict[str, Any]], None] | None = None,
62
  ) -> dict[str, Any]:
63
- """Upload an iterable of file info dictionaries.
64
-
65
- Each file_info must include `full_path` and `relative_path`. `size` is optional.
66
- """
67
  batch_size = batch_size or MAX_BATCH_SIZE
68
  uploaded = 0
69
  skipped = 0
70
  failed = 0
71
 
72
- # First pass: decide which files to skip and which to upload
73
  to_upload: List[Tuple[str, str, int]] = []
74
  for info in file_infos:
75
  full_path = str(info["full_path"])
@@ -87,7 +80,6 @@ class BatchUploader:
87
 
88
  to_upload.append((full_path, remote_path, int(info.get("size", 0))))
89
 
90
- # If no parallelism requested, perform sequential upload to preserve behavior
91
  if not self.max_workers or self.max_workers <= 1:
92
  for full_path, remote_path, size in to_upload:
93
  try:
@@ -107,7 +99,6 @@ class BatchUploader:
107
  on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
108
  return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
109
 
110
- # Parallel upload path
111
  def _worker(task: Tuple[str, str, int]) -> Tuple[str, str]:
112
  full_path, remote_path, size = task
113
  try:
@@ -134,5 +125,3 @@ class BatchUploader:
134
  on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
135
 
136
  return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
137
-
138
- return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
 
40
  last_exc: Exception | None = None
41
  while attempts <= self.max_retries:
42
  try:
 
 
43
  self._api.upload_file(path_or_file=local_path, path_in_repo=remote_path, repo_id=self.repo_id)
44
  return
45
  except Exception as exc: # pragma: no cover - network behavior
 
58
  batch_size: int | None = None,
59
  on_progress: Callable[[dict[str, Any]], None] | None = None,
60
  ) -> dict[str, Any]:
 
 
 
 
61
  batch_size = batch_size or MAX_BATCH_SIZE
62
  uploaded = 0
63
  skipped = 0
64
  failed = 0
65
 
 
66
  to_upload: List[Tuple[str, str, int]] = []
67
  for info in file_infos:
68
  full_path = str(info["full_path"])
 
80
 
81
  to_upload.append((full_path, remote_path, int(info.get("size", 0))))
82
 
 
83
  if not self.max_workers or self.max_workers <= 1:
84
  for full_path, remote_path, size in to_upload:
85
  try:
 
99
  on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
100
  return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
101
 
 
102
  def _worker(task: Tuple[str, str, int]) -> Tuple[str, str]:
103
  full_path, remote_path, size = task
104
  try:
 
125
  on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
126
 
127
  return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
 
 
src/{uploader_cli → uploader}/config.py RENAMED
@@ -28,7 +28,6 @@ def _resolve_path_from_env(env_name: str, default: Path) -> Path:
28
 
29
 
30
  def get_session_root() -> Path:
31
- """Resolve and create the session storage root directory."""
32
  data_root = os.getenv("BIRDNET_UPLOADER_DATA_DIR")
33
  if data_root:
34
  return _resolve_path_from_env("BIRDNET_UPLOADER_SESSION_DIR", Path(data_root).expanduser() / "sessions")
 
28
 
29
 
30
  def get_session_root() -> Path:
 
31
  data_root = os.getenv("BIRDNET_UPLOADER_DATA_DIR")
32
  if data_root:
33
  return _resolve_path_from_env("BIRDNET_UPLOADER_SESSION_DIR", Path(data_root).expanduser() / "sessions")
src/{uploader_cli → uploader}/deduplicator.py RENAMED
File without changes
src/{uploader_cli → uploader}/error_handler.py RENAMED
File without changes
src/{uploader_cli → uploader}/exceptions.py RENAMED
File without changes
src/{uploader_cli → uploader}/hash_utils.py RENAMED
File without changes
src/{uploader_cli → uploader}/main.py RENAMED
@@ -7,11 +7,11 @@ from typing import Any, Callable
7
 
8
  import click
9
 
10
- from src.uploader_cli.auth_service import AuthService
11
- from src.uploader_cli.error_handler import build_error_message
12
- from src.uploader_cli.repo_service import RepositoryService
13
- from src.uploader_cli.scanner import LocalScanner
14
- from src.uploader_cli.session_manager import SessionManager
15
 
16
 
17
  def handle_cli_errors(fn: Callable[..., Any]) -> Callable[..., Any]:
@@ -124,7 +124,7 @@ def upload_cmd(
124
  batch_size: int | None,
125
  workers: int | None,
126
  dry_run: bool,
127
- ) -> None:
128
  """Upload local segments (and optional CSV) into the HF dataset.
129
 
130
  Token resolution (in order of priority):
@@ -134,13 +134,11 @@ def upload_cmd(
134
 
135
  The command creates/uses a session checkpoint for resumable uploads.
136
  """
137
- # Resolve token from CLI, env var, or secure storage
138
  if token:
139
  hf_token = token
140
  else:
141
  hf_token = AuthService().require_token()
142
 
143
- # Scan local segments
144
  scanner = LocalScanner()
145
  summary = scanner.scan_folder(segments_dir)
146
  total_files = summary["total_files"]
@@ -163,7 +161,6 @@ def upload_cmd(
163
  click.echo(f"Would also upload CSV: {csv_file} -> index/detections.csv")
164
  return
165
 
166
- # Prepare API and session
167
  api = None
168
  try:
169
  from huggingface_hub import HfApi
@@ -172,28 +169,23 @@ def upload_cmd(
172
  except Exception as exc: # pragma: no cover - external
173
  raise click.ClickException(build_error_message(exc)) from exc
174
 
175
- # Validate dataset exists
176
  repo_service = RepositoryService(hf_token)
177
  validation = repo_service.validate_repo(repo_id)
178
  if not validation.get("is_valid"):
179
  click.echo(f"Warning: dataset {repo_id} may be missing structure: {validation.get('missing_prefixes')}")
180
 
181
- # Create deduplicator, session and uploader
182
- from src.uploader_cli.deduplicator import Deduplicator
183
- from src.uploader_cli.batch_uploader import BatchUploader
184
 
185
  dedup = Deduplicator(api=api, repo_id=repo_id)
186
  session = SessionManager(session_id=session_id) if session_id else SessionManager()
187
  uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_retries=None, initial_backoff=None, max_workers=workers)
188
 
189
- # Flatten file infos
190
  file_infos: list[dict] = []
191
  for species, items in summary["by_species"].items():
192
  for it in items:
193
- # keep the relative path as scanned (preserves species directories)
194
  file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
195
 
196
- # Upload CSV first if present
197
  if csv_file:
198
  click.echo(f"Uploading CSV to index/detections.csv...")
199
  try:
@@ -202,9 +194,8 @@ def upload_cmd(
202
  except Exception as exc:
203
  raise click.ClickException(f"CSV upload failed: {exc}") from exc
204
 
205
- # Build and upload manifest.json based on scan and optional CSV
206
  try:
207
- from src.uploader_cli.manifest import build_manifest_from_scan, manifest_to_bytes
208
 
209
  csv_rows = None
210
  if csv_file:
@@ -220,13 +211,12 @@ def upload_cmd(
220
  except Exception as exc: # pragma: no cover - external API
221
  raise click.ClickException(f"Manifest upload failed: {exc}") from exc
222
 
223
- # Generate and upload shards based on CSV rows (if any)
224
  try:
225
- from src.uploader_cli.manifest import write_shards_from_csv_rows
226
 
227
  if csv_file and csv_rows:
228
  click.echo("Generating index shards from CSV detections...")
229
- shards = write_shards_from_csv_rows(csv_rows, shard_size=int(__import__("src.uploader_cli.config", fromlist=["INDEX_SHARD_SIZE"]).INDEX_SHARD_SIZE))
230
  for shard_path in shards:
231
  shard_name = shard_path.name
232
  click.echo(f"Uploading shard: index/shards/{shard_name}")
 
7
 
8
  import click
9
 
10
+ from .auth_service import AuthService
11
+ from .error_handler import build_error_message
12
+ from .repo_service import RepositoryService
13
+ from .scanner import LocalScanner
14
+ from .session_manager import SessionManager
15
 
16
 
17
  def handle_cli_errors(fn: Callable[..., Any]) -> Callable[..., Any]:
 
124
  batch_size: int | None,
125
  workers: int | None,
126
  dry_run: bool,
127
+ ):
128
  """Upload local segments (and optional CSV) into the HF dataset.
129
 
130
  Token resolution (in order of priority):
 
134
 
135
  The command creates/uses a session checkpoint for resumable uploads.
136
  """
 
137
  if token:
138
  hf_token = token
139
  else:
140
  hf_token = AuthService().require_token()
141
 
 
142
  scanner = LocalScanner()
143
  summary = scanner.scan_folder(segments_dir)
144
  total_files = summary["total_files"]
 
161
  click.echo(f"Would also upload CSV: {csv_file} -> index/detections.csv")
162
  return
163
 
 
164
  api = None
165
  try:
166
  from huggingface_hub import HfApi
 
169
  except Exception as exc: # pragma: no cover - external
170
  raise click.ClickException(build_error_message(exc)) from exc
171
 
 
172
  repo_service = RepositoryService(hf_token)
173
  validation = repo_service.validate_repo(repo_id)
174
  if not validation.get("is_valid"):
175
  click.echo(f"Warning: dataset {repo_id} may be missing structure: {validation.get('missing_prefixes')}")
176
 
177
+ from .deduplicator import Deduplicator
178
+ from .batch_uploader import BatchUploader
 
179
 
180
  dedup = Deduplicator(api=api, repo_id=repo_id)
181
  session = SessionManager(session_id=session_id) if session_id else SessionManager()
182
  uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_retries=None, initial_backoff=None, max_workers=workers)
183
 
 
184
  file_infos: list[dict] = []
185
  for species, items in summary["by_species"].items():
186
  for it in items:
 
187
  file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
188
 
 
189
  if csv_file:
190
  click.echo(f"Uploading CSV to index/detections.csv...")
191
  try:
 
194
  except Exception as exc:
195
  raise click.ClickException(f"CSV upload failed: {exc}") from exc
196
 
 
197
  try:
198
+ from .manifest import build_manifest_from_scan, manifest_to_bytes
199
 
200
  csv_rows = None
201
  if csv_file:
 
211
  except Exception as exc: # pragma: no cover - external API
212
  raise click.ClickException(f"Manifest upload failed: {exc}") from exc
213
 
 
214
  try:
215
+ from .manifest import write_shards_from_csv_rows
216
 
217
  if csv_file and csv_rows:
218
  click.echo("Generating index shards from CSV detections...")
219
+ shards = write_shards_from_csv_rows(csv_rows, shard_size=int(__import__("src.uploader.config", fromlist=["INDEX_SHARD_SIZE"]).INDEX_SHARD_SIZE))
220
  for shard_path in shards:
221
  shard_name = shard_path.name
222
  click.echo(f"Uploading shard: index/shards/{shard_name}")
src/{uploader_cli → uploader}/manifest.py RENAMED
@@ -32,10 +32,8 @@ def build_manifest_from_scan(repo_id: str, scan_summary: dict[str, Any], *, csv_
32
  shards: list[str] = []
33
 
34
  if csv_rows is not None:
35
- # csv_rows expected to be an iterable of detection dicts with at least 'audio_file' key
36
  rows = list(csv_rows)
37
  total_detections = len(rows)
38
- # derive unique audio files referenced by CSV
39
  unique_files = {str(r.get("audio_file") or r.get("file") or "") for r in rows}
40
  unique_files = {p for p in unique_files if p}
41
  if unique_files:
@@ -61,13 +59,6 @@ def manifest_to_bytes(manifest: dict[str, Any]) -> bytes:
61
 
62
 
63
  def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: int = INDEX_SHARD_SIZE) -> List[Path]:
64
- """Write shards from CSV rows.
65
-
66
- Prefer Parquet (pandas/pyarrow). If pandas (and its parquet engine) is
67
- unavailable, fall back to newline-delimited JSON (.jsonl).
68
-
69
- Returns list of Path objects for shard files created.
70
- """
71
  rows_list = list(rows)
72
  if not rows_list:
73
  return []
@@ -83,7 +74,6 @@ def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: in
83
  df = pd.DataFrame.from_records(chunk)
84
  shard_name = f"shard-{shard_index:06d}.parquet"
85
  shard_path = tmpdir / shard_name
86
- # pandas will choose pyarrow or fastparquet if available
87
  df.to_parquet(shard_path, index=False)
88
  else:
89
  shard_name = f"shard-{shard_index:06d}.jsonl"
@@ -96,7 +86,6 @@ def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: in
96
 
97
  return out_paths
98
  except Exception:
99
- # If anything fails, attempt to cleanup created files then re-raise
100
  try:
101
  for p in tmpdir.iterdir():
102
  p.unlink()
 
32
  shards: list[str] = []
33
 
34
  if csv_rows is not None:
 
35
  rows = list(csv_rows)
36
  total_detections = len(rows)
 
37
  unique_files = {str(r.get("audio_file") or r.get("file") or "") for r in rows}
38
  unique_files = {p for p in unique_files if p}
39
  if unique_files:
 
59
 
60
 
61
  def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: int = INDEX_SHARD_SIZE) -> List[Path]:
 
 
 
 
 
 
 
62
  rows_list = list(rows)
63
  if not rows_list:
64
  return []
 
74
  df = pd.DataFrame.from_records(chunk)
75
  shard_name = f"shard-{shard_index:06d}.parquet"
76
  shard_path = tmpdir / shard_name
 
77
  df.to_parquet(shard_path, index=False)
78
  else:
79
  shard_name = f"shard-{shard_index:06d}.jsonl"
 
86
 
87
  return out_paths
88
  except Exception:
 
89
  try:
90
  for p in tmpdir.iterdir():
91
  p.unlink()
src/{uploader_cli → uploader}/repo_service.py RENAMED
File without changes
src/{uploader_cli → uploader}/scanner.py RENAMED
@@ -7,7 +7,6 @@ from typing import Any
7
  from .config import AUDIO_EXTENSIONS
8
  from .hash_utils import compute_file_hash
9
 
10
-
11
  class LocalScanner:
12
  """Scan local directories and group files by inferred species."""
13
 
@@ -59,4 +58,4 @@ class LocalScanner:
59
  return Path(relative_parts[0]).stem.split("_")[0] or "unknown"
60
 
61
  stem = file_path.stem
62
- return stem.split("_")[0] if "_" in stem else "unknown"
 
7
  from .config import AUDIO_EXTENSIONS
8
  from .hash_utils import compute_file_hash
9
 
 
10
  class LocalScanner:
11
  """Scan local directories and group files by inferred species."""
12
 
 
58
  return Path(relative_parts[0]).stem.split("_")[0] or "unknown"
59
 
60
  stem = file_path.stem
61
+ return stem.split("_")[0] if "_" in stem else "unknown"
src/{uploader_cli → uploader}/session_manager.py RENAMED
File without changes
src/{uploader_cli → uploader}/tui.py RENAMED
File without changes
src/uploader/uploader/__init__.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Compatibility shim package exporting uploader_cli modules under src.uploader."""
2
+
3
+ __all__ = [
4
+ "auth_service",
5
+ "batch_uploader",
6
+ "config",
7
+ "deduplicator",
8
+ "error_handler",
9
+ "exceptions",
10
+ "hash_utils",
11
+ "main",
12
+ "manifest",
13
+ "repo_service",
14
+ "scanner",
15
+ "session_manager",
16
+ "tui",
17
+ ]
src/uploader/uploader/auth_service.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+ import keyring
6
+ from huggingface_hub import HfApi
7
+
8
+ from .config import KEYRING_ACCOUNT, KEYRING_SERVICE
9
+ from .exceptions import AuthenticationError
10
+
11
+
12
+ class AuthService:
13
+ """Authenticate and securely store Hugging Face tokens."""
14
+
15
+ KEYRING_SERVICE = KEYRING_SERVICE
16
+ KEYRING_ACCOUNT = KEYRING_ACCOUNT
17
+
18
+ def authenticate(self, token: str) -> dict[str, Any]:
19
+ token = token.strip()
20
+ if not token:
21
+ raise AuthenticationError("Token is required")
22
+
23
+ try:
24
+ api = HfApi(token=token)
25
+ whoami = api.whoami()
26
+ except Exception as exc: # pragma: no cover - external API behavior
27
+ raise AuthenticationError(f"Token validation failed: {exc}") from exc
28
+
29
+ try:
30
+ keyring.set_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT, token)
31
+ except Exception as exc: # pragma: no cover - backend-specific behavior
32
+ raise AuthenticationError(f"Token validated, but could not be saved securely: {exc}") from exc
33
+
34
+ return {
35
+ "username": whoami.get("name") or "unknown",
36
+ "email": whoami.get("email", ""),
37
+ "user_id": whoami.get("id") or whoami.get("user_id", ""),
38
+ }
39
+
40
+ def get_token(self) -> str | None:
41
+ try:
42
+ return keyring.get_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
43
+ except Exception as exc: # pragma: no cover - backend-specific behavior
44
+ raise AuthenticationError(f"Could not read stored token: {exc}") from exc
45
+
46
+ def require_token(self) -> str:
47
+ token = self.get_token()
48
+ if not token:
49
+ raise AuthenticationError("No stored token found. Run 'birdnet-uploader login' first.")
50
+ return token
51
+
52
+ def clear_token(self) -> None:
53
+ try:
54
+ keyring.delete_password(self.KEYRING_SERVICE, self.KEYRING_ACCOUNT)
55
+ except keyring.errors.PasswordDeleteError:
56
+ return
57
+ except Exception as exc: # pragma: no cover - backend-specific behavior
58
+ raise AuthenticationError(f"Could not clear stored token: {exc}") from exc
59
+ else:
60
+ return
src/uploader/uploader/batch_uploader.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import time
4
+ from pathlib import Path
5
+ from typing import Any, Callable, Iterable
6
+
7
+ from huggingface_hub import HfApi
8
+
9
+ from .config import MAX_BATCH_SIZE, RETRY_MAX_ATTEMPTS, RETRY_INITIAL_BACKOFF_SECONDS
10
+ from .deduplicator import Deduplicator
11
+ from .session_manager import SessionManager
12
+
13
+
14
+ class BatchUploader:
15
+ """Orchestrate uploads with deduplication, retries and session checkpoints."""
16
+
17
+ def __init__(
18
+ self,
19
+ api: HfApi,
20
+ repo_id: str,
21
+ deduplicator: Deduplicator,
22
+ session: SessionManager | None = None,
23
+ *,
24
+ max_retries: int | None = None,
25
+ initial_backoff: float | None = None,
26
+ ) -> None:
27
+ self._api = api
28
+ self.repo_id = repo_id
29
+ self.deduplicator = deduplicator
30
+ self.session = session or SessionManager.create_session()
31
+ self.max_retries = max_retries or RETRY_MAX_ATTEMPTS
32
+ self.initial_backoff = initial_backoff or RETRY_INITIAL_BACKOFF_SECONDS
33
+
34
+ def _upload_file_with_retry(self, local_path: str | Path, remote_path: str) -> None:
35
+ attempts = 0
36
+ last_exc: Exception | None = None
37
+ while attempts <= self.max_retries:
38
+ try:
39
+ # The HfApi.upload_file signature varies across versions; callers/tests
40
+ # should accept the positional usage shown here and monkeypatch in tests.
41
+ self._api.upload_file(path_or_file=local_path, path_in_repo=remote_path, repo_id=self.repo_id)
42
+ return
43
+ except Exception as exc: # pragma: no cover - network behavior
44
+ last_exc = exc
45
+ attempts += 1
46
+ if attempts > self.max_retries:
47
+ raise
48
+ backoff = self.initial_backoff * (2 ** (attempts - 1))
49
+ time.sleep(backoff)
50
+
51
+ def upload_files(
52
+ self,
53
+ file_infos: Iterable[dict[str, Any]],
54
+ *,
55
+ remote_base: str = "",
56
+ batch_size: int | None = None,
57
+ on_progress: Callable[[dict[str, Any]], None] | None = None,
58
+ ) -> dict[str, Any]:
59
+ """Upload an iterable of file info dictionaries.
60
+
61
+ Each file_info must include `full_path` and `relative_path`. `size` is optional.
62
+ """
63
+ batch_size = batch_size or MAX_BATCH_SIZE
64
+ uploaded = 0
65
+ skipped = 0
66
+ failed = 0
67
+
68
+ for info in file_infos:
69
+ full_path = str(info["full_path"])
70
+ relative = info["relative_path"].lstrip("/")
71
+ remote_path = f"{remote_base.rstrip('/')}/{relative}" if remote_base else relative
72
+ remote_path = remote_path.lstrip("/")
73
+
74
+ decision = self.deduplicator.check_remote(remote_path, file_path=full_path)
75
+ if decision.get("status") == "skip":
76
+ skipped += 1
77
+ self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=0, status="skipped")
78
+ if on_progress:
79
+ on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
80
+ continue
81
+
82
+ try:
83
+ self._upload_file_with_retry(full_path, remote_path)
84
+ # mark remote as present in local dedup cache
85
+ try:
86
+ self.deduplicator.mark_uploaded(remote_path)
87
+ except Exception:
88
+ # non-fatal cache update failure should not break overall flow
89
+ pass
90
+
91
+ uploaded += 1
92
+ bytes_uploaded = int(info.get("size", 0))
93
+ self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=bytes_uploaded)
94
+ if on_progress:
95
+ on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
96
+ except Exception as exc:
97
+ failed += 1
98
+ self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
99
+ if on_progress:
100
+ on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
101
+
102
+ return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
src/uploader/uploader/config.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+
8
+ APP_NAME = "BirdNET Uploader"
9
+ SCHEMA_VERSION = "1.0.0"
10
+ AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
11
+ INDEX_SHARD_SIZE = 10_000
12
+
13
+ MAX_BATCH_SIZE = 10
14
+ RETRY_MAX_ATTEMPTS = 3
15
+ RETRY_INITIAL_BACKOFF_SECONDS = 1.0
16
+ RETRY_MAX_BACKOFF_SECONDS = 30.0
17
+
18
+ KEYRING_SERVICE = "birdnet-uploader"
19
+ KEYRING_ACCOUNT = "hf_token"
20
+
21
+
22
+ def _resolve_path_from_env(env_name: str, default: Path) -> Path:
23
+ raw = os.getenv(env_name)
24
+ value = Path(raw).expanduser() if raw else default
25
+ value.mkdir(parents=True, exist_ok=True)
26
+ return value
27
+
28
+
29
+ def get_session_root() -> Path:
30
+ """Resolve and create the session storage root directory."""
31
+ return _resolve_path_from_env(
32
+ "BIRDNET_UPLOADER_SESSION_DIR",
33
+ Path.home() / ".birdnet-uploader" / "sessions",
34
+ )
35
+
36
+
37
+ def get_cache_root() -> Path:
38
+ return _resolve_path_from_env(
39
+ "BIRDNET_UPLOADER_CACHE_DIR",
40
+ Path.home() / ".birdnet-uploader" / "cache",
41
+ )
42
+
43
+
44
+ def get_log_root() -> Path:
45
+ return _resolve_path_from_env(
46
+ "BIRDNET_UPLOADER_LOG_DIR",
47
+ Path.home() / ".birdnet-uploader" / "logs",
48
+ )
49
+
50
+
51
+ def get_runtime_config() -> dict[str, Any]:
52
+ return {
53
+ "session_root": str(get_session_root()),
54
+ "cache_root": str(get_cache_root()),
55
+ "log_root": str(get_log_root()),
56
+ "max_batch_size": MAX_BATCH_SIZE,
57
+ "retry_max_attempts": RETRY_MAX_ATTEMPTS,
58
+ "retry_initial_backoff_seconds": RETRY_INITIAL_BACKOFF_SECONDS,
59
+ "retry_max_backoff_seconds": RETRY_MAX_BACKOFF_SECONDS,
60
+ "index_shard_size": INDEX_SHARD_SIZE,
61
+ }
src/uploader/uploader/deduplicator.py ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import hashlib
5
+ from datetime import UTC, datetime
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from huggingface_hub import HfApi
10
+
11
+ from .config import get_cache_root
12
+ from .hash_utils import compute_file_hash
13
+ from .exceptions import ValidationError
14
+
15
+
16
+ class Deduplicator:
17
+ """Check remote file presence and cache the remote path index locally."""
18
+
19
+ def __init__(self, api: HfApi, repo_id: str, *, repo_type: str = "dataset") -> None:
20
+ self._api = api
21
+ self.repo_id = repo_id
22
+ self.repo_type = repo_type
23
+ self._remote_paths: set[str] | None = None
24
+ self._decision_cache: dict[str, dict[str, Any]] = {}
25
+ self.cache_path = self._build_cache_path()
26
+
27
+ def _build_cache_path(self) -> Path:
28
+ cache_root = get_cache_root() / "dedup"
29
+ cache_root.mkdir(parents=True, exist_ok=True)
30
+ digest = hashlib.sha256(f"{self.repo_type}:{self.repo_id}".encode("utf-8")).hexdigest()[:16]
31
+ return cache_root / f"{digest}.json"
32
+
33
+ def load_cached_index(self) -> set[str]:
34
+ if self._remote_paths is not None:
35
+ return self._remote_paths
36
+
37
+ if self.cache_path.exists():
38
+ raw = self.cache_path.read_text(encoding="utf-8")
39
+ try:
40
+ payload = json.loads(raw)
41
+ except json.JSONDecodeError as exc:
42
+ raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc
43
+
44
+ remote_paths = payload.get("remote_paths", [])
45
+ if not isinstance(remote_paths, list):
46
+ raise ValidationError(f"Invalid dedup cache format at {self.cache_path}")
47
+ self._remote_paths = {str(path) for path in remote_paths}
48
+ return self._remote_paths
49
+
50
+ self._remote_paths = self.refresh_remote_index()
51
+ return self._remote_paths
52
+
53
+ def refresh_remote_index(self) -> set[str]:
54
+ try:
55
+ repo_files = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
56
+ except Exception as exc: # pragma: no cover - external API behavior
57
+ raise ValidationError(f"Could not read remote file listing for {self.repo_id}: {exc}") from exc
58
+
59
+ remote_paths = {str(path) for path in repo_files}
60
+ self._remote_paths = remote_paths
61
+ self._write_cache(remote_paths)
62
+ return remote_paths
63
+
64
+ def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
65
+ cache_key = self._build_decision_key(remote_path, file_path=file_path)
66
+ cached = self._decision_cache.get(cache_key)
67
+ if cached is not None:
68
+ return dict(cached)
69
+
70
+ remote_paths = self.load_cached_index()
71
+ exists = remote_path in remote_paths
72
+
73
+ payload = {
74
+ "remote_path": remote_path,
75
+ "status": "skip" if exists else "upload",
76
+ "cached_remote_index": True,
77
+ }
78
+ if file_path is not None:
79
+ file_path_obj = Path(file_path)
80
+ payload["local_size"] = file_path_obj.stat().st_size
81
+ payload["local_sha256"] = compute_file_hash(file_path_obj)
82
+
83
+ self._decision_cache[cache_key] = dict(payload)
84
+ return payload
85
+
86
+ def mark_uploaded(self, remote_path: str) -> None:
87
+ remote_paths = self.load_cached_index()
88
+ if remote_path not in remote_paths:
89
+ remote_paths.add(remote_path)
90
+ self._write_cache(remote_paths)
91
+
92
+ keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path]
93
+ for key in keys_to_update:
94
+ self._decision_cache[key]["status"] = "skip"
95
+
96
+ def _write_cache(self, remote_paths: set[str]) -> None:
97
+ payload = {
98
+ "repo_id": self.repo_id,
99
+ "repo_type": self.repo_type,
100
+ "updated_at": datetime.now(UTC).isoformat(),
101
+ "remote_paths": sorted(remote_paths),
102
+ }
103
+ temp_path = self.cache_path.with_suffix(".json.tmp")
104
+ temp_path.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
105
+ temp_path.replace(self.cache_path)
106
+
107
+ def _build_decision_key(self, remote_path: str, *, file_path: str | Path | None) -> str:
108
+ if file_path is None:
109
+ return remote_path
110
+ file_path_obj = Path(file_path)
111
+ try:
112
+ stat = file_path_obj.stat()
113
+ except FileNotFoundError as exc:
114
+ raise ValidationError(f"Local file not found: {file_path_obj}") from exc
115
+ return f"{remote_path}:{file_path_obj.resolve()}:{stat.st_size}:{stat.st_mtime_ns}"
src/uploader/uploader/error_handler.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from .exceptions import (
4
+ AuthenticationError,
5
+ RepositoryError,
6
+ SessionError,
7
+ UploaderCliError,
8
+ UploadError,
9
+ ValidationError,
10
+ )
11
+
12
+
13
+ ERROR_HINTS: dict[type[UploaderCliError], str] = {
14
+ AuthenticationError: "Run 'birdnet-uploader login' and verify your Hugging Face token.",
15
+ RepositoryError: "Check repository id format (owner/name) and write permissions on Hugging Face.",
16
+ SessionError: "Verify session id and local checkpoint files under .birdnet-uploader/sessions.",
17
+ ValidationError: "Fix invalid paths or inputs and retry the command.",
18
+ UploadError: "Check your network and run the command again to resume from checkpoint.",
19
+ }
20
+
21
+
22
+ def build_error_message(error: Exception) -> str:
23
+ if isinstance(error, UploaderCliError):
24
+ hint = error.suggestion or ERROR_HINTS.get(type(error), "")
25
+ if hint:
26
+ return f"{error} Hint: {hint}"
27
+ return str(error)
28
+
29
+ return f"Unexpected error: {error}"
src/uploader/uploader/exceptions.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ class UploaderCliError(Exception):
2
+ """Base exception for uploader CLI."""
3
+
4
+ def __init__(self, message: str, *, suggestion: str = ""):
5
+ super().__init__(message)
6
+ self.suggestion = suggestion
7
+
8
+
9
+ class AuthenticationError(UploaderCliError):
10
+ """Raised when Hugging Face authentication fails."""
11
+
12
+
13
+ class RepositoryError(UploaderCliError):
14
+ """Raised for dataset repository failures."""
15
+
16
+
17
+ class SessionError(UploaderCliError):
18
+ """Raised when session state cannot be read or written."""
19
+
20
+
21
+ class ValidationError(UploaderCliError):
22
+ """Raised when user input or local resources are invalid."""
23
+
24
+
25
+ class UploadError(UploaderCliError):
26
+ """Raised when upload operations fail after retries."""
src/uploader/uploader/hash_utils.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ from pathlib import Path
5
+
6
+
7
+ def compute_file_hash(path: str | Path, *, chunk_size: int = 1024 * 1024) -> str:
8
+ """Compute a streaming SHA-256 hash for a file."""
9
+ digest = hashlib.sha256()
10
+ file_path = Path(path)
11
+
12
+ with file_path.open("rb") as handle:
13
+ for chunk in iter(lambda: handle.read(chunk_size), b""):
14
+ digest.update(chunk)
15
+
16
+ return digest.hexdigest()
17
+
18
+
19
+ def verify_file_integrity(path: str | Path, expected_hash: str, *, chunk_size: int = 1024 * 1024) -> bool:
20
+ """Return True when a file's SHA-256 matches the expected hash."""
21
+ return compute_file_hash(path, chunk_size=chunk_size) == expected_hash
src/uploader/uploader/main.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from functools import wraps
5
+ from pathlib import Path
6
+ from typing import Any, Callable
7
+
8
+ import click
9
+
10
+ from src.uploader.auth_service import AuthService
11
+ from src.uploader.error_handler import build_error_message
12
+ from src.uploader.repo_service import RepositoryService
13
+ from src.uploader.scanner import LocalScanner
14
+ from src.uploader.session_manager import SessionManager
15
+
16
+
17
+ def handle_cli_errors(fn: Callable[..., Any]) -> Callable[..., Any]:
18
+ @wraps(fn)
19
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
20
+ try:
21
+ return fn(*args, **kwargs)
22
+ except click.ClickException:
23
+ raise
24
+ except Exception as exc:
25
+ raise click.ClickException(build_error_message(exc)) from exc
26
+
27
+ return wrapper
28
+
29
+
30
+ @click.group(help="BirdNET local uploader CLI")
31
+ def cli() -> None:
32
+ pass
33
+
34
+
35
+ @cli.command("login")
36
+ @click.option("--token", prompt=True, hide_input=True, help="Hugging Face token")
37
+ @handle_cli_errors
38
+ def login_cmd(token: str) -> None:
39
+ service = AuthService()
40
+ user = service.authenticate(token)
41
+ click.echo(f"OK: authenticated as {user['username']}")
42
+
43
+
44
+ @cli.command("init-repo")
45
+ @click.option("--repo-id", required=True, help="Dataset repo id in owner/name format")
46
+ @click.option("--private/--public", "private_repo", default=True, show_default=True)
47
+ @handle_cli_errors
48
+ def init_repo_cmd(repo_id: str, private_repo: bool) -> None:
49
+ token = AuthService().require_token()
50
+
51
+ created = RepositoryService(token).create_dataset(repo_id, private=private_repo)
52
+ click.echo(f"OK: dataset ready at {created}")
53
+
54
+
55
+ @cli.command("scan")
56
+ @click.option("--segments", "segments_dir", required=True, type=click.Path(exists=True, file_okay=False))
57
+ @handle_cli_errors
58
+ def scan_cmd(segments_dir: str) -> None:
59
+ summary = LocalScanner().scan_folder(segments_dir)
60
+ click.echo(
61
+ json.dumps(
62
+ {
63
+ "total_files": summary["total_files"],
64
+ "total_size": summary["total_size"],
65
+ "species_count": len(summary["by_species"]),
66
+ },
67
+ ensure_ascii=True,
68
+ indent=2,
69
+ )
70
+ )
71
+
72
+
73
+ @cli.command("start")
74
+ @click.option("--repo-id", required=True)
75
+ @click.option("--segments", "segments_dir", required=True, type=click.Path(exists=True, file_okay=False))
76
+ @handle_cli_errors
77
+ def start_cmd(repo_id: str, segments_dir: str) -> None:
78
+ scanner = LocalScanner()
79
+ summary = scanner.scan_folder(segments_dir)
80
+ session = SessionManager()
81
+ payload = {
82
+ "repo_id": repo_id,
83
+ "segments_dir": str(Path(segments_dir).resolve()),
84
+ "total_files": summary["total_files"],
85
+ "total_size": summary["total_size"],
86
+ "uploaded": 0,
87
+ "failed": 0,
88
+ "status": "ready",
89
+ }
90
+ session.save_checkpoint(payload)
91
+ click.echo(f"OK: session created: {session.session_id}")
92
+ click.echo(f"Checkpoint: {session.checkpoint_path}")
93
+
94
+
95
+ @cli.command("resume")
96
+ @click.argument("session_id")
97
+ @handle_cli_errors
98
+ def resume_cmd(session_id: str) -> None:
99
+ session = SessionManager(session_id=session_id)
100
+ payload = session.load_checkpoint()
101
+ if not payload:
102
+ raise click.ClickException(f"Session has no checkpoint: {session_id}")
103
+ click.echo(json.dumps(payload, ensure_ascii=True, indent=2))
104
+
105
+
106
+ if __name__ == "__main__":
107
+ cli()
src/uploader/uploader/manifest.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from io import BytesIO
5
+ from pathlib import Path
6
+ from typing import Any, Iterable, List
7
+
8
+ from .config import SCHEMA_VERSION, INDEX_SHARD_SIZE
9
+
10
+
11
+ def build_manifest_from_scan(scan: dict[str, Any], repo_id: str | None = None) -> dict[str, Any]:
12
+ project_slug = None
13
+ if repo_id and "/" in repo_id:
14
+ project_slug = repo_id.split("/", 1)[1]
15
+
16
+ return {
17
+ "schema_version": SCHEMA_VERSION,
18
+ "project_slug": project_slug or "unknown",
19
+ "dataset_repo_id": repo_id,
20
+ "index": {
21
+ "total_detections": 0,
22
+ "total_audio_files": scan.get("total_files", 0),
23
+ "shard_size": INDEX_SHARD_SIZE,
24
+ "shards": [],
25
+ },
26
+ }
27
+
28
+
29
+ def manifest_to_bytes(manifest: dict[str, Any]) -> bytes:
30
+ return json.dumps(manifest, ensure_ascii=True, indent=2).encode("utf-8")
31
+
32
+
33
+ def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], out_dir: Path) -> List[Path]:
34
+ """Write simple JSONL shards from CSV rows. Returns list of shard paths."""
35
+ out_dir.mkdir(parents=True, exist_ok=True)
36
+ shard_paths: List[Path] = []
37
+ shard_index = 0
38
+ batch: List[dict[str, Any]] = []
39
+ for row in rows:
40
+ batch.append(row)
41
+ if len(batch) >= INDEX_SHARD_SIZE:
42
+ path = out_dir / f"shard-{shard_index:04d}.jsonl"
43
+ with path.open("w", encoding="utf-8") as fh:
44
+ for item in batch:
45
+ fh.write(json.dumps(item, ensure_ascii=True) + "\n")
46
+ shard_paths.append(path)
47
+ shard_index += 1
48
+ batch = []
49
+
50
+ if batch:
51
+ path = out_dir / f"shard-{shard_index:04d}.jsonl"
52
+ with path.open("w", encoding="utf-8") as fh:
53
+ for item in batch:
54
+ fh.write(json.dumps(item, ensure_ascii=True) + "\n")
55
+ shard_paths.append(path)
56
+
57
+ return shard_paths
58
+ from src.uploader_cli.manifest import *
src/uploader/uploader/repo_service.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from io import BytesIO
5
+ from typing import Any
6
+
7
+ from huggingface_hub import HfApi
8
+
9
+ from .config import INDEX_SHARD_SIZE, SCHEMA_VERSION
10
+ from .exceptions import RepositoryError
11
+
12
+
13
+ class RepositoryService:
14
+ """Create and initialize dataset repositories for uploader sessions."""
15
+
16
+ def __init__(self, token: str):
17
+ self._api = HfApi(token=token)
18
+
19
+ def create_dataset(self, repo_id: str, *, private: bool = True) -> str:
20
+ if not self._is_valid_repo_id(repo_id):
21
+ raise RepositoryError("Repository id must be in the form 'owner/name'")
22
+
23
+ try:
24
+ self._api.create_repo(repo_id=repo_id, repo_type="dataset", private=private, exist_ok=True)
25
+ self._init_structure(repo_id)
26
+ except Exception as exc: # pragma: no cover - external API behavior
27
+ raise RepositoryError(f"Could not create dataset: {exc}") from exc
28
+
29
+ return repo_id
30
+
31
+ def _init_structure(self, repo_id: str) -> None:
32
+ project_slug = self._project_slug_from_repo_id(repo_id)
33
+ placeholder_dirs = [
34
+ f"audio/{project_slug}/.gitkeep",
35
+ "index/shards/.gitkeep",
36
+ "validations/.gitkeep",
37
+ "audit/ingestion-runs/.gitkeep",
38
+ ]
39
+
40
+ for path_in_repo in placeholder_dirs:
41
+ self._api.upload_file(
42
+ path_or_fileobj=BytesIO(b""),
43
+ path_in_repo=path_in_repo,
44
+ repo_id=repo_id,
45
+ repo_type="dataset",
46
+ )
47
+
48
+ manifest = {
49
+ "schema_version": SCHEMA_VERSION,
50
+ "project_slug": project_slug,
51
+ "dataset_repo_id": repo_id,
52
+ "index": {
53
+ "total_detections": 0,
54
+ "total_audio_files": 0,
55
+ "shard_size": INDEX_SHARD_SIZE,
56
+ "shards": [],
57
+ },
58
+ }
59
+ self._api.upload_file(
60
+ path_or_fileobj=BytesIO(json.dumps(manifest, ensure_ascii=True, indent=2).encode("utf-8")),
61
+ path_in_repo="index/manifest.json",
62
+ repo_id=repo_id,
63
+ repo_type="dataset",
64
+ )
65
+
66
+ def validate_repo(self, repo_id: str) -> dict[str, Any]:
67
+ if not self._is_valid_repo_id(repo_id):
68
+ raise RepositoryError("Repository id must be in the form 'owner/name'")
69
+
70
+ try:
71
+ repo_files = self._api.list_repo_files(repo_id=repo_id, repo_type="dataset")
72
+ except Exception as exc: # pragma: no cover - external API behavior
73
+ raise RepositoryError(f"Could not list dataset files: {exc}") from exc
74
+
75
+ project_slug = self._project_slug_from_repo_id(repo_id)
76
+ required_prefixes = [
77
+ f"audio/{project_slug}/",
78
+ "index/shards/",
79
+ "validations/",
80
+ "audit/ingestion-runs/",
81
+ ]
82
+ missing_prefixes = [
83
+ prefix for prefix in required_prefixes if not any(path.startswith(prefix) for path in repo_files)
84
+ ]
85
+
86
+ has_manifest = "index/manifest.json" in repo_files
87
+ manifest_ok = has_manifest
88
+ manifest_error = ""
89
+
90
+ is_valid = not missing_prefixes and has_manifest
91
+ return {
92
+ "repo_id": repo_id,
93
+ "is_valid": is_valid,
94
+ "project_slug": project_slug,
95
+ "missing_prefixes": missing_prefixes,
96
+ "has_manifest": has_manifest,
97
+ "manifest_ok": manifest_ok if has_manifest else False,
98
+ "manifest_error": manifest_error,
99
+ }
100
+
101
+ @staticmethod
102
+ def _is_valid_repo_id(repo_id: str) -> bool:
103
+ if not repo_id or repo_id.count("/") != 1:
104
+ return False
105
+ owner, name = repo_id.split("/", maxsplit=1)
106
+ return bool(owner.strip()) and bool(name.strip()) and " " not in repo_id
107
+
108
+ @staticmethod
109
+ def _project_slug_from_repo_id(repo_id: str) -> str:
110
+ return repo_id.split("/", maxsplit=1)[1]
src/uploader/uploader/scanner.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+ from .config import AUDIO_EXTENSIONS
8
+ from .hash_utils import compute_file_hash
9
+
10
+
11
+ class LocalScanner:
12
+ """Scan local directories and group files by inferred species."""
13
+
14
+ def scan_folder(self, folder_path: str) -> dict[str, Any]:
15
+ root_path = Path(folder_path).resolve()
16
+ by_species: dict[str, list[dict[str, Any]]] = {}
17
+ total_files = 0
18
+ total_size = 0
19
+
20
+ for root, _, files in os.walk(root_path):
21
+ current_root = Path(root)
22
+ for filename in files:
23
+ ext = Path(filename).suffix.lower()
24
+ if ext not in AUDIO_EXTENSIONS:
25
+ continue
26
+
27
+ full_path = current_root / filename
28
+ relative_path = full_path.relative_to(root_path)
29
+ species = self._infer_species(root_path, full_path)
30
+ item = {
31
+ "name": filename,
32
+ "full_path": str(full_path),
33
+ "relative_path": relative_path.as_posix(),
34
+ "species": species,
35
+ "size": full_path.stat().st_size,
36
+ "sha256": compute_file_hash(full_path),
37
+ }
38
+ by_species.setdefault(species, []).append(item)
39
+ total_files += 1
40
+ total_size += item["size"]
41
+
42
+ return {
43
+ "total_files": total_files,
44
+ "total_size": total_size,
45
+ "by_species": by_species,
46
+ "root_path": str(root_path),
47
+ }
48
+
49
+ def _infer_species(self, root_path: Path, file_path: Path) -> str:
50
+ try:
51
+ relative_parts = file_path.relative_to(root_path).parts
52
+ except ValueError:
53
+ relative_parts = file_path.parts
54
+
55
+ if len(relative_parts) >= 2:
56
+ return relative_parts[0]
57
+
58
+ if len(relative_parts) == 1:
59
+ return Path(relative_parts[0]).stem.split("_")[0] or "unknown"
60
+
61
+ stem = file_path.stem
62
+ return stem.split("_")[0] if "_" in stem else "unknown"
src/uploader/uploader/session_manager.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import json
5
+ from datetime import UTC, datetime
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from .config import get_session_root
10
+ from .exceptions import SessionError
11
+
12
+
13
+ class SessionManager:
14
+ """Persist and recover upload checkpoints."""
15
+
16
+ def __init__(self, session_id: str | None = None):
17
+ if session_id is None:
18
+ now = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
19
+ session_id = f"upload-{now}"
20
+ self.session_id = session_id
21
+ self.session_dir = get_session_root() / session_id
22
+ self.session_dir.mkdir(parents=True, exist_ok=True)
23
+
24
+ @classmethod
25
+ def create_session(cls, metadata: dict[str, Any] | None = None) -> "SessionManager":
26
+ session = cls()
27
+ if metadata:
28
+ session.save_metadata(metadata)
29
+ return session
30
+
31
+ @property
32
+ def checkpoint_path(self) -> Path:
33
+ return self.session_dir / "checkpoint.json"
34
+
35
+ @property
36
+ def metadata_path(self) -> Path:
37
+ return self.session_dir / "metadata.json"
38
+
39
+ def _atomic_write_json(self, path: Path, payload: dict[str, Any]) -> None:
40
+ temp_path = path.with_suffix(path.suffix + ".tmp")
41
+ serialized = json.dumps(payload, ensure_ascii=True, indent=2) + "\n"
42
+ temp_path.write_text(serialized, encoding="utf-8")
43
+ with temp_path.open("a", encoding="utf-8") as handle:
44
+ handle.flush()
45
+ os.fsync(handle.fileno())
46
+ temp_path.replace(path)
47
+
48
+ def save_metadata(self, payload: dict[str, Any]) -> None:
49
+ self._atomic_write_json(self.metadata_path, payload)
50
+
51
+ def load_metadata(self) -> dict[str, Any]:
52
+ if not self.metadata_path.exists():
53
+ return {}
54
+ raw = self.metadata_path.read_text(encoding="utf-8")
55
+ try:
56
+ return json.loads(raw)
57
+ except json.JSONDecodeError as exc:
58
+ raise SessionError(f"Invalid metadata JSON for session '{self.session_id}'") from exc
59
+
60
+ def save_checkpoint(self, payload: dict[str, Any]) -> None:
61
+ self._atomic_write_json(self.checkpoint_path, payload)
62
+
63
+ def load_checkpoint(self) -> dict[str, Any]:
64
+ if not self.checkpoint_path.exists():
65
+ return {}
66
+ raw = self.checkpoint_path.read_text(encoding="utf-8")
67
+ try:
68
+ return json.loads(raw)
69
+ except json.JSONDecodeError as exc:
70
+ raise SessionError(f"Invalid checkpoint JSON for session '{self.session_id}'") from exc
71
+
72
+ def mark_file_done(self, *, remote_path: str, bytes_uploaded: int, status: str = "uploaded") -> dict[str, Any]:
73
+ checkpoint = self.load_checkpoint()
74
+ checkpoint.setdefault("uploaded_files", [])
75
+ checkpoint.setdefault("uploaded", 0)
76
+ checkpoint.setdefault("failed", 0)
77
+ checkpoint["uploaded"] = int(checkpoint["uploaded"]) + 1
78
+ checkpoint["status"] = status
79
+ checkpoint["last_completed_file"] = remote_path
80
+ checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
81
+ checkpoint["bytes_uploaded"] = int(checkpoint.get("bytes_uploaded", 0)) + int(bytes_uploaded)
82
+ checkpoint["uploaded_files"].append(remote_path)
83
+ self.save_checkpoint(checkpoint)
84
+ return checkpoint
85
+
86
+ def mark_file_failed(self, *, remote_path: str, error: str) -> dict[str, Any]:
87
+ checkpoint = self.load_checkpoint()
88
+ checkpoint.setdefault("failed_files", [])
89
+ checkpoint.setdefault("uploaded", 0)
90
+ checkpoint.setdefault("failed", 0)
91
+ checkpoint["failed"] = int(checkpoint["failed"]) + 1
92
+ checkpoint["status"] = "failed"
93
+ checkpoint["last_failed_file"] = remote_path
94
+ checkpoint["last_error"] = error
95
+ checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
96
+ checkpoint["failed_files"].append({"remote_path": remote_path, "error": error})
97
+ self.save_checkpoint(checkpoint)
98
+ return checkpoint
src/uploader/uploader/tui.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from textual.app import App, ComposeResult
4
+ from textual.containers import Container
5
+ from textual.widgets import Button, Static
6
+
7
+
8
+ class BirdNetUploaderTui(App[None]):
9
+ """Minimal shell used during sprint scaffolding."""
10
+
11
+ TITLE = "BirdNET Uploader"
12
+
13
+ def compose(self) -> ComposeResult:
14
+ yield Container(
15
+ Static("BirdNET Uploader\nLogin -> Repo -> Upload", id="title"),
16
+ Button("Exit", id="exit"),
17
+ id="main",
18
+ )
19
+
20
+ def on_button_pressed(self, event: Button.Pressed) -> None:
21
+ if event.button.id == "exit":
22
+ self.exit()
src/uploader/web_ui.py ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import shutil
5
+ import tempfile
6
+ from pathlib import Path
7
+ from typing import List
8
+
9
+ import gradio as gr
10
+
11
+ from huggingface_hub import HfApi
12
+
13
+ from .auth_service import AuthService
14
+ from .repo_service import RepositoryService
15
+ from .scanner import LocalScanner
16
+ from .deduplicator import Deduplicator
17
+ from .batch_uploader import BatchUploader
18
+ from .manifest import build_manifest_from_scan, manifest_to_bytes, write_shards_from_csv_rows
19
+ from .session_manager import SessionManager
20
+
21
+
22
+ def _store_uploaded_files(files: List, dst: Path) -> None:
23
+ dst.mkdir(parents=True, exist_ok=True)
24
+ if not files:
25
+ return
26
+ for f in files:
27
+ # Gradio may pass either a path string, a file-like object, or a dict
28
+ src_path = None
29
+ if isinstance(f, str) and os.path.exists(f):
30
+ src_path = f
31
+ elif hasattr(f, "name") and isinstance(getattr(f, "name"), str) and os.path.exists(getattr(f, "name")):
32
+ src_path = getattr(f, "name")
33
+ elif isinstance(f, dict) and f.get("name") and os.path.exists(f.get("name")):
34
+ src_path = f.get("name")
35
+
36
+ if src_path:
37
+ shutil.copy(src_path, dst / Path(src_path).name)
38
+ else:
39
+ # fallback: try reading bytes and write
40
+ try:
41
+ data = f.read()
42
+ out_path = dst / (getattr(f, "name", "uploaded_file"))
43
+ with open(out_path, "wb") as fh:
44
+ fh.write(data)
45
+ except Exception:
46
+ # ignore silently for best-effort behavior
47
+ continue
48
+
49
+
50
+ def _handle_upload(token: str, repo_id: str, files, csv_file, remote_base: str, workers: int | None):
51
+ # Basic validation
52
+ if not token or not repo_id:
53
+ return "Error: token and repo_id are required." , None
54
+
55
+ # Store uploaded files to a temp directory
56
+ tmpdir = Path(tempfile.mkdtemp(prefix="birdnet-upload-"))
57
+ try:
58
+ _store_uploaded_files(files or [], tmpdir)
59
+
60
+ csv_path = None
61
+ if csv_file:
62
+ csv_tmp = tmpdir / Path(csv_file.name).name if hasattr(csv_file, "name") else tmpdir / "detections.csv"
63
+ try:
64
+ # csv_file may be a dict or file-like
65
+ if isinstance(csv_file, str) and os.path.exists(csv_file):
66
+ shutil.copy(csv_file, csv_tmp)
67
+ elif hasattr(csv_file, "name") and os.path.exists(getattr(csv_file, "name")):
68
+ shutil.copy(getattr(csv_file, "name"), csv_tmp)
69
+ elif isinstance(csv_file, dict) and csv_file.get("name") and os.path.exists(csv_file.get("name")):
70
+ shutil.copy(csv_file.get("name"), csv_tmp)
71
+ else:
72
+ # try read() then write
73
+ data = csv_file.read()
74
+ with open(csv_tmp, "wb") as fh:
75
+ fh.write(data)
76
+ csv_path = str(csv_tmp)
77
+ except Exception:
78
+ csv_path = None
79
+
80
+ # Validate repo / create if necessary
81
+ api = HfApi(token=token)
82
+ repo_service = RepositoryService(token)
83
+ validation = repo_service.validate_repo(repo_id)
84
+ if not validation.get("is_valid"):
85
+ # attempt to create required structure
86
+ try:
87
+ repo_service.create_dataset(repo_id, private=True)
88
+ except Exception:
89
+ # fall back to warning
90
+ pass
91
+
92
+ # Build scan summary from files on disk
93
+ scanner = LocalScanner()
94
+ # LocalScanner expects a directory of segments; use tmpdir
95
+ summary = scanner.scan_folder(str(tmpdir))
96
+
97
+ # Upload CSV if present
98
+ try:
99
+ if csv_path:
100
+ api.upload_file(path_or_file=str(csv_path), path_in_repo="index/detections.csv", repo_id=repo_id, repo_type="dataset")
101
+ except Exception as exc:
102
+ return f"CSV upload failed: {exc}", None
103
+
104
+ # Build and upload manifest
105
+ try:
106
+ csv_rows = None
107
+ if csv_path:
108
+ import csv
109
+
110
+ with open(csv_path, newline="", encoding="utf-8") as fh:
111
+ reader = csv.DictReader(fh)
112
+ csv_rows = list(reader)
113
+
114
+ manifest = build_manifest_from_scan(repo_id, summary, csv_rows=csv_rows)
115
+ api.upload_file(path_or_fileobj=manifest_to_bytes(manifest), path_in_repo="index/manifest.json", repo_id=repo_id, repo_type="dataset")
116
+ except Exception as exc:
117
+ return f"Manifest upload failed: {exc}", None
118
+
119
+ # Generate and upload shards if CSV provided
120
+ try:
121
+ if csv_path and csv_rows:
122
+ shards = write_shards_from_csv_rows(csv_rows)
123
+ for shard_path in shards:
124
+ try:
125
+ api.upload_file(path_or_file=str(shard_path), path_in_repo=f"index/shards/{shard_path.name}", repo_id=repo_id, repo_type="dataset")
126
+ except Exception:
127
+ # continue with other shards
128
+ continue
129
+ except Exception:
130
+ pass
131
+
132
+ # Dedup + batch upload
133
+ dedup = Deduplicator(api=api, repo_id=repo_id)
134
+ session = SessionManager()
135
+ uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_workers=workers)
136
+
137
+ # Build file_infos from scanned summary
138
+ file_infos = []
139
+ for species, items in summary.get("by_species", {}).items():
140
+ for it in items:
141
+ file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
142
+
143
+ def on_progress(state: dict):
144
+ # for now, we don't stream progress; could integrate via websocket
145
+ pass
146
+
147
+ result = uploader.upload_files(file_infos, remote_base=remote_base or "audio", on_progress=on_progress)
148
+ return f"Upload finished: {result}", result
149
+ finally:
150
+ # cleanup tempdir
151
+ try:
152
+ shutil.rmtree(tmpdir)
153
+ except Exception:
154
+ pass
155
+
156
+
157
+ def create_uploader_app():
158
+ with gr.Blocks(title="BirdNET Uploader") as app:
159
+ gr.Markdown("""
160
+ # 🐦 BirdNET Uploader
161
+ Upload audio segments (multiple files) and optional CSV of detections to a Hugging Face dataset.
162
+
163
+ If you need to upload many files, download the Windows container and run the standalone uploader locally for best performance.
164
+ """)
165
+
166
+ with gr.Group("Authentication"):
167
+ token = gr.Textbox(label="Hugging Face Token", type="password", placeholder="hf_xxx...", interactive=True)
168
+
169
+ with gr.Group("Dataset"):
170
+ repo_id = gr.Textbox(label="Dataset repo id (owner/name)", placeholder="username/dataset-name")
171
+ remote_base = gr.Textbox(label="Remote base path (default: audio)", value="audio")
172
+
173
+ with gr.Group("Files"):
174
+ files = gr.File(label="Audio files (multiple)", file_count="multiple")
175
+ csv_file = gr.File(label="Optional detections CSV", file_count="single", file_types=[".csv"])
176
+
177
+ with gr.Row():
178
+ workers = gr.Number(label="Parallel workers (for large uploads)", value=4)
179
+ upload_btn = gr.Button("Start Upload")
180
+
181
+ with gr.Group("Progress"):
182
+ status = gr.Textbox(label="Status", interactive=False)
183
+
184
+ with gr.Group("Download"):
185
+ gr.Markdown("[Download Windows standalone uploader (zip)](https://github.com/jrrribeiro/BirdNET-Uploader-App/releases/latest/download/birdnet-uploader-windows.zip)")
186
+
187
+ def _start(token_val, repo_val, files_val, csv_val, remote_base_val, workers_val):
188
+ status_text, result = _handle_upload(token_val, repo_val, files_val, csv_val, remote_base_val, int(workers_val) if workers_val else None)
189
+ return status_text
190
+
191
+ upload_btn.click(_start, inputs=[token, repo_id, files, csv_file, remote_base, workers], outputs=[status])
192
+
193
+ return app
src/uploader_cli/__init__.py DELETED
@@ -1,7 +0,0 @@
1
- """BirdNET local uploader CLI package."""
2
-
3
- __all__ = [
4
- "__version__",
5
- ]
6
-
7
- __version__ = "0.1.0"