jrrribeiro commited on
Commit
7c83d76
·
1 Parent(s): 1f2c564

Add uploader release flow and integration test

Browse files
Dockerfile.windows ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM mcr.microsoft.com/windows/servercore:ltsc2022
2
+
3
+ SHELL ["powershell","-Command","$ErrorActionPreference = 'Stop';"]
4
+
5
+ WORKDIR C:\app
6
+
7
+ # Copy a one-folder PyInstaller bundle into the image at build time.
8
+ # Expect the build artifact layout: dist\birdnet-uploader\* (exe + _internal)
9
+ COPY dist\birdnet-uploader\ C:\app\
10
+
11
+ ENV HF_TOKEN=""
12
+
13
+ EXPOSE 0
14
+
15
+ ENTRYPOINT ["C:\\app\\birdnet-uploader.exe"]
scripts/smoke_test_upload.ps1 ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Param(
2
+ [string]$RepoId = "owner/dataset",
3
+ [string]$Segments = "C:\data\segments",
4
+ [string]$Token = $env:HF_TOKEN
5
+ )
6
+
7
+ if (-not $Token) {
8
+ Write-Error "HF_TOKEN environment variable must be set or pass --Token"
9
+ exit 2
10
+ }
11
+
12
+ Write-Host "Running smoke test upload against $RepoId using segments at $Segments"
13
+
14
+ # Dry run first
15
+ & .\birdnet-uploader.exe upload --repo-id $RepoId --segments $Segments --token $Token --dry-run
16
+
17
+ if ($LASTEXITCODE -ne 0) {
18
+ Write-Error "Dry run failed"
19
+ exit $LASTEXITCODE
20
+ }
21
+
22
+ Write-Host "Dry run OK. To perform a real small upload, run without --dry-run and mount a small folder."
src/uploader_cli/batch_uploader.py CHANGED
@@ -1,6 +1,8 @@
1
  from __future__ import annotations
2
 
3
  import time
 
 
4
  from pathlib import Path
5
  from typing import Any, Callable, Iterable
6
 
@@ -23,6 +25,7 @@ class BatchUploader:
23
  *,
24
  max_retries: int | None = None,
25
  initial_backoff: float | None = None,
 
26
  ) -> None:
27
  self._api = api
28
  self.repo_id = repo_id
@@ -30,6 +33,7 @@ class BatchUploader:
30
  self.session = session or SessionManager.create_session()
31
  self.max_retries = max_retries or RETRY_MAX_ATTEMPTS
32
  self.initial_backoff = initial_backoff or RETRY_INITIAL_BACKOFF_SECONDS
 
33
 
34
  def _upload_file_with_retry(self, local_path: str | Path, remote_path: str) -> None:
35
  attempts = 0
@@ -65,6 +69,8 @@ class BatchUploader:
65
  skipped = 0
66
  failed = 0
67
 
 
 
68
  for info in file_infos:
69
  full_path = str(info["full_path"])
70
  relative = info["relative_path"].lstrip("/")
@@ -79,24 +85,54 @@ class BatchUploader:
79
  on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
80
  continue
81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  try:
83
  self._upload_file_with_retry(full_path, remote_path)
84
- # mark remote as present in local dedup cache
85
  try:
86
  self.deduplicator.mark_uploaded(remote_path)
87
  except Exception:
88
- # non-fatal cache update failure should not break overall flow
89
  pass
90
-
91
- uploaded += 1
92
- bytes_uploaded = int(info.get("size", 0))
93
- self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=bytes_uploaded)
94
- if on_progress:
95
- on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
96
  except Exception as exc:
97
- failed += 1
98
  self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
 
 
 
 
 
 
 
 
 
 
99
  if on_progress:
100
  on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
101
 
102
  return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
 
 
 
1
  from __future__ import annotations
2
 
3
  import time
4
+ import concurrent.futures
5
+ from typing import List, Tuple
6
  from pathlib import Path
7
  from typing import Any, Callable, Iterable
8
 
 
25
  *,
26
  max_retries: int | None = None,
27
  initial_backoff: float | None = None,
28
+ max_workers: int | None = None,
29
  ) -> None:
30
  self._api = api
31
  self.repo_id = repo_id
 
33
  self.session = session or SessionManager.create_session()
34
  self.max_retries = max_retries or RETRY_MAX_ATTEMPTS
35
  self.initial_backoff = initial_backoff or RETRY_INITIAL_BACKOFF_SECONDS
36
+ self.max_workers = max_workers
37
 
38
  def _upload_file_with_retry(self, local_path: str | Path, remote_path: str) -> None:
39
  attempts = 0
 
69
  skipped = 0
70
  failed = 0
71
 
72
+ # First pass: decide which files to skip and which to upload
73
+ to_upload: List[Tuple[str, str, int]] = []
74
  for info in file_infos:
75
  full_path = str(info["full_path"])
76
  relative = info["relative_path"].lstrip("/")
 
85
  on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
86
  continue
87
 
88
+ to_upload.append((full_path, remote_path, int(info.get("size", 0))))
89
+
90
+ # If no parallelism requested, perform sequential upload to preserve behavior
91
+ if not self.max_workers or self.max_workers <= 1:
92
+ for full_path, remote_path, size in to_upload:
93
+ try:
94
+ self._upload_file_with_retry(full_path, remote_path)
95
+ try:
96
+ self.deduplicator.mark_uploaded(remote_path)
97
+ except Exception:
98
+ pass
99
+ uploaded += 1
100
+ self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=size)
101
+ if on_progress:
102
+ on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
103
+ except Exception as exc:
104
+ failed += 1
105
+ self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
106
+ if on_progress:
107
+ on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
108
+ return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
109
+
110
+ # Parallel upload path
111
+ def _worker(task: Tuple[str, str, int]) -> Tuple[str, str]:
112
+ full_path, remote_path, size = task
113
  try:
114
  self._upload_file_with_retry(full_path, remote_path)
 
115
  try:
116
  self.deduplicator.mark_uploaded(remote_path)
117
  except Exception:
 
118
  pass
119
+ self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=size)
120
+ return ("ok", remote_path)
 
 
 
 
121
  except Exception as exc:
 
122
  self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
123
+ return ("fail", remote_path)
124
+
125
+ with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as ex:
126
+ futures = {ex.submit(_worker, t): t for t in to_upload}
127
+ for fut in concurrent.futures.as_completed(futures):
128
+ res = fut.result()
129
+ if res[0] == "ok":
130
+ uploaded += 1
131
+ else:
132
+ failed += 1
133
  if on_progress:
134
  on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
135
 
136
  return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
137
+
138
+ return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
src/uploader_cli/deduplicator.py CHANGED
@@ -5,6 +5,7 @@ import hashlib
5
  from datetime import UTC, datetime
6
  from pathlib import Path
7
  from typing import Any
 
8
 
9
  from huggingface_hub import HfApi
10
 
@@ -23,6 +24,7 @@ class Deduplicator:
23
  self._remote_paths: set[str] | None = None
24
  self._decision_cache: dict[str, dict[str, Any]] = {}
25
  self.cache_path = self._build_cache_path()
 
26
 
27
  def _build_cache_path(self) -> Path:
28
  cache_root = get_cache_root() / "dedup"
@@ -33,23 +35,23 @@ class Deduplicator:
33
  def load_cached_index(self) -> set[str]:
34
  if self._remote_paths is not None:
35
  return self._remote_paths
36
-
37
- if self.cache_path.exists():
38
- raw = self.cache_path.read_text(encoding="utf-8")
39
- try:
40
- payload = json.loads(raw)
41
- except json.JSONDecodeError as exc:
42
- raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc
43
-
44
- remote_paths = payload.get("remote_paths", [])
45
- if not isinstance(remote_paths, list):
46
- raise ValidationError(f"Invalid dedup cache format at {self.cache_path}")
47
- self._remote_paths = {str(path) for path in remote_paths}
 
 
 
48
  return self._remote_paths
49
 
50
- self._remote_paths = self.refresh_remote_index()
51
- return self._remote_paths
52
-
53
  def refresh_remote_index(self) -> set[str]:
54
  try:
55
  repo_files = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
@@ -57,8 +59,9 @@ class Deduplicator:
57
  raise ValidationError(f"Could not read remote file listing for {self.repo_id}: {exc}") from exc
58
 
59
  remote_paths = {str(path) for path in repo_files}
60
- self._remote_paths = remote_paths
61
- self._write_cache(remote_paths)
 
62
  return remote_paths
63
 
64
  def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
@@ -84,14 +87,15 @@ class Deduplicator:
84
  return payload
85
 
86
  def mark_uploaded(self, remote_path: str) -> None:
87
- remote_paths = self.load_cached_index()
88
- if remote_path not in remote_paths:
89
- remote_paths.add(remote_path)
90
- self._write_cache(remote_paths)
91
-
92
- keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path]
93
- for key in keys_to_update:
94
- self._decision_cache[key]["status"] = "skip"
 
95
 
96
  def _write_cache(self, remote_paths: set[str]) -> None:
97
  payload = {
 
5
  from datetime import UTC, datetime
6
  from pathlib import Path
7
  from typing import Any
8
+ import threading
9
 
10
  from huggingface_hub import HfApi
11
 
 
24
  self._remote_paths: set[str] | None = None
25
  self._decision_cache: dict[str, dict[str, Any]] = {}
26
  self.cache_path = self._build_cache_path()
27
+ self._lock = threading.Lock()
28
 
29
  def _build_cache_path(self) -> Path:
30
  cache_root = get_cache_root() / "dedup"
 
35
  def load_cached_index(self) -> set[str]:
36
  if self._remote_paths is not None:
37
  return self._remote_paths
38
+ with self._lock:
39
+ if self.cache_path.exists():
40
+ raw = self.cache_path.read_text(encoding="utf-8")
41
+ try:
42
+ payload = json.loads(raw)
43
+ except json.JSONDecodeError as exc:
44
+ raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc
45
+
46
+ remote_paths = payload.get("remote_paths", [])
47
+ if not isinstance(remote_paths, list):
48
+ raise ValidationError(f"Invalid dedup cache format at {self.cache_path}")
49
+ self._remote_paths = {str(path) for path in remote_paths}
50
+ return self._remote_paths
51
+
52
+ self._remote_paths = self.refresh_remote_index()
53
  return self._remote_paths
54
 
 
 
 
55
  def refresh_remote_index(self) -> set[str]:
56
  try:
57
  repo_files = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
 
59
  raise ValidationError(f"Could not read remote file listing for {self.repo_id}: {exc}") from exc
60
 
61
  remote_paths = {str(path) for path in repo_files}
62
+ with self._lock:
63
+ self._remote_paths = remote_paths
64
+ self._write_cache(remote_paths)
65
  return remote_paths
66
 
67
  def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
 
87
  return payload
88
 
89
  def mark_uploaded(self, remote_path: str) -> None:
90
+ with self._lock:
91
+ remote_paths = self.load_cached_index()
92
+ if remote_path not in remote_paths:
93
+ remote_paths.add(remote_path)
94
+ self._write_cache(remote_paths)
95
+
96
+ keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path]
97
+ for key in keys_to_update:
98
+ self._decision_cache[key]["status"] = "skip"
99
 
100
  def _write_cache(self, remote_paths: set[str]) -> None:
101
  payload = {
src/uploader_cli/main.py CHANGED
@@ -103,5 +103,146 @@ def resume_cmd(session_id: str) -> None:
103
  click.echo(json.dumps(payload, ensure_ascii=True, indent=2))
104
 
105
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  if __name__ == "__main__":
107
  cli()
 
103
  click.echo(json.dumps(payload, ensure_ascii=True, indent=2))
104
 
105
 
106
+ @cli.command("upload")
107
+ @click.option("--repo-id", required=True, help="Dataset repo id in owner/name format")
108
+ @click.option("--segments", "segments_dir", required=True, type=click.Path(exists=True, file_okay=False), help="Local folder with audio segments")
109
+ @click.option("--csv", "csv_file", required=False, type=click.Path(exists=True, dir_okay=False), help="Optional detections CSV to upload to the dataset")
110
+ @click.option("--token", required=False, help="Hugging Face token (falls back to stored token)")
111
+ @click.option("--session-id", required=False, help="Use or create a session id for resumable uploads")
112
+ @click.option("--remote-base", default="audio", show_default=True, help="Remote base path inside the dataset")
113
+ @click.option("--batch-size", default=None, type=int, help="Override batch size for uploads")
114
+ @click.option("--workers", default=None, type=int, help="Number of parallel upload workers")
115
+ @click.option("--dry-run", is_flag=True, default=False, help="Scan and report what would be uploaded, do not perform uploads")
116
+ @handle_cli_errors
117
+ def upload_cmd(
118
+ repo_id: str,
119
+ segments_dir: str,
120
+ csv_file: str | None,
121
+ token: str | None,
122
+ session_id: str | None,
123
+ remote_base: str,
124
+ batch_size: int | None,
125
+ workers: int | None,
126
+ dry_run: bool,
127
+ ) -> None:
128
+ """Upload local segments (and optional CSV) into the HF dataset.
129
+
130
+ The command uses stored token if `--token` is not provided. It creates/uses
131
+ a session checkpoint under the configurable session directory so uploads
132
+ can be resumed from the host when the container is mounted.
133
+ """
134
+ # Resolve token
135
+ if token:
136
+ hf_token = token
137
+ else:
138
+ hf_token = AuthService().require_token()
139
+
140
+ # Scan local segments
141
+ scanner = LocalScanner()
142
+ summary = scanner.scan_folder(segments_dir)
143
+ total_files = summary["total_files"]
144
+ total_size = summary["total_size"]
145
+
146
+ click.echo(f"Found {total_files} audio files ({total_size} bytes) under {segments_dir}")
147
+
148
+ if dry_run:
149
+ click.echo("Dry run: no files will be uploaded. Showing first 20 files:")
150
+ count = 0
151
+ for species, items in summary["by_species"].items():
152
+ for it in items:
153
+ if count >= 20:
154
+ break
155
+ click.echo(f" - {it['relative_path']} ({it['size']} bytes)")
156
+ count += 1
157
+ if count >= 20:
158
+ break
159
+ if csv_file:
160
+ click.echo(f"Would also upload CSV: {csv_file} -> index/detections.csv")
161
+ return
162
+
163
+ # Prepare API and session
164
+ api = None
165
+ try:
166
+ from huggingface_hub import HfApi
167
+
168
+ api = HfApi(token=hf_token)
169
+ except Exception as exc: # pragma: no cover - external
170
+ raise click.ClickException(build_error_message(exc)) from exc
171
+
172
+ # Validate dataset exists
173
+ repo_service = RepositoryService(hf_token)
174
+ validation = repo_service.validate_repo(repo_id)
175
+ if not validation.get("is_valid"):
176
+ click.echo(f"Warning: dataset {repo_id} may be missing structure: {validation.get('missing_prefixes')}")
177
+
178
+ # Create deduplicator, session and uploader
179
+ from src.uploader_cli.deduplicator import Deduplicator
180
+ from src.uploader_cli.batch_uploader import BatchUploader
181
+
182
+ dedup = Deduplicator(api=api, repo_id=repo_id)
183
+ session = SessionManager(session_id=session_id) if session_id else SessionManager()
184
+ uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_retries=None, initial_backoff=None, max_workers=workers)
185
+
186
+ # Flatten file infos
187
+ file_infos: list[dict] = []
188
+ for species, items in summary["by_species"].items():
189
+ for it in items:
190
+ # keep the relative path as scanned (preserves species directories)
191
+ file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
192
+
193
+ # Upload CSV first if present
194
+ if csv_file:
195
+ click.echo(f"Uploading CSV to index/detections.csv...")
196
+ try:
197
+ api.upload_file(path_or_file=str(csv_file), path_in_repo="index/detections.csv", repo_id=repo_id, repo_type="dataset")
198
+ click.echo("CSV uploaded")
199
+ except Exception as exc:
200
+ raise click.ClickException(f"CSV upload failed: {exc}") from exc
201
+
202
+ # Build and upload manifest.json based on scan and optional CSV
203
+ try:
204
+ from src.uploader_cli.manifest import build_manifest_from_scan, manifest_to_bytes
205
+
206
+ csv_rows = None
207
+ if csv_file:
208
+ import csv
209
+
210
+ with open(csv_file, newline="", encoding="utf-8") as fh:
211
+ reader = csv.DictReader(fh)
212
+ csv_rows = list(reader)
213
+
214
+ manifest = build_manifest_from_scan(repo_id, summary, csv_rows=csv_rows)
215
+ api.upload_file(path_or_fileobj=manifest_to_bytes(manifest), path_in_repo="index/manifest.json", repo_id=repo_id, repo_type="dataset")
216
+ click.echo("Manifest uploaded: index/manifest.json")
217
+ except Exception as exc: # pragma: no cover - external API
218
+ raise click.ClickException(f"Manifest upload failed: {exc}") from exc
219
+
220
+ # Generate and upload shards based on CSV rows (if any)
221
+ try:
222
+ from src.uploader_cli.manifest import write_shards_from_csv_rows
223
+
224
+ if csv_file and csv_rows:
225
+ click.echo("Generating index shards from CSV detections...")
226
+ shards = write_shards_from_csv_rows(csv_rows, shard_size=int(__import__("src.uploader_cli.config", fromlist=["INDEX_SHARD_SIZE"]).INDEX_SHARD_SIZE))
227
+ for shard_path in shards:
228
+ shard_name = shard_path.name
229
+ click.echo(f"Uploading shard: index/shards/{shard_name}")
230
+ try:
231
+ api.upload_file(path_or_file=str(shard_path), path_in_repo=f"index/shards/{shard_name}", repo_id=repo_id, repo_type="dataset")
232
+ except Exception as exc:
233
+ click.echo(f"Warning: failed to upload shard {shard_name}: {exc}")
234
+ click.echo(f"Uploaded {len(shards)} shards")
235
+ except Exception as exc: # pragma: no cover - external API
236
+ raise click.ClickException(f"Shard generation/upload failed: {exc}") from exc
237
+
238
+ click.echo("Starting upload of audio files...")
239
+
240
+ def on_progress(state: dict) -> None:
241
+ click.echo(f"Progress: uploaded={state.get('uploaded')} skipped={state.get('skipped')} failed={state.get('failed')}", err=False)
242
+
243
+ result = uploader.upload_files(file_infos, remote_base=remote_base, batch_size=batch_size, on_progress=on_progress)
244
+ click.echo(json.dumps(result, ensure_ascii=True, indent=2))
245
+
246
+
247
  if __name__ == "__main__":
248
  cli()
src/uploader_cli/manifest.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any, Iterable
6
+
7
+ from .config import SCHEMA_VERSION, INDEX_SHARD_SIZE
8
+ import tempfile
9
+ import os
10
+ import json
11
+ from pathlib import Path
12
+ from typing import List
13
+
14
+ try:
15
+ import pandas as pd # type: ignore
16
+ _HAVE_PANDAS = True
17
+ except Exception:
18
+ _HAVE_PANDAS = False
19
+
20
+
21
+ def _project_from_repo(repo_id: str) -> str:
22
+ return repo_id.split("/", 1)[1]
23
+
24
+
25
+ def build_manifest_from_scan(repo_id: str, scan_summary: dict[str, Any], *, csv_rows: Iterable[dict[str, Any]] | None = None) -> dict[str, Any]:
26
+ project_slug = _project_from_repo(repo_id)
27
+ total_files = int(scan_summary.get("total_files", 0))
28
+ total_size = int(scan_summary.get("total_size", 0))
29
+
30
+ total_detections = 0
31
+ total_audio_files = total_files
32
+ shards: list[str] = []
33
+
34
+ if csv_rows is not None:
35
+ # csv_rows expected to be an iterable of detection dicts with at least 'audio_file' key
36
+ rows = list(csv_rows)
37
+ total_detections = len(rows)
38
+ # derive unique audio files referenced by CSV
39
+ unique_files = {str(r.get("audio_file") or r.get("file") or "") for r in rows}
40
+ unique_files = {p for p in unique_files if p}
41
+ if unique_files:
42
+ total_audio_files = len(unique_files)
43
+
44
+ manifest = {
45
+ "schema_version": SCHEMA_VERSION,
46
+ "project_slug": project_slug,
47
+ "dataset_repo_id": repo_id,
48
+ "index": {
49
+ "total_detections": int(total_detections),
50
+ "total_audio_files": int(total_audio_files),
51
+ "shard_size": INDEX_SHARD_SIZE,
52
+ "shards": shards,
53
+ },
54
+ }
55
+
56
+ return manifest
57
+
58
+
59
+ def manifest_to_bytes(manifest: dict[str, Any]) -> bytes:
60
+ return json.dumps(manifest, ensure_ascii=True, indent=2).encode("utf-8")
61
+
62
+
63
+ def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: int = INDEX_SHARD_SIZE) -> List[Path]:
64
+ """Write shards from CSV rows.
65
+
66
+ Prefer Parquet (pandas/pyarrow). If pandas (and its parquet engine) is
67
+ unavailable, fall back to newline-delimited JSON (.jsonl).
68
+
69
+ Returns list of Path objects for shard files created.
70
+ """
71
+ rows_list = list(rows)
72
+ if not rows_list:
73
+ return []
74
+
75
+ out_paths: List[Path] = []
76
+ tmpdir = Path(tempfile.mkdtemp(prefix="birdnet-shards-"))
77
+
78
+ try:
79
+ for i in range(0, len(rows_list), shard_size):
80
+ chunk = rows_list[i : i + shard_size]
81
+ shard_index = i // shard_size
82
+ if _HAVE_PANDAS:
83
+ df = pd.DataFrame.from_records(chunk)
84
+ shard_name = f"shard-{shard_index:06d}.parquet"
85
+ shard_path = tmpdir / shard_name
86
+ # pandas will choose pyarrow or fastparquet if available
87
+ df.to_parquet(shard_path, index=False)
88
+ else:
89
+ shard_name = f"shard-{shard_index:06d}.jsonl"
90
+ shard_path = tmpdir / shard_name
91
+ with shard_path.open("w", encoding="utf-8") as fh:
92
+ for r in chunk:
93
+ fh.write(json.dumps(r, ensure_ascii=True) + "\n")
94
+
95
+ out_paths.append(shard_path)
96
+
97
+ return out_paths
98
+ except Exception:
99
+ # If anything fails, attempt to cleanup created files then re-raise
100
+ try:
101
+ for p in tmpdir.iterdir():
102
+ p.unlink()
103
+ tmpdir.rmdir()
104
+ except Exception:
105
+ pass
106
+ raise
src/uploader_cli/session_manager.py CHANGED
@@ -5,6 +5,7 @@ import json
5
  from datetime import UTC, datetime
6
  from pathlib import Path
7
  from typing import Any
 
8
 
9
  from .config import get_session_root
10
  from .exceptions import SessionError
@@ -20,6 +21,7 @@ class SessionManager:
20
  self.session_id = session_id
21
  self.session_dir = get_session_root() / session_id
22
  self.session_dir.mkdir(parents=True, exist_ok=True)
 
23
 
24
  @classmethod
25
  def create_session(cls, metadata: dict[str, Any] | None = None) -> "SessionManager":
@@ -70,29 +72,31 @@ class SessionManager:
70
  raise SessionError(f"Invalid checkpoint JSON for session '{self.session_id}'") from exc
71
 
72
  def mark_file_done(self, *, remote_path: str, bytes_uploaded: int, status: str = "uploaded") -> dict[str, Any]:
73
- checkpoint = self.load_checkpoint()
74
- checkpoint.setdefault("uploaded_files", [])
75
- checkpoint.setdefault("uploaded", 0)
76
- checkpoint.setdefault("failed", 0)
77
- checkpoint["uploaded"] = int(checkpoint["uploaded"]) + 1
78
- checkpoint["status"] = status
79
- checkpoint["last_completed_file"] = remote_path
80
- checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
81
- checkpoint["bytes_uploaded"] = int(checkpoint.get("bytes_uploaded", 0)) + int(bytes_uploaded)
82
- checkpoint["uploaded_files"].append(remote_path)
83
- self.save_checkpoint(checkpoint)
84
- return checkpoint
 
85
 
86
  def mark_file_failed(self, *, remote_path: str, error: str) -> dict[str, Any]:
87
- checkpoint = self.load_checkpoint()
88
- checkpoint.setdefault("failed_files", [])
89
- checkpoint.setdefault("uploaded", 0)
90
- checkpoint.setdefault("failed", 0)
91
- checkpoint["failed"] = int(checkpoint["failed"]) + 1
92
- checkpoint["status"] = "failed"
93
- checkpoint["last_failed_file"] = remote_path
94
- checkpoint["last_error"] = error
95
- checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
96
- checkpoint["failed_files"].append({"remote_path": remote_path, "error": error})
97
- self.save_checkpoint(checkpoint)
98
- return checkpoint
 
 
5
  from datetime import UTC, datetime
6
  from pathlib import Path
7
  from typing import Any
8
+ import threading
9
 
10
  from .config import get_session_root
11
  from .exceptions import SessionError
 
21
  self.session_id = session_id
22
  self.session_dir = get_session_root() / session_id
23
  self.session_dir.mkdir(parents=True, exist_ok=True)
24
+ self._lock = threading.Lock()
25
 
26
  @classmethod
27
  def create_session(cls, metadata: dict[str, Any] | None = None) -> "SessionManager":
 
72
  raise SessionError(f"Invalid checkpoint JSON for session '{self.session_id}'") from exc
73
 
74
  def mark_file_done(self, *, remote_path: str, bytes_uploaded: int, status: str = "uploaded") -> dict[str, Any]:
75
+ with self._lock:
76
+ checkpoint = self.load_checkpoint()
77
+ checkpoint.setdefault("uploaded_files", [])
78
+ checkpoint.setdefault("uploaded", 0)
79
+ checkpoint.setdefault("failed", 0)
80
+ checkpoint["uploaded"] = int(checkpoint["uploaded"]) + 1
81
+ checkpoint["status"] = status
82
+ checkpoint["last_completed_file"] = remote_path
83
+ checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
84
+ checkpoint["bytes_uploaded"] = int(checkpoint.get("bytes_uploaded", 0)) + int(bytes_uploaded)
85
+ checkpoint["uploaded_files"].append(remote_path)
86
+ self.save_checkpoint(checkpoint)
87
+ return checkpoint
88
 
89
  def mark_file_failed(self, *, remote_path: str, error: str) -> dict[str, Any]:
90
+ with self._lock:
91
+ checkpoint = self.load_checkpoint()
92
+ checkpoint.setdefault("failed_files", [])
93
+ checkpoint.setdefault("uploaded", 0)
94
+ checkpoint.setdefault("failed", 0)
95
+ checkpoint["failed"] = int(checkpoint["failed"]) + 1
96
+ checkpoint["status"] = "failed"
97
+ checkpoint["last_failed_file"] = remote_path
98
+ checkpoint["last_error"] = error
99
+ checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
100
+ checkpoint["failed_files"].append({"remote_path": remote_path, "error": error})
101
+ self.save_checkpoint(checkpoint)
102
+ return checkpoint
tests/integration/test_upload_flow.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import csv
2
+ import os
3
+ from pathlib import Path
4
+ from click.testing import CliRunner
5
+
6
+ import pytest
7
+
8
+ from src.uploader_cli.main import cli
9
+
10
+
11
+ class FakeHfApi:
12
+ def __init__(self, token=None, storage_dir=None):
13
+ self.token = token
14
+ self.storage_dir = Path(storage_dir)
15
+ self.storage_dir.mkdir(parents=True, exist_ok=True)
16
+
17
+ def upload_file(self, path_or_file=None, path_or_fileobj=None, path_in_repo=None, repo_id=None, repo_type=None):
18
+ target = self.storage_dir / path_in_repo
19
+ target.parent.mkdir(parents=True, exist_ok=True)
20
+ if path_or_fileobj is not None:
21
+ # bytes or file-like
22
+ if hasattr(path_or_fileobj, "read"):
23
+ data = path_or_fileobj.read()
24
+ else:
25
+ data = path_or_fileobj
26
+ if isinstance(data, str):
27
+ data = data.encode("utf-8")
28
+ target.write_bytes(data)
29
+ return {"file": path_in_repo}
30
+ if path_or_file is not None:
31
+ src = Path(path_or_file)
32
+ if not src.exists():
33
+ raise FileNotFoundError(path_or_file)
34
+ target.write_bytes(src.read_bytes())
35
+ return {"file": path_in_repo}
36
+ raise ValueError("No file provided to upload_file")
37
+
38
+ def list_repo_files(self, repo_id=None, repo_type=None):
39
+ files = []
40
+ for p in self.storage_dir.rglob("*"):
41
+ if p.is_file():
42
+ files.append(str(p.relative_to(self.storage_dir)).replace("\\", "/"))
43
+ return files
44
+
45
+
46
+ @pytest.fixture()
47
+ def tmp_remote(tmp_path):
48
+ d = tmp_path / "remote"
49
+ d.mkdir()
50
+ return d
51
+
52
+
53
+ def test_upload_flow_csv_and_segments(tmp_path, monkeypatch, tmp_remote):
54
+ # prepare local segments
55
+ segments = tmp_path / "segments"
56
+ (segments / "proj").mkdir(parents=True)
57
+ a = segments / "proj" / "a.wav"
58
+ a.write_bytes(b"one")
59
+ b = segments / "proj" / "b.wav"
60
+ b.write_bytes(b"two")
61
+
62
+ # prepare CSV
63
+ csv_file = tmp_path / "detections.csv"
64
+ with csv_file.open("w", newline="", encoding="utf-8") as fh:
65
+ writer = csv.DictWriter(fh, fieldnames=["audio_file", "score"])
66
+ writer.writeheader()
67
+ writer.writerow({"audio_file": "proj/a.wav", "score": "0.9"})
68
+ writer.writerow({"audio_file": "proj/b.wav", "score": "0.8"})
69
+
70
+ # monkeypatch HfApi to our fake implementation
71
+ def _fake_hfapi_factory(token=None):
72
+ return FakeHfApi(token=token, storage_dir=tmp_remote)
73
+
74
+ monkeypatch.setattr("huggingface_hub.HfApi", _fake_hfapi_factory)
75
+ # avoid network call in RepositoryService.validate_repo by stubbing it
76
+ # avoid network call in RepositoryService.validate_repo by stubbing it
77
+ def _fake_validate(self, repo_id: str):
78
+ return {
79
+ "repo_id": repo_id,
80
+ "is_valid": True,
81
+ "project_slug": repo_id.split("/", 1)[1],
82
+ "missing_prefixes": [],
83
+ "has_manifest": True,
84
+ "manifest_ok": True,
85
+ "manifest_error": "",
86
+ }
87
+
88
+ monkeypatch.setattr("src.uploader_cli.main.RepositoryService.validate_repo", _fake_validate)
89
+
90
+ runner = CliRunner()
91
+ result = runner.invoke(
92
+ cli,
93
+ [
94
+ "upload",
95
+ "--repo-id",
96
+ "me/repo",
97
+ "--segments",
98
+ str(segments),
99
+ "--csv",
100
+ str(csv_file),
101
+ "--token",
102
+ "fake-token",
103
+ "--workers",
104
+ "2",
105
+ ],
106
+ )
107
+
108
+ assert result.exit_code == 0, result.output
109
+
110
+ # verify remote structure
111
+ remote_files = {p.relative_to(tmp_remote).as_posix() for p in tmp_remote.rglob("*") if p.is_file()}
112
+ assert "index/manifest.json" in remote_files
113
+ assert "index/detections.csv" in remote_files
114
+ # shards may be parquet or jsonl
115
+ shards = [p for p in remote_files if p.startswith("index/shards/")]
116
+ assert len(shards) >= 1
117
+ # audio files uploaded under audio/
118
+ assert "audio/proj/a.wav" in remote_files
119
+ assert "audio/proj/b.wav" in remote_files