Spaces:
Sleeping
Sleeping
Commit ·
7c83d76
1
Parent(s): 1f2c564
Add uploader release flow and integration test
Browse files- Dockerfile.windows +15 -0
- scripts/smoke_test_upload.ps1 +22 -0
- src/uploader_cli/batch_uploader.py +45 -9
- src/uploader_cli/deduplicator.py +29 -25
- src/uploader_cli/main.py +141 -0
- src/uploader_cli/manifest.py +106 -0
- src/uploader_cli/session_manager.py +28 -24
- tests/integration/test_upload_flow.py +119 -0
Dockerfile.windows
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM mcr.microsoft.com/windows/servercore:ltsc2022
|
| 2 |
+
|
| 3 |
+
SHELL ["powershell","-Command","$ErrorActionPreference = 'Stop';"]
|
| 4 |
+
|
| 5 |
+
WORKDIR C:\app
|
| 6 |
+
|
| 7 |
+
# Copy a one-folder PyInstaller bundle into the image at build time.
|
| 8 |
+
# Expect the build artifact layout: dist\birdnet-uploader\* (exe + _internal)
|
| 9 |
+
COPY dist\birdnet-uploader\ C:\app\
|
| 10 |
+
|
| 11 |
+
ENV HF_TOKEN=""
|
| 12 |
+
|
| 13 |
+
EXPOSE 0
|
| 14 |
+
|
| 15 |
+
ENTRYPOINT ["C:\\app\\birdnet-uploader.exe"]
|
scripts/smoke_test_upload.ps1
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Param(
|
| 2 |
+
[string]$RepoId = "owner/dataset",
|
| 3 |
+
[string]$Segments = "C:\data\segments",
|
| 4 |
+
[string]$Token = $env:HF_TOKEN
|
| 5 |
+
)
|
| 6 |
+
|
| 7 |
+
if (-not $Token) {
|
| 8 |
+
Write-Error "HF_TOKEN environment variable must be set or pass --Token"
|
| 9 |
+
exit 2
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
Write-Host "Running smoke test upload against $RepoId using segments at $Segments"
|
| 13 |
+
|
| 14 |
+
# Dry run first
|
| 15 |
+
& .\birdnet-uploader.exe upload --repo-id $RepoId --segments $Segments --token $Token --dry-run
|
| 16 |
+
|
| 17 |
+
if ($LASTEXITCODE -ne 0) {
|
| 18 |
+
Write-Error "Dry run failed"
|
| 19 |
+
exit $LASTEXITCODE
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
Write-Host "Dry run OK. To perform a real small upload, run without --dry-run and mount a small folder."
|
src/uploader_cli/batch_uploader.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import time
|
|
|
|
|
|
|
| 4 |
from pathlib import Path
|
| 5 |
from typing import Any, Callable, Iterable
|
| 6 |
|
|
@@ -23,6 +25,7 @@ class BatchUploader:
|
|
| 23 |
*,
|
| 24 |
max_retries: int | None = None,
|
| 25 |
initial_backoff: float | None = None,
|
|
|
|
| 26 |
) -> None:
|
| 27 |
self._api = api
|
| 28 |
self.repo_id = repo_id
|
|
@@ -30,6 +33,7 @@ class BatchUploader:
|
|
| 30 |
self.session = session or SessionManager.create_session()
|
| 31 |
self.max_retries = max_retries or RETRY_MAX_ATTEMPTS
|
| 32 |
self.initial_backoff = initial_backoff or RETRY_INITIAL_BACKOFF_SECONDS
|
|
|
|
| 33 |
|
| 34 |
def _upload_file_with_retry(self, local_path: str | Path, remote_path: str) -> None:
|
| 35 |
attempts = 0
|
|
@@ -65,6 +69,8 @@ class BatchUploader:
|
|
| 65 |
skipped = 0
|
| 66 |
failed = 0
|
| 67 |
|
|
|
|
|
|
|
| 68 |
for info in file_infos:
|
| 69 |
full_path = str(info["full_path"])
|
| 70 |
relative = info["relative_path"].lstrip("/")
|
|
@@ -79,24 +85,54 @@ class BatchUploader:
|
|
| 79 |
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 80 |
continue
|
| 81 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
try:
|
| 83 |
self._upload_file_with_retry(full_path, remote_path)
|
| 84 |
-
# mark remote as present in local dedup cache
|
| 85 |
try:
|
| 86 |
self.deduplicator.mark_uploaded(remote_path)
|
| 87 |
except Exception:
|
| 88 |
-
# non-fatal cache update failure should not break overall flow
|
| 89 |
pass
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
bytes_uploaded = int(info.get("size", 0))
|
| 93 |
-
self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=bytes_uploaded)
|
| 94 |
-
if on_progress:
|
| 95 |
-
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 96 |
except Exception as exc:
|
| 97 |
-
failed += 1
|
| 98 |
self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
if on_progress:
|
| 100 |
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 101 |
|
| 102 |
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
|
|
|
|
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import time
|
| 4 |
+
import concurrent.futures
|
| 5 |
+
from typing import List, Tuple
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any, Callable, Iterable
|
| 8 |
|
|
|
|
| 25 |
*,
|
| 26 |
max_retries: int | None = None,
|
| 27 |
initial_backoff: float | None = None,
|
| 28 |
+
max_workers: int | None = None,
|
| 29 |
) -> None:
|
| 30 |
self._api = api
|
| 31 |
self.repo_id = repo_id
|
|
|
|
| 33 |
self.session = session or SessionManager.create_session()
|
| 34 |
self.max_retries = max_retries or RETRY_MAX_ATTEMPTS
|
| 35 |
self.initial_backoff = initial_backoff or RETRY_INITIAL_BACKOFF_SECONDS
|
| 36 |
+
self.max_workers = max_workers
|
| 37 |
|
| 38 |
def _upload_file_with_retry(self, local_path: str | Path, remote_path: str) -> None:
|
| 39 |
attempts = 0
|
|
|
|
| 69 |
skipped = 0
|
| 70 |
failed = 0
|
| 71 |
|
| 72 |
+
# First pass: decide which files to skip and which to upload
|
| 73 |
+
to_upload: List[Tuple[str, str, int]] = []
|
| 74 |
for info in file_infos:
|
| 75 |
full_path = str(info["full_path"])
|
| 76 |
relative = info["relative_path"].lstrip("/")
|
|
|
|
| 85 |
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 86 |
continue
|
| 87 |
|
| 88 |
+
to_upload.append((full_path, remote_path, int(info.get("size", 0))))
|
| 89 |
+
|
| 90 |
+
# If no parallelism requested, perform sequential upload to preserve behavior
|
| 91 |
+
if not self.max_workers or self.max_workers <= 1:
|
| 92 |
+
for full_path, remote_path, size in to_upload:
|
| 93 |
+
try:
|
| 94 |
+
self._upload_file_with_retry(full_path, remote_path)
|
| 95 |
+
try:
|
| 96 |
+
self.deduplicator.mark_uploaded(remote_path)
|
| 97 |
+
except Exception:
|
| 98 |
+
pass
|
| 99 |
+
uploaded += 1
|
| 100 |
+
self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=size)
|
| 101 |
+
if on_progress:
|
| 102 |
+
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 103 |
+
except Exception as exc:
|
| 104 |
+
failed += 1
|
| 105 |
+
self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
|
| 106 |
+
if on_progress:
|
| 107 |
+
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 108 |
+
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
| 109 |
+
|
| 110 |
+
# Parallel upload path
|
| 111 |
+
def _worker(task: Tuple[str, str, int]) -> Tuple[str, str]:
|
| 112 |
+
full_path, remote_path, size = task
|
| 113 |
try:
|
| 114 |
self._upload_file_with_retry(full_path, remote_path)
|
|
|
|
| 115 |
try:
|
| 116 |
self.deduplicator.mark_uploaded(remote_path)
|
| 117 |
except Exception:
|
|
|
|
| 118 |
pass
|
| 119 |
+
self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=size)
|
| 120 |
+
return ("ok", remote_path)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
except Exception as exc:
|
|
|
|
| 122 |
self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
|
| 123 |
+
return ("fail", remote_path)
|
| 124 |
+
|
| 125 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as ex:
|
| 126 |
+
futures = {ex.submit(_worker, t): t for t in to_upload}
|
| 127 |
+
for fut in concurrent.futures.as_completed(futures):
|
| 128 |
+
res = fut.result()
|
| 129 |
+
if res[0] == "ok":
|
| 130 |
+
uploaded += 1
|
| 131 |
+
else:
|
| 132 |
+
failed += 1
|
| 133 |
if on_progress:
|
| 134 |
on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
|
| 135 |
|
| 136 |
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
| 137 |
+
|
| 138 |
+
return {"uploaded": uploaded, "skipped": skipped, "failed": failed}
|
src/uploader_cli/deduplicator.py
CHANGED
|
@@ -5,6 +5,7 @@ import hashlib
|
|
| 5 |
from datetime import UTC, datetime
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any
|
|
|
|
| 8 |
|
| 9 |
from huggingface_hub import HfApi
|
| 10 |
|
|
@@ -23,6 +24,7 @@ class Deduplicator:
|
|
| 23 |
self._remote_paths: set[str] | None = None
|
| 24 |
self._decision_cache: dict[str, dict[str, Any]] = {}
|
| 25 |
self.cache_path = self._build_cache_path()
|
|
|
|
| 26 |
|
| 27 |
def _build_cache_path(self) -> Path:
|
| 28 |
cache_root = get_cache_root() / "dedup"
|
|
@@ -33,23 +35,23 @@ class Deduplicator:
|
|
| 33 |
def load_cached_index(self) -> set[str]:
|
| 34 |
if self._remote_paths is not None:
|
| 35 |
return self._remote_paths
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
| 48 |
return self._remote_paths
|
| 49 |
|
| 50 |
-
self._remote_paths = self.refresh_remote_index()
|
| 51 |
-
return self._remote_paths
|
| 52 |
-
|
| 53 |
def refresh_remote_index(self) -> set[str]:
|
| 54 |
try:
|
| 55 |
repo_files = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
|
|
@@ -57,8 +59,9 @@ class Deduplicator:
|
|
| 57 |
raise ValidationError(f"Could not read remote file listing for {self.repo_id}: {exc}") from exc
|
| 58 |
|
| 59 |
remote_paths = {str(path) for path in repo_files}
|
| 60 |
-
self.
|
| 61 |
-
|
|
|
|
| 62 |
return remote_paths
|
| 63 |
|
| 64 |
def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
|
|
@@ -84,14 +87,15 @@ class Deduplicator:
|
|
| 84 |
return payload
|
| 85 |
|
| 86 |
def mark_uploaded(self, remote_path: str) -> None:
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
|
|
|
| 95 |
|
| 96 |
def _write_cache(self, remote_paths: set[str]) -> None:
|
| 97 |
payload = {
|
|
|
|
| 5 |
from datetime import UTC, datetime
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any
|
| 8 |
+
import threading
|
| 9 |
|
| 10 |
from huggingface_hub import HfApi
|
| 11 |
|
|
|
|
| 24 |
self._remote_paths: set[str] | None = None
|
| 25 |
self._decision_cache: dict[str, dict[str, Any]] = {}
|
| 26 |
self.cache_path = self._build_cache_path()
|
| 27 |
+
self._lock = threading.Lock()
|
| 28 |
|
| 29 |
def _build_cache_path(self) -> Path:
|
| 30 |
cache_root = get_cache_root() / "dedup"
|
|
|
|
| 35 |
def load_cached_index(self) -> set[str]:
|
| 36 |
if self._remote_paths is not None:
|
| 37 |
return self._remote_paths
|
| 38 |
+
with self._lock:
|
| 39 |
+
if self.cache_path.exists():
|
| 40 |
+
raw = self.cache_path.read_text(encoding="utf-8")
|
| 41 |
+
try:
|
| 42 |
+
payload = json.loads(raw)
|
| 43 |
+
except json.JSONDecodeError as exc:
|
| 44 |
+
raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc
|
| 45 |
+
|
| 46 |
+
remote_paths = payload.get("remote_paths", [])
|
| 47 |
+
if not isinstance(remote_paths, list):
|
| 48 |
+
raise ValidationError(f"Invalid dedup cache format at {self.cache_path}")
|
| 49 |
+
self._remote_paths = {str(path) for path in remote_paths}
|
| 50 |
+
return self._remote_paths
|
| 51 |
+
|
| 52 |
+
self._remote_paths = self.refresh_remote_index()
|
| 53 |
return self._remote_paths
|
| 54 |
|
|
|
|
|
|
|
|
|
|
| 55 |
def refresh_remote_index(self) -> set[str]:
|
| 56 |
try:
|
| 57 |
repo_files = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
|
|
|
|
| 59 |
raise ValidationError(f"Could not read remote file listing for {self.repo_id}: {exc}") from exc
|
| 60 |
|
| 61 |
remote_paths = {str(path) for path in repo_files}
|
| 62 |
+
with self._lock:
|
| 63 |
+
self._remote_paths = remote_paths
|
| 64 |
+
self._write_cache(remote_paths)
|
| 65 |
return remote_paths
|
| 66 |
|
| 67 |
def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
|
|
|
|
| 87 |
return payload
|
| 88 |
|
| 89 |
def mark_uploaded(self, remote_path: str) -> None:
|
| 90 |
+
with self._lock:
|
| 91 |
+
remote_paths = self.load_cached_index()
|
| 92 |
+
if remote_path not in remote_paths:
|
| 93 |
+
remote_paths.add(remote_path)
|
| 94 |
+
self._write_cache(remote_paths)
|
| 95 |
+
|
| 96 |
+
keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path]
|
| 97 |
+
for key in keys_to_update:
|
| 98 |
+
self._decision_cache[key]["status"] = "skip"
|
| 99 |
|
| 100 |
def _write_cache(self, remote_paths: set[str]) -> None:
|
| 101 |
payload = {
|
src/uploader_cli/main.py
CHANGED
|
@@ -103,5 +103,146 @@ def resume_cmd(session_id: str) -> None:
|
|
| 103 |
click.echo(json.dumps(payload, ensure_ascii=True, indent=2))
|
| 104 |
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
if __name__ == "__main__":
|
| 107 |
cli()
|
|
|
|
| 103 |
click.echo(json.dumps(payload, ensure_ascii=True, indent=2))
|
| 104 |
|
| 105 |
|
| 106 |
+
@cli.command("upload")
|
| 107 |
+
@click.option("--repo-id", required=True, help="Dataset repo id in owner/name format")
|
| 108 |
+
@click.option("--segments", "segments_dir", required=True, type=click.Path(exists=True, file_okay=False), help="Local folder with audio segments")
|
| 109 |
+
@click.option("--csv", "csv_file", required=False, type=click.Path(exists=True, dir_okay=False), help="Optional detections CSV to upload to the dataset")
|
| 110 |
+
@click.option("--token", required=False, help="Hugging Face token (falls back to stored token)")
|
| 111 |
+
@click.option("--session-id", required=False, help="Use or create a session id for resumable uploads")
|
| 112 |
+
@click.option("--remote-base", default="audio", show_default=True, help="Remote base path inside the dataset")
|
| 113 |
+
@click.option("--batch-size", default=None, type=int, help="Override batch size for uploads")
|
| 114 |
+
@click.option("--workers", default=None, type=int, help="Number of parallel upload workers")
|
| 115 |
+
@click.option("--dry-run", is_flag=True, default=False, help="Scan and report what would be uploaded, do not perform uploads")
|
| 116 |
+
@handle_cli_errors
|
| 117 |
+
def upload_cmd(
|
| 118 |
+
repo_id: str,
|
| 119 |
+
segments_dir: str,
|
| 120 |
+
csv_file: str | None,
|
| 121 |
+
token: str | None,
|
| 122 |
+
session_id: str | None,
|
| 123 |
+
remote_base: str,
|
| 124 |
+
batch_size: int | None,
|
| 125 |
+
workers: int | None,
|
| 126 |
+
dry_run: bool,
|
| 127 |
+
) -> None:
|
| 128 |
+
"""Upload local segments (and optional CSV) into the HF dataset.
|
| 129 |
+
|
| 130 |
+
The command uses stored token if `--token` is not provided. It creates/uses
|
| 131 |
+
a session checkpoint under the configurable session directory so uploads
|
| 132 |
+
can be resumed from the host when the container is mounted.
|
| 133 |
+
"""
|
| 134 |
+
# Resolve token
|
| 135 |
+
if token:
|
| 136 |
+
hf_token = token
|
| 137 |
+
else:
|
| 138 |
+
hf_token = AuthService().require_token()
|
| 139 |
+
|
| 140 |
+
# Scan local segments
|
| 141 |
+
scanner = LocalScanner()
|
| 142 |
+
summary = scanner.scan_folder(segments_dir)
|
| 143 |
+
total_files = summary["total_files"]
|
| 144 |
+
total_size = summary["total_size"]
|
| 145 |
+
|
| 146 |
+
click.echo(f"Found {total_files} audio files ({total_size} bytes) under {segments_dir}")
|
| 147 |
+
|
| 148 |
+
if dry_run:
|
| 149 |
+
click.echo("Dry run: no files will be uploaded. Showing first 20 files:")
|
| 150 |
+
count = 0
|
| 151 |
+
for species, items in summary["by_species"].items():
|
| 152 |
+
for it in items:
|
| 153 |
+
if count >= 20:
|
| 154 |
+
break
|
| 155 |
+
click.echo(f" - {it['relative_path']} ({it['size']} bytes)")
|
| 156 |
+
count += 1
|
| 157 |
+
if count >= 20:
|
| 158 |
+
break
|
| 159 |
+
if csv_file:
|
| 160 |
+
click.echo(f"Would also upload CSV: {csv_file} -> index/detections.csv")
|
| 161 |
+
return
|
| 162 |
+
|
| 163 |
+
# Prepare API and session
|
| 164 |
+
api = None
|
| 165 |
+
try:
|
| 166 |
+
from huggingface_hub import HfApi
|
| 167 |
+
|
| 168 |
+
api = HfApi(token=hf_token)
|
| 169 |
+
except Exception as exc: # pragma: no cover - external
|
| 170 |
+
raise click.ClickException(build_error_message(exc)) from exc
|
| 171 |
+
|
| 172 |
+
# Validate dataset exists
|
| 173 |
+
repo_service = RepositoryService(hf_token)
|
| 174 |
+
validation = repo_service.validate_repo(repo_id)
|
| 175 |
+
if not validation.get("is_valid"):
|
| 176 |
+
click.echo(f"Warning: dataset {repo_id} may be missing structure: {validation.get('missing_prefixes')}")
|
| 177 |
+
|
| 178 |
+
# Create deduplicator, session and uploader
|
| 179 |
+
from src.uploader_cli.deduplicator import Deduplicator
|
| 180 |
+
from src.uploader_cli.batch_uploader import BatchUploader
|
| 181 |
+
|
| 182 |
+
dedup = Deduplicator(api=api, repo_id=repo_id)
|
| 183 |
+
session = SessionManager(session_id=session_id) if session_id else SessionManager()
|
| 184 |
+
uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_retries=None, initial_backoff=None, max_workers=workers)
|
| 185 |
+
|
| 186 |
+
# Flatten file infos
|
| 187 |
+
file_infos: list[dict] = []
|
| 188 |
+
for species, items in summary["by_species"].items():
|
| 189 |
+
for it in items:
|
| 190 |
+
# keep the relative path as scanned (preserves species directories)
|
| 191 |
+
file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
|
| 192 |
+
|
| 193 |
+
# Upload CSV first if present
|
| 194 |
+
if csv_file:
|
| 195 |
+
click.echo(f"Uploading CSV to index/detections.csv...")
|
| 196 |
+
try:
|
| 197 |
+
api.upload_file(path_or_file=str(csv_file), path_in_repo="index/detections.csv", repo_id=repo_id, repo_type="dataset")
|
| 198 |
+
click.echo("CSV uploaded")
|
| 199 |
+
except Exception as exc:
|
| 200 |
+
raise click.ClickException(f"CSV upload failed: {exc}") from exc
|
| 201 |
+
|
| 202 |
+
# Build and upload manifest.json based on scan and optional CSV
|
| 203 |
+
try:
|
| 204 |
+
from src.uploader_cli.manifest import build_manifest_from_scan, manifest_to_bytes
|
| 205 |
+
|
| 206 |
+
csv_rows = None
|
| 207 |
+
if csv_file:
|
| 208 |
+
import csv
|
| 209 |
+
|
| 210 |
+
with open(csv_file, newline="", encoding="utf-8") as fh:
|
| 211 |
+
reader = csv.DictReader(fh)
|
| 212 |
+
csv_rows = list(reader)
|
| 213 |
+
|
| 214 |
+
manifest = build_manifest_from_scan(repo_id, summary, csv_rows=csv_rows)
|
| 215 |
+
api.upload_file(path_or_fileobj=manifest_to_bytes(manifest), path_in_repo="index/manifest.json", repo_id=repo_id, repo_type="dataset")
|
| 216 |
+
click.echo("Manifest uploaded: index/manifest.json")
|
| 217 |
+
except Exception as exc: # pragma: no cover - external API
|
| 218 |
+
raise click.ClickException(f"Manifest upload failed: {exc}") from exc
|
| 219 |
+
|
| 220 |
+
# Generate and upload shards based on CSV rows (if any)
|
| 221 |
+
try:
|
| 222 |
+
from src.uploader_cli.manifest import write_shards_from_csv_rows
|
| 223 |
+
|
| 224 |
+
if csv_file and csv_rows:
|
| 225 |
+
click.echo("Generating index shards from CSV detections...")
|
| 226 |
+
shards = write_shards_from_csv_rows(csv_rows, shard_size=int(__import__("src.uploader_cli.config", fromlist=["INDEX_SHARD_SIZE"]).INDEX_SHARD_SIZE))
|
| 227 |
+
for shard_path in shards:
|
| 228 |
+
shard_name = shard_path.name
|
| 229 |
+
click.echo(f"Uploading shard: index/shards/{shard_name}")
|
| 230 |
+
try:
|
| 231 |
+
api.upload_file(path_or_file=str(shard_path), path_in_repo=f"index/shards/{shard_name}", repo_id=repo_id, repo_type="dataset")
|
| 232 |
+
except Exception as exc:
|
| 233 |
+
click.echo(f"Warning: failed to upload shard {shard_name}: {exc}")
|
| 234 |
+
click.echo(f"Uploaded {len(shards)} shards")
|
| 235 |
+
except Exception as exc: # pragma: no cover - external API
|
| 236 |
+
raise click.ClickException(f"Shard generation/upload failed: {exc}") from exc
|
| 237 |
+
|
| 238 |
+
click.echo("Starting upload of audio files...")
|
| 239 |
+
|
| 240 |
+
def on_progress(state: dict) -> None:
|
| 241 |
+
click.echo(f"Progress: uploaded={state.get('uploaded')} skipped={state.get('skipped')} failed={state.get('failed')}", err=False)
|
| 242 |
+
|
| 243 |
+
result = uploader.upload_files(file_infos, remote_base=remote_base, batch_size=batch_size, on_progress=on_progress)
|
| 244 |
+
click.echo(json.dumps(result, ensure_ascii=True, indent=2))
|
| 245 |
+
|
| 246 |
+
|
| 247 |
if __name__ == "__main__":
|
| 248 |
cli()
|
src/uploader_cli/manifest.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any, Iterable
|
| 6 |
+
|
| 7 |
+
from .config import SCHEMA_VERSION, INDEX_SHARD_SIZE
|
| 8 |
+
import tempfile
|
| 9 |
+
import os
|
| 10 |
+
import json
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import List
|
| 13 |
+
|
| 14 |
+
try:
|
| 15 |
+
import pandas as pd # type: ignore
|
| 16 |
+
_HAVE_PANDAS = True
|
| 17 |
+
except Exception:
|
| 18 |
+
_HAVE_PANDAS = False
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _project_from_repo(repo_id: str) -> str:
|
| 22 |
+
return repo_id.split("/", 1)[1]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def build_manifest_from_scan(repo_id: str, scan_summary: dict[str, Any], *, csv_rows: Iterable[dict[str, Any]] | None = None) -> dict[str, Any]:
|
| 26 |
+
project_slug = _project_from_repo(repo_id)
|
| 27 |
+
total_files = int(scan_summary.get("total_files", 0))
|
| 28 |
+
total_size = int(scan_summary.get("total_size", 0))
|
| 29 |
+
|
| 30 |
+
total_detections = 0
|
| 31 |
+
total_audio_files = total_files
|
| 32 |
+
shards: list[str] = []
|
| 33 |
+
|
| 34 |
+
if csv_rows is not None:
|
| 35 |
+
# csv_rows expected to be an iterable of detection dicts with at least 'audio_file' key
|
| 36 |
+
rows = list(csv_rows)
|
| 37 |
+
total_detections = len(rows)
|
| 38 |
+
# derive unique audio files referenced by CSV
|
| 39 |
+
unique_files = {str(r.get("audio_file") or r.get("file") or "") for r in rows}
|
| 40 |
+
unique_files = {p for p in unique_files if p}
|
| 41 |
+
if unique_files:
|
| 42 |
+
total_audio_files = len(unique_files)
|
| 43 |
+
|
| 44 |
+
manifest = {
|
| 45 |
+
"schema_version": SCHEMA_VERSION,
|
| 46 |
+
"project_slug": project_slug,
|
| 47 |
+
"dataset_repo_id": repo_id,
|
| 48 |
+
"index": {
|
| 49 |
+
"total_detections": int(total_detections),
|
| 50 |
+
"total_audio_files": int(total_audio_files),
|
| 51 |
+
"shard_size": INDEX_SHARD_SIZE,
|
| 52 |
+
"shards": shards,
|
| 53 |
+
},
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
return manifest
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def manifest_to_bytes(manifest: dict[str, Any]) -> bytes:
|
| 60 |
+
return json.dumps(manifest, ensure_ascii=True, indent=2).encode("utf-8")
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def write_shards_from_csv_rows(rows: Iterable[dict[str, Any]], *, shard_size: int = INDEX_SHARD_SIZE) -> List[Path]:
|
| 64 |
+
"""Write shards from CSV rows.
|
| 65 |
+
|
| 66 |
+
Prefer Parquet (pandas/pyarrow). If pandas (and its parquet engine) is
|
| 67 |
+
unavailable, fall back to newline-delimited JSON (.jsonl).
|
| 68 |
+
|
| 69 |
+
Returns list of Path objects for shard files created.
|
| 70 |
+
"""
|
| 71 |
+
rows_list = list(rows)
|
| 72 |
+
if not rows_list:
|
| 73 |
+
return []
|
| 74 |
+
|
| 75 |
+
out_paths: List[Path] = []
|
| 76 |
+
tmpdir = Path(tempfile.mkdtemp(prefix="birdnet-shards-"))
|
| 77 |
+
|
| 78 |
+
try:
|
| 79 |
+
for i in range(0, len(rows_list), shard_size):
|
| 80 |
+
chunk = rows_list[i : i + shard_size]
|
| 81 |
+
shard_index = i // shard_size
|
| 82 |
+
if _HAVE_PANDAS:
|
| 83 |
+
df = pd.DataFrame.from_records(chunk)
|
| 84 |
+
shard_name = f"shard-{shard_index:06d}.parquet"
|
| 85 |
+
shard_path = tmpdir / shard_name
|
| 86 |
+
# pandas will choose pyarrow or fastparquet if available
|
| 87 |
+
df.to_parquet(shard_path, index=False)
|
| 88 |
+
else:
|
| 89 |
+
shard_name = f"shard-{shard_index:06d}.jsonl"
|
| 90 |
+
shard_path = tmpdir / shard_name
|
| 91 |
+
with shard_path.open("w", encoding="utf-8") as fh:
|
| 92 |
+
for r in chunk:
|
| 93 |
+
fh.write(json.dumps(r, ensure_ascii=True) + "\n")
|
| 94 |
+
|
| 95 |
+
out_paths.append(shard_path)
|
| 96 |
+
|
| 97 |
+
return out_paths
|
| 98 |
+
except Exception:
|
| 99 |
+
# If anything fails, attempt to cleanup created files then re-raise
|
| 100 |
+
try:
|
| 101 |
+
for p in tmpdir.iterdir():
|
| 102 |
+
p.unlink()
|
| 103 |
+
tmpdir.rmdir()
|
| 104 |
+
except Exception:
|
| 105 |
+
pass
|
| 106 |
+
raise
|
src/uploader_cli/session_manager.py
CHANGED
|
@@ -5,6 +5,7 @@ import json
|
|
| 5 |
from datetime import UTC, datetime
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any
|
|
|
|
| 8 |
|
| 9 |
from .config import get_session_root
|
| 10 |
from .exceptions import SessionError
|
|
@@ -20,6 +21,7 @@ class SessionManager:
|
|
| 20 |
self.session_id = session_id
|
| 21 |
self.session_dir = get_session_root() / session_id
|
| 22 |
self.session_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
| 23 |
|
| 24 |
@classmethod
|
| 25 |
def create_session(cls, metadata: dict[str, Any] | None = None) -> "SessionManager":
|
|
@@ -70,29 +72,31 @@ class SessionManager:
|
|
| 70 |
raise SessionError(f"Invalid checkpoint JSON for session '{self.session_id}'") from exc
|
| 71 |
|
| 72 |
def mark_file_done(self, *, remote_path: str, bytes_uploaded: int, status: str = "uploaded") -> dict[str, Any]:
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
|
|
|
| 85 |
|
| 86 |
def mark_file_failed(self, *, remote_path: str, error: str) -> dict[str, Any]:
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
|
|
|
|
|
| 5 |
from datetime import UTC, datetime
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any
|
| 8 |
+
import threading
|
| 9 |
|
| 10 |
from .config import get_session_root
|
| 11 |
from .exceptions import SessionError
|
|
|
|
| 21 |
self.session_id = session_id
|
| 22 |
self.session_dir = get_session_root() / session_id
|
| 23 |
self.session_dir.mkdir(parents=True, exist_ok=True)
|
| 24 |
+
self._lock = threading.Lock()
|
| 25 |
|
| 26 |
@classmethod
|
| 27 |
def create_session(cls, metadata: dict[str, Any] | None = None) -> "SessionManager":
|
|
|
|
| 72 |
raise SessionError(f"Invalid checkpoint JSON for session '{self.session_id}'") from exc
|
| 73 |
|
| 74 |
def mark_file_done(self, *, remote_path: str, bytes_uploaded: int, status: str = "uploaded") -> dict[str, Any]:
|
| 75 |
+
with self._lock:
|
| 76 |
+
checkpoint = self.load_checkpoint()
|
| 77 |
+
checkpoint.setdefault("uploaded_files", [])
|
| 78 |
+
checkpoint.setdefault("uploaded", 0)
|
| 79 |
+
checkpoint.setdefault("failed", 0)
|
| 80 |
+
checkpoint["uploaded"] = int(checkpoint["uploaded"]) + 1
|
| 81 |
+
checkpoint["status"] = status
|
| 82 |
+
checkpoint["last_completed_file"] = remote_path
|
| 83 |
+
checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
|
| 84 |
+
checkpoint["bytes_uploaded"] = int(checkpoint.get("bytes_uploaded", 0)) + int(bytes_uploaded)
|
| 85 |
+
checkpoint["uploaded_files"].append(remote_path)
|
| 86 |
+
self.save_checkpoint(checkpoint)
|
| 87 |
+
return checkpoint
|
| 88 |
|
| 89 |
def mark_file_failed(self, *, remote_path: str, error: str) -> dict[str, Any]:
|
| 90 |
+
with self._lock:
|
| 91 |
+
checkpoint = self.load_checkpoint()
|
| 92 |
+
checkpoint.setdefault("failed_files", [])
|
| 93 |
+
checkpoint.setdefault("uploaded", 0)
|
| 94 |
+
checkpoint.setdefault("failed", 0)
|
| 95 |
+
checkpoint["failed"] = int(checkpoint["failed"]) + 1
|
| 96 |
+
checkpoint["status"] = "failed"
|
| 97 |
+
checkpoint["last_failed_file"] = remote_path
|
| 98 |
+
checkpoint["last_error"] = error
|
| 99 |
+
checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
|
| 100 |
+
checkpoint["failed_files"].append({"remote_path": remote_path, "error": error})
|
| 101 |
+
self.save_checkpoint(checkpoint)
|
| 102 |
+
return checkpoint
|
tests/integration/test_upload_flow.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import csv
|
| 2 |
+
import os
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from click.testing import CliRunner
|
| 5 |
+
|
| 6 |
+
import pytest
|
| 7 |
+
|
| 8 |
+
from src.uploader_cli.main import cli
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class FakeHfApi:
|
| 12 |
+
def __init__(self, token=None, storage_dir=None):
|
| 13 |
+
self.token = token
|
| 14 |
+
self.storage_dir = Path(storage_dir)
|
| 15 |
+
self.storage_dir.mkdir(parents=True, exist_ok=True)
|
| 16 |
+
|
| 17 |
+
def upload_file(self, path_or_file=None, path_or_fileobj=None, path_in_repo=None, repo_id=None, repo_type=None):
|
| 18 |
+
target = self.storage_dir / path_in_repo
|
| 19 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 20 |
+
if path_or_fileobj is not None:
|
| 21 |
+
# bytes or file-like
|
| 22 |
+
if hasattr(path_or_fileobj, "read"):
|
| 23 |
+
data = path_or_fileobj.read()
|
| 24 |
+
else:
|
| 25 |
+
data = path_or_fileobj
|
| 26 |
+
if isinstance(data, str):
|
| 27 |
+
data = data.encode("utf-8")
|
| 28 |
+
target.write_bytes(data)
|
| 29 |
+
return {"file": path_in_repo}
|
| 30 |
+
if path_or_file is not None:
|
| 31 |
+
src = Path(path_or_file)
|
| 32 |
+
if not src.exists():
|
| 33 |
+
raise FileNotFoundError(path_or_file)
|
| 34 |
+
target.write_bytes(src.read_bytes())
|
| 35 |
+
return {"file": path_in_repo}
|
| 36 |
+
raise ValueError("No file provided to upload_file")
|
| 37 |
+
|
| 38 |
+
def list_repo_files(self, repo_id=None, repo_type=None):
|
| 39 |
+
files = []
|
| 40 |
+
for p in self.storage_dir.rglob("*"):
|
| 41 |
+
if p.is_file():
|
| 42 |
+
files.append(str(p.relative_to(self.storage_dir)).replace("\\", "/"))
|
| 43 |
+
return files
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@pytest.fixture()
|
| 47 |
+
def tmp_remote(tmp_path):
|
| 48 |
+
d = tmp_path / "remote"
|
| 49 |
+
d.mkdir()
|
| 50 |
+
return d
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def test_upload_flow_csv_and_segments(tmp_path, monkeypatch, tmp_remote):
|
| 54 |
+
# prepare local segments
|
| 55 |
+
segments = tmp_path / "segments"
|
| 56 |
+
(segments / "proj").mkdir(parents=True)
|
| 57 |
+
a = segments / "proj" / "a.wav"
|
| 58 |
+
a.write_bytes(b"one")
|
| 59 |
+
b = segments / "proj" / "b.wav"
|
| 60 |
+
b.write_bytes(b"two")
|
| 61 |
+
|
| 62 |
+
# prepare CSV
|
| 63 |
+
csv_file = tmp_path / "detections.csv"
|
| 64 |
+
with csv_file.open("w", newline="", encoding="utf-8") as fh:
|
| 65 |
+
writer = csv.DictWriter(fh, fieldnames=["audio_file", "score"])
|
| 66 |
+
writer.writeheader()
|
| 67 |
+
writer.writerow({"audio_file": "proj/a.wav", "score": "0.9"})
|
| 68 |
+
writer.writerow({"audio_file": "proj/b.wav", "score": "0.8"})
|
| 69 |
+
|
| 70 |
+
# monkeypatch HfApi to our fake implementation
|
| 71 |
+
def _fake_hfapi_factory(token=None):
|
| 72 |
+
return FakeHfApi(token=token, storage_dir=tmp_remote)
|
| 73 |
+
|
| 74 |
+
monkeypatch.setattr("huggingface_hub.HfApi", _fake_hfapi_factory)
|
| 75 |
+
# avoid network call in RepositoryService.validate_repo by stubbing it
|
| 76 |
+
# avoid network call in RepositoryService.validate_repo by stubbing it
|
| 77 |
+
def _fake_validate(self, repo_id: str):
|
| 78 |
+
return {
|
| 79 |
+
"repo_id": repo_id,
|
| 80 |
+
"is_valid": True,
|
| 81 |
+
"project_slug": repo_id.split("/", 1)[1],
|
| 82 |
+
"missing_prefixes": [],
|
| 83 |
+
"has_manifest": True,
|
| 84 |
+
"manifest_ok": True,
|
| 85 |
+
"manifest_error": "",
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
monkeypatch.setattr("src.uploader_cli.main.RepositoryService.validate_repo", _fake_validate)
|
| 89 |
+
|
| 90 |
+
runner = CliRunner()
|
| 91 |
+
result = runner.invoke(
|
| 92 |
+
cli,
|
| 93 |
+
[
|
| 94 |
+
"upload",
|
| 95 |
+
"--repo-id",
|
| 96 |
+
"me/repo",
|
| 97 |
+
"--segments",
|
| 98 |
+
str(segments),
|
| 99 |
+
"--csv",
|
| 100 |
+
str(csv_file),
|
| 101 |
+
"--token",
|
| 102 |
+
"fake-token",
|
| 103 |
+
"--workers",
|
| 104 |
+
"2",
|
| 105 |
+
],
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
assert result.exit_code == 0, result.output
|
| 109 |
+
|
| 110 |
+
# verify remote structure
|
| 111 |
+
remote_files = {p.relative_to(tmp_remote).as_posix() for p in tmp_remote.rglob("*") if p.is_file()}
|
| 112 |
+
assert "index/manifest.json" in remote_files
|
| 113 |
+
assert "index/detections.csv" in remote_files
|
| 114 |
+
# shards may be parquet or jsonl
|
| 115 |
+
shards = [p for p in remote_files if p.startswith("index/shards/")]
|
| 116 |
+
assert len(shards) >= 1
|
| 117 |
+
# audio files uploaded under audio/
|
| 118 |
+
assert "audio/proj/a.wav" in remote_files
|
| 119 |
+
assert "audio/proj/b.wav" in remote_files
|