Spaces:
Sleeping
Sleeping
Download src/uploader/deduplicator.py from jrrribeiro/BirdNET-Uploader-App: direct link, hf CLI and curl.
- Browser
- Download file 6.26 kB
-
https://huggingface.co/spaces/jrrribeiro/BirdNET-Uploader-App/resolve/main/src/uploader/deduplicator.py
- Command line
-
hf download hf://spaces/jrrribeiro/BirdNET-Uploader-App/src/uploader/deduplicator.py
-
curl -L -o deduplicator.py https://huggingface.co/spaces/jrrribeiro/BirdNET-Uploader-App/resolve/main/src/uploader/deduplicator.py
6.26 kB
| from __future__ import annotations | |
| import json | |
| import hashlib | |
| import os | |
| import logging | |
| import threading | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any, TYPE_CHECKING | |
| import threading | |
| if TYPE_CHECKING: | |
| from huggingface_hub import HfApi | |
| from .config import get_cache_root | |
| from .hash_utils import compute_file_hash | |
| from .exceptions import ValidationError | |
| logger = logging.getLogger("birdnet_uploader.deduplicator") | |
| class Deduplicator: | |
| """Check remote file presence and cache the remote path index locally.""" | |
| def __init__(self, api: Any, repo_id: str, *, repo_type: str = "dataset") -> None: | |
| self._api = api | |
| self.repo_id = repo_id | |
| self.repo_type = repo_type | |
| self._remote_paths: set[str] | None = None | |
| self._decision_cache: dict[str, dict[str, Any]] = {} | |
| self.cache_path = self._build_cache_path() | |
| # RLock avoids deadlock when refresh_remote_index is called while lock is already held. | |
| self._lock = threading.RLock() | |
| def _build_cache_path(self) -> Path: | |
| cache_root = get_cache_root() / "dedup" | |
| cache_root.mkdir(parents=True, exist_ok=True) | |
| digest = hashlib.sha256(f"{self.repo_type}:{self.repo_id}".encode("utf-8")).hexdigest()[:16] | |
| return cache_root / f"{digest}.json" | |
| def load_cached_index(self) -> set[str]: | |
| if self._remote_paths is not None: | |
| return self._remote_paths | |
| with self._lock: | |
| if self.cache_path.exists(): | |
| raw = self.cache_path.read_text(encoding="utf-8") | |
| try: | |
| payload = json.loads(raw) | |
| except json.JSONDecodeError as exc: | |
| raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc | |
| remote_paths = payload.get("remote_paths", []) | |
| if not isinstance(remote_paths, list): | |
| raise ValidationError(f"Invalid dedup cache format at {self.cache_path}") | |
| self._remote_paths = {str(path) for path in remote_paths} | |
| return self._remote_paths | |
| self._remote_paths = self.refresh_remote_index() | |
| return self._remote_paths | |
| def refresh_remote_index(self) -> set[str]: | |
| try: | |
| # Run list_repo_files in a short-lived thread so we don't block forever on network issues. | |
| timeout = float(os.getenv("BNU_LIST_REPO_TIMEOUT", "10")) | |
| result: dict[str, Any] = {"value": None, "exc": None} | |
| def _target() -> None: | |
| try: | |
| result["value"] = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type) | |
| except Exception as exc: # pragma: no cover - network behavior | |
| result["exc"] = exc | |
| t = threading.Thread(target=_target, daemon=True) | |
| t.start() | |
| t.join(timeout) | |
| if t.is_alive(): | |
| raise TimeoutError(f"list_repo_files timed out after {timeout}s") | |
| if result["exc"] is not None: | |
| raise result["exc"] | |
| repo_files = result["value"] | |
| except Exception as exc: # pragma: no cover - external API behavior | |
| # Fail-open: if listing remote files is unavailable, continue upload path | |
| # and rely on server-side conflict handling. This avoids blocking uploads | |
| # on flaky network/listing endpoints. | |
| logger.warning( | |
| "Could not read remote file listing for %s (continuing without dedup index): %s", | |
| self.repo_id, | |
| exc, | |
| ) | |
| with self._lock: | |
| self._remote_paths = set() | |
| return set() | |
| remote_paths = {str(path) for path in repo_files} | |
| with self._lock: | |
| self._remote_paths = remote_paths | |
| self._write_cache(remote_paths) | |
| return remote_paths | |
| def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]: | |
| cache_key = self._build_decision_key(remote_path, file_path=file_path) | |
| cached = self._decision_cache.get(cache_key) | |
| if cached is not None: | |
| return dict(cached) | |
| remote_paths = self.load_cached_index() | |
| exists = remote_path in remote_paths | |
| payload = { | |
| "remote_path": remote_path, | |
| "status": "skip" if exists else "upload", | |
| "cached_remote_index": True, | |
| } | |
| if file_path is not None: | |
| file_path_obj = Path(file_path) | |
| payload["local_size"] = file_path_obj.stat().st_size | |
| payload["local_sha256"] = compute_file_hash(file_path_obj) | |
| self._decision_cache[cache_key] = dict(payload) | |
| return payload | |
| def mark_uploaded(self, remote_path: str) -> None: | |
| with self._lock: | |
| remote_paths = self.load_cached_index() | |
| if remote_path not in remote_paths: | |
| remote_paths.add(remote_path) | |
| self._write_cache(remote_paths) | |
| keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path] | |
| for key in keys_to_update: | |
| self._decision_cache[key]["status"] = "skip" | |
| def _write_cache(self, remote_paths: set[str]) -> None: | |
| payload = { | |
| "repo_id": self.repo_id, | |
| "repo_type": self.repo_type, | |
| "updated_at": datetime.now(UTC).isoformat(), | |
| "remote_paths": sorted(remote_paths), | |
| } | |
| temp_path = self.cache_path.with_suffix(".json.tmp") | |
| temp_path.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8") | |
| temp_path.replace(self.cache_path) | |
| def _build_decision_key(self, remote_path: str, *, file_path: str | Path | None) -> str: | |
| if file_path is None: | |
| return remote_path | |
| file_path_obj = Path(file_path) | |
| try: | |
| stat = file_path_obj.stat() | |
| except FileNotFoundError as exc: | |
| raise ValidationError(f"Local file not found: {file_path_obj}") from exc | |
| return f"{remote_path}:{file_path_obj.resolve()}:{stat.st_size}:{stat.st_mtime_ns}" | |