from __future__ import annotations import json import hashlib import os import logging import threading from datetime import UTC, datetime from pathlib import Path from typing import Any, TYPE_CHECKING import threading if TYPE_CHECKING: from huggingface_hub import HfApi from .config import get_cache_root from .hash_utils import compute_file_hash from .exceptions import ValidationError logger = logging.getLogger("birdnet_uploader.deduplicator") class Deduplicator: """Check remote file presence and cache the remote path index locally.""" def __init__(self, api: Any, repo_id: str, *, repo_type: str = "dataset") -> None: self._api = api self.repo_id = repo_id self.repo_type = repo_type self._remote_paths: set[str] | None = None self._decision_cache: dict[str, dict[str, Any]] = {} self.cache_path = self._build_cache_path() # RLock avoids deadlock when refresh_remote_index is called while lock is already held. self._lock = threading.RLock() def _build_cache_path(self) -> Path: cache_root = get_cache_root() / "dedup" cache_root.mkdir(parents=True, exist_ok=True) digest = hashlib.sha256(f"{self.repo_type}:{self.repo_id}".encode("utf-8")).hexdigest()[:16] return cache_root / f"{digest}.json" def load_cached_index(self) -> set[str]: if self._remote_paths is not None: return self._remote_paths with self._lock: if self.cache_path.exists(): raw = self.cache_path.read_text(encoding="utf-8") try: payload = json.loads(raw) except json.JSONDecodeError as exc: raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc remote_paths = payload.get("remote_paths", []) if not isinstance(remote_paths, list): raise ValidationError(f"Invalid dedup cache format at {self.cache_path}") self._remote_paths = {str(path) for path in remote_paths} return self._remote_paths self._remote_paths = self.refresh_remote_index() return self._remote_paths def refresh_remote_index(self) -> set[str]: try: # Run list_repo_files in a short-lived thread so we don't block forever on network issues. timeout = float(os.getenv("BNU_LIST_REPO_TIMEOUT", "10")) result: dict[str, Any] = {"value": None, "exc": None} def _target() -> None: try: result["value"] = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type) except Exception as exc: # pragma: no cover - network behavior result["exc"] = exc t = threading.Thread(target=_target, daemon=True) t.start() t.join(timeout) if t.is_alive(): raise TimeoutError(f"list_repo_files timed out after {timeout}s") if result["exc"] is not None: raise result["exc"] repo_files = result["value"] except Exception as exc: # pragma: no cover - external API behavior # Fail-open: if listing remote files is unavailable, continue upload path # and rely on server-side conflict handling. This avoids blocking uploads # on flaky network/listing endpoints. logger.warning( "Could not read remote file listing for %s (continuing without dedup index): %s", self.repo_id, exc, ) with self._lock: self._remote_paths = set() return set() remote_paths = {str(path) for path in repo_files} with self._lock: self._remote_paths = remote_paths self._write_cache(remote_paths) return remote_paths def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]: cache_key = self._build_decision_key(remote_path, file_path=file_path) cached = self._decision_cache.get(cache_key) if cached is not None: return dict(cached) remote_paths = self.load_cached_index() exists = remote_path in remote_paths payload = { "remote_path": remote_path, "status": "skip" if exists else "upload", "cached_remote_index": True, } if file_path is not None: file_path_obj = Path(file_path) payload["local_size"] = file_path_obj.stat().st_size payload["local_sha256"] = compute_file_hash(file_path_obj) self._decision_cache[cache_key] = dict(payload) return payload def mark_uploaded(self, remote_path: str) -> None: with self._lock: remote_paths = self.load_cached_index() if remote_path not in remote_paths: remote_paths.add(remote_path) self._write_cache(remote_paths) keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path] for key in keys_to_update: self._decision_cache[key]["status"] = "skip" def _write_cache(self, remote_paths: set[str]) -> None: payload = { "repo_id": self.repo_id, "repo_type": self.repo_type, "updated_at": datetime.now(UTC).isoformat(), "remote_paths": sorted(remote_paths), } temp_path = self.cache_path.with_suffix(".json.tmp") temp_path.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8") temp_path.replace(self.cache_path) def _build_decision_key(self, remote_path: str, *, file_path: str | Path | None) -> str: if file_path is None: return remote_path file_path_obj = Path(file_path) try: stat = file_path_obj.stat() except FileNotFoundError as exc: raise ValidationError(f"Local file not found: {file_path_obj}") from exc return f"{remote_path}:{file_path_obj.resolve()}:{stat.st_size}:{stat.st_mtime_ns}"