Spaces:
Sleeping
Sleeping
File size: 6,258 Bytes
2f1a5fb 2b0510e 2f1a5fb 2b0510e 7c83d76 2f1a5fb 2b0510e 2f1a5fb 2b0510e 2f1a5fb 2b0510e 2f1a5fb 2b0510e 2f1a5fb 7c83d76 2f1a5fb 2b0510e 2f1a5fb 2b0510e 2f1a5fb 7c83d76 2f1a5fb 7c83d76 2f1a5fb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | from __future__ import annotations
import json
import hashlib
import os
import logging
import threading
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, TYPE_CHECKING
import threading
if TYPE_CHECKING:
from huggingface_hub import HfApi
from .config import get_cache_root
from .hash_utils import compute_file_hash
from .exceptions import ValidationError
logger = logging.getLogger("birdnet_uploader.deduplicator")
class Deduplicator:
"""Check remote file presence and cache the remote path index locally."""
def __init__(self, api: Any, repo_id: str, *, repo_type: str = "dataset") -> None:
self._api = api
self.repo_id = repo_id
self.repo_type = repo_type
self._remote_paths: set[str] | None = None
self._decision_cache: dict[str, dict[str, Any]] = {}
self.cache_path = self._build_cache_path()
# RLock avoids deadlock when refresh_remote_index is called while lock is already held.
self._lock = threading.RLock()
def _build_cache_path(self) -> Path:
cache_root = get_cache_root() / "dedup"
cache_root.mkdir(parents=True, exist_ok=True)
digest = hashlib.sha256(f"{self.repo_type}:{self.repo_id}".encode("utf-8")).hexdigest()[:16]
return cache_root / f"{digest}.json"
def load_cached_index(self) -> set[str]:
if self._remote_paths is not None:
return self._remote_paths
with self._lock:
if self.cache_path.exists():
raw = self.cache_path.read_text(encoding="utf-8")
try:
payload = json.loads(raw)
except json.JSONDecodeError as exc:
raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc
remote_paths = payload.get("remote_paths", [])
if not isinstance(remote_paths, list):
raise ValidationError(f"Invalid dedup cache format at {self.cache_path}")
self._remote_paths = {str(path) for path in remote_paths}
return self._remote_paths
self._remote_paths = self.refresh_remote_index()
return self._remote_paths
def refresh_remote_index(self) -> set[str]:
try:
# Run list_repo_files in a short-lived thread so we don't block forever on network issues.
timeout = float(os.getenv("BNU_LIST_REPO_TIMEOUT", "10"))
result: dict[str, Any] = {"value": None, "exc": None}
def _target() -> None:
try:
result["value"] = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
except Exception as exc: # pragma: no cover - network behavior
result["exc"] = exc
t = threading.Thread(target=_target, daemon=True)
t.start()
t.join(timeout)
if t.is_alive():
raise TimeoutError(f"list_repo_files timed out after {timeout}s")
if result["exc"] is not None:
raise result["exc"]
repo_files = result["value"]
except Exception as exc: # pragma: no cover - external API behavior
# Fail-open: if listing remote files is unavailable, continue upload path
# and rely on server-side conflict handling. This avoids blocking uploads
# on flaky network/listing endpoints.
logger.warning(
"Could not read remote file listing for %s (continuing without dedup index): %s",
self.repo_id,
exc,
)
with self._lock:
self._remote_paths = set()
return set()
remote_paths = {str(path) for path in repo_files}
with self._lock:
self._remote_paths = remote_paths
self._write_cache(remote_paths)
return remote_paths
def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
cache_key = self._build_decision_key(remote_path, file_path=file_path)
cached = self._decision_cache.get(cache_key)
if cached is not None:
return dict(cached)
remote_paths = self.load_cached_index()
exists = remote_path in remote_paths
payload = {
"remote_path": remote_path,
"status": "skip" if exists else "upload",
"cached_remote_index": True,
}
if file_path is not None:
file_path_obj = Path(file_path)
payload["local_size"] = file_path_obj.stat().st_size
payload["local_sha256"] = compute_file_hash(file_path_obj)
self._decision_cache[cache_key] = dict(payload)
return payload
def mark_uploaded(self, remote_path: str) -> None:
with self._lock:
remote_paths = self.load_cached_index()
if remote_path not in remote_paths:
remote_paths.add(remote_path)
self._write_cache(remote_paths)
keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path]
for key in keys_to_update:
self._decision_cache[key]["status"] = "skip"
def _write_cache(self, remote_paths: set[str]) -> None:
payload = {
"repo_id": self.repo_id,
"repo_type": self.repo_type,
"updated_at": datetime.now(UTC).isoformat(),
"remote_paths": sorted(remote_paths),
}
temp_path = self.cache_path.with_suffix(".json.tmp")
temp_path.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
temp_path.replace(self.cache_path)
def _build_decision_key(self, remote_path: str, *, file_path: str | Path | None) -> str:
if file_path is None:
return remote_path
file_path_obj = Path(file_path)
try:
stat = file_path_obj.stat()
except FileNotFoundError as exc:
raise ValidationError(f"Local file not found: {file_path_obj}") from exc
return f"{remote_path}:{file_path_obj.resolve()}:{stat.st_size}:{stat.st_mtime_ns}"
|