File size: 6,258 Bytes
2f1a5fb
 
 
 
2b0510e
 
 
2f1a5fb
 
2b0510e
7c83d76
2f1a5fb
2b0510e
 
2f1a5fb
 
 
 
 
 
2b0510e
 
 
2f1a5fb
 
 
2b0510e
2f1a5fb
 
 
 
 
 
2b0510e
 
2f1a5fb
 
 
 
 
 
 
 
 
 
7c83d76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2f1a5fb
 
 
 
2b0510e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2f1a5fb
2b0510e
 
 
 
 
 
 
 
 
 
 
2f1a5fb
 
7c83d76
 
 
2f1a5fb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7c83d76
 
 
 
 
 
 
 
 
2f1a5fb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
from __future__ import annotations

import json
import hashlib
import os
import logging
import threading
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, TYPE_CHECKING
import threading

if TYPE_CHECKING:
    from huggingface_hub import HfApi

from .config import get_cache_root
from .hash_utils import compute_file_hash
from .exceptions import ValidationError


logger = logging.getLogger("birdnet_uploader.deduplicator")


class Deduplicator:
    """Check remote file presence and cache the remote path index locally."""

    def __init__(self, api: Any, repo_id: str, *, repo_type: str = "dataset") -> None:
        self._api = api
        self.repo_id = repo_id
        self.repo_type = repo_type
        self._remote_paths: set[str] | None = None
        self._decision_cache: dict[str, dict[str, Any]] = {}
        self.cache_path = self._build_cache_path()
        # RLock avoids deadlock when refresh_remote_index is called while lock is already held.
        self._lock = threading.RLock()

    def _build_cache_path(self) -> Path:
        cache_root = get_cache_root() / "dedup"
        cache_root.mkdir(parents=True, exist_ok=True)
        digest = hashlib.sha256(f"{self.repo_type}:{self.repo_id}".encode("utf-8")).hexdigest()[:16]
        return cache_root / f"{digest}.json"

    def load_cached_index(self) -> set[str]:
        if self._remote_paths is not None:
            return self._remote_paths
        with self._lock:
            if self.cache_path.exists():
                raw = self.cache_path.read_text(encoding="utf-8")
                try:
                    payload = json.loads(raw)
                except json.JSONDecodeError as exc:
                    raise ValidationError(f"Invalid dedup cache JSON at {self.cache_path}") from exc

                remote_paths = payload.get("remote_paths", [])
                if not isinstance(remote_paths, list):
                    raise ValidationError(f"Invalid dedup cache format at {self.cache_path}")
                self._remote_paths = {str(path) for path in remote_paths}
                return self._remote_paths

            self._remote_paths = self.refresh_remote_index()
            return self._remote_paths

    def refresh_remote_index(self) -> set[str]:
        try:
            # Run list_repo_files in a short-lived thread so we don't block forever on network issues.
            timeout = float(os.getenv("BNU_LIST_REPO_TIMEOUT", "10"))
            result: dict[str, Any] = {"value": None, "exc": None}

            def _target() -> None:
                try:
                    result["value"] = self._api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type)
                except Exception as exc:  # pragma: no cover - network behavior
                    result["exc"] = exc

            t = threading.Thread(target=_target, daemon=True)
            t.start()
            t.join(timeout)
            if t.is_alive():
                raise TimeoutError(f"list_repo_files timed out after {timeout}s")
            if result["exc"] is not None:
                raise result["exc"]
            repo_files = result["value"]
        except Exception as exc:  # pragma: no cover - external API behavior
            # Fail-open: if listing remote files is unavailable, continue upload path
            # and rely on server-side conflict handling. This avoids blocking uploads
            # on flaky network/listing endpoints.
            logger.warning(
                "Could not read remote file listing for %s (continuing without dedup index): %s",
                self.repo_id,
                exc,
            )
            with self._lock:
                self._remote_paths = set()
            return set()

        remote_paths = {str(path) for path in repo_files}
        with self._lock:
            self._remote_paths = remote_paths
            self._write_cache(remote_paths)
        return remote_paths

    def check_remote(self, remote_path: str, *, file_path: str | Path | None = None) -> dict[str, Any]:
        cache_key = self._build_decision_key(remote_path, file_path=file_path)
        cached = self._decision_cache.get(cache_key)
        if cached is not None:
            return dict(cached)

        remote_paths = self.load_cached_index()
        exists = remote_path in remote_paths

        payload = {
            "remote_path": remote_path,
            "status": "skip" if exists else "upload",
            "cached_remote_index": True,
        }
        if file_path is not None:
            file_path_obj = Path(file_path)
            payload["local_size"] = file_path_obj.stat().st_size
            payload["local_sha256"] = compute_file_hash(file_path_obj)

        self._decision_cache[cache_key] = dict(payload)
        return payload

    def mark_uploaded(self, remote_path: str) -> None:
        with self._lock:
            remote_paths = self.load_cached_index()
            if remote_path not in remote_paths:
                remote_paths.add(remote_path)
                self._write_cache(remote_paths)

            keys_to_update = [key for key, value in self._decision_cache.items() if value.get("remote_path") == remote_path]
            for key in keys_to_update:
                self._decision_cache[key]["status"] = "skip"

    def _write_cache(self, remote_paths: set[str]) -> None:
        payload = {
            "repo_id": self.repo_id,
            "repo_type": self.repo_type,
            "updated_at": datetime.now(UTC).isoformat(),
            "remote_paths": sorted(remote_paths),
        }
        temp_path = self.cache_path.with_suffix(".json.tmp")
        temp_path.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
        temp_path.replace(self.cache_path)

    def _build_decision_key(self, remote_path: str, *, file_path: str | Path | None) -> str:
        if file_path is None:
            return remote_path
        file_path_obj = Path(file_path)
        try:
            stat = file_path_obj.stat()
        except FileNotFoundError as exc:
            raise ValidationError(f"Local file not found: {file_path_obj}") from exc
        return f"{remote_path}:{file_path_obj.resolve()}:{stat.st_size}:{stat.st_mtime_ns}"