File size: 13,068 Bytes
2f1a5fb
 
 
7c83d76
2b0510e
 
 
 
2f1a5fb
 
 
2b0510e
 
2f1a5fb
 
 
 
 
 
2b0510e
 
 
2f1a5fb
 
 
 
 
2b0510e
2f1a5fb
 
 
 
 
 
7c83d76
2f1a5fb
 
 
 
 
 
 
2b0510e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2f1a5fb
 
 
 
 
 
2b0510e
 
 
 
 
 
 
 
4e68920
2b0510e
4e68920
2b0510e
 
2f1a5fb
 
 
 
2b0510e
 
2f1a5fb
2b0510e
2f1a5fb
 
2b0510e
2f1a5fb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7c83d76
2f1a5fb
 
 
 
 
 
 
 
 
 
 
 
 
 
7c83d76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2b0510e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7c83d76
 
2f1a5fb
2b0510e
 
2f1a5fb
2b0510e
 
2f1a5fb
 
 
 
7c83d76
 
2f1a5fb
 
7c83d76
 
2b0510e
7c83d76
2b0510e
7c83d76
 
2b0510e
7c83d76
 
 
 
2b0510e
 
2f1a5fb
 
 
2b0510e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2f1a5fb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
from __future__ import annotations

import time
import concurrent.futures
import os
import logging
import threading
from typing import List, Tuple, Any, TYPE_CHECKING
from pathlib import Path
from typing import Any, Callable, Iterable

if TYPE_CHECKING:
    from huggingface_hub import HfApi

from .config import MAX_BATCH_SIZE, RETRY_MAX_ATTEMPTS, RETRY_INITIAL_BACKOFF_SECONDS
from .deduplicator import Deduplicator
from .session_manager import SessionManager


logger = logging.getLogger("birdnet_uploader.batch_uploader")


class BatchUploader:
    """Orchestrate uploads with deduplication, retries and session checkpoints."""

    def __init__(
        self,
        api: Any,
        repo_id: str,
        deduplicator: Deduplicator,
        session: SessionManager | None = None,
        *,
        max_retries: int | None = None,
        initial_backoff: float | None = None,
        max_workers: int | None = None,
    ) -> None:
        self._api = api
        self.repo_id = repo_id
        self.deduplicator = deduplicator
        self.session = session or SessionManager.create_session()
        self.max_retries = max_retries or RETRY_MAX_ATTEMPTS
        self.initial_backoff = initial_backoff or RETRY_INITIAL_BACKOFF_SECONDS
        # If not provided, choose a reasonable parallelism level for IO-bound uploads
        if max_workers is None:
            cpu = os.cpu_count() or 1
            # Use more threads than CPUs since uploads are IO-bound; cap to 32
            self.max_workers = min(32, max(4, cpu * 4))
        else:
            self.max_workers = max_workers

        # Max seconds to wait for an upload_folder call before falling back
        self.folder_upload_timeout = float(os.getenv("BNU_FOLDER_UPLOAD_TIMEOUT", "20"))
        # Max seconds to wait for an individual upload_file call
        self.upload_file_timeout = float(os.getenv("BNU_UPLOAD_FILE_TIMEOUT", "90"))

    @staticmethod
    def _run_with_timeout(func: Callable[..., Any], timeout_s: float, *args: Any, **kwargs: Any) -> Any:
        result: dict[str, Any] = {"value": None, "exc": None}

        def _target() -> None:
            try:
                result["value"] = func(*args, **kwargs)
            except Exception as exc:  # pragma: no cover - network behavior
                result["exc"] = exc

        t = threading.Thread(target=_target, daemon=True)
        t.start()
        t.join(timeout_s)
        if t.is_alive():
            raise TimeoutError(f"Operation timed out after {timeout_s}s")
        if result["exc"] is not None:
            raise result["exc"]
        return result["value"]

    def _upload_file_with_retry(self, local_path: str | Path, remote_path: str) -> None:
        attempts = 0
        last_exc: Exception | None = None
        while attempts <= self.max_retries:
            try:
                logger.info("Starting upload_file: %s -> %s (attempt %d)", local_path, remote_path, attempts + 1)
                start = time.time()
                # Execute in daemon thread with real timeout (no blocking shutdown wait).
                self._run_with_timeout(
                    self._api.upload_file,
                    self.upload_file_timeout,
                    path_or_fileobj=str(local_path),
                    path_in_repo=remote_path,
                    repo_id=self.repo_id,
                    repo_type="dataset",
                )
                duration = time.time() - start
                logger.info("Finished upload_file: %s -> %s in %.2fs", local_path, remote_path, duration)
                return
            except Exception as exc:  # pragma: no cover - network behavior
                last_exc = exc
                attempts += 1
                logger.warning("upload_file failed for %s -> %s (attempt %d): %s", local_path, remote_path, attempts, exc)
                logger.debug("Exception repr: %r", exc)
                if attempts > self.max_retries:
                    logger.error("Giving up upload_file for %s after %d attempts", remote_path, attempts)
                    raise
                backoff = self.initial_backoff * (2 ** (attempts - 1))
                logger.info("Backing off %.1fs before retry", backoff)
                time.sleep(backoff)

    def upload_files(
        self,
        file_infos: Iterable[dict[str, Any]],
        *,
        remote_base: str = "",
        batch_size: int | None = None,
        on_progress: Callable[[dict[str, Any]], None] | None = None,
    ) -> dict[str, Any]:
        batch_size = batch_size or MAX_BATCH_SIZE
        uploaded = 0
        skipped = 0
        failed = 0

        to_upload: List[Tuple[str, str, int]] = []
        for info in file_infos:
            full_path = str(info["full_path"])
            relative = info["relative_path"].lstrip("/")
            remote_path = f"{remote_base.rstrip('/')}/{relative}" if remote_base else relative
            remote_path = remote_path.lstrip("/")

            decision = self.deduplicator.check_remote(remote_path, file_path=full_path)
            if decision.get("status") == "skip":
                skipped += 1
                self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=0, status="skipped")
                if on_progress:
                    on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
                continue

            to_upload.append((full_path, remote_path, int(info.get("size", 0))))

        if not self.max_workers or self.max_workers <= 1:
            for full_path, remote_path, size in to_upload:
                try:
                    self._upload_file_with_retry(full_path, remote_path)
                    try:
                        self.deduplicator.mark_uploaded(remote_path)
                    except Exception:
                        pass
                    uploaded += 1
                    self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=size)
                    if on_progress:
                        on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
                except Exception as exc:
                    failed += 1
                    self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
                    if on_progress:
                        on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
            return {"uploaded": uploaded, "skipped": skipped, "failed": failed}

        # Optimization: group files by their local directory and use upload_folder
        # for directories with many files to reduce API calls.
        dir_groups: dict[str, list[Tuple[str, str, int]]] = {}
        for full_path, remote_path, size in to_upload:
            local_dir = str(Path(full_path).parent)
            dir_groups.setdefault(local_dir, []).append((full_path, remote_path, size))

        remaining_tasks: list[Tuple[str, str, int]] = []
        # Threshold to prefer folder upload over per-file uploads
        FOLDER_UPLOAD_THRESHOLD = 8

        for local_dir, items in dir_groups.items():
            if len(items) >= FOLDER_UPLOAD_THRESHOLD:
                # Determine remote target directory (use the remote path of first item)
                _, sample_remote, _ = items[0]
                remote_parent = str(Path(sample_remote).parent).replace('\\', '/')
                # Attempt folder upload with retries
                attempts = 0
                max_attempts = self.max_retries or 3
                last_exc = None
                while attempts <= max_attempts:
                    try:
                        logger.info(
                            "Starting upload_folder: %s -> %s (attempt %d, %d files)",
                            local_dir,
                            remote_parent,
                            attempts + 1,
                            len(items),
                        )
                        start = time.time()
                        # Execute in daemon thread with real timeout (no blocking shutdown wait).
                        self._run_with_timeout(
                            self._api.upload_folder,
                            self.folder_upload_timeout,
                            local_dir,
                            remote_parent,
                            repo_id=self.repo_id,
                            repo_type="dataset",
                        )
                        duration = time.time() - start
                        logger.info("Finished upload_folder: %s -> %s in %.2fs", local_dir, remote_parent, duration)
                        # Mark each file as uploaded
                        for full_path, remote_path, size in items:
                            try:
                                self.deduplicator.mark_uploaded(remote_path)
                            except Exception:
                                pass
                            self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=size)
                            uploaded += 1
                            if on_progress:
                                on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
                        break
                    except Exception as exc:
                        last_exc = exc
                        attempts += 1
                        logger.warning("upload_folder failed for %s -> %s (attempt %d): %s", local_dir, remote_parent, attempts, exc)
                        if attempts > max_attempts:
                            logger.error("upload_folder giving up after %d attempts, falling back to per-file uploads", attempts)
                            # Fall back to per-file upload for these items
                            remaining_tasks.extend(items)
                            break
                        backoff = self.initial_backoff * (2 ** (attempts - 1))
                        logger.info("Backing off %.1fs before retrying upload_folder", backoff)
                        time.sleep(backoff)
            else:
                remaining_tasks.extend(items)


        # Proceed with threaded per-file uploads for remaining tasks

        def _worker(task: Tuple[str, str, int]) -> Tuple[str, str]:
            full_path, remote_path, size = task
            try:
                logger.debug("Worker starting upload for %s", remote_path)
                start = time.time()
                self._upload_file_with_retry(full_path, remote_path)
                dur = time.time() - start
                logger.debug("Worker finished upload for %s in %.2fs", remote_path, dur)
                try:
                    self.deduplicator.mark_uploaded(remote_path)
                except Exception:
                    pass
                self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=size)
                return ("ok", remote_path)
            except Exception as exc:
                self.session.mark_file_failed(remote_path=remote_path, error=str(exc))
                return ("fail", remote_path)

        failed_tasks: list[Tuple[str, str, int]] = []
        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as ex:
            futures = {ex.submit(_worker, t): t for t in remaining_tasks}
            for fut in concurrent.futures.as_completed(futures):
                res = fut.result()
                task = futures.get(fut)
                if res[0] == "ok":
                    uploaded += 1
                else:
                    failed += 1
                    if task:
                        failed_tasks.append(task)
                if on_progress:
                    on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})

        # If we had failures with parallel uploads, attempt a sequential retry pass with reduced concurrency
        if failed_tasks:
            logger.warning("%d files failed in parallel upload. Retrying sequentially with reduced concurrency...", len(failed_tasks))
            seq_retry_backoff = max(self.initial_backoff, 1.0)
            # Attempt sequential retries for failed tasks
            for full_path, remote_path, size in failed_tasks:
                try:
                    time.sleep(seq_retry_backoff)
                    self._upload_file_with_retry(full_path, remote_path)
                    try:
                        self.deduplicator.mark_uploaded(remote_path)
                    except Exception:
                        pass
                    self.session.mark_file_done(remote_path=remote_path, bytes_uploaded=size)
                    uploaded += 1
                    failed -= 1
                    if on_progress:
                        on_progress({"uploaded": uploaded, "skipped": skipped, "failed": failed})
                except Exception as exc:
                    logger.warning("Sequential retry failed for %s: %s", remote_path, exc)
                    # leave failed count as-is

        return {"uploaded": uploaded, "skipped": skipped, "failed": failed}