File size: 15,834 Bytes
4e31d1f
 
 
 
ba0c4c4
4e31d1f
ba0c4c4
4e31d1f
 
 
 
 
2b0510e
 
4e31d1f
 
2b0510e
 
 
 
 
4e31d1f
 
 
 
 
 
ba0c4c4
4e31d1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ba0c4c4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4e31d1f
 
 
 
 
 
 
ba0c4c4
4e31d1f
 
ba0c4c4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4e31d1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0c612f8
b77c32b
 
 
 
0c612f8
 
b77c32b
 
0c612f8
b77c32b
0c612f8
b77c32b
0c612f8
 
 
 
 
 
b77c32b
 
 
 
0c612f8
b77c32b
0c612f8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b77c32b
0c612f8
4e31d1f
ba0c4c4
 
4e31d1f
ba0c4c4
 
 
 
 
4e31d1f
 
 
 
ba0c4c4
54ead58
4e31d1f
 
 
 
 
 
 
 
 
 
 
 
 
ba0c4c4
4e31d1f
ba0c4c4
4e31d1f
 
 
 
 
 
 
ba0c4c4
4e31d1f
ba0c4c4
4e31d1f
ba0c4c4
54ead58
4e31d1f
 
 
 
 
 
 
ba0c4c4
4e31d1f
 
 
 
 
 
 
 
 
 
ba0c4c4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4e31d1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ba0c4c4
 
 
 
 
 
4e31d1f
 
 
 
 
 
 
 
 
ba0c4c4
2b0510e
ba0c4c4
2b0510e
ba0c4c4
2b0510e
ba0c4c4
2b0510e
ba0c4c4
2b0510e
 
b77c32b
ba0c4c4
2b0510e
ba0c4c4
4e68920
ba0c4c4
54ead58
4e68920
2b0510e
 
 
 
 
ba0c4c4
4e31d1f
ba0c4c4
 
4e31d1f
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
from __future__ import annotations

import os
import shutil
import tarfile
import tempfile
import zipfile
from pathlib import Path
from typing import List

import gradio as gr

os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "5")
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "30")

from .auth_service import AuthService
from .hf_tuning import configure_hf_http_backoff

# Apply HF http_backoff tuning before importing modules that may import huggingface_hub
configure_hf_http_backoff()

from .repo_service import RepositoryService
from .scanner import LocalScanner
from .deduplicator import Deduplicator
from .batch_uploader import BatchUploader
from .manifest import build_manifest_from_scan, manifest_to_bytes, write_shards_from_csv_rows
from .session_manager import SessionManager
from .config import WEB_UI_MAX_SIZE_BYTES


def _store_uploaded_files(files: List, dst: Path) -> None:
    dst.mkdir(parents=True, exist_ok=True)
    if not files:
        return
    for f in files:
        # Gradio may pass either a path string, a file-like object, or a dict
        src_path = None
        if isinstance(f, str) and os.path.exists(f):
            src_path = f
        elif hasattr(f, "name") and isinstance(getattr(f, "name"), str) and os.path.exists(getattr(f, "name")):
            src_path = getattr(f, "name")
        elif isinstance(f, dict) and f.get("name") and os.path.exists(f.get("name")):
            src_path = f.get("name")

        if src_path:
            shutil.copy(src_path, dst / Path(src_path).name)
        else:
            # fallback: try reading bytes and write
            try:
                data = f.read()
                out_path = dst / (getattr(f, "name", "uploaded_file"))
                with open(out_path, "wb") as fh:
                    fh.write(data)
            except Exception:
                # ignore silently for best-effort behavior
                continue


def _extract_archive(archive_path: Path, extract_to: Path) -> bool:
    """Extract tar, tar.gz, or zip archives. Returns True if extraction succeeded."""
    try:
        if archive_path.suffix.lower() == ".zip":
            with zipfile.ZipFile(archive_path, "r") as zf:
                zf.extractall(extract_to)
            return True
        elif archive_path.suffix.lower() == ".gz":
            # Handle .tar.gz
            if archive_path.stem.endswith(".tar"):
                with tarfile.open(archive_path, "r:gz") as tf:
                    tf.extractall(extract_to)
                return True
            else:
                return False
        elif archive_path.suffix.lower() == ".tar":
            with tarfile.open(archive_path, "r") as tf:
                tf.extractall(extract_to)
            return True
        else:
            return False
    except Exception as exc:
        print(f"Archive extraction error: {exc}")
        return False


def _handle_upload(token: str, repo_id: str, files, csv_file, remote_base: str, workers: int | None, progress=gr.Progress()):
    # Basic validation
    if not token or not repo_id:
        return "Error: token and repo_id are required." , None

    # Store uploaded files to a temp directory
    tmpdir = Path(tempfile.mkdtemp(prefix="birdnet-upload-"))
    try:
        progress(0, desc="Initializing upload...")
        _store_uploaded_files(files or [], tmpdir)

        # Handle archive extraction: look for .tar, .tar.gz, .zip files
        audio_dir = tmpdir / "audio"
        audio_dir.mkdir(parents=True, exist_ok=True)
        
        for uploaded_file in tmpdir.iterdir():
            if uploaded_file.is_file():
                suffix = uploaded_file.suffix.lower()
                # Check for .tar.gz before .gz
                if uploaded_file.name.endswith(".tar.gz"):
                    progress(0.1, desc=f"Extracting {uploaded_file.name}...")
                    if _extract_archive(uploaded_file, audio_dir):
                        uploaded_file.unlink()  # Delete archive after extraction
                    else:
                        return f"Failed to extract {uploaded_file.name}", None
                elif suffix in {".tar", ".gz", ".zip"}:
                    progress(0.1, desc=f"Extracting {uploaded_file.name}...")
                    if _extract_archive(uploaded_file, audio_dir):
                        uploaded_file.unlink()  # Delete archive after extraction
                    else:
                        return f"Failed to extract {uploaded_file.name}", None
                elif suffix == ".csv":
                    # Keep CSV files as-is; handle them separately
                    pass
                else:
                    # Assume non-archive, non-CSV files are audio files
                    shutil.move(str(uploaded_file), str(audio_dir / uploaded_file.name))

        # Validate total size for web UI (1GB limit)
        total_size = sum(f.stat().st_size for f in audio_dir.rglob("*") if f.is_file())
        if total_size > WEB_UI_MAX_SIZE_BYTES:
            size_gb = total_size / (1024**3)
            return (
                f"Upload exceeds 1 GB limit ({size_gb:.2f} GB). "
                "Please download the Windows standalone uploader for large archives: "
                "https://github.com/jrrribeiro/BirdNET-Uploader-App/releases/latest/download/birdnet-uploader-windows.zip",
                None
            )

        csv_path = None
        if csv_file:
            csv_tmp = tmpdir / Path(csv_file.name).name if hasattr(csv_file, "name") else tmpdir / "detections.csv"
            try:
                # csv_file may be a dict or file-like
                if isinstance(csv_file, str) and os.path.exists(csv_file):
                    shutil.copy(csv_file, csv_tmp)
                elif hasattr(csv_file, "name") and os.path.exists(getattr(csv_file, "name")):
                    shutil.copy(getattr(csv_file, "name"), csv_tmp)
                elif isinstance(csv_file, dict) and csv_file.get("name") and os.path.exists(csv_file.get("name")):
                    shutil.copy(csv_file.get("name"), csv_tmp)
                else:
                    # try read() then write
                    data = csv_file.read()
                    with open(csv_tmp, "wb") as fh:
                        fh.write(data)
                csv_path = str(csv_tmp)
            except Exception:
                csv_path = None

        # Validate repo / create if necessary
        api = HfApi(token=token)
        repo_service = RepositoryService(token)
        progress(0.2, desc="Validating/creating repository...")
        
        try:
            validation = repo_service.validate_repo(repo_id)
            if not validation.get("is_valid"):
                # Attempt to create required structure
                progress(0.25, desc="Creating repository structure...")
                try:
                    repo_service.create_dataset(repo_id, private=True)
                    progress(0.27, desc="Repository created successfully!")
                except Exception as e:
                    error_msg = str(e)
                    return (
                        f"❌ Could not create/initialize dataset '{repo_id}'.\n"
                        f"Error: {error_msg[:150]}\n\n"
                        f"Possible causes:\n"
                        f"• Invalid repo_id format (should be 'username/dataset-name')\n"
                        f"• No write permission to the dataset\n"
                        f"• HuggingFace API error", 
                        None
                    )
        except Exception as e:
            error_msg = str(e)
            # Dataset doesn't exist - try to create it
            if "404" in error_msg or "Repository Not Found" in error_msg:
                progress(0.22, desc="Dataset not found. Creating...")
                try:
                    repo_service.create_dataset(repo_id, private=True)
                    progress(0.27, desc="Dataset created successfully!")
                except Exception as create_error:
                    create_msg = str(create_error)
                    return (
                        f"❌ Dataset '{repo_id}' not found and could not be created.\n"
                        f"Error: {create_msg[:150]}\n\n"
                        f"Please:\n"
                        f"1. Verify the repo_id format: 'your-username/dataset-name'\n"
                        f"2. Check your HuggingFace token has dataset creation permissions\n"
                        f"3. Try again",
                        None
                    )
            else:
                return f"❌ Repository validation failed: {error_msg[:150]}", None

        # Build scan summary from extracted/uploaded files
        progress(0.3, desc="Scanning files...")
        scanner = LocalScanner()
        # Scan the audio_dir where all files are now consolidated
        summary = scanner.scan_folder(str(audio_dir))
        
        total_files = summary.get("total_files", 0)
        progress(0.4, desc=f"Found {total_files} audio files. Processing metadata...")

        # Upload CSV if present
        try:
            if csv_path:
                progress(0.45, desc="Uploading detections CSV...")
                api.upload_file(path_or_fileobj=str(csv_path), path_in_repo="index/detections.csv", repo_id=repo_id, repo_type="dataset")
        except Exception as exc:
            return f"CSV upload failed: {exc}", None

        # Build and upload manifest
        try:
            csv_rows = None
            if csv_path:
                import csv

                with open(csv_path, newline="", encoding="utf-8") as fh:
                    reader = csv.DictReader(fh)
                    csv_rows = list(reader)

            progress(0.5, desc="Building manifest...")
            manifest = build_manifest_from_scan(repo_id, summary, csv_rows=csv_rows)
            progress(0.55, desc="Uploading manifest...")
            api.upload_file(path_or_fileobj=manifest_to_bytes(manifest), path_in_repo="index/manifest.json", repo_id=repo_id, repo_type="dataset")
        except Exception as exc:
            return f"Manifest upload failed: {exc}", None

        # Generate and upload shards if CSV provided
        try:
            if csv_path and csv_rows:
                progress(0.6, desc="Generating shards...")
                shards = write_shards_from_csv_rows(csv_rows)
                for i, shard_path in enumerate(shards):
                    try:
                        progress(0.6 + (0.1 * i / len(shards)), desc=f"Uploading shard {i+1}/{len(shards)}...")
                        api.upload_file(path_or_fileobj=str(shard_path), path_in_repo=f"index/shards/{shard_path.name}", repo_id=repo_id, repo_type="dataset")
                    except Exception:
                        # continue with other shards
                        continue
        except Exception:
            pass

        # Dedup + batch upload
        progress(0.7, desc="Preparing uploads...")
        dedup = Deduplicator(api=api, repo_id=repo_id)
        session = SessionManager()
        uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_workers=workers)

        # Build file_infos from scanned summary
        file_infos = []
        for species, items in summary.get("by_species", {}).items():
            for it in items:
                file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})

        uploaded_count = [0]  # Use list to allow mutation in nested function
        
        def on_progress_update(state: dict):
            uploaded = state.get("uploaded", 0)
            skipped = state.get("skipped", 0)
            failed = state.get("failed", 0)
            total = len(file_infos)
            progress(0.7 + (0.25 * uploaded / max(total, 1)), desc=f"Uploading: {uploaded}/{total} files")

        progress(0.7, desc="Starting file uploads...")
        result = uploader.upload_files(file_infos, remote_base=remote_base or "audio", on_progress=on_progress_update)
        progress(0.95, desc="Finalizing...")
        
        uploaded = result.get("uploaded", 0)
        skipped = result.get("skipped", 0)
        failed = result.get("failed", 0)
        status_msg = f"Upload finished! Uploaded: {uploaded}, Skipped: {skipped}, Failed: {failed}"
        progress(1.0, desc="Complete")
        return status_msg, result
    finally:
        # cleanup tempdir
        try:
            shutil.rmtree(tmpdir)
        except Exception:
            pass


def create_uploader_app():
    with gr.Blocks(title="BirdNET Uploader") as app:
        gr.Markdown("""
        # 🐦 BirdNET Uploader
        Upload audio segments (multiple files) and optional CSV of detections to a Hugging Face dataset.

        If you need to upload many files, download the Windows container and run the standalone uploader locally for best performance.
        """)

        with gr.Group("Authentication"):
            token = gr.Textbox(label="Hugging Face Token", type="password", placeholder="hf_xxx...", interactive=True)

        with gr.Group("Dataset"):
            repo_id = gr.Textbox(label="Dataset repo id (owner/name)", placeholder="username/dataset-name")
            remote_base = gr.Textbox(label="Remote base path (default: audio)", value="audio")

        with gr.Group("Files"):
            files = gr.File(
                label="Audio files or archive (.tar/.tar.gz/.zip) with optional CSV",
                file_count="multiple",
                file_types=[".wav", ".mp3", ".flac", ".ogg", ".m4a", ".tar", ".tar.gz", ".zip"]
            )
            csv_file = gr.File(label="Optional detections CSV (or include in archive)", file_count="single", file_types=[".csv"])

        with gr.Row():
            workers = gr.Number(label="Parallel workers (for large uploads)", value=4)
            upload_btn = gr.Button("Start Upload")

        with gr.Group("Progress"):
            status = gr.Textbox(label="Status", interactive=False)

        with gr.Group("Download"):
            gr.Markdown("""
## [DOWNLOAD] Windows Portable Download

Para uploads maiores (>1 GB) ou melhor performance, baixe o executavel portatil:

**[Download birdnet-uploader-1.0.4-windows.zip](https://huggingface.co/datasets/jrrribeiro/birdnet-uploader-releases/resolve/main/releases/v1.0.4/birdnet-uploader-1.0.4-windows.zip)**

- **Tamanho**: ~109 MB (sem Python necessario)
- **Performance**: Upload ilimitado via CLI
- **Seguranca**: Checksum disponivel [aqui](https://huggingface.co/datasets/jrrribeiro/birdnet-uploader-releases/resolve/main/releases/v1.0.4/birdnet-uploader-1.0.4-windows.zip.sha256)
- **Instrucoes**: [Setup Guide](https://github.com/jrrribeiro/BirdNET-Uploader-App/blob/main/WINDOWS_PORTABLE_SETUP.md)
- **Troubleshooting**: [Guide](https://github.com/jrrribeiro/BirdNET-Uploader-App/blob/main/TROUBLESHOOTING.md)

### [INFO] Checksum SHA256
```
71d126edf726298ed25d4b72b5364e85fa6b1e76157c4f1ee0ab76b4a653f359
```

### ⚡ Novidades na v1.0.4
- [OK] **CRITICO**: Gradio agora incluido no exe (antes faltava!)
- [OK] **Melhorado**: Timeout aumentado para uploads longos
- [OK] **Corrigido**: Parametro API de batch_uploader (path_or_file -> path_or_fileobj)
- [OK] Melhorado: Tratamento de erros durante uploads
- [OK] Adicionado: test_imports.py para diagnostico
""")

        def _start(token_val, repo_val, files_val, csv_val, remote_base_val, workers_val, progress=gr.Progress()):
            status_text, result = _handle_upload(token_val, repo_val, files_val, csv_val, remote_base_val, int(workers_val) if workers_val else None, progress=progress)
            return status_text

        upload_btn.click(_start, inputs=[token, repo_id, files, csv_file, remote_base, workers], outputs=[status])

    return app