Spaces:
Sleeping
Sleeping
File size: 15,834 Bytes
4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f 2b0510e 4e31d1f 2b0510e 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f 0c612f8 b77c32b 0c612f8 b77c32b 0c612f8 b77c32b 0c612f8 b77c32b 0c612f8 b77c32b 0c612f8 b77c32b 0c612f8 b77c32b 0c612f8 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f ba0c4c4 54ead58 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f ba0c4c4 54ead58 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f ba0c4c4 4e31d1f ba0c4c4 2b0510e ba0c4c4 2b0510e ba0c4c4 2b0510e ba0c4c4 2b0510e ba0c4c4 2b0510e b77c32b ba0c4c4 2b0510e ba0c4c4 4e68920 ba0c4c4 54ead58 4e68920 2b0510e ba0c4c4 4e31d1f ba0c4c4 4e31d1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 | from __future__ import annotations
import os
import shutil
import tarfile
import tempfile
import zipfile
from pathlib import Path
from typing import List
import gradio as gr
os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "5")
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "30")
from .auth_service import AuthService
from .hf_tuning import configure_hf_http_backoff
# Apply HF http_backoff tuning before importing modules that may import huggingface_hub
configure_hf_http_backoff()
from .repo_service import RepositoryService
from .scanner import LocalScanner
from .deduplicator import Deduplicator
from .batch_uploader import BatchUploader
from .manifest import build_manifest_from_scan, manifest_to_bytes, write_shards_from_csv_rows
from .session_manager import SessionManager
from .config import WEB_UI_MAX_SIZE_BYTES
def _store_uploaded_files(files: List, dst: Path) -> None:
dst.mkdir(parents=True, exist_ok=True)
if not files:
return
for f in files:
# Gradio may pass either a path string, a file-like object, or a dict
src_path = None
if isinstance(f, str) and os.path.exists(f):
src_path = f
elif hasattr(f, "name") and isinstance(getattr(f, "name"), str) and os.path.exists(getattr(f, "name")):
src_path = getattr(f, "name")
elif isinstance(f, dict) and f.get("name") and os.path.exists(f.get("name")):
src_path = f.get("name")
if src_path:
shutil.copy(src_path, dst / Path(src_path).name)
else:
# fallback: try reading bytes and write
try:
data = f.read()
out_path = dst / (getattr(f, "name", "uploaded_file"))
with open(out_path, "wb") as fh:
fh.write(data)
except Exception:
# ignore silently for best-effort behavior
continue
def _extract_archive(archive_path: Path, extract_to: Path) -> bool:
"""Extract tar, tar.gz, or zip archives. Returns True if extraction succeeded."""
try:
if archive_path.suffix.lower() == ".zip":
with zipfile.ZipFile(archive_path, "r") as zf:
zf.extractall(extract_to)
return True
elif archive_path.suffix.lower() == ".gz":
# Handle .tar.gz
if archive_path.stem.endswith(".tar"):
with tarfile.open(archive_path, "r:gz") as tf:
tf.extractall(extract_to)
return True
else:
return False
elif archive_path.suffix.lower() == ".tar":
with tarfile.open(archive_path, "r") as tf:
tf.extractall(extract_to)
return True
else:
return False
except Exception as exc:
print(f"Archive extraction error: {exc}")
return False
def _handle_upload(token: str, repo_id: str, files, csv_file, remote_base: str, workers: int | None, progress=gr.Progress()):
# Basic validation
if not token or not repo_id:
return "Error: token and repo_id are required." , None
# Store uploaded files to a temp directory
tmpdir = Path(tempfile.mkdtemp(prefix="birdnet-upload-"))
try:
progress(0, desc="Initializing upload...")
_store_uploaded_files(files or [], tmpdir)
# Handle archive extraction: look for .tar, .tar.gz, .zip files
audio_dir = tmpdir / "audio"
audio_dir.mkdir(parents=True, exist_ok=True)
for uploaded_file in tmpdir.iterdir():
if uploaded_file.is_file():
suffix = uploaded_file.suffix.lower()
# Check for .tar.gz before .gz
if uploaded_file.name.endswith(".tar.gz"):
progress(0.1, desc=f"Extracting {uploaded_file.name}...")
if _extract_archive(uploaded_file, audio_dir):
uploaded_file.unlink() # Delete archive after extraction
else:
return f"Failed to extract {uploaded_file.name}", None
elif suffix in {".tar", ".gz", ".zip"}:
progress(0.1, desc=f"Extracting {uploaded_file.name}...")
if _extract_archive(uploaded_file, audio_dir):
uploaded_file.unlink() # Delete archive after extraction
else:
return f"Failed to extract {uploaded_file.name}", None
elif suffix == ".csv":
# Keep CSV files as-is; handle them separately
pass
else:
# Assume non-archive, non-CSV files are audio files
shutil.move(str(uploaded_file), str(audio_dir / uploaded_file.name))
# Validate total size for web UI (1GB limit)
total_size = sum(f.stat().st_size for f in audio_dir.rglob("*") if f.is_file())
if total_size > WEB_UI_MAX_SIZE_BYTES:
size_gb = total_size / (1024**3)
return (
f"Upload exceeds 1 GB limit ({size_gb:.2f} GB). "
"Please download the Windows standalone uploader for large archives: "
"https://github.com/jrrribeiro/BirdNET-Uploader-App/releases/latest/download/birdnet-uploader-windows.zip",
None
)
csv_path = None
if csv_file:
csv_tmp = tmpdir / Path(csv_file.name).name if hasattr(csv_file, "name") else tmpdir / "detections.csv"
try:
# csv_file may be a dict or file-like
if isinstance(csv_file, str) and os.path.exists(csv_file):
shutil.copy(csv_file, csv_tmp)
elif hasattr(csv_file, "name") and os.path.exists(getattr(csv_file, "name")):
shutil.copy(getattr(csv_file, "name"), csv_tmp)
elif isinstance(csv_file, dict) and csv_file.get("name") and os.path.exists(csv_file.get("name")):
shutil.copy(csv_file.get("name"), csv_tmp)
else:
# try read() then write
data = csv_file.read()
with open(csv_tmp, "wb") as fh:
fh.write(data)
csv_path = str(csv_tmp)
except Exception:
csv_path = None
# Validate repo / create if necessary
api = HfApi(token=token)
repo_service = RepositoryService(token)
progress(0.2, desc="Validating/creating repository...")
try:
validation = repo_service.validate_repo(repo_id)
if not validation.get("is_valid"):
# Attempt to create required structure
progress(0.25, desc="Creating repository structure...")
try:
repo_service.create_dataset(repo_id, private=True)
progress(0.27, desc="Repository created successfully!")
except Exception as e:
error_msg = str(e)
return (
f"❌ Could not create/initialize dataset '{repo_id}'.\n"
f"Error: {error_msg[:150]}\n\n"
f"Possible causes:\n"
f"• Invalid repo_id format (should be 'username/dataset-name')\n"
f"• No write permission to the dataset\n"
f"• HuggingFace API error",
None
)
except Exception as e:
error_msg = str(e)
# Dataset doesn't exist - try to create it
if "404" in error_msg or "Repository Not Found" in error_msg:
progress(0.22, desc="Dataset not found. Creating...")
try:
repo_service.create_dataset(repo_id, private=True)
progress(0.27, desc="Dataset created successfully!")
except Exception as create_error:
create_msg = str(create_error)
return (
f"❌ Dataset '{repo_id}' not found and could not be created.\n"
f"Error: {create_msg[:150]}\n\n"
f"Please:\n"
f"1. Verify the repo_id format: 'your-username/dataset-name'\n"
f"2. Check your HuggingFace token has dataset creation permissions\n"
f"3. Try again",
None
)
else:
return f"❌ Repository validation failed: {error_msg[:150]}", None
# Build scan summary from extracted/uploaded files
progress(0.3, desc="Scanning files...")
scanner = LocalScanner()
# Scan the audio_dir where all files are now consolidated
summary = scanner.scan_folder(str(audio_dir))
total_files = summary.get("total_files", 0)
progress(0.4, desc=f"Found {total_files} audio files. Processing metadata...")
# Upload CSV if present
try:
if csv_path:
progress(0.45, desc="Uploading detections CSV...")
api.upload_file(path_or_fileobj=str(csv_path), path_in_repo="index/detections.csv", repo_id=repo_id, repo_type="dataset")
except Exception as exc:
return f"CSV upload failed: {exc}", None
# Build and upload manifest
try:
csv_rows = None
if csv_path:
import csv
with open(csv_path, newline="", encoding="utf-8") as fh:
reader = csv.DictReader(fh)
csv_rows = list(reader)
progress(0.5, desc="Building manifest...")
manifest = build_manifest_from_scan(repo_id, summary, csv_rows=csv_rows)
progress(0.55, desc="Uploading manifest...")
api.upload_file(path_or_fileobj=manifest_to_bytes(manifest), path_in_repo="index/manifest.json", repo_id=repo_id, repo_type="dataset")
except Exception as exc:
return f"Manifest upload failed: {exc}", None
# Generate and upload shards if CSV provided
try:
if csv_path and csv_rows:
progress(0.6, desc="Generating shards...")
shards = write_shards_from_csv_rows(csv_rows)
for i, shard_path in enumerate(shards):
try:
progress(0.6 + (0.1 * i / len(shards)), desc=f"Uploading shard {i+1}/{len(shards)}...")
api.upload_file(path_or_fileobj=str(shard_path), path_in_repo=f"index/shards/{shard_path.name}", repo_id=repo_id, repo_type="dataset")
except Exception:
# continue with other shards
continue
except Exception:
pass
# Dedup + batch upload
progress(0.7, desc="Preparing uploads...")
dedup = Deduplicator(api=api, repo_id=repo_id)
session = SessionManager()
uploader = BatchUploader(api=api, repo_id=repo_id, deduplicator=dedup, session=session, max_workers=workers)
# Build file_infos from scanned summary
file_infos = []
for species, items in summary.get("by_species", {}).items():
for it in items:
file_infos.append({"full_path": it["full_path"], "relative_path": it["relative_path"], "size": it.get("size", 0)})
uploaded_count = [0] # Use list to allow mutation in nested function
def on_progress_update(state: dict):
uploaded = state.get("uploaded", 0)
skipped = state.get("skipped", 0)
failed = state.get("failed", 0)
total = len(file_infos)
progress(0.7 + (0.25 * uploaded / max(total, 1)), desc=f"Uploading: {uploaded}/{total} files")
progress(0.7, desc="Starting file uploads...")
result = uploader.upload_files(file_infos, remote_base=remote_base or "audio", on_progress=on_progress_update)
progress(0.95, desc="Finalizing...")
uploaded = result.get("uploaded", 0)
skipped = result.get("skipped", 0)
failed = result.get("failed", 0)
status_msg = f"Upload finished! Uploaded: {uploaded}, Skipped: {skipped}, Failed: {failed}"
progress(1.0, desc="Complete")
return status_msg, result
finally:
# cleanup tempdir
try:
shutil.rmtree(tmpdir)
except Exception:
pass
def create_uploader_app():
with gr.Blocks(title="BirdNET Uploader") as app:
gr.Markdown("""
# 🐦 BirdNET Uploader
Upload audio segments (multiple files) and optional CSV of detections to a Hugging Face dataset.
If you need to upload many files, download the Windows container and run the standalone uploader locally for best performance.
""")
with gr.Group("Authentication"):
token = gr.Textbox(label="Hugging Face Token", type="password", placeholder="hf_xxx...", interactive=True)
with gr.Group("Dataset"):
repo_id = gr.Textbox(label="Dataset repo id (owner/name)", placeholder="username/dataset-name")
remote_base = gr.Textbox(label="Remote base path (default: audio)", value="audio")
with gr.Group("Files"):
files = gr.File(
label="Audio files or archive (.tar/.tar.gz/.zip) with optional CSV",
file_count="multiple",
file_types=[".wav", ".mp3", ".flac", ".ogg", ".m4a", ".tar", ".tar.gz", ".zip"]
)
csv_file = gr.File(label="Optional detections CSV (or include in archive)", file_count="single", file_types=[".csv"])
with gr.Row():
workers = gr.Number(label="Parallel workers (for large uploads)", value=4)
upload_btn = gr.Button("Start Upload")
with gr.Group("Progress"):
status = gr.Textbox(label="Status", interactive=False)
with gr.Group("Download"):
gr.Markdown("""
## [DOWNLOAD] Windows Portable Download
Para uploads maiores (>1 GB) ou melhor performance, baixe o executavel portatil:
**[Download birdnet-uploader-1.0.4-windows.zip](https://huggingface.co/datasets/jrrribeiro/birdnet-uploader-releases/resolve/main/releases/v1.0.4/birdnet-uploader-1.0.4-windows.zip)**
- **Tamanho**: ~109 MB (sem Python necessario)
- **Performance**: Upload ilimitado via CLI
- **Seguranca**: Checksum disponivel [aqui](https://huggingface.co/datasets/jrrribeiro/birdnet-uploader-releases/resolve/main/releases/v1.0.4/birdnet-uploader-1.0.4-windows.zip.sha256)
- **Instrucoes**: [Setup Guide](https://github.com/jrrribeiro/BirdNET-Uploader-App/blob/main/WINDOWS_PORTABLE_SETUP.md)
- **Troubleshooting**: [Guide](https://github.com/jrrribeiro/BirdNET-Uploader-App/blob/main/TROUBLESHOOTING.md)
### [INFO] Checksum SHA256
```
71d126edf726298ed25d4b72b5364e85fa6b1e76157c4f1ee0ab76b4a653f359
```
### ⚡ Novidades na v1.0.4
- [OK] **CRITICO**: Gradio agora incluido no exe (antes faltava!)
- [OK] **Melhorado**: Timeout aumentado para uploads longos
- [OK] **Corrigido**: Parametro API de batch_uploader (path_or_file -> path_or_fileobj)
- [OK] Melhorado: Tratamento de erros durante uploads
- [OK] Adicionado: test_imports.py para diagnostico
""")
def _start(token_val, repo_val, files_val, csv_val, remote_base_val, workers_val, progress=gr.Progress()):
status_text, result = _handle_upload(token_val, repo_val, files_val, csv_val, remote_base_val, int(workers_val) if workers_val else None, progress=progress)
return status_text
upload_btn.click(_start, inputs=[token, repo_id, files, csv_file, remote_base, workers], outputs=[status])
return app
|