Spaces:
Sleeping
Sleeping
Commit ·
90a0223
1
Parent(s): 2b3cbf3
feat: add direct segments folder upload to hf dataset via ui button
Browse files- cli/hf_dataset_cli.py +79 -0
- src/ui/upload_app_factory.py +65 -0
cli/hf_dataset_cli.py
CHANGED
|
@@ -817,6 +817,85 @@ def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any]
|
|
| 817 |
return errors
|
| 818 |
|
| 819 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 820 |
def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
|
| 821 |
repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 822 |
if "manifest.json" not in repo_files:
|
|
|
|
| 817 |
return errors
|
| 818 |
|
| 819 |
|
| 820 |
+
def upload_segments_to_hf(
|
| 821 |
+
api: HfApi,
|
| 822 |
+
project_slug: str,
|
| 823 |
+
dataset_repo: str,
|
| 824 |
+
segments_root: str,
|
| 825 |
+
batch_size: int = 50,
|
| 826 |
+
) -> dict[str, Any]:
|
| 827 |
+
"""Upload local segments folder to dataset repo maintaining directory structure."""
|
| 828 |
+
segments_path = Path(segments_root)
|
| 829 |
+
if not segments_path.exists() or not segments_path.is_dir():
|
| 830 |
+
raise FileNotFoundError(f"Segments root not found: {segments_root}")
|
| 831 |
+
|
| 832 |
+
ensure_project_dataset_structure(
|
| 833 |
+
api=api,
|
| 834 |
+
project_slug=project_slug,
|
| 835 |
+
dataset_repo=dataset_repo,
|
| 836 |
+
create_private_repo=False,
|
| 837 |
+
)
|
| 838 |
+
|
| 839 |
+
all_audio_files: list[Path] = []
|
| 840 |
+
for ext in SUPPORTED_AUDIO_EXTENSIONS:
|
| 841 |
+
all_audio_files.extend(segments_path.rglob(f"*{ext}"))
|
| 842 |
+
all_audio_files = sorted(all_audio_files)
|
| 843 |
+
|
| 844 |
+
if not all_audio_files:
|
| 845 |
+
return {
|
| 846 |
+
"project_slug": project_slug,
|
| 847 |
+
"dataset_repo": dataset_repo,
|
| 848 |
+
"mode": "upload-segments-only",
|
| 849 |
+
"total_files": 0,
|
| 850 |
+
"uploaded": 0,
|
| 851 |
+
"failed": 0,
|
| 852 |
+
}
|
| 853 |
+
|
| 854 |
+
uploaded_count = 0
|
| 855 |
+
failed_count = 0
|
| 856 |
+
failed_files: list[str] = []
|
| 857 |
+
|
| 858 |
+
batches = _chunk_items(all_audio_files, batch_size)
|
| 859 |
+
for batch_idx, batch in enumerate(batches, start=1):
|
| 860 |
+
for local_file in batch:
|
| 861 |
+
relative_path = local_file.relative_to(segments_path)
|
| 862 |
+
repo_path = f"audio/{project_slug}/{relative_path.as_posix()}"
|
| 863 |
+
|
| 864 |
+
try:
|
| 865 |
+
api.upload_file(
|
| 866 |
+
path_or_fileobj=str(local_file),
|
| 867 |
+
path_in_repo=repo_path,
|
| 868 |
+
repo_id=dataset_repo,
|
| 869 |
+
repo_type="dataset",
|
| 870 |
+
)
|
| 871 |
+
uploaded_count += 1
|
| 872 |
+
except Exception as exc:
|
| 873 |
+
failed_count += 1
|
| 874 |
+
failed_files.append(f"{relative_path.as_posix()}: {exc}")
|
| 875 |
+
|
| 876 |
+
print(
|
| 877 |
+
json.dumps(
|
| 878 |
+
{
|
| 879 |
+
"event": "upload-batch",
|
| 880 |
+
"batch": batch_idx,
|
| 881 |
+
"total_batches": len(batches),
|
| 882 |
+
"uploaded_so_far": uploaded_count,
|
| 883 |
+
"failed_so_far": failed_count,
|
| 884 |
+
}
|
| 885 |
+
)
|
| 886 |
+
)
|
| 887 |
+
|
| 888 |
+
return {
|
| 889 |
+
"project_slug": project_slug,
|
| 890 |
+
"dataset_repo": dataset_repo,
|
| 891 |
+
"mode": "upload-segments-only",
|
| 892 |
+
"total_files": len(all_audio_files),
|
| 893 |
+
"uploaded": uploaded_count,
|
| 894 |
+
"failed": failed_count,
|
| 895 |
+
"failed_samples": failed_files[:10],
|
| 896 |
+
}
|
| 897 |
+
|
| 898 |
+
|
| 899 |
def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
|
| 900 |
repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
|
| 901 |
if "manifest.json" not in repo_files:
|
src/ui/upload_app_factory.py
CHANGED
|
@@ -62,6 +62,40 @@ def _pick_local_folder(current_value: str) -> tuple[str, str]:
|
|
| 62 |
return current_value or "", f"Erro ao selecionar pasta local: {exc}"
|
| 63 |
|
| 64 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
def build_upload_app() -> gr.Blocks:
|
| 66 |
with gr.Blocks(title="BirdNET Segments Uploader") as demo:
|
| 67 |
gr.Markdown("# BirdNET Segments Uploader")
|
|
@@ -90,6 +124,25 @@ def build_upload_app() -> gr.Blocks:
|
|
| 90 |
"Use o campo ZIP para enviar a pasta de segmentos."
|
| 91 |
)
|
| 92 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
with gr.Accordion("Configuracao avancada", open=False):
|
| 94 |
hf_token = gr.Textbox(
|
| 95 |
label="HF Token (opcional)",
|
|
@@ -226,6 +279,18 @@ def build_upload_app() -> gr.Blocks:
|
|
| 226 |
outputs=[segments_root, status],
|
| 227 |
)
|
| 228 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 229 |
upload_button.click(
|
| 230 |
fn=run_upload,
|
| 231 |
inputs=[
|
|
|
|
| 62 |
return current_value or "", f"Erro ao selecionar pasta local: {exc}"
|
| 63 |
|
| 64 |
|
| 65 |
+
def _run_segments_upload(
|
| 66 |
+
dataset_repo: str,
|
| 67 |
+
segments_path: str,
|
| 68 |
+
token: str,
|
| 69 |
+
) -> tuple[str, str]:
|
| 70 |
+
dataset_repo = (dataset_repo or "").strip()
|
| 71 |
+
segments_path = (segments_path or "").strip()
|
| 72 |
+
|
| 73 |
+
if not dataset_repo or "/" not in dataset_repo:
|
| 74 |
+
return "❌ Informe dataset repo no formato owner/repo", "{}"
|
| 75 |
+
if not segments_path or not Path(segments_path).exists():
|
| 76 |
+
return "❌ Selecione a pasta de segmentos valida", "{}"
|
| 77 |
+
|
| 78 |
+
try:
|
| 79 |
+
from cli.hf_dataset_cli import upload_segments_to_hf
|
| 80 |
+
project_slug = dataset_repo.split("/")[1].replace("-dataset", "")
|
| 81 |
+
api = _build_api(token)
|
| 82 |
+
result = upload_segments_to_hf(
|
| 83 |
+
api=api,
|
| 84 |
+
project_slug=project_slug,
|
| 85 |
+
dataset_repo=dataset_repo,
|
| 86 |
+
segments_root=segments_path,
|
| 87 |
+
)
|
| 88 |
+
summary = (
|
| 89 |
+
f"✅ Upload de segmentos completo | "
|
| 90 |
+
f"Total: {result['total_files']} | "
|
| 91 |
+
f"Enviados: {result['uploaded']} | "
|
| 92 |
+
f"Falhas: {result['failed']}"
|
| 93 |
+
)
|
| 94 |
+
return summary, _as_pretty_json(result)
|
| 95 |
+
except Exception as exc:
|
| 96 |
+
return f"❌ Upload de segmentos falhou: {exc}", "{}"
|
| 97 |
+
|
| 98 |
+
|
| 99 |
def build_upload_app() -> gr.Blocks:
|
| 100 |
with gr.Blocks(title="BirdNET Segments Uploader") as demo:
|
| 101 |
gr.Markdown("# BirdNET Segments Uploader")
|
|
|
|
| 124 |
"Use o campo ZIP para enviar a pasta de segmentos."
|
| 125 |
)
|
| 126 |
|
| 127 |
+
with gr.Group():
|
| 128 |
+
gr.Markdown(
|
| 129 |
+
"#### Ou: Enviar segmentos inteiros (50GB+) antes de processar deteccoes\n"
|
| 130 |
+
"Use esta secao para fazer upload de toda a pasta de segmentos para um dataset HF. "
|
| 131 |
+
"Recomendado executar primeira vez, antes de ingerir deteccoes CSV."
|
| 132 |
+
)
|
| 133 |
+
with gr.Row():
|
| 134 |
+
segments_upload_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo-dataset")
|
| 135 |
+
segments_upload_path_input = gr.Textbox(label="Pasta de Segmentos (local)")
|
| 136 |
+
pick_segments_upload_button = gr.Button("Selecionar pasta para upload", variant="secondary")
|
| 137 |
+
segments_upload_token = gr.Textbox(
|
| 138 |
+
label="HF Token (opcional)",
|
| 139 |
+
type="password",
|
| 140 |
+
placeholder="Se vazio, usa sessao ja autenticada",
|
| 141 |
+
)
|
| 142 |
+
segments_upload_button = gr.Button("Enviar Segmentos para Dataset", variant="primary")
|
| 143 |
+
segments_upload_status = gr.Markdown(value="Pronto")
|
| 144 |
+
segments_upload_result = gr.Code(label="Resultado JSON", language="json")
|
| 145 |
+
|
| 146 |
with gr.Accordion("Configuracao avancada", open=False):
|
| 147 |
hf_token = gr.Textbox(
|
| 148 |
label="HF Token (opcional)",
|
|
|
|
| 279 |
outputs=[segments_root, status],
|
| 280 |
)
|
| 281 |
|
| 282 |
+
pick_segments_upload_button.click(
|
| 283 |
+
fn=_pick_local_folder,
|
| 284 |
+
inputs=[segments_upload_path_input],
|
| 285 |
+
outputs=[segments_upload_path_input, segments_upload_status],
|
| 286 |
+
)
|
| 287 |
+
|
| 288 |
+
segments_upload_button.click(
|
| 289 |
+
fn=_run_segments_upload,
|
| 290 |
+
inputs=[segments_upload_repo, segments_upload_path_input, segments_upload_token],
|
| 291 |
+
outputs=[segments_upload_status, segments_upload_result],
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
upload_button.click(
|
| 295 |
fn=run_upload,
|
| 296 |
inputs=[
|