jrrribeiro commited on
Commit
90a0223
·
1 Parent(s): 2b3cbf3

feat: add direct segments folder upload to hf dataset via ui button

Browse files
cli/hf_dataset_cli.py CHANGED
@@ -817,6 +817,85 @@ def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any]
817
  return errors
818
 
819
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
820
  def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
821
  repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
822
  if "manifest.json" not in repo_files:
 
817
  return errors
818
 
819
 
820
+ def upload_segments_to_hf(
821
+ api: HfApi,
822
+ project_slug: str,
823
+ dataset_repo: str,
824
+ segments_root: str,
825
+ batch_size: int = 50,
826
+ ) -> dict[str, Any]:
827
+ """Upload local segments folder to dataset repo maintaining directory structure."""
828
+ segments_path = Path(segments_root)
829
+ if not segments_path.exists() or not segments_path.is_dir():
830
+ raise FileNotFoundError(f"Segments root not found: {segments_root}")
831
+
832
+ ensure_project_dataset_structure(
833
+ api=api,
834
+ project_slug=project_slug,
835
+ dataset_repo=dataset_repo,
836
+ create_private_repo=False,
837
+ )
838
+
839
+ all_audio_files: list[Path] = []
840
+ for ext in SUPPORTED_AUDIO_EXTENSIONS:
841
+ all_audio_files.extend(segments_path.rglob(f"*{ext}"))
842
+ all_audio_files = sorted(all_audio_files)
843
+
844
+ if not all_audio_files:
845
+ return {
846
+ "project_slug": project_slug,
847
+ "dataset_repo": dataset_repo,
848
+ "mode": "upload-segments-only",
849
+ "total_files": 0,
850
+ "uploaded": 0,
851
+ "failed": 0,
852
+ }
853
+
854
+ uploaded_count = 0
855
+ failed_count = 0
856
+ failed_files: list[str] = []
857
+
858
+ batches = _chunk_items(all_audio_files, batch_size)
859
+ for batch_idx, batch in enumerate(batches, start=1):
860
+ for local_file in batch:
861
+ relative_path = local_file.relative_to(segments_path)
862
+ repo_path = f"audio/{project_slug}/{relative_path.as_posix()}"
863
+
864
+ try:
865
+ api.upload_file(
866
+ path_or_fileobj=str(local_file),
867
+ path_in_repo=repo_path,
868
+ repo_id=dataset_repo,
869
+ repo_type="dataset",
870
+ )
871
+ uploaded_count += 1
872
+ except Exception as exc:
873
+ failed_count += 1
874
+ failed_files.append(f"{relative_path.as_posix()}: {exc}")
875
+
876
+ print(
877
+ json.dumps(
878
+ {
879
+ "event": "upload-batch",
880
+ "batch": batch_idx,
881
+ "total_batches": len(batches),
882
+ "uploaded_so_far": uploaded_count,
883
+ "failed_so_far": failed_count,
884
+ }
885
+ )
886
+ )
887
+
888
+ return {
889
+ "project_slug": project_slug,
890
+ "dataset_repo": dataset_repo,
891
+ "mode": "upload-segments-only",
892
+ "total_files": len(all_audio_files),
893
+ "uploaded": uploaded_count,
894
+ "failed": failed_count,
895
+ "failed_samples": failed_files[:10],
896
+ }
897
+
898
+
899
  def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
900
  repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
901
  if "manifest.json" not in repo_files:
src/ui/upload_app_factory.py CHANGED
@@ -62,6 +62,40 @@ def _pick_local_folder(current_value: str) -> tuple[str, str]:
62
  return current_value or "", f"Erro ao selecionar pasta local: {exc}"
63
 
64
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  def build_upload_app() -> gr.Blocks:
66
  with gr.Blocks(title="BirdNET Segments Uploader") as demo:
67
  gr.Markdown("# BirdNET Segments Uploader")
@@ -90,6 +124,25 @@ def build_upload_app() -> gr.Blocks:
90
  "Use o campo ZIP para enviar a pasta de segmentos."
91
  )
92
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
  with gr.Accordion("Configuracao avancada", open=False):
94
  hf_token = gr.Textbox(
95
  label="HF Token (opcional)",
@@ -226,6 +279,18 @@ def build_upload_app() -> gr.Blocks:
226
  outputs=[segments_root, status],
227
  )
228
 
 
 
 
 
 
 
 
 
 
 
 
 
229
  upload_button.click(
230
  fn=run_upload,
231
  inputs=[
 
62
  return current_value or "", f"Erro ao selecionar pasta local: {exc}"
63
 
64
 
65
+ def _run_segments_upload(
66
+ dataset_repo: str,
67
+ segments_path: str,
68
+ token: str,
69
+ ) -> tuple[str, str]:
70
+ dataset_repo = (dataset_repo or "").strip()
71
+ segments_path = (segments_path or "").strip()
72
+
73
+ if not dataset_repo or "/" not in dataset_repo:
74
+ return "❌ Informe dataset repo no formato owner/repo", "{}"
75
+ if not segments_path or not Path(segments_path).exists():
76
+ return "❌ Selecione a pasta de segmentos valida", "{}"
77
+
78
+ try:
79
+ from cli.hf_dataset_cli import upload_segments_to_hf
80
+ project_slug = dataset_repo.split("/")[1].replace("-dataset", "")
81
+ api = _build_api(token)
82
+ result = upload_segments_to_hf(
83
+ api=api,
84
+ project_slug=project_slug,
85
+ dataset_repo=dataset_repo,
86
+ segments_root=segments_path,
87
+ )
88
+ summary = (
89
+ f"✅ Upload de segmentos completo | "
90
+ f"Total: {result['total_files']} | "
91
+ f"Enviados: {result['uploaded']} | "
92
+ f"Falhas: {result['failed']}"
93
+ )
94
+ return summary, _as_pretty_json(result)
95
+ except Exception as exc:
96
+ return f"❌ Upload de segmentos falhou: {exc}", "{}"
97
+
98
+
99
  def build_upload_app() -> gr.Blocks:
100
  with gr.Blocks(title="BirdNET Segments Uploader") as demo:
101
  gr.Markdown("# BirdNET Segments Uploader")
 
124
  "Use o campo ZIP para enviar a pasta de segmentos."
125
  )
126
 
127
+ with gr.Group():
128
+ gr.Markdown(
129
+ "#### Ou: Enviar segmentos inteiros (50GB+) antes de processar deteccoes\n"
130
+ "Use esta secao para fazer upload de toda a pasta de segmentos para um dataset HF. "
131
+ "Recomendado executar primeira vez, antes de ingerir deteccoes CSV."
132
+ )
133
+ with gr.Row():
134
+ segments_upload_repo = gr.Textbox(label="Dataset Repo (HF)", placeholder="owner/repo-dataset")
135
+ segments_upload_path_input = gr.Textbox(label="Pasta de Segmentos (local)")
136
+ pick_segments_upload_button = gr.Button("Selecionar pasta para upload", variant="secondary")
137
+ segments_upload_token = gr.Textbox(
138
+ label="HF Token (opcional)",
139
+ type="password",
140
+ placeholder="Se vazio, usa sessao ja autenticada",
141
+ )
142
+ segments_upload_button = gr.Button("Enviar Segmentos para Dataset", variant="primary")
143
+ segments_upload_status = gr.Markdown(value="Pronto")
144
+ segments_upload_result = gr.Code(label="Resultado JSON", language="json")
145
+
146
  with gr.Accordion("Configuracao avancada", open=False):
147
  hf_token = gr.Textbox(
148
  label="HF Token (opcional)",
 
279
  outputs=[segments_root, status],
280
  )
281
 
282
+ pick_segments_upload_button.click(
283
+ fn=_pick_local_folder,
284
+ inputs=[segments_upload_path_input],
285
+ outputs=[segments_upload_path_input, segments_upload_status],
286
+ )
287
+
288
+ segments_upload_button.click(
289
+ fn=_run_segments_upload,
290
+ inputs=[segments_upload_repo, segments_upload_path_input, segments_upload_token],
291
+ outputs=[segments_upload_status, segments_upload_result],
292
+ )
293
+
294
  upload_button.click(
295
  fn=run_upload,
296
  inputs=[