jrrribeiro commited on
Commit
e87da82
·
1 Parent(s): 20568c9

feat(space-upload): add pause/resume/cancel with background progress and auto-skip resume

Browse files
Files changed (2) hide show
  1. cli/hf_dataset_cli.py +85 -8
  2. src/ui/upload_app_factory.py +207 -44
cli/hf_dataset_cli.py CHANGED
@@ -9,7 +9,7 @@ from dataclasses import asdict, dataclass
9
  from datetime import UTC, datetime
10
  from io import BytesIO
11
  from pathlib import Path
12
- from typing import Any
13
 
14
  import pandas as pd
15
  from huggingface_hub import HfApi, hf_hub_download
@@ -823,12 +823,20 @@ def upload_segments_to_hf(
823
  dataset_repo: str,
824
  segments_root: str,
825
  batch_size: int = 50,
 
 
 
 
 
826
  ) -> dict[str, Any]:
827
  """Upload local segments folder to dataset repo maintaining directory structure."""
828
  segments_path = Path(segments_root)
829
  if not segments_path.exists() or not segments_path.is_dir():
830
  raise FileNotFoundError(f"Segments root not found: {segments_root}")
831
 
 
 
 
832
  ensure_project_dataset_structure(
833
  api=api,
834
  project_slug=project_slug,
@@ -841,6 +849,17 @@ def upload_segments_to_hf(
841
  all_audio_files.extend(segments_path.rglob(f"*{ext}"))
842
  all_audio_files = sorted(all_audio_files)
843
 
 
 
 
 
 
 
 
 
 
 
 
844
  if not all_audio_files:
845
  return {
846
  "project_slug": project_slug,
@@ -848,31 +867,74 @@ def upload_segments_to_hf(
848
  "mode": "upload-segments-only",
849
  "total_files": 0,
850
  "uploaded": 0,
 
851
  "failed": 0,
 
852
  }
853
 
854
  uploaded_count = 0
855
  failed_count = 0
 
856
  failed_files: list[str] = []
857
 
858
- batches = _chunk_items(all_audio_files, batch_size)
 
 
 
 
 
 
 
 
 
 
 
859
  for batch_idx, batch in enumerate(batches, start=1):
860
- for local_file in batch:
 
 
 
 
 
 
 
 
 
 
 
 
 
861
  relative_path = local_file.relative_to(segments_path)
862
- repo_path = f"audio/{project_slug}/{relative_path.as_posix()}"
863
 
864
  try:
865
- api.upload_file(
866
- path_or_fileobj=str(local_file),
 
 
867
  path_in_repo=repo_path,
868
- repo_id=dataset_repo,
869
- repo_type="dataset",
870
  )
871
  uploaded_count += 1
872
  except Exception as exc:
873
  failed_count += 1
874
  failed_files.append(f"{relative_path.as_posix()}: {exc}")
875
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
876
  print(
877
  json.dumps(
878
  {
@@ -881,17 +943,32 @@ def upload_segments_to_hf(
881
  "total_batches": len(batches),
882
  "uploaded_so_far": uploaded_count,
883
  "failed_so_far": failed_count,
 
884
  }
885
  )
886
  )
887
 
 
 
 
 
 
 
 
 
 
 
 
888
  return {
889
  "project_slug": project_slug,
890
  "dataset_repo": dataset_repo,
891
  "mode": "upload-segments-only",
892
  "total_files": len(all_audio_files),
 
893
  "uploaded": uploaded_count,
 
894
  "failed": failed_count,
 
895
  "failed_samples": failed_files[:10],
896
  }
897
 
 
9
  from datetime import UTC, datetime
10
  from io import BytesIO
11
  from pathlib import Path
12
+ from typing import Any, Callable
13
 
14
  import pandas as pd
15
  from huggingface_hub import HfApi, hf_hub_download
 
823
  dataset_repo: str,
824
  segments_root: str,
825
  batch_size: int = 50,
826
+ max_retries: int = 3,
827
+ retry_backoff_seconds: float = 1.0,
828
+ should_pause: Callable[[], bool] | None = None,
829
+ should_cancel: Callable[[], bool] | None = None,
830
+ progress_callback: Callable[[dict[str, Any]], None] | None = None,
831
  ) -> dict[str, Any]:
832
  """Upload local segments folder to dataset repo maintaining directory structure."""
833
  segments_path = Path(segments_root)
834
  if not segments_path.exists() or not segments_path.is_dir():
835
  raise FileNotFoundError(f"Segments root not found: {segments_root}")
836
 
837
+ if batch_size <= 0:
838
+ raise ValueError("batch_size must be greater than zero")
839
+
840
  ensure_project_dataset_structure(
841
  api=api,
842
  project_slug=project_slug,
 
849
  all_audio_files.extend(segments_path.rglob(f"*{ext}"))
850
  all_audio_files = sorted(all_audio_files)
851
 
852
+ remote_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
853
+ pending: list[tuple[Path, str]] = []
854
+ skipped_existing = 0
855
+ for local_file in all_audio_files:
856
+ relative_path = local_file.relative_to(segments_path)
857
+ repo_path = f"audio/{project_slug}/{relative_path.as_posix()}"
858
+ if repo_path in remote_files:
859
+ skipped_existing += 1
860
+ continue
861
+ pending.append((local_file, repo_path))
862
+
863
  if not all_audio_files:
864
  return {
865
  "project_slug": project_slug,
 
867
  "mode": "upload-segments-only",
868
  "total_files": 0,
869
  "uploaded": 0,
870
+ "skipped_existing": 0,
871
  "failed": 0,
872
+ "cancelled": False,
873
  }
874
 
875
  uploaded_count = 0
876
  failed_count = 0
877
+ cancelled = False
878
  failed_files: list[str] = []
879
 
880
+ batches = _chunk_items(pending, batch_size) if pending else []
881
+ total_pending = len(pending)
882
+ if progress_callback:
883
+ progress_callback(
884
+ {
885
+ "event": "upload-start",
886
+ "total_files": len(all_audio_files),
887
+ "pending": total_pending,
888
+ "skipped_existing": skipped_existing,
889
+ }
890
+ )
891
+
892
  for batch_idx, batch in enumerate(batches, start=1):
893
+ for local_file, repo_path in batch:
894
+ if should_cancel and should_cancel():
895
+ cancelled = True
896
+ break
897
+
898
+ while should_pause and should_pause():
899
+ time.sleep(0.5)
900
+ if should_cancel and should_cancel():
901
+ cancelled = True
902
+ break
903
+
904
+ if cancelled:
905
+ break
906
+
907
  relative_path = local_file.relative_to(segments_path)
 
908
 
909
  try:
910
+ _upload_audio_with_retry(
911
+ api=api,
912
+ dataset_repo=dataset_repo,
913
+ local_path=local_file,
914
  path_in_repo=repo_path,
915
+ max_retries=max_retries,
916
+ retry_backoff_seconds=retry_backoff_seconds,
917
  )
918
  uploaded_count += 1
919
  except Exception as exc:
920
  failed_count += 1
921
  failed_files.append(f"{relative_path.as_posix()}: {exc}")
922
 
923
+ if progress_callback:
924
+ progress_callback(
925
+ {
926
+ "event": "upload-progress",
927
+ "uploaded": uploaded_count,
928
+ "failed": failed_count,
929
+ "skipped_existing": skipped_existing,
930
+ "processed_pending": uploaded_count + failed_count,
931
+ "pending_total": total_pending,
932
+ }
933
+ )
934
+
935
+ if cancelled:
936
+ break
937
+
938
  print(
939
  json.dumps(
940
  {
 
943
  "total_batches": len(batches),
944
  "uploaded_so_far": uploaded_count,
945
  "failed_so_far": failed_count,
946
+ "skipped_existing": skipped_existing,
947
  }
948
  )
949
  )
950
 
951
+ if progress_callback:
952
+ progress_callback(
953
+ {
954
+ "event": "upload-complete",
955
+ "uploaded": uploaded_count,
956
+ "failed": failed_count,
957
+ "skipped_existing": skipped_existing,
958
+ "cancelled": cancelled,
959
+ }
960
+ )
961
+
962
  return {
963
  "project_slug": project_slug,
964
  "dataset_repo": dataset_repo,
965
  "mode": "upload-segments-only",
966
  "total_files": len(all_audio_files),
967
+ "pending_files": total_pending,
968
  "uploaded": uploaded_count,
969
+ "skipped_existing": skipped_existing,
970
  "failed": failed_count,
971
+ "cancelled": cancelled,
972
  "failed_samples": failed_files[:10],
973
  }
974
 
src/ui/upload_app_factory.py CHANGED
@@ -1,7 +1,8 @@
1
  import json
2
- import os
3
  import tarfile
4
  import tempfile
 
 
5
  import zipfile
6
  from pathlib import Path
7
  from typing import Any
@@ -59,44 +60,176 @@ def _resolve_segments_root(
59
  return None, None
60
 
61
 
62
- def _run_segments_upload(
63
- dataset_repo: str,
64
- archive_path: str | None,
65
- token: str,
66
- ) -> tuple[str, str]:
67
- dataset_repo = (dataset_repo or "").strip()
68
- if not dataset_repo or "/" not in dataset_repo:
69
- return "❌ Informe dataset repo no formato owner/repo", "{}"
70
- if not archive_path:
71
- return "❌ Envie um arquivo (.tar, .tar.gz, ou .zip) de segmentos", "{}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
 
73
- try:
74
- segments_root, temp_dir = _extract_compressed_segments(archive_path)
75
- if segments_root is None:
76
- return "❌ Arquivo compactado invalido", "{}"
77
-
78
- try:
79
- from cli.hf_dataset_cli import upload_segments_to_hf
80
- project_slug = dataset_repo.split("/")[1].replace("-dataset", "")
81
- api = _build_api(token)
82
- result = upload_segments_to_hf(
83
- api=api,
84
- project_slug=project_slug,
85
- dataset_repo=dataset_repo,
86
- segments_root=str(segments_root),
87
- )
88
- summary = (
89
- f"✅ Upload de segmentos completo | "
90
- f"Total: {result['total_files']} | "
91
- f"Enviados: {result['uploaded']} | "
92
- f"Falhas: {result['failed']}"
93
- )
94
- return summary, _as_pretty_json(result)
95
- finally:
96
- if temp_dir is not None:
97
- temp_dir.cleanup()
98
- except Exception as exc:
99
- return f"❌ Upload falhou: {exc}", "{}"
100
 
101
 
102
  def build_upload_app() -> gr.Blocks:
@@ -140,7 +273,13 @@ def build_upload_app() -> gr.Blocks:
140
  type="password",
141
  placeholder="Se vazio, usa sessao ja autenticada",
142
  )
143
- segments_upload_button = gr.Button("Enviar Segmentos para Dataset", variant="primary")
 
 
 
 
 
 
144
  segments_upload_status = gr.Markdown(value="Pronto")
145
  segments_upload_result = gr.Code(label="Resultado JSON", language="json")
146
 
@@ -189,7 +328,7 @@ def build_upload_app() -> gr.Blocks:
189
 
190
  segments_root, temp_dir = _resolve_segments_root(segments_path, segments_zip_path)
191
  if segments_root is None:
192
- return "❌ Pasta de segmentos invalida. No Space, envie ZIP da pasta.", "{}"
193
 
194
  try:
195
  result = run_ingest_segments_dry_run(
@@ -237,7 +376,7 @@ def build_upload_app() -> gr.Blocks:
237
 
238
  segments_root, temp_dir = _resolve_segments_root(segments_path, segments_zip_path)
239
  if segments_root is None:
240
- return "❌ Pasta de segmentos invalida. No Space, envie ZIP da pasta.", "{}"
241
 
242
  try:
243
  api = _build_api(token)
@@ -274,9 +413,33 @@ def build_upload_app() -> gr.Blocks:
274
  outputs=[status, result_json],
275
  )
276
 
277
- segments_upload_button.click(
278
- fn=_run_segments_upload,
279
- inputs=[segments_upload_repo, segments_upload_file, segments_upload_token],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
280
  outputs=[segments_upload_status, segments_upload_result],
281
  )
282
 
 
1
  import json
 
2
  import tarfile
3
  import tempfile
4
+ import threading
5
+ import time
6
  import zipfile
7
  from pathlib import Path
8
  from typing import Any
 
60
  return None, None
61
 
62
 
63
+ class SegmentsUploadSession:
64
+ def __init__(self) -> None:
65
+ self._lock = threading.Lock()
66
+ self._thread: threading.Thread | None = None
67
+ self._pause = False
68
+ self._cancel = False
69
+ self._status = "idle"
70
+ self._message = "Pronto"
71
+ self._progress: dict[str, Any] = {}
72
+ self._result: dict[str, Any] = {}
73
+ self._temp_dir: tempfile.TemporaryDirectory[str] | None = None
74
+
75
+ def _is_running(self) -> bool:
76
+ return self._thread is not None and self._thread.is_alive()
77
+
78
+ def _on_progress(self, payload: dict[str, Any]) -> None:
79
+ with self._lock:
80
+ self._progress = payload
81
+
82
+ def _should_pause(self) -> bool:
83
+ with self._lock:
84
+ return self._pause
85
+
86
+ def _should_cancel(self) -> bool:
87
+ with self._lock:
88
+ return self._cancel
89
+
90
+ def start(self, dataset_repo: str, archive_path: str | None, token: str, batch_size: int) -> tuple[str, str]:
91
+ repo = (dataset_repo or "").strip()
92
+ if not repo or "/" not in repo:
93
+ return "❌ Informe dataset repo no formato owner/repo", "{}"
94
+ if not archive_path:
95
+ return "❌ Envie um arquivo (.tar, .tar.gz, ou .zip) de segmentos", "{}"
96
+
97
+ with self._lock:
98
+ if self._is_running():
99
+ return "⏳ Ja existe upload em andamento", _as_pretty_json(self.snapshot())
100
+
101
+ segments_root, temp_dir = _extract_compressed_segments(archive_path)
102
+ if segments_root is None or temp_dir is None:
103
+ return "❌ Arquivo compactado invalido", "{}"
104
+
105
+ self._pause = False
106
+ self._cancel = False
107
+ self._status = "running"
108
+ self._message = "Upload iniciado"
109
+ self._progress = {"event": "upload-start"}
110
+ self._result = {}
111
+ self._temp_dir = temp_dir
112
+
113
+ def worker() -> None:
114
+ try:
115
+ from cli.hf_dataset_cli import upload_segments_to_hf
116
+
117
+ project_slug = repo.split("/")[1].replace("-dataset", "")
118
+ api = _build_api(token)
119
+ result = upload_segments_to_hf(
120
+ api=api,
121
+ project_slug=project_slug,
122
+ dataset_repo=repo,
123
+ segments_root=str(segments_root),
124
+ batch_size=int(batch_size),
125
+ should_pause=self._should_pause,
126
+ should_cancel=self._should_cancel,
127
+ progress_callback=self._on_progress,
128
+ )
129
+ with self._lock:
130
+ if result.get("cancelled"):
131
+ self._status = "cancelled"
132
+ self._message = "Upload cancelado"
133
+ else:
134
+ self._status = "completed"
135
+ self._message = "Upload finalizado"
136
+ self._result = result
137
+ except Exception as exc:
138
+ with self._lock:
139
+ self._status = "failed"
140
+ self._message = f"Upload falhou: {exc}"
141
+ self._result = {"error": str(exc)}
142
+ finally:
143
+ with self._lock:
144
+ temp = self._temp_dir
145
+ self._temp_dir = None
146
+ if temp is not None:
147
+ temp.cleanup()
148
+
149
+ self._thread = threading.Thread(target=worker, daemon=True)
150
+ self._thread.start()
151
+
152
+ return "🚀 Upload iniciado em background", _as_pretty_json(self.snapshot())
153
+
154
+ def pause(self) -> tuple[str, str]:
155
+ with self._lock:
156
+ if not self._is_running():
157
+ return "ℹ️ Nenhum upload em andamento", _as_pretty_json(self.snapshot())
158
+ self._pause = True
159
+ self._status = "paused"
160
+ self._message = "Upload pausado"
161
+ return "⏸️ Upload pausado", _as_pretty_json(self.snapshot())
162
+
163
+ def resume(self) -> tuple[str, str]:
164
+ with self._lock:
165
+ if not self._is_running():
166
+ return "ℹ️ Nenhum upload em andamento", _as_pretty_json(self.snapshot())
167
+ self._pause = False
168
+ self._status = "running"
169
+ self._message = "Upload retomado"
170
+ return "▶️ Upload retomado", _as_pretty_json(self.snapshot())
171
+
172
+ def cancel(self) -> tuple[str, str]:
173
+ with self._lock:
174
+ if not self._is_running():
175
+ return "ℹ️ Nenhum upload em andamento", _as_pretty_json(self.snapshot())
176
+ self._cancel = True
177
+ self._pause = False
178
+ self._status = "cancelling"
179
+ self._message = "Cancelamento solicitado"
180
+ return "🛑 Cancelamento solicitado", _as_pretty_json(self.snapshot())
181
+
182
+ def snapshot(self) -> dict[str, Any]:
183
+ return {
184
+ "status": self._status,
185
+ "message": self._message,
186
+ "progress": self._progress,
187
+ "result": self._result,
188
+ "is_running": self._is_running(),
189
+ "timestamp": int(time.time()),
190
+ }
191
+
192
+ def status(self) -> tuple[str, str]:
193
+ snap = self.snapshot()
194
+ state = snap.get("status", "idle")
195
+ if state == "running":
196
+ msg = "⏳ Upload em andamento"
197
+ elif state == "paused":
198
+ msg = "⏸️ Upload pausado"
199
+ elif state == "completed":
200
+ msg = "✅ Upload concluido"
201
+ elif state == "failed":
202
+ msg = "❌ Upload falhou"
203
+ elif state == "cancelled":
204
+ msg = "🛑 Upload cancelado"
205
+ elif state == "cancelling":
206
+ msg = "🛑 Cancelando upload..."
207
+ else:
208
+ msg = "ℹ️ Pronto"
209
+ return msg, _as_pretty_json(snap)
210
+
211
+
212
+ _SEGMENTS_UPLOAD_SESSION = SegmentsUploadSession()
213
+
214
+
215
+ def _start_segments_upload(dataset_repo: str, archive_path: str | None, token: str, batch_size: float) -> tuple[str, str]:
216
+ return _SEGMENTS_UPLOAD_SESSION.start(dataset_repo, archive_path, token, int(batch_size))
217
+
218
+
219
+ def _pause_segments_upload() -> tuple[str, str]:
220
+ return _SEGMENTS_UPLOAD_SESSION.pause()
221
+
222
+
223
+ def _resume_segments_upload() -> tuple[str, str]:
224
+ return _SEGMENTS_UPLOAD_SESSION.resume()
225
+
226
+
227
+ def _cancel_segments_upload() -> tuple[str, str]:
228
+ return _SEGMENTS_UPLOAD_SESSION.cancel()
229
 
230
+
231
+ def _refresh_segments_upload_status() -> tuple[str, str]:
232
+ return _SEGMENTS_UPLOAD_SESSION.status()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
 
234
 
235
  def build_upload_app() -> gr.Blocks:
 
273
  type="password",
274
  placeholder="Se vazio, usa sessao ja autenticada",
275
  )
276
+ segments_upload_batch_size = gr.Number(label="Batch size upload segmentos", value=50, precision=0)
277
+ with gr.Row():
278
+ segments_upload_start = gr.Button("Iniciar Upload", variant="primary")
279
+ segments_upload_pause = gr.Button("Pausar", variant="secondary")
280
+ segments_upload_resume = gr.Button("Retomar", variant="secondary")
281
+ segments_upload_cancel = gr.Button("Cancelar", variant="stop")
282
+ segments_upload_refresh = gr.Button("Atualizar Status", variant="secondary")
283
  segments_upload_status = gr.Markdown(value="Pronto")
284
  segments_upload_result = gr.Code(label="Resultado JSON", language="json")
285
 
 
328
 
329
  segments_root, temp_dir = _resolve_segments_root(segments_path, segments_zip_path)
330
  if segments_root is None:
331
+ return "❌ Pasta de segmentos invalida. No Space, envie .tar.gz, .tar ou .zip.", "{}"
332
 
333
  try:
334
  result = run_ingest_segments_dry_run(
 
376
 
377
  segments_root, temp_dir = _resolve_segments_root(segments_path, segments_zip_path)
378
  if segments_root is None:
379
+ return "❌ Pasta de segmentos invalida. No Space, envie .tar.gz, .tar ou .zip.", "{}"
380
 
381
  try:
382
  api = _build_api(token)
 
413
  outputs=[status, result_json],
414
  )
415
 
416
+ segments_upload_start.click(
417
+ fn=_start_segments_upload,
418
+ inputs=[segments_upload_repo, segments_upload_file, segments_upload_token, segments_upload_batch_size],
419
+ outputs=[segments_upload_status, segments_upload_result],
420
+ )
421
+
422
+ segments_upload_pause.click(
423
+ fn=_pause_segments_upload,
424
+ inputs=[],
425
+ outputs=[segments_upload_status, segments_upload_result],
426
+ )
427
+
428
+ segments_upload_resume.click(
429
+ fn=_resume_segments_upload,
430
+ inputs=[],
431
+ outputs=[segments_upload_status, segments_upload_result],
432
+ )
433
+
434
+ segments_upload_cancel.click(
435
+ fn=_cancel_segments_upload,
436
+ inputs=[],
437
+ outputs=[segments_upload_status, segments_upload_result],
438
+ )
439
+
440
+ segments_upload_refresh.click(
441
+ fn=_refresh_segments_upload_status,
442
+ inputs=[],
443
  outputs=[segments_upload_status, segments_upload_result],
444
  )
445