Quantumbraid commited on
Commit
e327872
Β·
verified Β·
1 Parent(s): 61d2843

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +50 -13
app.py CHANGED
@@ -84,6 +84,7 @@ def get_user_buckets(custom_token: str | None = None) -> List[str]:
84
  def list_files_in_bucket(
85
  bucket_input: str,
86
  prefix_filter: str,
 
87
  custom_token: str | None = None
88
  ) -> Tuple[Any, str, Dict[str, Any]]:
89
  """List files in the selected source bucket and prepare CheckboxGroup choices."""
@@ -100,7 +101,7 @@ def list_files_in_bucket(
100
  except Exception as e:
101
  return gr.update(choices=[], value=[]), f"❌ Error accessing bucket '{bucket_id}': {str(e)}", {}
102
 
103
- # Filter only file items
104
  files_map = {}
105
  choices = []
106
  total_size = 0
@@ -114,6 +115,11 @@ def list_files_in_bucket(
114
  size = getattr(item, "size", 0)
115
  xet_hash = getattr(item, "xet_hash", None)
116
 
 
 
 
 
 
117
  total_size += (size or 0)
118
  display_label = f"{path} ({format_size(size)})"
119
  choices.append((display_label, path))
@@ -125,10 +131,11 @@ def list_files_in_bucket(
125
  }
126
 
127
  count = len(choices)
 
128
  if count == 0:
129
- msg = f"ℹ️ Bucket '{bucket_id}' contains no files" + (f" matching prefix '{prefix}'." if prefix else ".")
130
  else:
131
- msg = f"βœ… Found {count} file{'s' if count != 1 else ''} ({format_size(total_size)}) in bucket `{bucket_id}`."
132
 
133
  return gr.update(choices=choices, value=[]), msg, files_map
134
 
@@ -153,6 +160,7 @@ def transfer_files(
153
  preserve_structure: bool,
154
  create_dest_if_missing: bool,
155
  is_private_dest: bool,
 
156
  custom_token: str | None = None
157
  ) -> Generator[str, None, None]:
158
  """Execute transfer of files from source bucket to target bucket, dataset, or model."""
@@ -185,6 +193,8 @@ def transfer_files(
185
  return "\n".join(logs)
186
 
187
  yield log(f"πŸš€ Initializing transfer of {len(selected_files)} file(s) from bucket `{src_bucket}` to {dest_type} `{dest_id}`...")
 
 
188
 
189
  # Step 1: Ensure destination exists or create it
190
  if create_dest_if_missing:
@@ -203,7 +213,7 @@ def transfer_files(
203
  temp_dir = tempfile.mkdtemp(prefix="hf2hf_transfer_")
204
 
205
  total_files = len(selected_files)
206
- success_count = 0
207
  fail_count = 0
208
  start_time = time.time()
209
 
@@ -261,7 +271,7 @@ def transfer_files(
261
  except Exception:
262
  pass
263
 
264
- success_count += 1
265
 
266
  # Scenario B: Destination is Dataset or Model repo
267
  else:
@@ -278,7 +288,7 @@ def transfer_files(
278
  repo_type=dest_kind,
279
  commit_message=f"HF2HF: Transfer {target_rel_path} from bucket {src_bucket}"
280
  )
281
- success_count += 1
282
  yield log(f" Committed and uploaded to {dest_kind} `{dest_id}`!")
283
  except Exception as err:
284
  fail_count += 1
@@ -295,10 +305,25 @@ def transfer_files(
295
  # Clean up staging directory
296
  shutil.rmtree(temp_dir, ignore_errors=True)
297
 
 
 
 
 
 
 
 
 
 
 
 
 
 
298
  elapsed = time.time() - start_time
299
  yield log("\n" + "=" * 50)
300
  yield log(f"πŸŽ‰ Transfer Completed in {elapsed:.1f}s!")
301
- yield log(f" Successfully transferred: {success_count}/{total_files} file(s)")
 
 
302
  if fail_count > 0:
303
  yield log(f"⚠️ Failed: {fail_count} file(s)")
304
 
@@ -330,10 +355,17 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
330
  )
331
  refresh_buckets_btn = gr.Button("πŸ”„", scale=1, min_width=50)
332
 
333
- prefix_input = gr.Textbox(
334
- label="Prefix / Subfolder Filter (Optional)",
335
- placeholder="e.g. checkpoints/ or data/ (leave empty for entire bucket)",
336
- )
 
 
 
 
 
 
 
337
  list_btn = gr.Button("πŸ” List Files in Bucket", variant="secondary")
338
 
339
  source_status = gr.Markdown("Select or enter a source bucket and click **List Files**.")
@@ -355,7 +387,7 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
355
  )
356
  with gr.Row():
357
  preserve_structure = gr.Checkbox(
358
- label="Preserve source directory structure",
359
  value=True,
360
  )
361
  create_missing = gr.Checkbox(
@@ -366,6 +398,10 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
366
  label="Make new destination private",
367
  value=True,
368
  )
 
 
 
 
369
 
370
  gr.Markdown("### 3️⃣ File Selection")
371
  with gr.Row():
@@ -409,7 +445,7 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
409
 
410
  list_btn.click(
411
  fn=list_files_in_bucket,
412
- inputs=[src_bucket, prefix_input, token_input],
413
  outputs=[file_checkboxes, source_status, files_state],
414
  )
415
 
@@ -437,6 +473,7 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
437
  preserve_structure,
438
  create_missing,
439
  is_private,
 
440
  token_input,
441
  ],
442
  outputs=[log_output],
 
84
  def list_files_in_bucket(
85
  bucket_input: str,
86
  prefix_filter: str,
87
+ only_safetensors: bool = True,
88
  custom_token: str | None = None
89
  ) -> Tuple[Any, str, Dict[str, Any]]:
90
  """List files in the selected source bucket and prepare CheckboxGroup choices."""
 
101
  except Exception as e:
102
  return gr.update(choices=[], value=[]), f"❌ Error accessing bucket '{bucket_id}': {str(e)}", {}
103
 
104
+ # Filter only file items and .safetensors if enabled
105
  files_map = {}
106
  choices = []
107
  total_size = 0
 
115
  size = getattr(item, "size", 0)
116
  xet_hash = getattr(item, "xet_hash", None)
117
 
118
+ # Filter .safetensors files
119
+ path_lower = path.lower()
120
+ if only_safetensors and not (path_lower.endswith(".safetensors") or path_lower.endswith(".safetensor")):
121
+ continue
122
+
123
  total_size += (size or 0)
124
  display_label = f"{path} ({format_size(size)})"
125
  choices.append((display_label, path))
 
131
  }
132
 
133
  count = len(choices)
134
+ filter_label = ".safetensors " if only_safetensors else ""
135
  if count == 0:
136
+ msg = f"ℹ️ Bucket '{bucket_id}' contains no {filter_label}files" + (f" matching prefix '{prefix}'." if prefix else ".")
137
  else:
138
+ msg = f"βœ… Found {count} {filter_label}file{'s' if count != 1 else ''} ({format_size(total_size)}) in bucket `{bucket_id}`."
139
 
140
  return gr.update(choices=choices, value=[]), msg, files_map
141
 
 
160
  preserve_structure: bool,
161
  create_dest_if_missing: bool,
162
  is_private_dest: bool,
163
+ delete_source_files: bool = False,
164
  custom_token: str | None = None
165
  ) -> Generator[str, None, None]:
166
  """Execute transfer of files from source bucket to target bucket, dataset, or model."""
 
193
  return "\n".join(logs)
194
 
195
  yield log(f"πŸš€ Initializing transfer of {len(selected_files)} file(s) from bucket `{src_bucket}` to {dest_type} `{dest_id}`...")
196
+ if delete_source_files:
197
+ yield log("⚠️ Post-transfer deletion enabled: Successfully transferred files will be deleted from the source bucket.")
198
 
199
  # Step 1: Ensure destination exists or create it
200
  if create_dest_if_missing:
 
213
  temp_dir = tempfile.mkdtemp(prefix="hf2hf_transfer_")
214
 
215
  total_files = len(selected_files)
216
+ successful_transfers: List[str] = []
217
  fail_count = 0
218
  start_time = time.time()
219
 
 
271
  except Exception:
272
  pass
273
 
274
+ successful_transfers.append(file_rel_path)
275
 
276
  # Scenario B: Destination is Dataset or Model repo
277
  else:
 
288
  repo_type=dest_kind,
289
  commit_message=f"HF2HF: Transfer {target_rel_path} from bucket {src_bucket}"
290
  )
291
+ successful_transfers.append(file_rel_path)
292
  yield log(f" Committed and uploaded to {dest_kind} `{dest_id}`!")
293
  except Exception as err:
294
  fail_count += 1
 
305
  # Clean up staging directory
306
  shutil.rmtree(temp_dir, ignore_errors=True)
307
 
308
+ # Step 3: Optional deletion from original bucket
309
+ if delete_source_files and successful_transfers:
310
+ yield log(f"\nπŸ—‘οΈ Deleting {len(successful_transfers)} transferred file(s) from source bucket `{src_bucket}`...")
311
+ try:
312
+ api.batch_bucket_files(
313
+ bucket_id=src_bucket,
314
+ delete=successful_transfers,
315
+ token=token
316
+ )
317
+ yield log(f"βœ… Successfully deleted {len(successful_transfers)} file(s) from original bucket `{src_bucket}`.")
318
+ except Exception as del_err:
319
+ yield log(f"⚠️ Warning: Could not delete source files ({del_err}).")
320
+
321
  elapsed = time.time() - start_time
322
  yield log("\n" + "=" * 50)
323
  yield log(f"πŸŽ‰ Transfer Completed in {elapsed:.1f}s!")
324
+ yield log(f" Successfully transferred: {len(successful_transfers)}/{total_files} file(s)")
325
+ if delete_source_files and successful_transfers:
326
+ yield log(f" Deleted from source: {len(successful_transfers)} file(s)")
327
  if fail_count > 0:
328
  yield log(f"⚠️ Failed: {fail_count} file(s)")
329
 
 
355
  )
356
  refresh_buckets_btn = gr.Button("πŸ”„", scale=1, min_width=50)
357
 
358
+ with gr.Row():
359
+ prefix_input = gr.Textbox(
360
+ label="Prefix / Subfolder Filter (Optional)",
361
+ placeholder="e.g. checkpoints/ or data/ (leave empty for entire bucket)",
362
+ scale=3,
363
+ )
364
+ only_safetensors = gr.Checkbox(
365
+ label="Only .safetensors files",
366
+ value=True,
367
+ scale=2,
368
+ )
369
  list_btn = gr.Button("πŸ” List Files in Bucket", variant="secondary")
370
 
371
  source_status = gr.Markdown("Select or enter a source bucket and click **List Files**.")
 
387
  )
388
  with gr.Row():
389
  preserve_structure = gr.Checkbox(
390
+ label="Preserve directory structure",
391
  value=True,
392
  )
393
  create_missing = gr.Checkbox(
 
398
  label="Make new destination private",
399
  value=True,
400
  )
401
+ delete_source_files = gr.Checkbox(
402
+ label="πŸ—‘οΈ Delete transferred files from source bucket after transfer",
403
+ value=False,
404
+ )
405
 
406
  gr.Markdown("### 3️⃣ File Selection")
407
  with gr.Row():
 
445
 
446
  list_btn.click(
447
  fn=list_files_in_bucket,
448
+ inputs=[src_bucket, prefix_input, only_safetensors, token_input],
449
  outputs=[file_checkboxes, source_status, files_state],
450
  )
451
 
 
473
  preserve_structure,
474
  create_missing,
475
  is_private,
476
+ delete_source_files,
477
  token_input,
478
  ],
479
  outputs=[log_output],