Spaces:
Running
Running
Upload app.py with huggingface_hub
Browse files
app.py
CHANGED
|
@@ -84,6 +84,7 @@ def get_user_buckets(custom_token: str | None = None) -> List[str]:
|
|
| 84 |
def list_files_in_bucket(
|
| 85 |
bucket_input: str,
|
| 86 |
prefix_filter: str,
|
|
|
|
| 87 |
custom_token: str | None = None
|
| 88 |
) -> Tuple[Any, str, Dict[str, Any]]:
|
| 89 |
"""List files in the selected source bucket and prepare CheckboxGroup choices."""
|
|
@@ -100,7 +101,7 @@ def list_files_in_bucket(
|
|
| 100 |
except Exception as e:
|
| 101 |
return gr.update(choices=[], value=[]), f"β Error accessing bucket '{bucket_id}': {str(e)}", {}
|
| 102 |
|
| 103 |
-
# Filter only file items
|
| 104 |
files_map = {}
|
| 105 |
choices = []
|
| 106 |
total_size = 0
|
|
@@ -114,6 +115,11 @@ def list_files_in_bucket(
|
|
| 114 |
size = getattr(item, "size", 0)
|
| 115 |
xet_hash = getattr(item, "xet_hash", None)
|
| 116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
total_size += (size or 0)
|
| 118 |
display_label = f"{path} ({format_size(size)})"
|
| 119 |
choices.append((display_label, path))
|
|
@@ -125,10 +131,11 @@ def list_files_in_bucket(
|
|
| 125 |
}
|
| 126 |
|
| 127 |
count = len(choices)
|
|
|
|
| 128 |
if count == 0:
|
| 129 |
-
msg = f"βΉοΈ Bucket '{bucket_id}' contains no files" + (f" matching prefix '{prefix}'." if prefix else ".")
|
| 130 |
else:
|
| 131 |
-
msg = f"β
Found {count} file{'s' if count != 1 else ''} ({format_size(total_size)}) in bucket `{bucket_id}`."
|
| 132 |
|
| 133 |
return gr.update(choices=choices, value=[]), msg, files_map
|
| 134 |
|
|
@@ -153,6 +160,7 @@ def transfer_files(
|
|
| 153 |
preserve_structure: bool,
|
| 154 |
create_dest_if_missing: bool,
|
| 155 |
is_private_dest: bool,
|
|
|
|
| 156 |
custom_token: str | None = None
|
| 157 |
) -> Generator[str, None, None]:
|
| 158 |
"""Execute transfer of files from source bucket to target bucket, dataset, or model."""
|
|
@@ -185,6 +193,8 @@ def transfer_files(
|
|
| 185 |
return "\n".join(logs)
|
| 186 |
|
| 187 |
yield log(f"π Initializing transfer of {len(selected_files)} file(s) from bucket `{src_bucket}` to {dest_type} `{dest_id}`...")
|
|
|
|
|
|
|
| 188 |
|
| 189 |
# Step 1: Ensure destination exists or create it
|
| 190 |
if create_dest_if_missing:
|
|
@@ -203,7 +213,7 @@ def transfer_files(
|
|
| 203 |
temp_dir = tempfile.mkdtemp(prefix="hf2hf_transfer_")
|
| 204 |
|
| 205 |
total_files = len(selected_files)
|
| 206 |
-
|
| 207 |
fail_count = 0
|
| 208 |
start_time = time.time()
|
| 209 |
|
|
@@ -261,7 +271,7 @@ def transfer_files(
|
|
| 261 |
except Exception:
|
| 262 |
pass
|
| 263 |
|
| 264 |
-
|
| 265 |
|
| 266 |
# Scenario B: Destination is Dataset or Model repo
|
| 267 |
else:
|
|
@@ -278,7 +288,7 @@ def transfer_files(
|
|
| 278 |
repo_type=dest_kind,
|
| 279 |
commit_message=f"HF2HF: Transfer {target_rel_path} from bucket {src_bucket}"
|
| 280 |
)
|
| 281 |
-
|
| 282 |
yield log(f" Committed and uploaded to {dest_kind} `{dest_id}`!")
|
| 283 |
except Exception as err:
|
| 284 |
fail_count += 1
|
|
@@ -295,10 +305,25 @@ def transfer_files(
|
|
| 295 |
# Clean up staging directory
|
| 296 |
shutil.rmtree(temp_dir, ignore_errors=True)
|
| 297 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
elapsed = time.time() - start_time
|
| 299 |
yield log("\n" + "=" * 50)
|
| 300 |
yield log(f"π Transfer Completed in {elapsed:.1f}s!")
|
| 301 |
-
yield log(f" Successfully transferred: {
|
|
|
|
|
|
|
| 302 |
if fail_count > 0:
|
| 303 |
yield log(f"β οΈ Failed: {fail_count} file(s)")
|
| 304 |
|
|
@@ -330,10 +355,17 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
|
|
| 330 |
)
|
| 331 |
refresh_buckets_btn = gr.Button("π", scale=1, min_width=50)
|
| 332 |
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
list_btn = gr.Button("π List Files in Bucket", variant="secondary")
|
| 338 |
|
| 339 |
source_status = gr.Markdown("Select or enter a source bucket and click **List Files**.")
|
|
@@ -355,7 +387,7 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
|
|
| 355 |
)
|
| 356 |
with gr.Row():
|
| 357 |
preserve_structure = gr.Checkbox(
|
| 358 |
-
label="Preserve
|
| 359 |
value=True,
|
| 360 |
)
|
| 361 |
create_missing = gr.Checkbox(
|
|
@@ -366,6 +398,10 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
|
|
| 366 |
label="Make new destination private",
|
| 367 |
value=True,
|
| 368 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 369 |
|
| 370 |
gr.Markdown("### 3οΈβ£ File Selection")
|
| 371 |
with gr.Row():
|
|
@@ -409,7 +445,7 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
|
|
| 409 |
|
| 410 |
list_btn.click(
|
| 411 |
fn=list_files_in_bucket,
|
| 412 |
-
inputs=[src_bucket, prefix_input, token_input],
|
| 413 |
outputs=[file_checkboxes, source_status, files_state],
|
| 414 |
)
|
| 415 |
|
|
@@ -437,6 +473,7 @@ with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
|
|
| 437 |
preserve_structure,
|
| 438 |
create_missing,
|
| 439 |
is_private,
|
|
|
|
| 440 |
token_input,
|
| 441 |
],
|
| 442 |
outputs=[log_output],
|
|
|
|
| 84 |
def list_files_in_bucket(
|
| 85 |
bucket_input: str,
|
| 86 |
prefix_filter: str,
|
| 87 |
+
only_safetensors: bool = True,
|
| 88 |
custom_token: str | None = None
|
| 89 |
) -> Tuple[Any, str, Dict[str, Any]]:
|
| 90 |
"""List files in the selected source bucket and prepare CheckboxGroup choices."""
|
|
|
|
| 101 |
except Exception as e:
|
| 102 |
return gr.update(choices=[], value=[]), f"β Error accessing bucket '{bucket_id}': {str(e)}", {}
|
| 103 |
|
| 104 |
+
# Filter only file items and .safetensors if enabled
|
| 105 |
files_map = {}
|
| 106 |
choices = []
|
| 107 |
total_size = 0
|
|
|
|
| 115 |
size = getattr(item, "size", 0)
|
| 116 |
xet_hash = getattr(item, "xet_hash", None)
|
| 117 |
|
| 118 |
+
# Filter .safetensors files
|
| 119 |
+
path_lower = path.lower()
|
| 120 |
+
if only_safetensors and not (path_lower.endswith(".safetensors") or path_lower.endswith(".safetensor")):
|
| 121 |
+
continue
|
| 122 |
+
|
| 123 |
total_size += (size or 0)
|
| 124 |
display_label = f"{path} ({format_size(size)})"
|
| 125 |
choices.append((display_label, path))
|
|
|
|
| 131 |
}
|
| 132 |
|
| 133 |
count = len(choices)
|
| 134 |
+
filter_label = ".safetensors " if only_safetensors else ""
|
| 135 |
if count == 0:
|
| 136 |
+
msg = f"βΉοΈ Bucket '{bucket_id}' contains no {filter_label}files" + (f" matching prefix '{prefix}'." if prefix else ".")
|
| 137 |
else:
|
| 138 |
+
msg = f"β
Found {count} {filter_label}file{'s' if count != 1 else ''} ({format_size(total_size)}) in bucket `{bucket_id}`."
|
| 139 |
|
| 140 |
return gr.update(choices=choices, value=[]), msg, files_map
|
| 141 |
|
|
|
|
| 160 |
preserve_structure: bool,
|
| 161 |
create_dest_if_missing: bool,
|
| 162 |
is_private_dest: bool,
|
| 163 |
+
delete_source_files: bool = False,
|
| 164 |
custom_token: str | None = None
|
| 165 |
) -> Generator[str, None, None]:
|
| 166 |
"""Execute transfer of files from source bucket to target bucket, dataset, or model."""
|
|
|
|
| 193 |
return "\n".join(logs)
|
| 194 |
|
| 195 |
yield log(f"π Initializing transfer of {len(selected_files)} file(s) from bucket `{src_bucket}` to {dest_type} `{dest_id}`...")
|
| 196 |
+
if delete_source_files:
|
| 197 |
+
yield log("β οΈ Post-transfer deletion enabled: Successfully transferred files will be deleted from the source bucket.")
|
| 198 |
|
| 199 |
# Step 1: Ensure destination exists or create it
|
| 200 |
if create_dest_if_missing:
|
|
|
|
| 213 |
temp_dir = tempfile.mkdtemp(prefix="hf2hf_transfer_")
|
| 214 |
|
| 215 |
total_files = len(selected_files)
|
| 216 |
+
successful_transfers: List[str] = []
|
| 217 |
fail_count = 0
|
| 218 |
start_time = time.time()
|
| 219 |
|
|
|
|
| 271 |
except Exception:
|
| 272 |
pass
|
| 273 |
|
| 274 |
+
successful_transfers.append(file_rel_path)
|
| 275 |
|
| 276 |
# Scenario B: Destination is Dataset or Model repo
|
| 277 |
else:
|
|
|
|
| 288 |
repo_type=dest_kind,
|
| 289 |
commit_message=f"HF2HF: Transfer {target_rel_path} from bucket {src_bucket}"
|
| 290 |
)
|
| 291 |
+
successful_transfers.append(file_rel_path)
|
| 292 |
yield log(f" Committed and uploaded to {dest_kind} `{dest_id}`!")
|
| 293 |
except Exception as err:
|
| 294 |
fail_count += 1
|
|
|
|
| 305 |
# Clean up staging directory
|
| 306 |
shutil.rmtree(temp_dir, ignore_errors=True)
|
| 307 |
|
| 308 |
+
# Step 3: Optional deletion from original bucket
|
| 309 |
+
if delete_source_files and successful_transfers:
|
| 310 |
+
yield log(f"\nποΈ Deleting {len(successful_transfers)} transferred file(s) from source bucket `{src_bucket}`...")
|
| 311 |
+
try:
|
| 312 |
+
api.batch_bucket_files(
|
| 313 |
+
bucket_id=src_bucket,
|
| 314 |
+
delete=successful_transfers,
|
| 315 |
+
token=token
|
| 316 |
+
)
|
| 317 |
+
yield log(f"β
Successfully deleted {len(successful_transfers)} file(s) from original bucket `{src_bucket}`.")
|
| 318 |
+
except Exception as del_err:
|
| 319 |
+
yield log(f"β οΈ Warning: Could not delete source files ({del_err}).")
|
| 320 |
+
|
| 321 |
elapsed = time.time() - start_time
|
| 322 |
yield log("\n" + "=" * 50)
|
| 323 |
yield log(f"π Transfer Completed in {elapsed:.1f}s!")
|
| 324 |
+
yield log(f" Successfully transferred: {len(successful_transfers)}/{total_files} file(s)")
|
| 325 |
+
if delete_source_files and successful_transfers:
|
| 326 |
+
yield log(f" Deleted from source: {len(successful_transfers)} file(s)")
|
| 327 |
if fail_count > 0:
|
| 328 |
yield log(f"β οΈ Failed: {fail_count} file(s)")
|
| 329 |
|
|
|
|
| 355 |
)
|
| 356 |
refresh_buckets_btn = gr.Button("π", scale=1, min_width=50)
|
| 357 |
|
| 358 |
+
with gr.Row():
|
| 359 |
+
prefix_input = gr.Textbox(
|
| 360 |
+
label="Prefix / Subfolder Filter (Optional)",
|
| 361 |
+
placeholder="e.g. checkpoints/ or data/ (leave empty for entire bucket)",
|
| 362 |
+
scale=3,
|
| 363 |
+
)
|
| 364 |
+
only_safetensors = gr.Checkbox(
|
| 365 |
+
label="Only .safetensors files",
|
| 366 |
+
value=True,
|
| 367 |
+
scale=2,
|
| 368 |
+
)
|
| 369 |
list_btn = gr.Button("π List Files in Bucket", variant="secondary")
|
| 370 |
|
| 371 |
source_status = gr.Markdown("Select or enter a source bucket and click **List Files**.")
|
|
|
|
| 387 |
)
|
| 388 |
with gr.Row():
|
| 389 |
preserve_structure = gr.Checkbox(
|
| 390 |
+
label="Preserve directory structure",
|
| 391 |
value=True,
|
| 392 |
)
|
| 393 |
create_missing = gr.Checkbox(
|
|
|
|
| 398 |
label="Make new destination private",
|
| 399 |
value=True,
|
| 400 |
)
|
| 401 |
+
delete_source_files = gr.Checkbox(
|
| 402 |
+
label="ποΈ Delete transferred files from source bucket after transfer",
|
| 403 |
+
value=False,
|
| 404 |
+
)
|
| 405 |
|
| 406 |
gr.Markdown("### 3οΈβ£ File Selection")
|
| 407 |
with gr.Row():
|
|
|
|
| 445 |
|
| 446 |
list_btn.click(
|
| 447 |
fn=list_files_in_bucket,
|
| 448 |
+
inputs=[src_bucket, prefix_input, only_safetensors, token_input],
|
| 449 |
outputs=[file_checkboxes, source_status, files_state],
|
| 450 |
)
|
| 451 |
|
|
|
|
| 473 |
preserve_structure,
|
| 474 |
create_missing,
|
| 475 |
is_private,
|
| 476 |
+
delete_source_files,
|
| 477 |
token_input,
|
| 478 |
],
|
| 479 |
outputs=[log_output],
|