File size: 932 Bytes
6c5e1e0 9177f64 6c5e1e0 9177f64 6c5e1e0 184fd20 6c5e1e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/usr/bin/env python3
"""Resume the M1 cache upload until it completes."""
import socket
import time
from huggingface_hub import HfApi
_getaddrinfo = socket.getaddrinfo
def _prefer_responsive_s3_edge(host, *args, **kwargs):
if host == "hf-hub-lfs-us-east-1.s3-accelerate.amazonaws.com":
host = "3.173.240.184"
return _getaddrinfo(host, *args, **kwargs)
socket.getaddrinfo = _prefer_responsive_s3_edge
api = HfApi()
while True:
try:
api.whoami()
api.upload_large_folder(
repo_id="xixibuxixi/plot-checkpoints",
repo_type="model",
folder_path="/data/huangyh/hxh/.hf_upload_m1_cache_chunks15",
num_workers=8,
print_report=True,
print_report_every=300,
)
break
except Exception as error:
print(f"upload pass failed: {type(error).__name__}; retry in 30s", flush=True)
time.sleep(30)
|