| #!/usr/bin/env python3 | |
| """Resume the M1 cache upload until it completes.""" | |
| import socket | |
| import time | |
| from huggingface_hub import HfApi | |
| _getaddrinfo = socket.getaddrinfo | |
| def _prefer_responsive_s3_edge(host, *args, **kwargs): | |
| if host == "hf-hub-lfs-us-east-1.s3-accelerate.amazonaws.com": | |
| host = "3.173.240.184" | |
| return _getaddrinfo(host, *args, **kwargs) | |
| socket.getaddrinfo = _prefer_responsive_s3_edge | |
| api = HfApi() | |
| while True: | |
| try: | |
| api.whoami() | |
| api.upload_large_folder( | |
| repo_id="xixibuxixi/plot-checkpoints", | |
| repo_type="model", | |
| folder_path="/data/huangyh/hxh/.hf_upload_m1_cache_chunks15", | |
| num_workers=8, | |
| print_report=True, | |
| print_report_every=300, | |
| ) | |
| break | |
| except Exception as error: | |
| print(f"upload pass failed: {type(error).__name__}; retry in 30s", flush=True) | |
| time.sleep(30) | |