Download scripts/data/download.py from Tinghong-Ye22/code-Psi0-yth-v19: direct link, hf CLI and curl.
- Browser
- Download file 2.98 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v19/resolve/main/scripts/data/download.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v19/scripts/data/download.py
-
curl -L -o download.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v19/resolve/main/scripts/data/download.py
2.98 kB
| import os | |
| from dotenv import load_dotenv | |
| from huggingface_hub import snapshot_download, hf_hub_download, list_repo_files | |
| import tyro | |
| load_dotenv() | |
| def download_from_huggingface(repo_id: str, remote_dir: str, repo_type: str= "model", local_dir: str=".data", move_to_local_root: bool=True) -> str: | |
| """Download files or folders from Hugging Face repository. | |
| Args: | |
| repo_id: Repository ID on Hugging Face | |
| repo_dir: File or folder path within the repository to download | |
| repo_type: Type of the repository (e.g., "model", "dataset") | |
| local_dir: Local directory to download files to | |
| move_to_local_root: If True and downloading a folder, move files to root local_dir | |
| """ | |
| try: | |
| # List all files in the repository to check if repo_dir is a file or folder | |
| repo_files = list_repo_files(repo_id=repo_id, repo_type=repo_type, token=os.getenv("HF_TOKEN")) | |
| # Check if repo_dir is an exact file match | |
| is_file = remote_dir in repo_files | |
| if is_file: | |
| # Download single file | |
| local_path = hf_hub_download( | |
| repo_id=repo_id, | |
| filename=remote_dir, | |
| local_dir=local_dir, | |
| repo_type=repo_type, | |
| token=os.getenv("HF_TOKEN") | |
| ) | |
| print("Downloaded file to:", local_path) | |
| else: | |
| # Download all files under remote_dir directly into local_dir (flatten structure) | |
| local_path = snapshot_download( | |
| repo_id=repo_id, | |
| allow_patterns=f"{remote_dir}/**", | |
| local_dir=local_dir, | |
| repo_type=repo_type, | |
| token=os.getenv("HF_TOKEN"), | |
| ignore_patterns=None, | |
| # flatten: remove remote_dir prefix from local files | |
| # This will place files directly under local_dir | |
| # If snapshot_download does not support flatten, move files after download | |
| ) | |
| if move_to_local_root: | |
| # Move files from local_dir/remote_dir/* to local_dir/* | |
| import shutil, glob | |
| src_dir = os.path.join(local_dir, remote_dir) | |
| if os.path.exists(src_dir): | |
| for f in glob.glob(os.path.join(src_dir, "**"), recursive=True): | |
| if os.path.isfile(f): | |
| rel_path = os.path.relpath(f, src_dir) | |
| dest_path = os.path.join(local_dir, rel_path) | |
| os.makedirs(os.path.dirname(dest_path), exist_ok=True) | |
| shutil.move(f, dest_path) | |
| shutil.rmtree(src_dir) | |
| print("Downloaded folder to:", local_dir) | |
| except Exception as e: | |
| print(f"Error downloading {remote_dir}: {e}") | |
| raise | |
| return local_path | |
| if __name__ == "__main__": | |
| tyro.cli(download_from_huggingface) | |