code-Psi0-yth-v18 / scripts /data /download.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
3cdcff6 verified
Raw History Blame Contribute Delete
2.98 kB
import os
from dotenv import load_dotenv
from huggingface_hub import snapshot_download, hf_hub_download, list_repo_files
import tyro
load_dotenv()
def download_from_huggingface(repo_id: str, remote_dir: str, repo_type: str= "model", local_dir: str=".data", move_to_local_root: bool=True) -> str:
"""Download files or folders from Hugging Face repository.
Args:
repo_id: Repository ID on Hugging Face
repo_dir: File or folder path within the repository to download
repo_type: Type of the repository (e.g., "model", "dataset")
local_dir: Local directory to download files to
move_to_local_root: If True and downloading a folder, move files to root local_dir
"""
try:
# List all files in the repository to check if repo_dir is a file or folder
repo_files = list_repo_files(repo_id=repo_id, repo_type=repo_type, token=os.getenv("HF_TOKEN"))
# Check if repo_dir is an exact file match
is_file = remote_dir in repo_files
if is_file:
# Download single file
local_path = hf_hub_download(
repo_id=repo_id,
filename=remote_dir,
local_dir=local_dir,
repo_type=repo_type,
token=os.getenv("HF_TOKEN")
)
print("Downloaded file to:", local_path)
else:
# Download all files under remote_dir directly into local_dir (flatten structure)
local_path = snapshot_download(
repo_id=repo_id,
allow_patterns=f"{remote_dir}/**",
local_dir=local_dir,
repo_type=repo_type,
token=os.getenv("HF_TOKEN"),
ignore_patterns=None,
# flatten: remove remote_dir prefix from local files
# This will place files directly under local_dir
# If snapshot_download does not support flatten, move files after download
)
if move_to_local_root:
# Move files from local_dir/remote_dir/* to local_dir/*
import shutil, glob
src_dir = os.path.join(local_dir, remote_dir)
if os.path.exists(src_dir):
for f in glob.glob(os.path.join(src_dir, "**"), recursive=True):
if os.path.isfile(f):
rel_path = os.path.relpath(f, src_dir)
dest_path = os.path.join(local_dir, rel_path)
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
shutil.move(f, dest_path)
shutil.rmtree(src_dir)
print("Downloaded folder to:", local_dir)
except Exception as e:
print(f"Error downloading {remote_dir}: {e}")
raise
return local_path
if __name__ == "__main__":
tyro.cli(download_from_huggingface)