File size: 2,980 Bytes
2d3265e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import os
from dotenv import load_dotenv
from huggingface_hub import snapshot_download, hf_hub_download, list_repo_files
import tyro

load_dotenv()

def download_from_huggingface(repo_id: str, remote_dir: str, repo_type: str= "model", local_dir: str=".data", move_to_local_root: bool=True) -> str:
    """Download files or folders from Hugging Face repository.
    
    Args:
        repo_id: Repository ID on Hugging Face
        repo_dir: File or folder path within the repository to download
        repo_type: Type of the repository (e.g., "model", "dataset")
        local_dir: Local directory to download files to
        move_to_local_root: If True and downloading a folder, move files to root local_dir 
    """
    
    try:
        # List all files in the repository to check if repo_dir is a file or folder
        repo_files = list_repo_files(repo_id=repo_id, repo_type=repo_type, token=os.getenv("HF_TOKEN"))
        
        # Check if repo_dir is an exact file match
        is_file = remote_dir in repo_files
        
        if is_file:
            # Download single file
            local_path = hf_hub_download(
                repo_id=repo_id,
                filename=remote_dir,
                local_dir=local_dir,
                repo_type=repo_type,
                token=os.getenv("HF_TOKEN")
            )
            print("Downloaded file to:", local_path)
        else:
            # Download all files under remote_dir directly into local_dir (flatten structure)
            local_path = snapshot_download(
                repo_id=repo_id,
                allow_patterns=f"{remote_dir}/**",
                local_dir=local_dir,
                repo_type=repo_type,
                token=os.getenv("HF_TOKEN"),
                ignore_patterns=None,
                # flatten: remove remote_dir prefix from local files
                # This will place files directly under local_dir
                # If snapshot_download does not support flatten, move files after download
            )
            if move_to_local_root:
                # Move files from local_dir/remote_dir/* to local_dir/*
                import shutil, glob
                src_dir = os.path.join(local_dir, remote_dir)
                if os.path.exists(src_dir):
                    for f in glob.glob(os.path.join(src_dir, "**"), recursive=True):
                        if os.path.isfile(f):
                            rel_path = os.path.relpath(f, src_dir)
                            dest_path = os.path.join(local_dir, rel_path)
                            os.makedirs(os.path.dirname(dest_path), exist_ok=True)
                            shutil.move(f, dest_path)
                    shutil.rmtree(src_dir)
                print("Downloaded folder to:", local_dir)
            
    except Exception as e:
        print(f"Error downloading {remote_dir}: {e}")
        raise
    
    return local_path

if __name__ == "__main__":
    tyro.cli(download_from_huggingface)