File size: 3,025 Bytes
501bc8c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
import os
import shutil
import logging
from pathlib import Path
from huggingface_hub import snapshot_download

from src.engine.config import config

# Configure Logger
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger("DataSync")

def sync_from_huggingface(target_dir: str = "."):
    """
    Synchronizes the local directory with the Hugging Face Dataset repository.
    Downloads only the 'data' folder.
    """
    hf_token = config.hf_token
    repo_id = config.hf_dataset_repo

    if not hf_token or not repo_id:
        logger.warning("HF_TOKEN or HF_DATASET_REPO not found. Skipping data sync.")
        return

    logger.info(f"Checking for updates from {repo_id}...")
    
    from huggingface_hub import HfApi
    from requests.exceptions import RequestException
    import time
    
    api = HfApi(token=hf_token)
    try:
        remote_info = api.repo_info(repo_id=repo_id, repo_type="dataset")
        remote_sha = remote_info.sha
        
        last_sync_file = Path(target_dir) / ".last_sync_commit"
        if last_sync_file.exists():
            local_sha = last_sync_file.read_text().strip()
            if local_sha == remote_sha:
                # Check if target directories actually exist. If not, we still need to sync.
                if (Path(target_dir) / "data").exists():
                    logger.info("Local data is already up-to-date with remote commit. Skipping sync.")
                    return True # Success (already up to date)
        
    except Exception as e:
        logger.warning(f"Failed to check remote repo info: {e}. Proceeding with standard sync.")
        remote_sha = None

    logger.info(f"Starting data sync from {repo_id} to {target_dir}...")
    
    max_retries = 5
    retry_delay = 60 # seconds

    for attempt in range(max_retries):
        try:
            snapshot_download(
                repo_id=repo_id,
                repo_type="dataset",
                local_dir=target_dir,
                token=hf_token,
                allow_patterns=["data/**"],
            )
            
            # Save the SHA after successful sync
            if remote_sha:
                last_sync_file = Path(target_dir) / ".last_sync_commit"
                last_sync_file.write_text(remote_sha)
                
            logger.info("Data sync completed successfully.")
            break
        except Exception as e:
            logger.warning(f"Sync attempt {attempt + 1}/{max_retries} failed: {e}")
            if "429" in str(e) or isinstance(e, RequestException):
                logger.info(f"Rate limited or network error. Retrying in {retry_delay} seconds...")
                time.sleep(retry_delay)
            else:
                logger.error("Non-recoverable error during sync.")
                time.sleep(10)
    else:
        logger.error("Data sync failed after maximum retries.")
        return False



if __name__ == "__main__":
    sync_from_huggingface()