import os import shutil import logging from pathlib import Path from huggingface_hub import snapshot_download from src.engine.config import config # Configure Logger logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger("DataSync") def sync_from_huggingface(target_dir: str = "."): """ Synchronizes the local directory with the Hugging Face Dataset repository. Downloads only the 'data' folder. """ hf_token = config.hf_token repo_id = config.hf_dataset_repo if not hf_token or not repo_id: logger.warning("HF_TOKEN or HF_DATASET_REPO not found. Skipping data sync.") return logger.info(f"Checking for updates from {repo_id}...") from huggingface_hub import HfApi from requests.exceptions import RequestException import time api = HfApi(token=hf_token) try: remote_info = api.repo_info(repo_id=repo_id, repo_type="dataset") remote_sha = remote_info.sha last_sync_file = Path(target_dir) / ".last_sync_commit" if last_sync_file.exists(): local_sha = last_sync_file.read_text().strip() if local_sha == remote_sha: # Check if target directories actually exist. If not, we still need to sync. if (Path(target_dir) / "data").exists(): logger.info("Local data is already up-to-date with remote commit. Skipping sync.") return True # Success (already up to date) except Exception as e: logger.warning(f"Failed to check remote repo info: {e}. Proceeding with standard sync.") remote_sha = None logger.info(f"Starting data sync from {repo_id} to {target_dir}...") max_retries = 5 retry_delay = 60 # seconds for attempt in range(max_retries): try: snapshot_download( repo_id=repo_id, repo_type="dataset", local_dir=target_dir, token=hf_token, allow_patterns=["data/**"], ) # Save the SHA after successful sync if remote_sha: last_sync_file = Path(target_dir) / ".last_sync_commit" last_sync_file.write_text(remote_sha) logger.info("Data sync completed successfully.") break except Exception as e: logger.warning(f"Sync attempt {attempt + 1}/{max_retries} failed: {e}") if "429" in str(e) or isinstance(e, RequestException): logger.info(f"Rate limited or network error. Retrying in {retry_delay} seconds...") time.sleep(retry_delay) else: logger.error("Non-recoverable error during sync.") time.sleep(10) else: logger.error("Data sync failed after maximum retries.") return False if __name__ == "__main__": sync_from_huggingface()