Download src/data_sync.py from YGCXLab/ux-research-agent-slackbot: direct link, hf CLI and curl.
- Browser
- Download file 3.03 kB
-
https://huggingface.co/spaces/YGCXLab/ux-research-agent-slackbot/resolve/main/src/data_sync.py
- Command line
-
hf download hf://spaces/YGCXLab/ux-research-agent-slackbot/src/data_sync.py
-
curl -L -o data_sync.py https://huggingface.co/spaces/YGCXLab/ux-research-agent-slackbot/resolve/main/src/data_sync.py
3.03 kB
| import os | |
| import shutil | |
| import logging | |
| from pathlib import Path | |
| from huggingface_hub import snapshot_download | |
| from src.engine.config import config | |
| # Configure Logger | |
| logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') | |
| logger = logging.getLogger("DataSync") | |
| def sync_from_huggingface(target_dir: str = "."): | |
| """ | |
| Synchronizes the local directory with the Hugging Face Dataset repository. | |
| Downloads only the 'data' folder. | |
| """ | |
| hf_token = config.hf_token | |
| repo_id = config.hf_dataset_repo | |
| if not hf_token or not repo_id: | |
| logger.warning("HF_TOKEN or HF_DATASET_REPO not found. Skipping data sync.") | |
| return | |
| logger.info(f"Checking for updates from {repo_id}...") | |
| from huggingface_hub import HfApi | |
| from requests.exceptions import RequestException | |
| import time | |
| api = HfApi(token=hf_token) | |
| try: | |
| remote_info = api.repo_info(repo_id=repo_id, repo_type="dataset") | |
| remote_sha = remote_info.sha | |
| last_sync_file = Path(target_dir) / ".last_sync_commit" | |
| if last_sync_file.exists(): | |
| local_sha = last_sync_file.read_text().strip() | |
| if local_sha == remote_sha: | |
| # Check if target directories actually exist. If not, we still need to sync. | |
| if (Path(target_dir) / "data").exists(): | |
| logger.info("Local data is already up-to-date with remote commit. Skipping sync.") | |
| return True # Success (already up to date) | |
| except Exception as e: | |
| logger.warning(f"Failed to check remote repo info: {e}. Proceeding with standard sync.") | |
| remote_sha = None | |
| logger.info(f"Starting data sync from {repo_id} to {target_dir}...") | |
| max_retries = 5 | |
| retry_delay = 60 # seconds | |
| for attempt in range(max_retries): | |
| try: | |
| snapshot_download( | |
| repo_id=repo_id, | |
| repo_type="dataset", | |
| local_dir=target_dir, | |
| token=hf_token, | |
| allow_patterns=["data/**"], | |
| ) | |
| # Save the SHA after successful sync | |
| if remote_sha: | |
| last_sync_file = Path(target_dir) / ".last_sync_commit" | |
| last_sync_file.write_text(remote_sha) | |
| logger.info("Data sync completed successfully.") | |
| break | |
| except Exception as e: | |
| logger.warning(f"Sync attempt {attempt + 1}/{max_retries} failed: {e}") | |
| if "429" in str(e) or isinstance(e, RequestException): | |
| logger.info(f"Rate limited or network error. Retrying in {retry_delay} seconds...") | |
| time.sleep(retry_delay) | |
| else: | |
| logger.error("Non-recoverable error during sync.") | |
| time.sleep(10) | |
| else: | |
| logger.error("Data sync failed after maximum retries.") | |
| return False | |
| if __name__ == "__main__": | |
| sync_from_huggingface() | |