debug-log
Claude Opus 4.6
feat: Insight Repository โ€” GitLab ์ดˆ๊ธฐํ™” + HF ์ž๋™ ๋ฐฐํฌ
501bc8c
Raw History Blame Contribute Delete
3.03 kB
import os
import shutil
import logging
from pathlib import Path
from huggingface_hub import snapshot_download
from src.engine.config import config
# Configure Logger
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger("DataSync")
def sync_from_huggingface(target_dir: str = "."):
"""
Synchronizes the local directory with the Hugging Face Dataset repository.
Downloads only the 'data' folder.
"""
hf_token = config.hf_token
repo_id = config.hf_dataset_repo
if not hf_token or not repo_id:
logger.warning("HF_TOKEN or HF_DATASET_REPO not found. Skipping data sync.")
return
logger.info(f"Checking for updates from {repo_id}...")
from huggingface_hub import HfApi
from requests.exceptions import RequestException
import time
api = HfApi(token=hf_token)
try:
remote_info = api.repo_info(repo_id=repo_id, repo_type="dataset")
remote_sha = remote_info.sha
last_sync_file = Path(target_dir) / ".last_sync_commit"
if last_sync_file.exists():
local_sha = last_sync_file.read_text().strip()
if local_sha == remote_sha:
# Check if target directories actually exist. If not, we still need to sync.
if (Path(target_dir) / "data").exists():
logger.info("Local data is already up-to-date with remote commit. Skipping sync.")
return True # Success (already up to date)
except Exception as e:
logger.warning(f"Failed to check remote repo info: {e}. Proceeding with standard sync.")
remote_sha = None
logger.info(f"Starting data sync from {repo_id} to {target_dir}...")
max_retries = 5
retry_delay = 60 # seconds
for attempt in range(max_retries):
try:
snapshot_download(
repo_id=repo_id,
repo_type="dataset",
local_dir=target_dir,
token=hf_token,
allow_patterns=["data/**"],
)
# Save the SHA after successful sync
if remote_sha:
last_sync_file = Path(target_dir) / ".last_sync_commit"
last_sync_file.write_text(remote_sha)
logger.info("Data sync completed successfully.")
break
except Exception as e:
logger.warning(f"Sync attempt {attempt + 1}/{max_retries} failed: {e}")
if "429" in str(e) or isinstance(e, RequestException):
logger.info(f"Rate limited or network error. Retrying in {retry_delay} seconds...")
time.sleep(retry_delay)
else:
logger.error("Non-recoverable error during sync.")
time.sleep(10)
else:
logger.error("Data sync failed after maximum retries.")
return False
if __name__ == "__main__":
sync_from_huggingface()