#!/usr/bin/env python3 """ Hugging Face Dataset Storage Synchronization Script for OmniRoute. Syncs persistent state (/data) with a Hugging Face Dataset repository. """ import os import sys import shutil import subprocess from pathlib import Path def get_config(): token = ( os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or os.environ.get("HF_ACCESS_TOKEN") ) dataset_repo = os.environ.get("HF_DATASET_REPO", "userbymahadi/OmniRoute-storage") data_dir = Path(os.environ.get("DATA_DIR", "/data")).resolve() return token, dataset_repo, data_dir def checkpoint_sqlite(data_dir: Path, mode: str = "PASSIVE"): """Flush WAL journal changes to disk prior to synchronization.""" db_file = data_dir / "storage.sqlite" if db_file.exists(): try: subprocess.run( ["sqlite3", str(db_file), f"PRAGMA wal_checkpoint({mode});"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15, ) except Exception as e: print(f"[HF-Sync] SQLite checkpoint note: {e}", file=sys.stderr) def download_data(): token, dataset_repo, data_dir = get_config() if not token: print("[HF-Sync] No HF_TOKEN supplied; relying on local/attached persistent disk.", flush=True) return 0 data_dir.mkdir(parents=True, exist_ok=True) print(f"[HF-Sync] Checking remote storage repository: {dataset_repo}...", flush=True) try: from huggingface_hub import HfApi, snapshot_download from huggingface_hub.utils import RepositoryNotFoundError except ImportError: print("[HF-Sync] huggingface_hub not installed; skipping sync download.", file=sys.stderr) return 0 api = HfApi(token=token) try: api.repo_info(repo_id=dataset_repo, repo_type="dataset") except RepositoryNotFoundError: print(f"[HF-Sync] Remote dataset {dataset_repo} not found. Creating private repository...", flush=True) try: api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True) print(f"[HF-Sync] Created private dataset repo: {dataset_repo}", flush=True) return 0 except Exception as err: print(f"[HF-Sync] Failed to initialize remote repo {dataset_repo}: {err}", file=sys.stderr) return 0 except Exception as err: print(f"[HF-Sync] Remote repository probe notice: {err}", file=sys.stderr) try: print(f"[HF-Sync] Pulling persistent data into {data_dir}...", flush=True) snapshot_download( repo_id=dataset_repo, repo_type="dataset", local_dir=str(data_dir), token=token, ignore_patterns=[".git*", "*.tmp", "lost+found*"], ) print("[HF-Sync] Download completed successfully.", flush=True) except Exception as err: print(f"[HF-Sync] Snapshot download notice: {err}", file=sys.stderr) return 0 def upload_data(checkpoint_mode: str = "PASSIVE"): token, dataset_repo, data_dir = get_config() if not token: return 0 if not data_dir.exists(): return 0 try: from huggingface_hub import HfApi, upload_folder except ImportError: return 0 checkpoint_sqlite(data_dir, mode=checkpoint_mode) print(f"[HF-Sync] Synchronizing {data_dir} to dataset {dataset_repo}...", flush=True) try: api = HfApi(token=token) api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True) upload_folder( repo_id=dataset_repo, repo_type="dataset", folder_path=str(data_dir), token=token, commit_message="OmniRoute state automated backup", ignore_patterns=[ "*.sock", "*.tmp", "lost+found*", "storage.sqlite-shm", ".cache*", ], ) print("[HF-Sync] Upload to Hugging Face Dataset completed successfully.", flush=True) except Exception as err: print(f"[HF-Sync] Sync upload notice: {err}", file=sys.stderr) return 0 if __name__ == "__main__": action = sys.argv[1] if len(sys.argv) > 1 else "download" if action == "download": sys.exit(download_data()) elif action == "upload": checkpoint = sys.argv[2] if len(sys.argv) > 2 else "PASSIVE" sys.exit(upload_data(checkpoint)) else: print(f"Unknown action: {action}") sys.exit(1)