FreeLLMAPI / sync.py
MD Mahadi
Deploy OmniRoute gateway on Hugging Face Docker Space with dataset persistence
c7ba288
Raw History Blame Contribute Delete
4.67 kB
#!/usr/bin/env python3
"""
Hugging Face Dataset Storage Synchronization Script for OmniRoute.
Syncs persistent state (/data) with a Hugging Face Dataset repository.
"""
import os
import sys
import shutil
import subprocess
from pathlib import Path
def get_config():
token = (
os.environ.get("HF_TOKEN")
or os.environ.get("HUGGINGFACE_TOKEN")
or os.environ.get("HUGGING_FACE_HUB_TOKEN")
or os.environ.get("HF_ACCESS_TOKEN")
)
dataset_repo = os.environ.get("HF_DATASET_REPO", "userbymahadi/OmniRoute-storage")
data_dir = Path(os.environ.get("DATA_DIR", "/data")).resolve()
return token, dataset_repo, data_dir
def checkpoint_sqlite(data_dir: Path, mode: str = "PASSIVE"):
"""Flush WAL journal changes to disk prior to synchronization."""
db_file = data_dir / "storage.sqlite"
if db_file.exists():
try:
subprocess.run(
["sqlite3", str(db_file), f"PRAGMA wal_checkpoint({mode});"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=15,
)
except Exception as e:
print(f"[HF-Sync] SQLite checkpoint note: {e}", file=sys.stderr)
def download_data():
token, dataset_repo, data_dir = get_config()
if not token:
print("[HF-Sync] No HF_TOKEN supplied; relying on local/attached persistent disk.", flush=True)
return 0
data_dir.mkdir(parents=True, exist_ok=True)
print(f"[HF-Sync] Checking remote storage repository: {dataset_repo}...", flush=True)
try:
from huggingface_hub import HfApi, snapshot_download
from huggingface_hub.utils import RepositoryNotFoundError
except ImportError:
print("[HF-Sync] huggingface_hub not installed; skipping sync download.", file=sys.stderr)
return 0
api = HfApi(token=token)
try:
api.repo_info(repo_id=dataset_repo, repo_type="dataset")
except RepositoryNotFoundError:
print(f"[HF-Sync] Remote dataset {dataset_repo} not found. Creating private repository...", flush=True)
try:
api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True)
print(f"[HF-Sync] Created private dataset repo: {dataset_repo}", flush=True)
return 0
except Exception as err:
print(f"[HF-Sync] Failed to initialize remote repo {dataset_repo}: {err}", file=sys.stderr)
return 0
except Exception as err:
print(f"[HF-Sync] Remote repository probe notice: {err}", file=sys.stderr)
try:
print(f"[HF-Sync] Pulling persistent data into {data_dir}...", flush=True)
snapshot_download(
repo_id=dataset_repo,
repo_type="dataset",
local_dir=str(data_dir),
token=token,
ignore_patterns=[".git*", "*.tmp", "lost+found*"],
)
print("[HF-Sync] Download completed successfully.", flush=True)
except Exception as err:
print(f"[HF-Sync] Snapshot download notice: {err}", file=sys.stderr)
return 0
def upload_data(checkpoint_mode: str = "PASSIVE"):
token, dataset_repo, data_dir = get_config()
if not token:
return 0
if not data_dir.exists():
return 0
try:
from huggingface_hub import HfApi, upload_folder
except ImportError:
return 0
checkpoint_sqlite(data_dir, mode=checkpoint_mode)
print(f"[HF-Sync] Synchronizing {data_dir} to dataset {dataset_repo}...", flush=True)
try:
api = HfApi(token=token)
api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True)
upload_folder(
repo_id=dataset_repo,
repo_type="dataset",
folder_path=str(data_dir),
token=token,
commit_message="OmniRoute state automated backup",
ignore_patterns=[
"*.sock",
"*.tmp",
"lost+found*",
"storage.sqlite-shm",
".cache*",
],
)
print("[HF-Sync] Upload to Hugging Face Dataset completed successfully.", flush=True)
except Exception as err:
print(f"[HF-Sync] Sync upload notice: {err}", file=sys.stderr)
return 0
if __name__ == "__main__":
action = sys.argv[1] if len(sys.argv) > 1 else "download"
if action == "download":
sys.exit(download_data())
elif action == "upload":
checkpoint = sys.argv[2] if len(sys.argv) > 2 else "PASSIVE"
sys.exit(upload_data(checkpoint))
else:
print(f"Unknown action: {action}")
sys.exit(1)