Spaces:
Paused
Paused
File size: 4,669 Bytes
c7ba288 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | #!/usr/bin/env python3
"""
Hugging Face Dataset Storage Synchronization Script for OmniRoute.
Syncs persistent state (/data) with a Hugging Face Dataset repository.
"""
import os
import sys
import shutil
import subprocess
from pathlib import Path
def get_config():
token = (
os.environ.get("HF_TOKEN")
or os.environ.get("HUGGINGFACE_TOKEN")
or os.environ.get("HUGGING_FACE_HUB_TOKEN")
or os.environ.get("HF_ACCESS_TOKEN")
)
dataset_repo = os.environ.get("HF_DATASET_REPO", "userbymahadi/OmniRoute-storage")
data_dir = Path(os.environ.get("DATA_DIR", "/data")).resolve()
return token, dataset_repo, data_dir
def checkpoint_sqlite(data_dir: Path, mode: str = "PASSIVE"):
"""Flush WAL journal changes to disk prior to synchronization."""
db_file = data_dir / "storage.sqlite"
if db_file.exists():
try:
subprocess.run(
["sqlite3", str(db_file), f"PRAGMA wal_checkpoint({mode});"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=15,
)
except Exception as e:
print(f"[HF-Sync] SQLite checkpoint note: {e}", file=sys.stderr)
def download_data():
token, dataset_repo, data_dir = get_config()
if not token:
print("[HF-Sync] No HF_TOKEN supplied; relying on local/attached persistent disk.", flush=True)
return 0
data_dir.mkdir(parents=True, exist_ok=True)
print(f"[HF-Sync] Checking remote storage repository: {dataset_repo}...", flush=True)
try:
from huggingface_hub import HfApi, snapshot_download
from huggingface_hub.utils import RepositoryNotFoundError
except ImportError:
print("[HF-Sync] huggingface_hub not installed; skipping sync download.", file=sys.stderr)
return 0
api = HfApi(token=token)
try:
api.repo_info(repo_id=dataset_repo, repo_type="dataset")
except RepositoryNotFoundError:
print(f"[HF-Sync] Remote dataset {dataset_repo} not found. Creating private repository...", flush=True)
try:
api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True)
print(f"[HF-Sync] Created private dataset repo: {dataset_repo}", flush=True)
return 0
except Exception as err:
print(f"[HF-Sync] Failed to initialize remote repo {dataset_repo}: {err}", file=sys.stderr)
return 0
except Exception as err:
print(f"[HF-Sync] Remote repository probe notice: {err}", file=sys.stderr)
try:
print(f"[HF-Sync] Pulling persistent data into {data_dir}...", flush=True)
snapshot_download(
repo_id=dataset_repo,
repo_type="dataset",
local_dir=str(data_dir),
token=token,
ignore_patterns=[".git*", "*.tmp", "lost+found*"],
)
print("[HF-Sync] Download completed successfully.", flush=True)
except Exception as err:
print(f"[HF-Sync] Snapshot download notice: {err}", file=sys.stderr)
return 0
def upload_data(checkpoint_mode: str = "PASSIVE"):
token, dataset_repo, data_dir = get_config()
if not token:
return 0
if not data_dir.exists():
return 0
try:
from huggingface_hub import HfApi, upload_folder
except ImportError:
return 0
checkpoint_sqlite(data_dir, mode=checkpoint_mode)
print(f"[HF-Sync] Synchronizing {data_dir} to dataset {dataset_repo}...", flush=True)
try:
api = HfApi(token=token)
api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True)
upload_folder(
repo_id=dataset_repo,
repo_type="dataset",
folder_path=str(data_dir),
token=token,
commit_message="OmniRoute state automated backup",
ignore_patterns=[
"*.sock",
"*.tmp",
"lost+found*",
"storage.sqlite-shm",
".cache*",
],
)
print("[HF-Sync] Upload to Hugging Face Dataset completed successfully.", flush=True)
except Exception as err:
print(f"[HF-Sync] Sync upload notice: {err}", file=sys.stderr)
return 0
if __name__ == "__main__":
action = sys.argv[1] if len(sys.argv) > 1 else "download"
if action == "download":
sys.exit(download_data())
elif action == "upload":
checkpoint = sys.argv[2] if len(sys.argv) > 2 else "PASSIVE"
sys.exit(upload_data(checkpoint))
else:
print(f"Unknown action: {action}")
sys.exit(1)
|