Spaces:
Paused
Paused
Download sync.py from immahadi/FreeLLMAPI: direct link, hf CLI and curl.
- Browser
- Download file 4.67 kB
-
https://huggingface.co/spaces/immahadi/FreeLLMAPI/resolve/main/sync.py
- Command line
-
hf download hf://spaces/immahadi/FreeLLMAPI/sync.py
-
curl -L -o sync.py https://huggingface.co/spaces/immahadi/FreeLLMAPI/resolve/main/sync.py
4.67 kB
| #!/usr/bin/env python3 | |
| """ | |
| Hugging Face Dataset Storage Synchronization Script for OmniRoute. | |
| Syncs persistent state (/data) with a Hugging Face Dataset repository. | |
| """ | |
| import os | |
| import sys | |
| import shutil | |
| import subprocess | |
| from pathlib import Path | |
| def get_config(): | |
| token = ( | |
| os.environ.get("HF_TOKEN") | |
| or os.environ.get("HUGGINGFACE_TOKEN") | |
| or os.environ.get("HUGGING_FACE_HUB_TOKEN") | |
| or os.environ.get("HF_ACCESS_TOKEN") | |
| ) | |
| dataset_repo = os.environ.get("HF_DATASET_REPO", "userbymahadi/OmniRoute-storage") | |
| data_dir = Path(os.environ.get("DATA_DIR", "/data")).resolve() | |
| return token, dataset_repo, data_dir | |
| def checkpoint_sqlite(data_dir: Path, mode: str = "PASSIVE"): | |
| """Flush WAL journal changes to disk prior to synchronization.""" | |
| db_file = data_dir / "storage.sqlite" | |
| if db_file.exists(): | |
| try: | |
| subprocess.run( | |
| ["sqlite3", str(db_file), f"PRAGMA wal_checkpoint({mode});"], | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.DEVNULL, | |
| timeout=15, | |
| ) | |
| except Exception as e: | |
| print(f"[HF-Sync] SQLite checkpoint note: {e}", file=sys.stderr) | |
| def download_data(): | |
| token, dataset_repo, data_dir = get_config() | |
| if not token: | |
| print("[HF-Sync] No HF_TOKEN supplied; relying on local/attached persistent disk.", flush=True) | |
| return 0 | |
| data_dir.mkdir(parents=True, exist_ok=True) | |
| print(f"[HF-Sync] Checking remote storage repository: {dataset_repo}...", flush=True) | |
| try: | |
| from huggingface_hub import HfApi, snapshot_download | |
| from huggingface_hub.utils import RepositoryNotFoundError | |
| except ImportError: | |
| print("[HF-Sync] huggingface_hub not installed; skipping sync download.", file=sys.stderr) | |
| return 0 | |
| api = HfApi(token=token) | |
| try: | |
| api.repo_info(repo_id=dataset_repo, repo_type="dataset") | |
| except RepositoryNotFoundError: | |
| print(f"[HF-Sync] Remote dataset {dataset_repo} not found. Creating private repository...", flush=True) | |
| try: | |
| api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True) | |
| print(f"[HF-Sync] Created private dataset repo: {dataset_repo}", flush=True) | |
| return 0 | |
| except Exception as err: | |
| print(f"[HF-Sync] Failed to initialize remote repo {dataset_repo}: {err}", file=sys.stderr) | |
| return 0 | |
| except Exception as err: | |
| print(f"[HF-Sync] Remote repository probe notice: {err}", file=sys.stderr) | |
| try: | |
| print(f"[HF-Sync] Pulling persistent data into {data_dir}...", flush=True) | |
| snapshot_download( | |
| repo_id=dataset_repo, | |
| repo_type="dataset", | |
| local_dir=str(data_dir), | |
| token=token, | |
| ignore_patterns=[".git*", "*.tmp", "lost+found*"], | |
| ) | |
| print("[HF-Sync] Download completed successfully.", flush=True) | |
| except Exception as err: | |
| print(f"[HF-Sync] Snapshot download notice: {err}", file=sys.stderr) | |
| return 0 | |
| def upload_data(checkpoint_mode: str = "PASSIVE"): | |
| token, dataset_repo, data_dir = get_config() | |
| if not token: | |
| return 0 | |
| if not data_dir.exists(): | |
| return 0 | |
| try: | |
| from huggingface_hub import HfApi, upload_folder | |
| except ImportError: | |
| return 0 | |
| checkpoint_sqlite(data_dir, mode=checkpoint_mode) | |
| print(f"[HF-Sync] Synchronizing {data_dir} to dataset {dataset_repo}...", flush=True) | |
| try: | |
| api = HfApi(token=token) | |
| api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True) | |
| upload_folder( | |
| repo_id=dataset_repo, | |
| repo_type="dataset", | |
| folder_path=str(data_dir), | |
| token=token, | |
| commit_message="OmniRoute state automated backup", | |
| ignore_patterns=[ | |
| "*.sock", | |
| "*.tmp", | |
| "lost+found*", | |
| "storage.sqlite-shm", | |
| ".cache*", | |
| ], | |
| ) | |
| print("[HF-Sync] Upload to Hugging Face Dataset completed successfully.", flush=True) | |
| except Exception as err: | |
| print(f"[HF-Sync] Sync upload notice: {err}", file=sys.stderr) | |
| return 0 | |
| if __name__ == "__main__": | |
| action = sys.argv[1] if len(sys.argv) > 1 else "download" | |
| if action == "download": | |
| sys.exit(download_data()) | |
| elif action == "upload": | |
| checkpoint = sys.argv[2] if len(sys.argv) > 2 else "PASSIVE" | |
| sys.exit(upload_data(checkpoint)) | |
| else: | |
| print(f"Unknown action: {action}") | |
| sys.exit(1) | |