File size: 4,669 Bytes
c7ba288
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
#!/usr/bin/env python3
"""
Hugging Face Dataset Storage Synchronization Script for OmniRoute.
Syncs persistent state (/data) with a Hugging Face Dataset repository.
"""
import os
import sys
import shutil
import subprocess
from pathlib import Path

def get_config():
    token = (
        os.environ.get("HF_TOKEN")
        or os.environ.get("HUGGINGFACE_TOKEN")
        or os.environ.get("HUGGING_FACE_HUB_TOKEN")
        or os.environ.get("HF_ACCESS_TOKEN")
    )
    dataset_repo = os.environ.get("HF_DATASET_REPO", "userbymahadi/OmniRoute-storage")
    data_dir = Path(os.environ.get("DATA_DIR", "/data")).resolve()
    return token, dataset_repo, data_dir

def checkpoint_sqlite(data_dir: Path, mode: str = "PASSIVE"):
    """Flush WAL journal changes to disk prior to synchronization."""
    db_file = data_dir / "storage.sqlite"
    if db_file.exists():
        try:
            subprocess.run(
                ["sqlite3", str(db_file), f"PRAGMA wal_checkpoint({mode});"],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
                timeout=15,
            )
        except Exception as e:
            print(f"[HF-Sync] SQLite checkpoint note: {e}", file=sys.stderr)

def download_data():
    token, dataset_repo, data_dir = get_config()
    if not token:
        print("[HF-Sync] No HF_TOKEN supplied; relying on local/attached persistent disk.", flush=True)
        return 0

    data_dir.mkdir(parents=True, exist_ok=True)
    print(f"[HF-Sync] Checking remote storage repository: {dataset_repo}...", flush=True)

    try:
        from huggingface_hub import HfApi, snapshot_download
        from huggingface_hub.utils import RepositoryNotFoundError
    except ImportError:
        print("[HF-Sync] huggingface_hub not installed; skipping sync download.", file=sys.stderr)
        return 0

    api = HfApi(token=token)
    try:
        api.repo_info(repo_id=dataset_repo, repo_type="dataset")
    except RepositoryNotFoundError:
        print(f"[HF-Sync] Remote dataset {dataset_repo} not found. Creating private repository...", flush=True)
        try:
            api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True)
            print(f"[HF-Sync] Created private dataset repo: {dataset_repo}", flush=True)
            return 0
        except Exception as err:
            print(f"[HF-Sync] Failed to initialize remote repo {dataset_repo}: {err}", file=sys.stderr)
            return 0
    except Exception as err:
        print(f"[HF-Sync] Remote repository probe notice: {err}", file=sys.stderr)

    try:
        print(f"[HF-Sync] Pulling persistent data into {data_dir}...", flush=True)
        snapshot_download(
            repo_id=dataset_repo,
            repo_type="dataset",
            local_dir=str(data_dir),
            token=token,
            ignore_patterns=[".git*", "*.tmp", "lost+found*"],
        )
        print("[HF-Sync] Download completed successfully.", flush=True)
    except Exception as err:
        print(f"[HF-Sync] Snapshot download notice: {err}", file=sys.stderr)

    return 0

def upload_data(checkpoint_mode: str = "PASSIVE"):
    token, dataset_repo, data_dir = get_config()
    if not token:
        return 0

    if not data_dir.exists():
        return 0

    try:
        from huggingface_hub import HfApi, upload_folder
    except ImportError:
        return 0

    checkpoint_sqlite(data_dir, mode=checkpoint_mode)

    print(f"[HF-Sync] Synchronizing {data_dir} to dataset {dataset_repo}...", flush=True)
    try:
        api = HfApi(token=token)
        api.create_repo(repo_id=dataset_repo, repo_type="dataset", private=True, exist_ok=True)
        upload_folder(
            repo_id=dataset_repo,
            repo_type="dataset",
            folder_path=str(data_dir),
            token=token,
            commit_message="OmniRoute state automated backup",
            ignore_patterns=[
                "*.sock",
                "*.tmp",
                "lost+found*",
                "storage.sqlite-shm",
                ".cache*",
            ],
        )
        print("[HF-Sync] Upload to Hugging Face Dataset completed successfully.", flush=True)
    except Exception as err:
        print(f"[HF-Sync] Sync upload notice: {err}", file=sys.stderr)

    return 0

if __name__ == "__main__":
    action = sys.argv[1] if len(sys.argv) > 1 else "download"
    if action == "download":
        sys.exit(download_data())
    elif action == "upload":
        checkpoint = sys.argv[2] if len(sys.argv) > 2 else "PASSIVE"
        sys.exit(upload_data(checkpoint))
    else:
        print(f"Unknown action: {action}")
        sys.exit(1)