saadpie commited on
Commit
9f48675
·
verified ·
1 Parent(s): 21f7eff

Create hermes-sync.py

Browse files
Files changed (1) hide show
  1. hermes-sync.py +362 -0
hermes-sync.py ADDED
@@ -0,0 +1,362 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """HuggingMes Hermes state backup via Hugging Face Datasets."""
3
+
4
+ import hashlib
5
+ import json
6
+ import logging
7
+ import os
8
+ import shutil
9
+ import signal
10
+ import random
11
+ import socket
12
+ import stat
13
+ import sys
14
+ import tempfile
15
+ import threading
16
+ import time
17
+ from pathlib import Path
18
+
19
+ os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
20
+ os.environ.setdefault("HF_HUB_VERBOSITY", "error")
21
+ os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300") # 5 min — default 10s causes timeout on large state
22
+ os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1") # faster multipart upload/download (replaces deprecated HF_HUB_ENABLE_HF_TRANSFER)
23
+
24
+ from huggingface_hub import HfApi, snapshot_download, upload_folder
25
+ from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError
26
+
27
+ logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
28
+
29
+ HERMES_HOME = Path(os.environ.get("HERMES_HOME", "/opt/data"))
30
+ STATUS_FILE = Path("/tmp/huggingmes-sync-status.json")
31
+ STATE_FILE = HERMES_HOME / ".huggingmes-sync-state.json"
32
+ INTERVAL = int(os.environ.get("SYNC_INTERVAL", "600"))
33
+ INITIAL_DELAY = int(os.environ.get("SYNC_START_DELAY", "10"))
34
+ HF_TOKEN = os.environ.get("HF_TOKEN", "").strip()
35
+ HF_USERNAME = os.environ.get("HF_USERNAME", "").strip()
36
+ SPACE_AUTHOR_NAME = os.environ.get("SPACE_AUTHOR_NAME", "").strip()
37
+ BACKUP_DATASET_NAME = os.environ.get("BACKUP_DATASET_NAME", "huggingmes-backup").strip()
38
+ INCLUDE_ENV = os.environ.get("SYNC_INCLUDE_ENV", "").strip().lower() in {"1", "true", "yes"}
39
+ MAX_FILE_SIZE_BYTES = int(os.environ.get("SYNC_MAX_FILE_BYTES", str(50 * 1024 * 1024)))
40
+
41
+ EXCLUDED_DIRS = {
42
+ ".cache",
43
+ ".git",
44
+ ".npm",
45
+ ".venv",
46
+ "__pycache__",
47
+ "node_modules",
48
+ "venv",
49
+ }
50
+ EXCLUDED_TOP_LEVEL = {"logs", STATE_FILE.name}
51
+ if not INCLUDE_ENV:
52
+ EXCLUDED_TOP_LEVEL.add(".env")
53
+
54
+ HF_API = HfApi(token=HF_TOKEN) if HF_TOKEN else None
55
+ STOP_EVENT = threading.Event()
56
+ _REPO_ID_CACHE: str | None = None
57
+
58
+
59
+ def write_status(status: str, message: str, fingerprint: str | None = None, marker: tuple[int, int, int] | None = None) -> None:
60
+ timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
61
+ payload = {
62
+ "status": status,
63
+ "message": message,
64
+ "timestamp": timestamp,
65
+ }
66
+
67
+ # Update temporary status file for health checks/dashboard
68
+ tmp_path = STATUS_FILE.with_suffix(".tmp")
69
+ try:
70
+ tmp_path.write_text(json.dumps(payload), encoding="utf-8")
71
+ tmp_path.replace(STATUS_FILE)
72
+ except OSError:
73
+ pass
74
+
75
+ # Update persistent state file in HERMES_HOME
76
+ if fingerprint or marker:
77
+ state = {}
78
+ if STATE_FILE.exists():
79
+ try:
80
+ state = json.loads(STATE_FILE.read_text(encoding="utf-8"))
81
+ except Exception:
82
+ pass
83
+
84
+ if fingerprint:
85
+ state["last_fingerprint"] = fingerprint
86
+ if marker:
87
+ state["last_marker"] = list(marker)
88
+ state["last_sync"] = timestamp
89
+
90
+ try:
91
+ STATE_FILE.write_text(json.dumps(state), encoding="utf-8")
92
+ except OSError:
93
+ pass
94
+
95
+
96
+ def resolve_backup_repo() -> str:
97
+ global _REPO_ID_CACHE
98
+ if _REPO_ID_CACHE:
99
+ return _REPO_ID_CACHE
100
+
101
+ namespace = HF_USERNAME or SPACE_AUTHOR_NAME
102
+ if not namespace and HF_API is not None:
103
+ whoami = HF_API.whoami()
104
+ namespace = whoami.get("name") or whoami.get("user") or ""
105
+
106
+ namespace = str(namespace).strip()
107
+ if not namespace:
108
+ raise RuntimeError("Could not determine HF username. Set HF_USERNAME or use an account HF_TOKEN.")
109
+
110
+ _REPO_ID_CACHE = f"{namespace}/{BACKUP_DATASET_NAME}"
111
+ return _REPO_ID_CACHE
112
+
113
+
114
+ def ensure_repo_exists() -> str:
115
+ repo_id = resolve_backup_repo()
116
+ try:
117
+ HF_API.repo_info(repo_id=repo_id, repo_type="dataset")
118
+ except RepositoryNotFoundError:
119
+ HF_API.create_repo(repo_id=repo_id, repo_type="dataset", private=True)
120
+ return repo_id
121
+
122
+
123
+ def should_exclude(rel_posix: str, path: Path) -> bool:
124
+ parts = Path(rel_posix).parts
125
+ if not parts:
126
+ return False
127
+ if parts[0] in EXCLUDED_TOP_LEVEL:
128
+ return True
129
+ if any(part in EXCLUDED_DIRS for part in parts):
130
+ return True
131
+ # Exclude SQLite temporary files (shm, wal, journal)
132
+ if path.is_file():
133
+ name_lower = path.name.lower()
134
+ if name_lower.endswith((".db-shm", ".db-wal", ".db-journal")):
135
+ return True
136
+ try:
137
+ return path.stat().st_size > MAX_FILE_SIZE_BYTES
138
+ except OSError:
139
+ return True
140
+ return False
141
+
142
+
143
+ def metadata_marker(root: Path) -> tuple[int, int, int]:
144
+ if not root.exists():
145
+ return (0, 0, 0)
146
+ file_count = 0
147
+ total_size = 0
148
+ newest_mtime = 0
149
+ for path in root.rglob("*"):
150
+ if not path.is_file():
151
+ continue
152
+ rel = path.relative_to(root).as_posix()
153
+ if should_exclude(rel, path):
154
+ continue
155
+ try:
156
+ file_stat = path.stat()
157
+ except OSError:
158
+ continue
159
+ file_count += 1
160
+ total_size += int(file_stat.st_size)
161
+ newest_mtime = max(newest_mtime, int(file_stat.st_mtime_ns))
162
+ return (file_count, total_size, newest_mtime)
163
+
164
+
165
+ def fingerprint_dir(root: Path) -> str:
166
+ hasher = hashlib.sha256()
167
+ if not root.exists():
168
+ return hasher.hexdigest()
169
+ for path in sorted(p for p in root.rglob("*") if p.is_file()):
170
+ rel = path.relative_to(root).as_posix()
171
+ if should_exclude(rel, path):
172
+ continue
173
+ hasher.update(rel.encode("utf-8"))
174
+ with path.open("rb") as handle:
175
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
176
+ hasher.update(chunk)
177
+ return hasher.hexdigest()
178
+
179
+
180
+ def create_snapshot_dir(source_root: Path) -> Path:
181
+ staging_root = Path(tempfile.mkdtemp(prefix="huggingmes-sync-"))
182
+ for path in sorted(source_root.rglob("*")):
183
+ rel = path.relative_to(source_root)
184
+ rel_posix = rel.as_posix()
185
+ if should_exclude(rel_posix, path):
186
+ continue
187
+ target = staging_root / rel
188
+ if path.is_dir():
189
+ target.mkdir(parents=True, exist_ok=True)
190
+ continue
191
+ target.parent.mkdir(parents=True, exist_ok=True)
192
+ try:
193
+ shutil.copy2(path, target)
194
+ except OSError:
195
+ # File may have been deleted by the application since enumeration
196
+ continue
197
+ return staging_root
198
+
199
+
200
+ def _make_writable(path: Path) -> None:
201
+ """Recursively add owner-write permission so rmtree can delete read-only trees."""
202
+ for dirpath, dirs, files in os.walk(str(path)):
203
+ for name in dirs + files:
204
+ try:
205
+ full = os.path.join(dirpath, name)
206
+ os.chmod(full, os.stat(full).st_mode | stat.S_IWUSR)
207
+ except OSError:
208
+ pass
209
+ try:
210
+ os.chmod(str(path), os.stat(str(path)).st_mode | stat.S_IRWXU)
211
+ except OSError:
212
+ pass
213
+
214
+
215
+ def restore() -> bool:
216
+ if not HF_TOKEN:
217
+ write_status("disabled", "HF_TOKEN is not configured.")
218
+ return False
219
+
220
+ repo_id = resolve_backup_repo()
221
+ write_status("restoring", f"Restoring Hermes state from {repo_id}")
222
+ try:
223
+ with tempfile.TemporaryDirectory() as tmpdir:
224
+ snapshot_download(repo_id=repo_id, repo_type="dataset", token=HF_TOKEN, local_dir=tmpdir)
225
+ tmp_path = Path(tmpdir)
226
+ if not any(tmp_path.iterdir()):
227
+ write_status("fresh", "Backup dataset is empty. Starting fresh.")
228
+ return True
229
+
230
+ HERMES_HOME.mkdir(parents=True, exist_ok=True)
231
+ for child in tmp_path.iterdir():
232
+ if should_exclude(child.name, child):
233
+ continue
234
+ target = HERMES_HOME / child.name
235
+ if target.is_symlink() or target.is_file():
236
+ target.unlink()
237
+ elif target.is_dir():
238
+ _make_writable(target)
239
+ shutil.rmtree(target, ignore_errors=True)
240
+ if child.is_dir():
241
+ shutil.copytree(child, target, dirs_exist_ok=True)
242
+ else:
243
+ shutil.copy2(child, target)
244
+
245
+ write_status("restored", f"Restored Hermes state from {repo_id}")
246
+ return True
247
+ except RepositoryNotFoundError:
248
+ write_status("fresh", f"Backup dataset {repo_id} does not exist yet.")
249
+ return True
250
+ except HfHubHTTPError as exc:
251
+ if exc.response is not None and exc.response.status_code == 404:
252
+ write_status("fresh", f"Backup dataset {repo_id} does not exist yet.")
253
+ return True
254
+ write_status("error", f"Restore failed: {exc}")
255
+ print(f"Restore failed: {exc}", file=sys.stderr)
256
+ return False
257
+ except Exception as exc:
258
+ write_status("error", f"Restore failed: {exc}")
259
+ print(f"Restore failed: {exc}", file=sys.stderr)
260
+ return False
261
+
262
+
263
+ def sync_once(last_fingerprint: str | None = None, last_marker: tuple[int, int, int] | None = None):
264
+ # If no state provided, try to load from persistent state file
265
+ if last_fingerprint is None and last_marker is None:
266
+ if STATE_FILE.exists():
267
+ try:
268
+ state = json.loads(STATE_FILE.read_text(encoding="utf-8"))
269
+ last_fingerprint = state.get("last_fingerprint")
270
+ m = state.get("last_marker")
271
+ if m and len(m) == 3:
272
+ last_marker = tuple(m)
273
+ except Exception:
274
+ pass
275
+
276
+ repo_id = ensure_repo_exists()
277
+ current_marker = metadata_marker(HERMES_HOME)
278
+ if last_marker is not None and current_marker == last_marker:
279
+ write_status("synced", "No Hermes state changes detected (marker match).")
280
+ return (last_fingerprint or "", current_marker)
281
+
282
+ current_fingerprint = fingerprint_dir(HERMES_HOME)
283
+ if last_fingerprint is not None and current_fingerprint == last_fingerprint:
284
+ write_status("synced", "No Hermes state changes detected (fingerprint match).")
285
+ return (last_fingerprint, current_marker)
286
+
287
+ hostname = socket.gethostname()
288
+ write_status("syncing", f"Uploading Hermes state to {repo_id} from {hostname}")
289
+ snapshot_dir = create_snapshot_dir(HERMES_HOME)
290
+ try:
291
+ upload_folder(
292
+ folder_path=str(snapshot_dir),
293
+ repo_id=repo_id,
294
+ repo_type="dataset",
295
+ token=HF_TOKEN,
296
+ commit_message=f"HuggingMes sync [{hostname}] {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
297
+ ignore_patterns=[".git/*", ".git"],
298
+ )
299
+ finally:
300
+ shutil.rmtree(snapshot_dir, ignore_errors=True)
301
+
302
+ write_status("success", f"Uploaded Hermes state to {repo_id}", fingerprint=current_fingerprint, marker=current_marker)
303
+ return (current_fingerprint, current_marker)
304
+
305
+
306
+ def handle_signal(_sig, _frame) -> None:
307
+ STOP_EVENT.set()
308
+
309
+
310
+ def loop() -> int:
311
+ signal.signal(signal.SIGTERM, handle_signal)
312
+ signal.signal(signal.SIGINT, handle_signal)
313
+ try:
314
+ repo_id = resolve_backup_repo()
315
+ write_status("configured", f"Backup loop active for {repo_id} with {INTERVAL}s interval.")
316
+ except Exception as exc:
317
+ write_status("error", str(exc))
318
+ print(f"Hermes sync error: {exc}")
319
+ return 1
320
+
321
+ last_fingerprint = fingerprint_dir(HERMES_HOME)
322
+ last_marker = metadata_marker(HERMES_HOME)
323
+ time.sleep(INITIAL_DELAY)
324
+ print(f"Hermes state sync started: every {INTERVAL}s -> {repo_id}")
325
+
326
+ while not STOP_EVENT.is_set():
327
+ try:
328
+ last_fingerprint, last_marker = sync_once(last_fingerprint, last_marker)
329
+ except Exception as exc:
330
+ write_status("error", f"Sync failed: {exc}")
331
+ print(f"Hermes sync failed: {exc}")
332
+
333
+ # Add 10% jitter to interval to avoid synchronized commits from multiple containers
334
+ jitter = random.uniform(0.9, 1.1)
335
+ if STOP_EVENT.wait(INTERVAL * jitter):
336
+ break
337
+ return 0
338
+
339
+
340
+ def main() -> int:
341
+ HERMES_HOME.mkdir(parents=True, exist_ok=True)
342
+ if len(sys.argv) < 2:
343
+ return loop()
344
+ command = sys.argv[1]
345
+ if command == "restore":
346
+ return 0 if restore() else 1
347
+ if command == "sync-once":
348
+ try:
349
+ sync_once()
350
+ return 0
351
+ except Exception as exc:
352
+ write_status("error", f"Shutdown sync failed: {exc}")
353
+ print(f"Hermes sync: shutdown sync failed: {exc}")
354
+ return 1
355
+ if command == "loop":
356
+ return loop()
357
+ print(f"Unknown command: {command}", file=sys.stderr)
358
+ return 1
359
+
360
+
361
+ if __name__ == "__main__":
362
+ raise SystemExit(main())