#!/usr/bin/env python3 """ Lightweight in-system backup & sync daemon for /data/apps. - Syncs durable configs, skills, memories, and SQLite dumps between /home/appuser and /data/apps/ - Uses SQLite `.backup` API so live WAL/SHM locks NEVER touch FUSE S3 bucket - Strictly excludes all temp directories, cache, node_modules, logs, and locks """ import os import sys import time import shutil import sqlite3 import hashlib from pathlib import Path from datetime import datetime DATA_APPS = Path("/data/apps") LOCAL_HERMES = Path("/home/appuser/.hermes") LOCAL_EKKO = Path("/home/appuser/.hermes-web-ui") SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", 60)) DURABLE_DIRS = [ "memories", "skills", "cron", "script", "plugins", "extensions", "prompts", "knowledge", "profiles", ] DURABLE_FILES = [ "config.yaml", ".env", "auth.json", "SOUL.md", "channel_directory.json", ] HERMES_DBS = [ "state.db", "shared-state.db", "runs_idempotency.db", "response_store.db", "kanban.db", ] EXCLUDE_PATTERNS = { "__pycache__", ".git", "node_modules", ".cache", "tmp", "scratch", "logs", "gateway.lock", "gateway.pid", "gateway_state.json", ".db-shm", ".db-wal", "bridge.log", "server.log", ".login-lock.json" } def should_exclude(src: str, names: list) -> list: ignored = [] for n in names: if n in EXCLUDE_PATTERNS or any(n.endswith(ext) for ext in [".log", ".lock", ".pid", ".tmp", ".db-shm", ".db-wal"]): ignored.append(n) if "request_dump" in n or "scratch" in n or "audio_cache" in n: ignored.append(n) # Exclude broken/dangling symlinks from copytree traversal try: full_p = os.path.join(src, n) if os.path.islink(full_p) and not os.path.exists(full_p): ignored.append(n) except Exception: pass return ignored def safe_sqlite_backup(src_db_path: Path, dst_db_path: Path): """Perform atomic online backup of SQLite database without locking FUSE.""" if not src_db_path.exists() or src_db_path.stat().st_size == 0: return try: dst_db_path.parent.mkdir(parents=True, exist_ok=True) tmp_dst = dst_db_path.with_suffix(".tmp") src_conn = sqlite3.connect(f"file:{src_db_path}?mode=ro", uri=True) dst_conn = sqlite3.connect(tmp_dst) with dst_conn: src_conn.backup(dst_conn) src_conn.close() dst_conn.close() if tmp_dst.exists() and tmp_dst.stat().st_size > 0: tmp_dst.replace(dst_db_path) except Exception as e: print(f"[sync] SQLite backup error ({src_db_path.name}): {e}", file=sys.stderr) def safe_sqlite_restore(src_db_path: Path, dst_db_path: Path): """Restore SQLite database from backup onto local fast disk.""" if not src_db_path.exists() or src_db_path.stat().st_size == 0: return try: dst_db_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src_db_path, dst_db_path) print(f"[sync] Restored SQLite database: {dst_db_path.name}") except Exception as e: print(f"[sync] SQLite restore error ({src_db_path.name}): {e}", file=sys.stderr) def robust_copytree(src: Path, dst: Path): """Recursively copy directory tree, resilient to broken symlinks, vanished files, or FUSE I/O hiccups.""" if not src.exists(): return dst.mkdir(parents=True, exist_ok=True) try: shutil.copytree(src, dst, dirs_exist_ok=True, ignore=should_exclude, ignore_dangling_symlinks=True) except Exception as e: # Fallback manual file-by-file copy ignoring vanishing/broken files for root, dirs, files in os.walk(src): rel = os.path.relpath(root, src) target_dir = dst / rel if rel != "." else dst target_dir.mkdir(parents=True, exist_ok=True) for f in files: sf = os.path.join(root, f) df = target_dir / f try: if os.path.islink(sf): if not os.path.exists(sf): continue # skip dangling symlink link_target = os.readlink(sf) if df.is_symlink() or df.exists(): df.unlink(missing_ok=True) os.symlink(link_target, df) elif os.path.isfile(sf): shutil.copy2(sf, df) except Exception: pass def restore_all(): """Run on startup: restore /data/apps/hermes and /data/apps/ekko into local storage.""" print("🔄 Restoring durable configs & databases from /data/apps/...", flush=True) # 1. Restore Hermes hermes_backup = DATA_APPS / "hermes" if hermes_backup.exists(): for d in DURABLE_DIRS: if d == "skills": # Skills are bundled at build in container image; only copy non-existing user additions continue src_d = hermes_backup / d dst_d = LOCAL_HERMES / d if src_d.exists(): try: robust_copytree(src_d, dst_d) except Exception as e: print(f"[sync] Warning restoring dir {d}: {e}", file=sys.stderr, flush=True) for f in DURABLE_FILES: src_f = hermes_backup / f dst_f = LOCAL_HERMES / f if src_f.exists(): try: shutil.copy2(src_f, dst_f) except Exception as e: print(f"[sync] Warning restoring file {f}: {e}", file=sys.stderr, flush=True) for db_name in HERMES_DBS: safe_sqlite_restore(hermes_backup / db_name, LOCAL_HERMES / db_name) safe_sqlite_restore(hermes_backup / "cron" / "executions.db", LOCAL_HERMES / "cron" / "executions.db") # 2. Restore Ekko ekko_backup = DATA_APPS / "ekko" if ekko_backup.exists(): safe_sqlite_restore(ekko_backup / "hermes-web-ui.db", LOCAL_EKKO / "hermes-web-ui.db") for f in ["device-identity.json", "startup-tasks.json"]: src_f = ekko_backup / f dst_f = LOCAL_EKKO / f if src_f.exists(): try: shutil.copy2(src_f, dst_f) except Exception: pass print("✅ Restore from /data/apps/ complete!", flush=True) def backup_all(): """Periodic backup of durable data and clean SQLite states to /data/apps/.""" # 1. Backup Hermes hermes_backup = DATA_APPS / "hermes" hermes_backup.mkdir(parents=True, exist_ok=True) for d in DURABLE_DIRS: src_d = LOCAL_HERMES / d dst_d = hermes_backup / d if src_d.exists(): try: robust_copytree(src_d, dst_d) except Exception as e: print(f"[sync] Warning backing up dir {d}: {e}", file=sys.stderr) for f in DURABLE_FILES: src_f = LOCAL_HERMES / f dst_f = hermes_backup / f if src_f.exists(): try: shutil.copy2(src_f, dst_f) except Exception: pass for db_name in HERMES_DBS: safe_sqlite_backup(LOCAL_HERMES / db_name, hermes_backup / db_name) safe_sqlite_backup(LOCAL_HERMES / "cron" / "executions.db", hermes_backup / "cron" / "executions.db") # 2. Backup Ekko ekko_backup = DATA_APPS / "ekko" ekko_backup.mkdir(parents=True, exist_ok=True) safe_sqlite_backup(LOCAL_EKKO / "hermes-web-ui.db", ekko_backup / "hermes-web-ui.db") for f in ["device-identity.json", "startup-tasks.json"]: src_f = LOCAL_EKKO / f dst_f = ekko_backup / f if src_f.exists(): try: shutil.copy2(src_f, dst_f) except Exception: pass def daemon_loop(): print(f"🔄 Apps sync daemon started (Interval: {SYNC_INTERVAL}s)", flush=True) # Perform immediate initial backup after restore so /data/apps is never left stale try: backup_all() print("[sync] Initial backup completed.", flush=True) except Exception as e: print(f"[sync] Initial backup error: {e}", file=sys.stderr, flush=True) last_hash = "" while True: try: # Check if any durable files modified hasher = hashlib.md5() for root, _, files in os.walk(LOCAL_HERMES): if any(x in root for x in ["scratch", "logs", "tmp", "cache", "__pycache__"]): continue for f in files: if not any(f.endswith(x) for x in [".log", ".lock", ".pid", ".wal", ".shm"]): fp = os.path.join(root, f) try: hasher.update(f"{f}:{os.path.getmtime(fp)}".encode()) except Exception: pass web_db = LOCAL_EKKO / "hermes-web-ui.db" if web_db.exists(): try: hasher.update(f"web_db:{web_db.stat().st_mtime}".encode()) except Exception: pass cur_hash = hasher.hexdigest() if cur_hash != last_hash: backup_all() last_hash = cur_hash except Exception as e: print(f"[sync] Daemon error: {e}", file=sys.stderr) time.sleep(SYNC_INTERVAL) if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "restore": restore_all() elif len(sys.argv) > 1 and sys.argv[1] == "backup": backup_all() else: daemon_loop()