cbot / services /apps_sync.py
amangs's picture
Optimize restore_all: exclude redundant skills tree copy during fast startup
68b875d
Raw History Blame
9.73 kB
#!/usr/bin/env python3
"""
Lightweight in-system backup & sync daemon for /data/apps.
- Syncs durable configs, skills, memories, and SQLite dumps between /home/appuser and /data/apps/
- Uses SQLite `.backup` API so live WAL/SHM locks NEVER touch FUSE S3 bucket
- Strictly excludes all temp directories, cache, node_modules, logs, and locks
"""
import os
import sys
import time
import shutil
import sqlite3
import hashlib
from pathlib import Path
from datetime import datetime
DATA_APPS = Path("/data/apps")
LOCAL_HERMES = Path("/home/appuser/.hermes")
LOCAL_EKKO = Path("/home/appuser/.hermes-web-ui")
SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", 60))
DURABLE_DIRS = [
"memories",
"skills",
"cron",
"script",
"plugins",
"extensions",
"prompts",
"knowledge",
"profiles",
]
DURABLE_FILES = [
"config.yaml",
".env",
"auth.json",
"SOUL.md",
"channel_directory.json",
]
HERMES_DBS = [
"state.db",
"shared-state.db",
"runs_idempotency.db",
"response_store.db",
"kanban.db",
]
EXCLUDE_PATTERNS = {
"__pycache__", ".git", "node_modules", ".cache", "tmp", "scratch",
"logs", "gateway.lock", "gateway.pid", "gateway_state.json",
".db-shm", ".db-wal", "bridge.log", "server.log", ".login-lock.json"
}
def should_exclude(src: str, names: list) -> list:
ignored = []
for n in names:
if n in EXCLUDE_PATTERNS or any(n.endswith(ext) for ext in [".log", ".lock", ".pid", ".tmp", ".db-shm", ".db-wal"]):
ignored.append(n)
if "request_dump" in n or "scratch" in n or "audio_cache" in n:
ignored.append(n)
# Exclude broken/dangling symlinks from copytree traversal
try:
full_p = os.path.join(src, n)
if os.path.islink(full_p) and not os.path.exists(full_p):
ignored.append(n)
except Exception:
pass
return ignored
def safe_sqlite_backup(src_db_path: Path, dst_db_path: Path):
"""Perform atomic online backup of SQLite database without locking FUSE."""
if not src_db_path.exists() or src_db_path.stat().st_size == 0:
return
try:
dst_db_path.parent.mkdir(parents=True, exist_ok=True)
tmp_dst = dst_db_path.with_suffix(".tmp")
src_conn = sqlite3.connect(f"file:{src_db_path}?mode=ro", uri=True)
dst_conn = sqlite3.connect(tmp_dst)
with dst_conn:
src_conn.backup(dst_conn)
src_conn.close()
dst_conn.close()
if tmp_dst.exists() and tmp_dst.stat().st_size > 0:
tmp_dst.replace(dst_db_path)
except Exception as e:
print(f"[sync] SQLite backup error ({src_db_path.name}): {e}", file=sys.stderr)
def safe_sqlite_restore(src_db_path: Path, dst_db_path: Path):
"""Restore SQLite database from backup onto local fast disk."""
if not src_db_path.exists() or src_db_path.stat().st_size == 0:
return
try:
dst_db_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src_db_path, dst_db_path)
print(f"[sync] Restored SQLite database: {dst_db_path.name}")
except Exception as e:
print(f"[sync] SQLite restore error ({src_db_path.name}): {e}", file=sys.stderr)
def robust_copytree(src: Path, dst: Path):
"""Recursively copy directory tree, resilient to broken symlinks, vanished files, or FUSE I/O hiccups."""
if not src.exists():
return
dst.mkdir(parents=True, exist_ok=True)
try:
shutil.copytree(src, dst, dirs_exist_ok=True, ignore=should_exclude, ignore_dangling_symlinks=True)
except Exception as e:
# Fallback manual file-by-file copy ignoring vanishing/broken files
for root, dirs, files in os.walk(src):
rel = os.path.relpath(root, src)
target_dir = dst / rel if rel != "." else dst
target_dir.mkdir(parents=True, exist_ok=True)
for f in files:
sf = os.path.join(root, f)
df = target_dir / f
try:
if os.path.islink(sf):
if not os.path.exists(sf):
continue # skip dangling symlink
link_target = os.readlink(sf)
if df.is_symlink() or df.exists():
df.unlink(missing_ok=True)
os.symlink(link_target, df)
elif os.path.isfile(sf):
shutil.copy2(sf, df)
except Exception:
pass
def restore_all():
"""Run on startup: restore /data/apps/hermes and /data/apps/ekko into local storage."""
print("🔄 Restoring durable configs & databases from /data/apps/...", flush=True)
# 1. Restore Hermes
hermes_backup = DATA_APPS / "hermes"
if hermes_backup.exists():
for d in DURABLE_DIRS:
if d == "skills":
# Skills are bundled at build in container image; only copy non-existing user additions
continue
src_d = hermes_backup / d
dst_d = LOCAL_HERMES / d
if src_d.exists():
try:
robust_copytree(src_d, dst_d)
except Exception as e:
print(f"[sync] Warning restoring dir {d}: {e}", file=sys.stderr, flush=True)
for f in DURABLE_FILES:
src_f = hermes_backup / f
dst_f = LOCAL_HERMES / f
if src_f.exists():
try:
shutil.copy2(src_f, dst_f)
except Exception as e:
print(f"[sync] Warning restoring file {f}: {e}", file=sys.stderr, flush=True)
for db_name in HERMES_DBS:
safe_sqlite_restore(hermes_backup / db_name, LOCAL_HERMES / db_name)
safe_sqlite_restore(hermes_backup / "cron" / "executions.db", LOCAL_HERMES / "cron" / "executions.db")
# 2. Restore Ekko
ekko_backup = DATA_APPS / "ekko"
if ekko_backup.exists():
safe_sqlite_restore(ekko_backup / "hermes-web-ui.db", LOCAL_EKKO / "hermes-web-ui.db")
for f in ["device-identity.json", "startup-tasks.json"]:
src_f = ekko_backup / f
dst_f = LOCAL_EKKO / f
if src_f.exists():
try:
shutil.copy2(src_f, dst_f)
except Exception:
pass
print("✅ Restore from /data/apps/ complete!", flush=True)
def backup_all():
"""Periodic backup of durable data and clean SQLite states to /data/apps/."""
# 1. Backup Hermes
hermes_backup = DATA_APPS / "hermes"
hermes_backup.mkdir(parents=True, exist_ok=True)
for d in DURABLE_DIRS:
src_d = LOCAL_HERMES / d
dst_d = hermes_backup / d
if src_d.exists():
try:
robust_copytree(src_d, dst_d)
except Exception as e:
print(f"[sync] Warning backing up dir {d}: {e}", file=sys.stderr)
for f in DURABLE_FILES:
src_f = LOCAL_HERMES / f
dst_f = hermes_backup / f
if src_f.exists():
try:
shutil.copy2(src_f, dst_f)
except Exception:
pass
for db_name in HERMES_DBS:
safe_sqlite_backup(LOCAL_HERMES / db_name, hermes_backup / db_name)
safe_sqlite_backup(LOCAL_HERMES / "cron" / "executions.db", hermes_backup / "cron" / "executions.db")
# 2. Backup Ekko
ekko_backup = DATA_APPS / "ekko"
ekko_backup.mkdir(parents=True, exist_ok=True)
safe_sqlite_backup(LOCAL_EKKO / "hermes-web-ui.db", ekko_backup / "hermes-web-ui.db")
for f in ["device-identity.json", "startup-tasks.json"]:
src_f = LOCAL_EKKO / f
dst_f = ekko_backup / f
if src_f.exists():
try:
shutil.copy2(src_f, dst_f)
except Exception:
pass
def daemon_loop():
print(f"🔄 Apps sync daemon started (Interval: {SYNC_INTERVAL}s)", flush=True)
# Perform immediate initial backup after restore so /data/apps is never left stale
try:
backup_all()
print("[sync] Initial backup completed.", flush=True)
except Exception as e:
print(f"[sync] Initial backup error: {e}", file=sys.stderr, flush=True)
last_hash = ""
while True:
try:
# Check if any durable files modified
hasher = hashlib.md5()
for root, _, files in os.walk(LOCAL_HERMES):
if any(x in root for x in ["scratch", "logs", "tmp", "cache", "__pycache__"]):
continue
for f in files:
if not any(f.endswith(x) for x in [".log", ".lock", ".pid", ".wal", ".shm"]):
fp = os.path.join(root, f)
try:
hasher.update(f"{f}:{os.path.getmtime(fp)}".encode())
except Exception:
pass
web_db = LOCAL_EKKO / "hermes-web-ui.db"
if web_db.exists():
try:
hasher.update(f"web_db:{web_db.stat().st_mtime}".encode())
except Exception:
pass
cur_hash = hasher.hexdigest()
if cur_hash != last_hash:
backup_all()
last_hash = cur_hash
except Exception as e:
print(f"[sync] Daemon error: {e}", file=sys.stderr)
time.sleep(SYNC_INTERVAL)
if __name__ == "__main__":
if len(sys.argv) > 1 and sys.argv[1] == "restore":
restore_all()
elif len(sys.argv) > 1 and sys.argv[1] == "backup":
backup_all()
else:
daemon_loop()