amangs commited on
Commit
5c78eb5
·
1 Parent(s): 9c5ee69

Implement safe in-system /data/apps backup daemon: auto-syncs configs, skills, and clean SQLite snapshots

Browse files
entrypoint.sh CHANGED
@@ -5,13 +5,13 @@ echo "🚀 Starting REA + Codeman Mission Control Space..."
5
  echo "================================================="
6
 
7
  # 1. Directories:
8
- # - /data (S3 bucket): ONLY workspace code files, services, opencode config
9
  # - /tmp (ephemeral local disk): logs, caches, scratch files
10
  # - /home/appuser (container overlayfs): SQLite databases (hermes-web-ui.db, state.db)
11
- mkdir -p /data/{workspace,code-server,services,config/opencode}
12
  mkdir -p /tmp/{logs,.cache,code-server-cache}
13
  mkdir -p /home/appuser/{.hermes,.hermes-web-ui}
14
- chmod -R 755 /data/services
15
 
16
  # Ensure opencode global configuration persists on /data
17
  mkdir -p /root/.config
@@ -41,17 +41,16 @@ ln -sfn /data/config/opencode/opencode.json /data/workspace/.opencode/opencode.j
41
  rm -rf /data/logs /data/tmp /data/.cache /data/openclaw /data/script/codeman /data/.hermes /data/.hermes-web-ui /data/.ekko-studio /data/.ekko 2>/dev/null || true
42
 
43
  # Hermes & Ekko Studio run purely on local fast container storage (/home/appuser)
44
- mkdir -p /home/appuser/{.hermes,.hermes-web-ui}
45
  ln -sfn /home/appuser/.hermes /root/.hermes 2>/dev/null || true
46
  ln -sfn /home/appuser/.hermes-web-ui /root/.hermes-web-ui 2>/dev/null || true
47
  ln -sfn /home/appuser/.hermes-web-ui /root/.ekko 2>/dev/null || true
48
 
49
- # Restore persistent database and configs from HF Dataset backup (if dataset configured)
50
- if [ -n "$HF_DATASET_REPO" ]; then
51
- echo "🔄 Restoring persistent database and user configs from $HF_DATASET_REPO..."
52
- python3 -m src.data_sync restore || true
53
- python3 -m src.data_sync daemon &
54
- fi
55
 
56
  # Deploy clean opencode-router with assets and public UI into /data/script/opencode-router
57
  mkdir -p /data/script/opencode-router
 
5
  echo "================================================="
6
 
7
  # 1. Directories:
8
+ # - /data (S3 bucket): workspace code, services, opencode config, and /data/apps/ backups
9
  # - /tmp (ephemeral local disk): logs, caches, scratch files
10
  # - /home/appuser (container overlayfs): SQLite databases (hermes-web-ui.db, state.db)
11
+ mkdir -p /data/{workspace,code-server,services,config/opencode,apps/hermes,apps/ekko}
12
  mkdir -p /tmp/{logs,.cache,code-server-cache}
13
  mkdir -p /home/appuser/{.hermes,.hermes-web-ui}
14
+ chmod -R 755 /data/services /data/apps
15
 
16
  # Ensure opencode global configuration persists on /data
17
  mkdir -p /root/.config
 
41
  rm -rf /data/logs /data/tmp /data/.cache /data/openclaw /data/script/codeman /data/.hermes /data/.hermes-web-ui /data/.ekko-studio /data/.ekko 2>/dev/null || true
42
 
43
  # Hermes & Ekko Studio run purely on local fast container storage (/home/appuser)
44
+ mkdir -p /home/appuser/{.hermes,.hermes-web-ui} /data/apps/{hermes,ekko}
45
  ln -sfn /home/appuser/.hermes /root/.hermes 2>/dev/null || true
46
  ln -sfn /home/appuser/.hermes-web-ui /root/.hermes-web-ui 2>/dev/null || true
47
  ln -sfn /home/appuser/.hermes-web-ui /root/.ekko 2>/dev/null || true
48
 
49
+ # Restore persistent database, skills, and configs from /data/apps/ backup
50
+ python3 /app/services/apps_sync.py restore || true
51
+
52
+ # Launch continuous local background sync daemon (updates /data/apps every 60s on change)
53
+ nohup python3 /app/services/apps_sync.py > /tmp/logs/apps-sync.log 2>&1 &
 
54
 
55
  # Deploy clean opencode-router with assets and public UI into /data/script/opencode-router
56
  mkdir -p /data/script/opencode-router
services/28-hermes-agent.sh CHANGED
@@ -25,7 +25,11 @@ export GATEWAY_ALLOW_ALL_USERS=true
25
  # Seed hermes config if not present
26
  mkdir -p /home/appuser/.hermes
27
  if [ ! -f /home/appuser/.hermes/config.yaml ]; then
28
- cp /app/config/hermes.yaml /home/appuser/.hermes/config.yaml
 
 
 
 
29
  fi
30
 
31
  # Ensure custom telegram api base is set in config
 
25
  # Seed hermes config if not present
26
  mkdir -p /home/appuser/.hermes
27
  if [ ! -f /home/appuser/.hermes/config.yaml ]; then
28
+ if [ -f /data/apps/hermes/config.yaml ]; then
29
+ cp /data/apps/hermes/config.yaml /home/appuser/.hermes/config.yaml
30
+ else
31
+ cp /app/config/hermes.yaml /home/appuser/.hermes/config.yaml
32
+ fi
33
  fi
34
 
35
  # Ensure custom telegram api base is set in config
services/apps_sync.py ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Lightweight in-system backup & sync daemon for /data/apps.
4
+ - Syncs durable configs, skills, memories, and SQLite dumps between /home/appuser and /data/apps/
5
+ - Uses SQLite `.backup` API so live WAL/SHM locks NEVER touch FUSE S3 bucket
6
+ - Strictly excludes all temp directories, cache, node_modules, logs, and locks
7
+ """
8
+
9
+ import os
10
+ import sys
11
+ import time
12
+ import shutil
13
+ import sqlite3
14
+ import hashlib
15
+ from pathlib import Path
16
+ from datetime import datetime
17
+
18
+ DATA_APPS = Path("/data/apps")
19
+ LOCAL_HERMES = Path("/home/appuser/.hermes")
20
+ LOCAL_EKKO = Path("/home/appuser/.hermes-web-ui")
21
+
22
+ SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", 60))
23
+
24
+ DURABLE_DIRS = [
25
+ "memories",
26
+ "skills",
27
+ "cron",
28
+ "script",
29
+ "plugins",
30
+ "extensions",
31
+ "prompts",
32
+ "knowledge",
33
+ "profiles",
34
+ ]
35
+
36
+ DURABLE_FILES = [
37
+ "config.yaml",
38
+ ".env",
39
+ "auth.json",
40
+ "SOUL.md",
41
+ "channel_directory.json",
42
+ ]
43
+
44
+ EXCLUDE_PATTERNS = {
45
+ "__pycache__", ".git", "node_modules", ".cache", "tmp", "scratch",
46
+ "logs", "gateway.lock", "gateway.pid", "gateway_state.json",
47
+ ".db-shm", ".db-wal", "bridge.log", "server.log", ".login-lock.json"
48
+ }
49
+
50
+ def should_exclude(src: str, names: list) -> list:
51
+ ignored = []
52
+ for n in names:
53
+ if n in EXCLUDE_PATTERNS or any(n.endswith(ext) for ext in [".log", ".lock", ".pid", ".tmp", ".db-shm", ".db-wal"]):
54
+ ignored.append(n)
55
+ if "request_dump" in n or "scratch" in n or "audio_cache" in n:
56
+ ignored.append(n)
57
+ return ignored
58
+
59
+ def safe_sqlite_backup(src_db_path: Path, dst_db_path: Path):
60
+ """Perform atomic online backup of SQLite database without locking FUSE."""
61
+ if not src_db_path.exists() or src_db_path.stat().st_size == 0:
62
+ return
63
+ try:
64
+ dst_db_path.parent.mkdir(parents=True, exist_ok=True)
65
+ tmp_dst = dst_db_path.with_suffix(".tmp")
66
+ src_conn = sqlite3.connect(f"file:{src_db_path}?mode=ro", uri=True)
67
+ dst_conn = sqlite3.connect(tmp_dst)
68
+ with dst_conn:
69
+ src_conn.backup(dst_conn)
70
+ src_conn.close()
71
+ dst_conn.close()
72
+ if tmp_dst.exists() and tmp_dst.stat().st_size > 0:
73
+ tmp_dst.replace(dst_db_path)
74
+ except Exception as e:
75
+ print(f"[sync] SQLite backup error ({src_db_path.name}): {e}", file=sys.stderr)
76
+
77
+ def safe_sqlite_restore(src_db_path: Path, dst_db_path: Path):
78
+ """Restore SQLite database from backup onto local fast disk."""
79
+ if not src_db_path.exists() or src_db_path.stat().st_size == 0:
80
+ return
81
+ try:
82
+ dst_db_path.parent.mkdir(parents=True, exist_ok=True)
83
+ shutil.copy2(src_db_path, dst_db_path)
84
+ print(f"[sync] Restored SQLite database: {dst_db_path.name}")
85
+ except Exception as e:
86
+ print(f"[sync] SQLite restore error ({src_db_path.name}): {e}", file=sys.stderr)
87
+
88
+ def restore_all():
89
+ """Run on startup: restore /data/apps/hermes and /data/apps/ekko into local storage."""
90
+ print("🔄 Restoring durable configs & databases from /data/apps/...")
91
+
92
+ # 1. Restore Hermes
93
+ hermes_backup = DATA_APPS / "hermes"
94
+ if hermes_backup.exists():
95
+ for d in DURABLE_DIRS:
96
+ src_d = hermes_backup / d
97
+ dst_d = LOCAL_HERMES / d
98
+ if src_d.exists():
99
+ shutil.copytree(src_d, dst_d, dirs_exist_ok=True, ignore=should_exclude)
100
+ for f in DURABLE_FILES:
101
+ src_f = hermes_backup / f
102
+ dst_f = LOCAL_HERMES / f
103
+ if src_f.exists() and not dst_f.exists():
104
+ shutil.copy2(src_f, dst_f)
105
+ safe_sqlite_restore(hermes_backup / "state.db", LOCAL_HERMES / "state.db")
106
+
107
+ # 2. Restore Ekko
108
+ ekko_backup = DATA_APPS / "ekko"
109
+ if ekko_backup.exists():
110
+ safe_sqlite_restore(ekko_backup / "hermes-web-ui.db", LOCAL_EKKO / "hermes-web-ui.db")
111
+ for f in ["device-identity.json", "startup-tasks.json"]:
112
+ src_f = ekko_backup / f
113
+ dst_f = LOCAL_EKKO / f
114
+ if src_f.exists() and not dst_f.exists():
115
+ shutil.copy2(src_f, dst_f)
116
+
117
+ print("✅ Restore from /data/apps/ complete!")
118
+
119
+ def backup_all():
120
+ """Periodic backup of durable data and clean SQLite states to /data/apps/."""
121
+ # 1. Backup Hermes
122
+ hermes_backup = DATA_APPS / "hermes"
123
+ hermes_backup.mkdir(parents=True, exist_ok=True)
124
+
125
+ for d in DURABLE_DIRS:
126
+ src_d = LOCAL_HERMES / d
127
+ dst_d = hermes_backup / d
128
+ if src_d.exists():
129
+ shutil.copytree(src_d, dst_d, dirs_exist_ok=True, ignore=should_exclude)
130
+
131
+ for f in DURABLE_FILES:
132
+ src_f = LOCAL_HERMES / f
133
+ dst_f = hermes_backup / f
134
+ if src_f.exists():
135
+ try:
136
+ shutil.copy2(src_f, dst_f)
137
+ except Exception:
138
+ pass
139
+
140
+ safe_sqlite_backup(LOCAL_HERMES / "state.db", hermes_backup / "state.db")
141
+
142
+ # 2. Backup Ekko
143
+ ekko_backup = DATA_APPS / "ekko"
144
+ ekko_backup.mkdir(parents=True, exist_ok=True)
145
+
146
+ safe_sqlite_backup(LOCAL_EKKO / "hermes-web-ui.db", ekko_backup / "hermes-web-ui.db")
147
+ for f in ["device-identity.json", "startup-tasks.json"]:
148
+ src_f = LOCAL_EKKO / f
149
+ dst_f = ekko_backup / f
150
+ if src_f.exists():
151
+ try:
152
+ shutil.copy2(src_f, dst_f)
153
+ except Exception:
154
+ pass
155
+
156
+ def daemon_loop():
157
+ print(f"🔄 Apps sync daemon started (Interval: {SYNC_INTERVAL}s)")
158
+ last_hash = ""
159
+ while True:
160
+ try:
161
+ # Check if any durable files modified
162
+ hasher = hashlib.md5()
163
+ for root, _, files in os.walk(LOCAL_HERMES):
164
+ if any(x in root for x in ["scratch", "logs", "tmp", "cache", "__pycache__"]):
165
+ continue
166
+ for f in files:
167
+ if not any(f.endswith(x) for x in [".log", ".lock", ".pid", ".wal", ".shm"]):
168
+ fp = os.path.join(root, f)
169
+ try:
170
+ hasher.update(f"{f}:{os.path.getmtime(fp)}".encode())
171
+ except Exception:
172
+ pass
173
+
174
+ web_db = LOCAL_EKKO / "hermes-web-ui.db"
175
+ if web_db.exists():
176
+ try:
177
+ hasher.update(f"web_db:{web_db.stat().st_mtime}".encode())
178
+ except Exception:
179
+ pass
180
+
181
+ cur_hash = hasher.hexdigest()
182
+ if cur_hash != last_hash:
183
+ backup_all()
184
+ last_hash = cur_hash
185
+ except Exception as e:
186
+ print(f"[sync] Daemon error: {e}", file=sys.stderr)
187
+
188
+ time.sleep(SYNC_INTERVAL)
189
+
190
+ if __name__ == "__main__":
191
+ if len(sys.argv) > 1 and sys.argv[1] == "restore":
192
+ restore_all()
193
+ elif len(sys.argv) > 1 and sys.argv[1] == "backup":
194
+ backup_all()
195
+ else:
196
+ restore_all()
197
+ daemon_loop()