amangs commited on
Commit
9c5ee69
·
1 Parent(s): 7a19fe4

Integrate data_sync daemon to persist SQLite databases to HF Dataset instead of corrupting FUSE bucket

Browse files
Dockerfile CHANGED
@@ -154,6 +154,9 @@ ENV REA_UNBLOB_COMMAND="/usr/local/bin/unblob"
154
  ENV REA_FIRMWARE_PRLIMIT_COMMAND="/usr/bin/prlimit"
155
  ENV REA_PWNTOOLS_PYTHON="/usr/bin/python3"
156
 
 
 
 
157
  # ------------------------------------------
158
  # 8. Hermes Agent Core & Caddy Server
159
  # ------------------------------------------
@@ -232,6 +235,7 @@ RUN npm config set ignore-scripts false && \
232
  WORKDIR /app
233
  COPY config/ /app/config/
234
  COPY services/ /app/services/
 
235
  COPY entrypoint.sh /app/entrypoint.sh
236
  RUN chmod +x /app/entrypoint.sh /app/services/*.sh && mkdir -p /data
237
 
 
154
  ENV REA_FIRMWARE_PRLIMIT_COMMAND="/usr/bin/prlimit"
155
  ENV REA_PWNTOOLS_PYTHON="/usr/bin/python3"
156
 
157
+ # Install loguru for data_sync
158
+ RUN pip3 install --no-cache-dir --break-system-packages loguru
159
+
160
  # ------------------------------------------
161
  # 8. Hermes Agent Core & Caddy Server
162
  # ------------------------------------------
 
235
  WORKDIR /app
236
  COPY config/ /app/config/
237
  COPY services/ /app/services/
238
+ COPY src/ /app/src/
239
  COPY entrypoint.sh /app/entrypoint.sh
240
  RUN chmod +x /app/entrypoint.sh /app/services/*.sh && mkdir -p /data
241
 
entrypoint.sh CHANGED
@@ -46,6 +46,13 @@ ln -sfn /home/appuser/.hermes /root/.hermes 2>/dev/null || true
46
  ln -sfn /home/appuser/.hermes-web-ui /root/.hermes-web-ui 2>/dev/null || true
47
  ln -sfn /home/appuser/.hermes-web-ui /root/.ekko 2>/dev/null || true
48
 
 
 
 
 
 
 
 
49
  # Deploy clean opencode-router with assets and public UI into /data/script/opencode-router
50
  mkdir -p /data/script/opencode-router
51
  cp -rn /opt/opencode-router/* /data/script/opencode-router/ 2>/dev/null || true
 
46
  ln -sfn /home/appuser/.hermes-web-ui /root/.hermes-web-ui 2>/dev/null || true
47
  ln -sfn /home/appuser/.hermes-web-ui /root/.ekko 2>/dev/null || true
48
 
49
+ # Restore persistent database and configs from HF Dataset backup (if dataset configured)
50
+ if [ -n "$HF_DATASET_REPO" ]; then
51
+ echo "🔄 Restoring persistent database and user configs from $HF_DATASET_REPO..."
52
+ python3 -m src.data_sync restore || true
53
+ python3 -m src.data_sync daemon &
54
+ fi
55
+
56
  # Deploy clean opencode-router with assets and public UI into /data/script/opencode-router
57
  mkdir -p /data/script/opencode-router
58
  cp -rn /opt/opencode-router/* /data/script/opencode-router/ 2>/dev/null || true
src/__init__.py ADDED
File without changes
src/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (129 Bytes). View file
 
src/__pycache__/data_sync.cpython-311.pyc ADDED
Binary file (16.7 kB). View file
 
src/data_sync.py ADDED
@@ -0,0 +1,283 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Hermes Agent Data Sync Service
4
+ Handles data persistence to/from Hugging Face Dataset strictly on SYNC_INTERVAL schedule
5
+ """
6
+
7
+ import os
8
+ import sys
9
+ import time
10
+ import json
11
+ import shutil
12
+ import hashlib
13
+ import argparse
14
+ from pathlib import Path
15
+ from datetime import datetime
16
+ from typing import Optional, Dict, List
17
+
18
+ from huggingface_hub import HfApi, hf_hub_download, upload_folder
19
+ from loguru import logger
20
+
21
+
22
+ class DatasetManager:
23
+ """Manages data synchronization with Hugging Face Dataset"""
24
+
25
+ def __init__(self, dataset_repo: Optional[str] = None, token: Optional[str] = None):
26
+ self.dataset_repo = dataset_repo or os.environ.get('HF_DATASET_REPO')
27
+ self.token = token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')
28
+ self.api = HfApi(token=self.token)
29
+ self.hermes_home = Path(os.environ.get('HERMES_HOME', '/home/appuser/.hermes'))
30
+ self.webui_home = Path(os.environ.get('HERMES_WEB_UI_HOME', '/home/appuser/.hermes-web-ui'))
31
+ self.temp_dir = Path('/tmp/hermes_sync')
32
+ self.last_sync_hash = None
33
+
34
+ # Directories that represent REAL persistent user data
35
+ self.backup_dirs = [
36
+ 'memories',
37
+ 'skills',
38
+ 'sessions',
39
+ 'cron',
40
+ 'script',
41
+ 'plugins',
42
+ 'extensions',
43
+ 'prompts',
44
+ 'knowledge',
45
+ ]
46
+
47
+ # Files that represent REAL persistent user config
48
+ self.backup_files = [
49
+ 'config.yaml',
50
+ '.env',
51
+ 'auth.json',
52
+ 'SOUL.md',
53
+ 'state.db',
54
+ 'channel_directory.json',
55
+ ]
56
+
57
+ # STRICT exclusion: never commit volatile logs, locks, or temporary files
58
+ self.exclude_patterns = {
59
+ '__pycache__', '.git', 'node_modules', '.cache', 'tmp', 'reports', 'src',
60
+ 'logs', 'gateway.lock', 'gateway.pid', 'gateway_state.json',
61
+ '*.log', '*.db-shm', '*.db-wal', 'bridge.log', 'server.log',
62
+ '.login-lock.json', 'preview-action.log'
63
+ }
64
+
65
+ def validate(self) -> bool:
66
+ if not self.dataset_repo:
67
+ logger.error("HF_DATASET_REPO not set")
68
+ return False
69
+ if not self.token:
70
+ logger.warning("HF_TOKEN not set")
71
+ return True
72
+
73
+ def calculate_data_hash(self) -> str:
74
+ """Calculate hash of actual user data (sessions, memories, skills, config, webui users)"""
75
+ hasher = hashlib.md5()
76
+
77
+ for fn in self.backup_files:
78
+ fp = self.hermes_home / fn
79
+ if fp.exists() and fp.is_file():
80
+ try:
81
+ stat = fp.stat()
82
+ hasher.update(f"{fn}:{stat.st_size}:{stat.st_mtime}".encode())
83
+ except:
84
+ pass
85
+
86
+ for dname in self.backup_dirs:
87
+ dp = self.hermes_home / dname
88
+ if dp.exists():
89
+ for root, _, files in os.walk(dp):
90
+ for f in sorted(files):
91
+ if any(f.endswith(ext) for ext in ['.log', '.lock', '.pid', '.tmp', '.json']):
92
+ # session request dumps / temporary jsons shouldn't trigger commit loops
93
+ if 'request_dump' in f:
94
+ continue
95
+ try:
96
+ stat = os.stat(os.path.join(root, f))
97
+ hasher.update(f"{f}:{stat.st_size}:{stat.st_mtime}".encode())
98
+ except:
99
+ pass
100
+
101
+ webui_db = self.webui_home / 'hermes-web-ui.db'
102
+ if webui_db.exists():
103
+ try:
104
+ stat = webui_db.stat()
105
+ hasher.update(f"webui_db:{stat.st_size}".encode())
106
+ except:
107
+ pass
108
+
109
+ return hasher.hexdigest()
110
+
111
+ def prepare_backup_data(self) -> Path:
112
+ if self.temp_dir.exists():
113
+ shutil.rmtree(self.temp_dir)
114
+ self.temp_dir.mkdir(parents=True)
115
+
116
+ for dirname in self.backup_dirs:
117
+ (self.temp_dir / dirname).mkdir(exist_ok=True)
118
+
119
+ def should_exclude(src: str, names: list) -> bool:
120
+ ignored = []
121
+ for n in names:
122
+ if n in self.exclude_patterns or any(n.endswith(ext) for ext in ['.log', '.lock', '.pid', '.db-shm', '.db-wal']):
123
+ ignored.append(n)
124
+ if 'request_dump' in n:
125
+ ignored.append(n)
126
+ return ignored
127
+
128
+ try:
129
+ for dirname in self.backup_dirs:
130
+ src_dir = self.hermes_home / dirname
131
+ if src_dir.exists():
132
+ shutil.copytree(src_dir, self.temp_dir / dirname, dirs_exist_ok=True, ignore=should_exclude)
133
+
134
+ for filename in self.backup_files:
135
+ src_file = self.hermes_home / filename
136
+ if src_file.exists():
137
+ shutil.copy2(src_file, self.temp_dir / filename)
138
+
139
+ # Backup Web UI sqlite db and tokens only (exclude logs/locks)
140
+ if self.webui_home.exists():
141
+ webui_dst = self.temp_dir / 'webui_data'
142
+ webui_dst.mkdir(parents=True, exist_ok=True)
143
+ for f in os.listdir(self.webui_home):
144
+ if f.endswith(('.db', '.token')) and not f.endswith(('.db-shm', '.db-wal', '.lock.json')):
145
+ src_p = self.webui_home / f
146
+ if src_p.is_file():
147
+ shutil.copy2(src_p, webui_dst / f)
148
+
149
+ return self.temp_dir
150
+
151
+ except Exception as e:
152
+ logger.error(f"Failed to prepare backup: {e}")
153
+ raise
154
+
155
+ def upload_to_dataset(self, force: bool = False) -> bool:
156
+ try:
157
+ current_hash = self.calculate_data_hash()
158
+ if not force and self.last_sync_hash is not None and current_hash == self.last_sync_hash:
159
+ logger.info("No data changes detected. Skipping backup upload.")
160
+ return True
161
+
162
+ backup_dir = self.prepare_backup_data()
163
+ logger.info(f"Uploading scheduled backup to dataset: {self.dataset_repo}")
164
+
165
+ self.api.upload_folder(
166
+ folder_path=str(backup_dir),
167
+ repo_id=self.dataset_repo,
168
+ repo_type="dataset",
169
+ commit_message=f"Hermes Agent backup - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
170
+ )
171
+
172
+ # Squash commit history so the dataset repo always stays at 1 commit
173
+ try:
174
+ self.api.super_squash_history(
175
+ repo_id=self.dataset_repo,
176
+ repo_type="dataset",
177
+ commit_message=f"Hermes Agent backup - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
178
+ )
179
+ except Exception as se:
180
+ logger.warning(f"Squash history failed: {se}")
181
+
182
+ self.last_sync_hash = current_hash
183
+ logger.success("Backup uploaded successfully")
184
+ return True
185
+
186
+ except Exception as e:
187
+ logger.error(f"Failed to upload to dataset: {e}")
188
+ return False
189
+
190
+ def download_from_dataset(self) -> bool:
191
+ try:
192
+ logger.info(f"Downloading from dataset: {self.dataset_repo}")
193
+ download_dir = Path('/tmp/hermes_download')
194
+ if download_dir.exists():
195
+ shutil.rmtree(download_dir)
196
+ download_dir.mkdir(parents=True)
197
+
198
+ self.api.snapshot_download(
199
+ repo_id=self.dataset_repo,
200
+ repo_type="dataset",
201
+ local_dir=str(download_dir)
202
+ )
203
+
204
+ self.restore_from_download(download_dir)
205
+ return True
206
+ except Exception as e:
207
+ logger.error(f"Failed to download from dataset: {e}")
208
+ return False
209
+
210
+ def restore_from_download(self, download_dir: Path):
211
+ self.hermes_home.mkdir(parents=True, exist_ok=True)
212
+ self.webui_home.mkdir(parents=True, exist_ok=True)
213
+
214
+ skip_restore = os.environ.get('SKIP_CONFIG_RESTORE', 'true').lower() in ('true', '1', 'yes')
215
+
216
+ restore_list = []
217
+ for dirname in self.backup_dirs:
218
+ restore_list.append((dirname, self.hermes_home / dirname))
219
+ for filename in self.backup_files:
220
+ restore_list.append((filename, self.hermes_home / filename))
221
+
222
+ if not skip_restore:
223
+ restore_list.append(('config.yaml', self.hermes_home / 'config.yaml'))
224
+ else:
225
+ restored_path = self.hermes_home / 'config.yaml.restored'
226
+ src = download_dir / 'config.yaml'
227
+ if src.exists():
228
+ shutil.copy2(src, restored_path)
229
+
230
+ for src_rel, dst in restore_list:
231
+ src = download_dir / src_rel
232
+ if src.exists():
233
+ try:
234
+ if src.is_file():
235
+ dst.parent.mkdir(parents=True, exist_ok=True)
236
+ shutil.copy2(src, dst)
237
+ elif src.is_dir():
238
+ if dst.exists():
239
+ shutil.rmtree(dst)
240
+ shutil.copytree(src, dst)
241
+ except Exception as e:
242
+ logger.error(f"Failed to restore {src_rel}: {e}")
243
+
244
+ webui_src = download_dir / 'webui_data'
245
+ if webui_src.exists():
246
+ shutil.copytree(webui_src, self.webui_home, dirs_exist_ok=True)
247
+
248
+ logger.success("Data restoration completed")
249
+
250
+
251
+ def run_daemon():
252
+ logger.info("Starting data sync daemon...")
253
+ sync_interval = int(os.environ.get('SYNC_INTERVAL', '1800'))
254
+ manager = DatasetManager()
255
+
256
+ if not manager.validate():
257
+ logger.error("Configuration invalid, exiting")
258
+ sys.exit(1)
259
+
260
+ logger.info(f"Sync interval strictly set to: {sync_interval} seconds")
261
+
262
+ try:
263
+ while True:
264
+ time.sleep(sync_interval)
265
+ # Only commits on the strict schedule, AND only if real user data changed!
266
+ manager.upload_to_dataset(force=False)
267
+ except KeyboardInterrupt:
268
+ pass
269
+
270
+
271
+ def main():
272
+ parser = argparse.ArgumentParser(description='Hermes Agent Data Sync')
273
+ parser.add_argument('action', choices=['backup', 'restore', 'daemon'], help='Action')
274
+ parser.add_argument('--force', '-f', action='store_true', help='Force')
275
+ args = parser.parse_args()
276
+ manager = DatasetManager()
277
+ if not manager.validate(): sys.exit(1)
278
+ if args.action == 'backup': sys.exit(0 if manager.upload_to_dataset(force=args.force) else 1)
279
+ elif args.action == 'restore': sys.exit(0 if manager.download_from_dataset() else 1)
280
+ elif args.action == 'daemon': run_daemon()
281
+
282
+ if __name__ == '__main__':
283
+ main()