amangs commited on
Commit
4b6e8c0
·
verified ·
1 Parent(s): be3a29e

Enforce strict SYNC_INTERVAL schedule and disable rogue real-time watchdog

Browse files
Files changed (1) hide show
  1. src/data_sync.py +15 -65
src/data_sync.py CHANGED
@@ -1,7 +1,7 @@
1
  #!/usr/bin/env python3
2
  """
3
  Hermes Agent Data Sync Service
4
- Handles data persistence to/from Hugging Face Dataset
5
  """
6
 
7
  import os
@@ -18,14 +18,6 @@ from typing import Optional, Dict, List
18
  from huggingface_hub import HfApi, hf_hub_download, upload_folder
19
  from loguru import logger
20
 
21
- try:
22
- from watchdog.observers import Observer
23
- from watchdog.events import FileSystemEventHandler
24
- WATCHDOG_AVAILABLE = True
25
- except ImportError:
26
- WATCHDOG_AVAILABLE = False
27
- logger.warning("watchdog not installed, file change detection disabled")
28
-
29
 
30
  class DatasetManager:
31
  """Manages data synchronization with Hugging Face Dataset"""
@@ -62,7 +54,7 @@ class DatasetManager:
62
  'channel_directory.json',
63
  ]
64
 
65
- # STRICT exclusion: never commit volatile logs, locks, or cache
66
  self.exclude_patterns = {
67
  '__pycache__', '.git', 'node_modules', '.cache', 'tmp', 'reports', 'src',
68
  'logs', 'gateway.lock', 'gateway.pid', 'gateway_state.json',
@@ -96,8 +88,10 @@ class DatasetManager:
96
  if dp.exists():
97
  for root, _, files in os.walk(dp):
98
  for f in sorted(files):
99
- if any(f.endswith(ext) for ext in ['.log', '.lock', '.pid', '.tmp']):
100
- continue
 
 
101
  try:
102
  stat = os.stat(os.path.join(root, f))
103
  hasher.update(f"{f}:{stat.st_size}:{stat.st_mtime}".encode())
@@ -108,7 +102,7 @@ class DatasetManager:
108
  if webui_db.exists():
109
  try:
110
  stat = webui_db.stat()
111
- hasher.update(f"webui_db:{stat.st_size}:{stat.st_mtime}".encode())
112
  except:
113
  pass
114
 
@@ -127,6 +121,8 @@ class DatasetManager:
127
  for n in names:
128
  if n in self.exclude_patterns or any(n.endswith(ext) for ext in ['.log', '.lock', '.pid', '.db-shm', '.db-wal']):
129
  ignored.append(n)
 
 
130
  return ignored
131
 
132
  try:
@@ -145,7 +141,7 @@ class DatasetManager:
145
  webui_dst = self.temp_dir / 'webui_data'
146
  webui_dst.mkdir(parents=True, exist_ok=True)
147
  for f in os.listdir(self.webui_home):
148
- if f.endswith(('.db', '.token', '.json')) and not f.endswith(('.db-shm', '.db-wal', '.lock.json')):
149
  src_p = self.webui_home / f
150
  if src_p.is_file():
151
  shutil.copy2(src_p, webui_dst / f)
@@ -160,11 +156,11 @@ class DatasetManager:
160
  try:
161
  current_hash = self.calculate_data_hash()
162
  if not force and self.last_sync_hash is not None and current_hash == self.last_sync_hash:
163
- # No data changed — do not spam Hugging Face commits!
164
  return True
165
 
166
  backup_dir = self.prepare_backup_data()
167
- logger.info(f"Uploading changes to dataset: {self.dataset_repo}")
168
 
169
  self.api.upload_folder(
170
  folder_path=str(backup_dir),
@@ -242,70 +238,24 @@ class DatasetManager:
242
  logger.success("Data restoration completed")
243
 
244
 
245
- class ConfigFileHandler(FileSystemEventHandler):
246
- """Watches config and user files for real-time dataset sync"""
247
- STARTUP_GRACE_PERIOD = 45
248
-
249
- def __init__(self, manager: DatasetManager):
250
- self.manager = manager
251
- self.last_backup_time = 0
252
- self.backup_cooldown = 20
253
- self.start_time = time.time()
254
-
255
- def on_modified(self, event):
256
- if event.is_directory:
257
- return
258
-
259
- elapsed = time.time() - self.start_time
260
- if elapsed < self.STARTUP_GRACE_PERIOD:
261
- return
262
-
263
- watched_files = ['config.yaml', '.env', 'auth.json', 'hermes-web-ui.db', 'SOUL.md']
264
- if any(event.src_path.endswith(f) for f in watched_files):
265
- current_time = time.time()
266
- if current_time - self.last_backup_time > self.backup_cooldown:
267
- try:
268
- self.manager.upload_to_dataset(force=False)
269
- self.last_backup_time = current_time
270
- except Exception as e:
271
- logger.error(f"Change backup failed: {e}")
272
-
273
-
274
  def run_daemon():
275
  logger.info("Starting data sync daemon...")
276
- sync_interval = int(os.environ.get('SYNC_INTERVAL', '60'))
277
  manager = DatasetManager()
278
 
279
  if not manager.validate():
280
  logger.error("Configuration invalid, exiting")
281
  sys.exit(1)
282
 
283
- logger.info(f"Sync interval: {sync_interval} seconds")
284
-
285
- observer = None
286
- if WATCHDOG_AVAILABLE:
287
- try:
288
- event_handler = ConfigFileHandler(manager)
289
- observer = Observer()
290
- observer.schedule(event_handler, str(manager.hermes_home), recursive=False)
291
- if manager.webui_home.exists():
292
- observer.schedule(event_handler, str(manager.webui_home), recursive=False)
293
- observer.start()
294
- except Exception as e:
295
- logger.error(f"Failed to start file watcher: {e}")
296
- observer = None
297
 
298
  try:
299
  while True:
300
  time.sleep(sync_interval)
301
- # Only commits when real user data or config has changed!
302
  manager.upload_to_dataset(force=False)
303
  except KeyboardInterrupt:
304
  pass
305
- finally:
306
- if observer:
307
- observer.stop()
308
- observer.join()
309
 
310
 
311
  def main():
 
1
  #!/usr/bin/env python3
2
  """
3
  Hermes Agent Data Sync Service
4
+ Handles data persistence to/from Hugging Face Dataset strictly on SYNC_INTERVAL schedule
5
  """
6
 
7
  import os
 
18
  from huggingface_hub import HfApi, hf_hub_download, upload_folder
19
  from loguru import logger
20
 
 
 
 
 
 
 
 
 
21
 
22
  class DatasetManager:
23
  """Manages data synchronization with Hugging Face Dataset"""
 
54
  'channel_directory.json',
55
  ]
56
 
57
+ # STRICT exclusion: never commit volatile logs, locks, or temporary files
58
  self.exclude_patterns = {
59
  '__pycache__', '.git', 'node_modules', '.cache', 'tmp', 'reports', 'src',
60
  'logs', 'gateway.lock', 'gateway.pid', 'gateway_state.json',
 
88
  if dp.exists():
89
  for root, _, files in os.walk(dp):
90
  for f in sorted(files):
91
+ if any(f.endswith(ext) for ext in ['.log', '.lock', '.pid', '.tmp', '.json']):
92
+ # session request dumps / temporary jsons shouldn't trigger commit loops
93
+ if 'request_dump' in f:
94
+ continue
95
  try:
96
  stat = os.stat(os.path.join(root, f))
97
  hasher.update(f"{f}:{stat.st_size}:{stat.st_mtime}".encode())
 
102
  if webui_db.exists():
103
  try:
104
  stat = webui_db.stat()
105
+ hasher.update(f"webui_db:{stat.st_size}".encode())
106
  except:
107
  pass
108
 
 
121
  for n in names:
122
  if n in self.exclude_patterns or any(n.endswith(ext) for ext in ['.log', '.lock', '.pid', '.db-shm', '.db-wal']):
123
  ignored.append(n)
124
+ if 'request_dump' in n:
125
+ ignored.append(n)
126
  return ignored
127
 
128
  try:
 
141
  webui_dst = self.temp_dir / 'webui_data'
142
  webui_dst.mkdir(parents=True, exist_ok=True)
143
  for f in os.listdir(self.webui_home):
144
+ if f.endswith(('.db', '.token')) and not f.endswith(('.db-shm', '.db-wal', '.lock.json')):
145
  src_p = self.webui_home / f
146
  if src_p.is_file():
147
  shutil.copy2(src_p, webui_dst / f)
 
156
  try:
157
  current_hash = self.calculate_data_hash()
158
  if not force and self.last_sync_hash is not None and current_hash == self.last_sync_hash:
159
+ logger.info("No data changes detected. Skipping backup upload.")
160
  return True
161
 
162
  backup_dir = self.prepare_backup_data()
163
+ logger.info(f"Uploading scheduled backup to dataset: {self.dataset_repo}")
164
 
165
  self.api.upload_folder(
166
  folder_path=str(backup_dir),
 
238
  logger.success("Data restoration completed")
239
 
240
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
  def run_daemon():
242
  logger.info("Starting data sync daemon...")
243
+ sync_interval = int(os.environ.get('SYNC_INTERVAL', '1800'))
244
  manager = DatasetManager()
245
 
246
  if not manager.validate():
247
  logger.error("Configuration invalid, exiting")
248
  sys.exit(1)
249
 
250
+ logger.info(f"Sync interval strictly set to: {sync_interval} seconds")
 
 
 
 
 
 
 
 
 
 
 
 
 
251
 
252
  try:
253
  while True:
254
  time.sleep(sync_interval)
255
+ # Only commits on the strict schedule, AND only if real user data changed!
256
  manager.upload_to_dataset(force=False)
257
  except KeyboardInterrupt:
258
  pass
 
 
 
 
259
 
260
 
261
  def main():