amangs commited on
Commit
577ccfd
Β·
1 Parent(s): 3d09fff

Fix apps_sync daemon: make copytree resilient to broken symlinks and run daemon_loop without duplicate blocking restore

Browse files
Files changed (1) hide show
  1. services/apps_sync.py +60 -6
services/apps_sync.py CHANGED
@@ -62,6 +62,13 @@ def should_exclude(src: str, names: list) -> list:
62
  ignored.append(n)
63
  if "request_dump" in n or "scratch" in n or "audio_cache" in n:
64
  ignored.append(n)
 
 
 
 
 
 
 
65
  return ignored
66
 
67
  def safe_sqlite_backup(src_db_path: Path, dst_db_path: Path):
@@ -93,6 +100,35 @@ def safe_sqlite_restore(src_db_path: Path, dst_db_path: Path):
93
  except Exception as e:
94
  print(f"[sync] SQLite restore error ({src_db_path.name}): {e}", file=sys.stderr)
95
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  def restore_all():
97
  """Run on startup: restore /data/apps/hermes and /data/apps/ekko into local storage."""
98
  print("πŸ”„ Restoring durable configs & databases from /data/apps/...")
@@ -104,12 +140,18 @@ def restore_all():
104
  src_d = hermes_backup / d
105
  dst_d = LOCAL_HERMES / d
106
  if src_d.exists():
107
- shutil.copytree(src_d, dst_d, dirs_exist_ok=True, ignore=should_exclude)
 
 
 
108
  for f in DURABLE_FILES:
109
  src_f = hermes_backup / f
110
  dst_f = LOCAL_HERMES / f
111
  if src_f.exists():
112
- shutil.copy2(src_f, dst_f)
 
 
 
113
  for db_name in HERMES_DBS:
114
  safe_sqlite_restore(hermes_backup / db_name, LOCAL_HERMES / db_name)
115
  safe_sqlite_restore(hermes_backup / "cron" / "executions.db", LOCAL_HERMES / "cron" / "executions.db")
@@ -122,7 +164,10 @@ def restore_all():
122
  src_f = ekko_backup / f
123
  dst_f = LOCAL_EKKO / f
124
  if src_f.exists():
125
- shutil.copy2(src_f, dst_f)
 
 
 
126
 
127
  print("βœ… Restore from /data/apps/ complete!")
128
 
@@ -136,7 +181,10 @@ def backup_all():
136
  src_d = LOCAL_HERMES / d
137
  dst_d = hermes_backup / d
138
  if src_d.exists():
139
- shutil.copytree(src_d, dst_d, dirs_exist_ok=True, ignore=should_exclude)
 
 
 
140
 
141
  for f in DURABLE_FILES:
142
  src_f = LOCAL_HERMES / f
@@ -166,7 +214,14 @@ def backup_all():
166
  pass
167
 
168
  def daemon_loop():
169
- print(f"πŸ”„ Apps sync daemon started (Interval: {SYNC_INTERVAL}s)")
 
 
 
 
 
 
 
170
  last_hash = ""
171
  while True:
172
  try:
@@ -205,5 +260,4 @@ if __name__ == "__main__":
205
  elif len(sys.argv) > 1 and sys.argv[1] == "backup":
206
  backup_all()
207
  else:
208
- restore_all()
209
  daemon_loop()
 
62
  ignored.append(n)
63
  if "request_dump" in n or "scratch" in n or "audio_cache" in n:
64
  ignored.append(n)
65
+ # Exclude broken/dangling symlinks from copytree traversal
66
+ try:
67
+ full_p = os.path.join(src, n)
68
+ if os.path.islink(full_p) and not os.path.exists(full_p):
69
+ ignored.append(n)
70
+ except Exception:
71
+ pass
72
  return ignored
73
 
74
  def safe_sqlite_backup(src_db_path: Path, dst_db_path: Path):
 
100
  except Exception as e:
101
  print(f"[sync] SQLite restore error ({src_db_path.name}): {e}", file=sys.stderr)
102
 
103
+ def robust_copytree(src: Path, dst: Path):
104
+ """Recursively copy directory tree, resilient to broken symlinks, vanished files, or FUSE I/O hiccups."""
105
+ if not src.exists():
106
+ return
107
+ dst.mkdir(parents=True, exist_ok=True)
108
+ try:
109
+ shutil.copytree(src, dst, dirs_exist_ok=True, ignore=should_exclude, ignore_dangling_symlinks=True)
110
+ except Exception as e:
111
+ # Fallback manual file-by-file copy ignoring vanishing/broken files
112
+ for root, dirs, files in os.walk(src):
113
+ rel = os.path.relpath(root, src)
114
+ target_dir = dst / rel if rel != "." else dst
115
+ target_dir.mkdir(parents=True, exist_ok=True)
116
+ for f in files:
117
+ sf = os.path.join(root, f)
118
+ df = target_dir / f
119
+ try:
120
+ if os.path.islink(sf):
121
+ if not os.path.exists(sf):
122
+ continue # skip dangling symlink
123
+ link_target = os.readlink(sf)
124
+ if df.is_symlink() or df.exists():
125
+ df.unlink(missing_ok=True)
126
+ os.symlink(link_target, df)
127
+ elif os.path.isfile(sf):
128
+ shutil.copy2(sf, df)
129
+ except Exception:
130
+ pass
131
+
132
  def restore_all():
133
  """Run on startup: restore /data/apps/hermes and /data/apps/ekko into local storage."""
134
  print("πŸ”„ Restoring durable configs & databases from /data/apps/...")
 
140
  src_d = hermes_backup / d
141
  dst_d = LOCAL_HERMES / d
142
  if src_d.exists():
143
+ try:
144
+ robust_copytree(src_d, dst_d)
145
+ except Exception as e:
146
+ print(f"[sync] Warning restoring dir {d}: {e}", file=sys.stderr)
147
  for f in DURABLE_FILES:
148
  src_f = hermes_backup / f
149
  dst_f = LOCAL_HERMES / f
150
  if src_f.exists():
151
+ try:
152
+ shutil.copy2(src_f, dst_f)
153
+ except Exception as e:
154
+ print(f"[sync] Warning restoring file {f}: {e}", file=sys.stderr)
155
  for db_name in HERMES_DBS:
156
  safe_sqlite_restore(hermes_backup / db_name, LOCAL_HERMES / db_name)
157
  safe_sqlite_restore(hermes_backup / "cron" / "executions.db", LOCAL_HERMES / "cron" / "executions.db")
 
164
  src_f = ekko_backup / f
165
  dst_f = LOCAL_EKKO / f
166
  if src_f.exists():
167
+ try:
168
+ shutil.copy2(src_f, dst_f)
169
+ except Exception:
170
+ pass
171
 
172
  print("βœ… Restore from /data/apps/ complete!")
173
 
 
181
  src_d = LOCAL_HERMES / d
182
  dst_d = hermes_backup / d
183
  if src_d.exists():
184
+ try:
185
+ robust_copytree(src_d, dst_d)
186
+ except Exception as e:
187
+ print(f"[sync] Warning backing up dir {d}: {e}", file=sys.stderr)
188
 
189
  for f in DURABLE_FILES:
190
  src_f = LOCAL_HERMES / f
 
214
  pass
215
 
216
  def daemon_loop():
217
+ print(f"πŸ”„ Apps sync daemon started (Interval: {SYNC_INTERVAL}s)", flush=True)
218
+ # Perform immediate initial backup after restore so /data/apps is never left stale
219
+ try:
220
+ backup_all()
221
+ print("[sync] Initial backup completed.", flush=True)
222
+ except Exception as e:
223
+ print(f"[sync] Initial backup error: {e}", file=sys.stderr, flush=True)
224
+
225
  last_hash = ""
226
  while True:
227
  try:
 
260
  elif len(sys.argv) > 1 and sys.argv[1] == "backup":
261
  backup_all()
262
  else:
 
263
  daemon_loop()