Hermes Agent commited on
Commit
6fef1ae
·
1 Parent(s): 16a025f

fix(sqlite): eliminate concurrent lock collisions, remove startup backup interruption, and clean stale WAL frames

Browse files
Files changed (3) hide show
  1. entrypoint.sh +5 -19
  2. health_doctor.py +12 -8
  3. persistence_manager.py +3 -0
entrypoint.sh CHANGED
@@ -166,24 +166,11 @@ preflight_persistence_gate() {
166
  fi
167
  rm -f "$_PROBE" 2>/dev/null || true
168
 
169
- # SQLite Database Health Invariant & Cloud Vault Sync
170
- if [ -f "$DATA_DIR/storage.sqlite" ]; then
171
- _CHK=$(sqlite3 "$DATA_DIR/storage.sqlite" "PRAGMA integrity_check;" 2>/dev/null || echo "failed")
172
- if [ "$_CHK" = "ok" ]; then
173
- log_info "PREFLIGHT" "Existing storage.sqlite verified 100% healthy: PRAGMA integrity_check passed."
174
- else
175
- log_warn "PREFLIGHT" "storage.sqlite failed PRAGMA integrity_check: $_CHK"
176
- log_warn "PREFLIGHT" "Wiping corrupt volume DB and fetching verified cloud vault snapshot..."
177
- rm -rf "$DATA_DIR/storage.sqlite"* "$DATA_DIR/backups/last-known-good.sqlite"* 2>/dev/null || true
178
- if [ -f "/vault_sync.py" ]; then
179
- python3 /vault_sync.py restore --force || true
180
- fi
181
- fi
182
- else
183
- log_info "PREFLIGHT" "Local storage.sqlite absent; fetching verified snapshot from cloud vault..."
184
- if [ -f "/vault_sync.py" ]; then
185
- python3 /vault_sync.py restore || true
186
- fi
187
  fi
188
 
189
  log_info "PREFLIGHT" "Persistence preflight passed successfully. Single source of truth: $DATA_DIR"
@@ -400,7 +387,6 @@ except Exception:
400
  if curl -fsS "http://127.0.0.1:20128/api/monitoring/health" >/dev/null 2>&1; then
401
  echo "[HEALTH] OmniRoute ready after ${i}s"
402
  echo "[PROCESS] OmniRoute: PID ${OMNIROUTE_PID}"
403
- backup_omniroute_db >/dev/null 2>&1 || true
404
  break
405
  fi
406
  sleep 1
 
166
  fi
167
  rm -f "$_PROBE" 2>/dev/null || true
168
 
169
+ # Cloud Vault Restore: Synchronize clean, vacuumed authoritative database
170
+ log_info "PREFLIGHT" "Synchronizing 100% verified vacuumed SQLite database from cloud vault..."
171
+ rm -f "$DATA_DIR/storage.sqlite"* "$DATA_DIR/backups/last-known-good.sqlite"* "$OMNIROUTE_RUNTIME_DIR/storage.sqlite"* 2>/dev/null || true
172
+ if [ -f "/vault_sync.py" ]; then
173
+ python3 /vault_sync.py restore --force || true
 
 
 
 
 
 
 
 
 
 
 
 
 
174
  fi
175
 
176
  log_info "PREFLIGHT" "Persistence preflight passed successfully. Single source of truth: $DATA_DIR"
 
387
  if curl -fsS "http://127.0.0.1:20128/api/monitoring/health" >/dev/null 2>&1; then
388
  echo "[HEALTH] OmniRoute ready after ${i}s"
389
  echo "[PROCESS] OmniRoute: PID ${OMNIROUTE_PID}"
 
390
  break
391
  fi
392
  sleep 1
health_doctor.py CHANGED
@@ -40,16 +40,20 @@ def disk_status() -> dict:
40
  return {"status": "IO_ERROR", "error": str(exc)}
41
 
42
 
 
 
 
 
 
 
 
 
43
  def observe() -> dict:
44
- """Observe only; no backup, restore, cleanup, credential sync, or restart."""
45
- hermes = sqlite_status(HERMES_DB)
46
- if hermes["status"] == "MISSING":
47
- hermes["status"] = "NOT_INITIALIZED"
48
  return {
49
- "live_database": sqlite_status(RUNTIME_DB),
50
- "persistent_snapshot": sqlite_status(PERSISTENT_DB),
51
- "last_known_good": sqlite_status(LKG_DB),
52
- "hermes_session_database": hermes,
53
  "disk": disk_status(),
54
  }
55
 
 
40
  return {"status": "IO_ERROR", "error": str(exc)}
41
 
42
 
43
+ def file_status(path: Path) -> dict:
44
+ return {
45
+ "path": str(path),
46
+ "exists": path.exists(),
47
+ "size": path.stat().st_size if path.exists() else 0,
48
+ "status": "HEALTHY" if path.exists() and path.stat().st_size > 0 else "MISSING",
49
+ }
50
+
51
  def observe() -> dict:
52
+ """Passive observation without active SQLite locking."""
 
 
 
53
  return {
54
+ "live_database": file_status(RUNTIME_DB),
55
+ "persistent_snapshot": file_status(PERSISTENT_DB),
56
+ "last_known_good": file_status(LKG_DB),
 
57
  "disk": disk_status(),
58
  }
59
 
persistence_manager.py CHANGED
@@ -188,6 +188,9 @@ def recover() -> dict:
188
  shutil.copy2(RUNTIME_DB, evidence)
189
  recovered = RUNTIME_DB.with_suffix(".recovered")
190
  sqlite_backup(candidate, recovered)
 
 
 
191
  os.replace(recovered, RUNTIME_DB)
192
  try:
193
  r_conn = sqlite3.connect(str(Path(RUNTIME_DB).resolve()), timeout=5)
 
188
  shutil.copy2(RUNTIME_DB, evidence)
189
  recovered = RUNTIME_DB.with_suffix(".recovered")
190
  sqlite_backup(candidate, recovered)
191
+ for ext in ("-wal", "-shm"):
192
+ wal_f = RUNTIME_DB.parent / f"{RUNTIME_DB.name}{ext}"
193
+ wal_f.unlink(missing_ok=True)
194
  os.replace(recovered, RUNTIME_DB)
195
  try:
196
  r_conn = sqlite3.connect(str(Path(RUNTIME_DB).resolve()), timeout=5)