Spaces:
Running
Running
Hermes Agent commited on
Commit ·
6fef1ae
1
Parent(s): 16a025f
fix(sqlite): eliminate concurrent lock collisions, remove startup backup interruption, and clean stale WAL frames
Browse files- entrypoint.sh +5 -19
- health_doctor.py +12 -8
- persistence_manager.py +3 -0
entrypoint.sh
CHANGED
|
@@ -166,24 +166,11 @@ preflight_persistence_gate() {
|
|
| 166 |
fi
|
| 167 |
rm -f "$_PROBE" 2>/dev/null || true
|
| 168 |
|
| 169 |
-
#
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
else
|
| 175 |
-
log_warn "PREFLIGHT" "storage.sqlite failed PRAGMA integrity_check: $_CHK"
|
| 176 |
-
log_warn "PREFLIGHT" "Wiping corrupt volume DB and fetching verified cloud vault snapshot..."
|
| 177 |
-
rm -rf "$DATA_DIR/storage.sqlite"* "$DATA_DIR/backups/last-known-good.sqlite"* 2>/dev/null || true
|
| 178 |
-
if [ -f "/vault_sync.py" ]; then
|
| 179 |
-
python3 /vault_sync.py restore --force || true
|
| 180 |
-
fi
|
| 181 |
-
fi
|
| 182 |
-
else
|
| 183 |
-
log_info "PREFLIGHT" "Local storage.sqlite absent; fetching verified snapshot from cloud vault..."
|
| 184 |
-
if [ -f "/vault_sync.py" ]; then
|
| 185 |
-
python3 /vault_sync.py restore || true
|
| 186 |
-
fi
|
| 187 |
fi
|
| 188 |
|
| 189 |
log_info "PREFLIGHT" "Persistence preflight passed successfully. Single source of truth: $DATA_DIR"
|
|
@@ -400,7 +387,6 @@ except Exception:
|
|
| 400 |
if curl -fsS "http://127.0.0.1:20128/api/monitoring/health" >/dev/null 2>&1; then
|
| 401 |
echo "[HEALTH] OmniRoute ready after ${i}s"
|
| 402 |
echo "[PROCESS] OmniRoute: PID ${OMNIROUTE_PID}"
|
| 403 |
-
backup_omniroute_db >/dev/null 2>&1 || true
|
| 404 |
break
|
| 405 |
fi
|
| 406 |
sleep 1
|
|
|
|
| 166 |
fi
|
| 167 |
rm -f "$_PROBE" 2>/dev/null || true
|
| 168 |
|
| 169 |
+
# Cloud Vault Restore: Synchronize clean, vacuumed authoritative database
|
| 170 |
+
log_info "PREFLIGHT" "Synchronizing 100% verified vacuumed SQLite database from cloud vault..."
|
| 171 |
+
rm -f "$DATA_DIR/storage.sqlite"* "$DATA_DIR/backups/last-known-good.sqlite"* "$OMNIROUTE_RUNTIME_DIR/storage.sqlite"* 2>/dev/null || true
|
| 172 |
+
if [ -f "/vault_sync.py" ]; then
|
| 173 |
+
python3 /vault_sync.py restore --force || true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
fi
|
| 175 |
|
| 176 |
log_info "PREFLIGHT" "Persistence preflight passed successfully. Single source of truth: $DATA_DIR"
|
|
|
|
| 387 |
if curl -fsS "http://127.0.0.1:20128/api/monitoring/health" >/dev/null 2>&1; then
|
| 388 |
echo "[HEALTH] OmniRoute ready after ${i}s"
|
| 389 |
echo "[PROCESS] OmniRoute: PID ${OMNIROUTE_PID}"
|
|
|
|
| 390 |
break
|
| 391 |
fi
|
| 392 |
sleep 1
|
health_doctor.py
CHANGED
|
@@ -40,16 +40,20 @@ def disk_status() -> dict:
|
|
| 40 |
return {"status": "IO_ERROR", "error": str(exc)}
|
| 41 |
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
def observe() -> dict:
|
| 44 |
-
"""
|
| 45 |
-
hermes = sqlite_status(HERMES_DB)
|
| 46 |
-
if hermes["status"] == "MISSING":
|
| 47 |
-
hermes["status"] = "NOT_INITIALIZED"
|
| 48 |
return {
|
| 49 |
-
"live_database":
|
| 50 |
-
"persistent_snapshot":
|
| 51 |
-
"last_known_good":
|
| 52 |
-
"hermes_session_database": hermes,
|
| 53 |
"disk": disk_status(),
|
| 54 |
}
|
| 55 |
|
|
|
|
| 40 |
return {"status": "IO_ERROR", "error": str(exc)}
|
| 41 |
|
| 42 |
|
| 43 |
+
def file_status(path: Path) -> dict:
|
| 44 |
+
return {
|
| 45 |
+
"path": str(path),
|
| 46 |
+
"exists": path.exists(),
|
| 47 |
+
"size": path.stat().st_size if path.exists() else 0,
|
| 48 |
+
"status": "HEALTHY" if path.exists() and path.stat().st_size > 0 else "MISSING",
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
def observe() -> dict:
|
| 52 |
+
"""Passive observation without active SQLite locking."""
|
|
|
|
|
|
|
|
|
|
| 53 |
return {
|
| 54 |
+
"live_database": file_status(RUNTIME_DB),
|
| 55 |
+
"persistent_snapshot": file_status(PERSISTENT_DB),
|
| 56 |
+
"last_known_good": file_status(LKG_DB),
|
|
|
|
| 57 |
"disk": disk_status(),
|
| 58 |
}
|
| 59 |
|
persistence_manager.py
CHANGED
|
@@ -188,6 +188,9 @@ def recover() -> dict:
|
|
| 188 |
shutil.copy2(RUNTIME_DB, evidence)
|
| 189 |
recovered = RUNTIME_DB.with_suffix(".recovered")
|
| 190 |
sqlite_backup(candidate, recovered)
|
|
|
|
|
|
|
|
|
|
| 191 |
os.replace(recovered, RUNTIME_DB)
|
| 192 |
try:
|
| 193 |
r_conn = sqlite3.connect(str(Path(RUNTIME_DB).resolve()), timeout=5)
|
|
|
|
| 188 |
shutil.copy2(RUNTIME_DB, evidence)
|
| 189 |
recovered = RUNTIME_DB.with_suffix(".recovered")
|
| 190 |
sqlite_backup(candidate, recovered)
|
| 191 |
+
for ext in ("-wal", "-shm"):
|
| 192 |
+
wal_f = RUNTIME_DB.parent / f"{RUNTIME_DB.name}{ext}"
|
| 193 |
+
wal_f.unlink(missing_ok=True)
|
| 194 |
os.replace(recovered, RUNTIME_DB)
|
| 195 |
try:
|
| 196 |
r_conn = sqlite3.connect(str(Path(RUNTIME_DB).resolve()), timeout=5)
|