fix(persistence): eliminate container startup hang with tmp flock, fast backup, and corrupted DB auto-recovery

entrypoint.sh

@@ -182,9 +182,11 @@ preflight_persistence_gate() {
                 log_info "PREFLIGHT" "Existing storage.sqlite integrity verified: PRAGMA quick_check passed."
             else
                 log_warn "PREFLIGHT" "storage.sqlite failed quick_check: $_CHK"
+                mv "$DATA_DIR/storage.sqlite" "$DATA_DIR/storage.sqlite.corrupt-$(date +%s)" 2>/dev/null || true
+                rm -f "$DATA_DIR/storage.sqlite-wal" "$DATA_DIR/storage.sqlite-shm" 2>/dev/null || true
                 if [ -f "/vault_sync.py" ]; then
                     log_warn "PREFLIGHT" "Attempting cloud vault restore as fallback..."
-                    python3 /vault_sync.py restore || true
+                    python3 /vault_sync.py restore --force || true
                 fi
             fi
         fi

persistence_manager.py

@@ -23,7 +23,7 @@ PERSISTENT_DB = DATA_DIR / "storage.sqlite"
 BACKUP_DIR = DATA_DIR / "backups"
 INCOMING_DIR = DATA_DIR / ".incoming"
 LKG_DB = BACKUP_DIR / "last-known-good.sqlite"
-LOCK_FILE = DATA_DIR / ".persistence.lock"
+LOCK_FILE = Path(os.getenv("OMNIROUTE_LOCK_FILE", "/tmp/.persistence.lock"))
 ROLLING_LIMIT = max(1, int(os.getenv("OMNIROUTE_ROLLING_BACKUPS", "5")))
 log = logging.getLogger("persistence")
 
@@ -70,7 +70,7 @@ def sqlite_status(path: Path, deep: bool = False) -> dict:
         return result
     connection = None
     try:
-        connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5)
+        connection = sqlite3.connect(f"file:{path}?mode=ro&nolock=1", uri=True, timeout=5)
         quick = connection.execute("PRAGMA quick_check").fetchone()
         result["quick_check"] = quick[0] if quick else "empty result"
         if result["quick_check"] != "ok":
@@ -123,10 +123,10 @@ def sqlite_backup(source: Path, destination: Path) -> dict:
     started = time.monotonic()
     temporary = destination.with_name(f"{destination.name}.tmp-{os.getpid()}-{time.time_ns()}")
     try:
-        source_connection = sqlite3.connect(f"file:{source}?mode=ro", uri=True, timeout=10)
+        source_connection = sqlite3.connect(f"file:{source}?mode=ro&nolock=1", uri=True, timeout=10)
         destination_connection = sqlite3.connect(temporary, timeout=10)
         try:
-            source_connection.backup(destination_connection, pages=256, sleep=0.05)
+            source_connection.backup(destination_connection, pages=-1)
             destination_connection.commit()
         finally:
             destination_connection.close()
@@ -182,7 +182,7 @@ def recover() -> dict:
         candidates = [PERSISTENT_DB, LKG_DB]
         candidates.extend(sorted(BACKUP_DIR.glob("storage-*.sqlite"), key=lambda p: p.stat().st_mtime, reverse=True))
         for candidate in dict.fromkeys(candidates):
-            if sqlite_status(candidate, deep=True)["status"] != "HEALTHY":
+            if sqlite_status(candidate, deep=False)["status"] != "HEALTHY":
                 continue
             if RUNTIME_DB.exists():
                 evidence = BACKUP_DIR / f"runtime-corrupt-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}.sqlite"

vault_sync.py

@@ -39,15 +39,24 @@ def get_hf_api():
         logger.error(f"huggingface_hub import error: {exc}")
         return None, f"huggingface_hub import error: {exc}"
 
-def restore_from_vault() -> tuple[bool, str]:
+def restore_from_vault(force: bool = False) -> tuple[bool, str]:
     """Restores storage.sqlite from private HF Dataset if local database is absent or empty."""
     os.makedirs(DATA_DIR, exist_ok=True)
     
-    # If a valid local database already exists with positive size, keep local state as authoritative
-    if os.path.exists(DB_PATH) and os.path.getsize(DB_PATH) > 0:
-        msg = f"Authoritative local database found at {DB_PATH} ({os.path.getsize(DB_PATH)} bytes)."
-        logger.info(msg)
-        return True, msg
+    # If a valid local database already exists with positive size AND passes quick_check, keep local state
+    if not force and os.path.exists(DB_PATH) and os.path.getsize(DB_PATH) > 0:
+        try:
+            chk_conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro&nolock=1", uri=True, timeout=3.0)
+            res = chk_conn.execute("PRAGMA quick_check;").fetchone()
+            chk_conn.close()
+            if res and res[0] == "ok":
+                msg = f"Authoritative local database found at {DB_PATH} ({os.path.getsize(DB_PATH)} bytes) and verified healthy."
+                logger.info(msg)
+                return True, msg
+            else:
+                logger.warning(f"Local database exists but failed quick_check: {res}. Forcing restore from cloud vault...")
+        except Exception as e:
+            logger.warning(f"Local database check threw exception: {e}. Forcing restore from cloud vault...")
 
     token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_API_KEY")
     if not token:
@@ -143,7 +152,7 @@ def backup_to_vault() -> tuple[bool, str]:
 if __name__ == "__main__":
     action = sys.argv[1] if len(sys.argv) > 1 else "backup"
     if action == "restore":
-        restore_from_vault()
+        restore_from_vault(force=("--force" in sys.argv))
     elif action == "backup":
         backup_to_vault()
     else: