code-HFD / src /psi /data /seed /omni_sqlite_cache.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
5.66 kB
from __future__ import annotations
import contextlib
import fcntl
import hashlib
import os
import shutil
import time
from pathlib import Path
def _enabled() -> bool:
value = os.environ.get("PSI_OMNI_SQLITE_LOCAL_CACHE", "0").strip().lower()
return value not in {"0", "false", "no", "off"}
def _cache_root() -> Path:
return Path(os.environ.get("PSI_OMNI_SQLITE_LOCAL_CACHE_DIR", "/tmp/psi0_omni_sqlite_index_cache"))
def _fallback_root() -> Path | None:
value = os.environ.get("PSI_OMNI_SQLITE_FALLBACK_ROOT", "").strip()
return Path(value) if value else None
def _fallback_path(db_path: Path) -> Path | None:
root = _fallback_root()
if root is None:
return None
parts = db_path.parts
try:
tasks_index = len(parts) - 1 - tuple(reversed(parts)).index("tasks")
task_name = parts[tasks_index + 1]
except (ValueError, IndexError):
return None
return root / task_name / db_path.name
def _is_valid_sqlite_file(path: Path, *, expected_size: int | None = None) -> bool:
try:
stat = path.stat()
if stat.st_size < 16:
return False
if expected_size is not None and stat.st_size != int(expected_size):
return False
with path.open("rb") as handle:
return handle.read(16) == b"SQLite format 3\x00"
except OSError:
return False
@contextlib.contextmanager
def _cache_lock(lock_path: Path):
try:
timeout = max(float(os.environ.get("PSI_OMNI_SQLITE_LOCAL_CACHE_LOCK_TIMEOUT", "1800")), 1.0)
except ValueError:
timeout = 1800.0
deadline = time.monotonic() + timeout
lock_file = lock_path.open("a+b")
try:
while True:
try:
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except BlockingIOError:
if time.monotonic() >= deadline:
raise TimeoutError(f"Timed out waiting for SQLite cache lock: {lock_path}")
time.sleep(0.1)
yield
finally:
try:
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
finally:
lock_file.close()
def local_sqlite_path(db_path: Path, *, expected_size: int | None = None) -> Path:
"""Return a verified read-only SQLite source, optionally staged in /tmp.
The OSS/FUSE source must never be opened in SQLite read-write mode. A
missing, empty, truncated, or non-SQLite source is rejected before it can
replace a valid local cache. Task-level indexes may fall back to a
separately preserved CPFS copy via ``PSI_OMNI_SQLITE_FALLBACK_ROOT``.
"""
db_path = Path(db_path)
fallback_path = _fallback_path(db_path)
if _is_valid_sqlite_file(db_path, expected_size=expected_size):
source_path = db_path
elif fallback_path is not None and _is_valid_sqlite_file(
fallback_path, expected_size=expected_size
):
source_path = fallback_path
if int(os.environ.get("RANK", "0")) == 0:
print(
f"[OMNI SQLITE FALLBACK] invalid source={db_path}; using={fallback_path}",
flush=True,
)
else:
source_path = None
digest = hashlib.sha1(str(db_path.parent).encode("utf-8")).hexdigest()[:16]
local_dir = _cache_root() / digest
local_path = local_dir / db_path.name
# An already verified local copy is preferable to returning a corrupt OSS
# source, even when the preserved CPFS fallback is temporarily unavailable.
if source_path is None:
if _is_valid_sqlite_file(local_path, expected_size=expected_size):
return local_path
raise RuntimeError(
"No valid SQLite index is available: "
f"source={db_path}, fallback={fallback_path}, local={local_path}, "
f"expected_size={expected_size}"
)
if not _enabled():
return source_path
source_stat = source_path.stat()
local_dir.mkdir(parents=True, exist_ok=True)
try:
local_stat = local_path.stat()
if (
local_stat.st_size == source_stat.st_size
and local_stat.st_mtime >= source_stat.st_mtime
and _is_valid_sqlite_file(local_path, expected_size=expected_size)
):
return local_path
except OSError:
pass
lock_path = local_path.with_suffix(local_path.suffix + ".lock")
with _cache_lock(lock_path):
try:
local_stat = local_path.stat()
if (
local_stat.st_size == source_stat.st_size
and local_stat.st_mtime >= source_stat.st_mtime
and _is_valid_sqlite_file(local_path, expected_size=expected_size)
):
return local_path
except OSError:
pass
tmp_path = local_path.with_suffix(local_path.suffix + f".tmp.{os.getpid()}")
try:
shutil.copy2(source_path, tmp_path)
if not _is_valid_sqlite_file(tmp_path, expected_size=expected_size):
raise RuntimeError(
f"SQLite cache copy failed validation: {source_path} -> {tmp_path}"
)
if tmp_path.stat().st_size != source_stat.st_size:
raise RuntimeError(
f"SQLite cache copy size mismatch: source={source_stat.st_size}, "
f"copy={tmp_path.stat().st_size}, path={source_path}"
)
os.replace(tmp_path, local_path)
finally:
try:
tmp_path.unlink()
except OSError:
pass
return local_path