Download src/psi/data/seed/omni_sqlite_cache.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.66 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/omni_sqlite_cache.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/seed/omni_sqlite_cache.py
-
curl -L -o omni_sqlite_cache.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/omni_sqlite_cache.py
5.66 kB
| from __future__ import annotations | |
| import contextlib | |
| import fcntl | |
| import hashlib | |
| import os | |
| import shutil | |
| import time | |
| from pathlib import Path | |
| def _enabled() -> bool: | |
| value = os.environ.get("PSI_OMNI_SQLITE_LOCAL_CACHE", "0").strip().lower() | |
| return value not in {"0", "false", "no", "off"} | |
| def _cache_root() -> Path: | |
| return Path(os.environ.get("PSI_OMNI_SQLITE_LOCAL_CACHE_DIR", "/tmp/psi0_omni_sqlite_index_cache")) | |
| def _fallback_root() -> Path | None: | |
| value = os.environ.get("PSI_OMNI_SQLITE_FALLBACK_ROOT", "").strip() | |
| return Path(value) if value else None | |
| def _fallback_path(db_path: Path) -> Path | None: | |
| root = _fallback_root() | |
| if root is None: | |
| return None | |
| parts = db_path.parts | |
| try: | |
| tasks_index = len(parts) - 1 - tuple(reversed(parts)).index("tasks") | |
| task_name = parts[tasks_index + 1] | |
| except (ValueError, IndexError): | |
| return None | |
| return root / task_name / db_path.name | |
| def _is_valid_sqlite_file(path: Path, *, expected_size: int | None = None) -> bool: | |
| try: | |
| stat = path.stat() | |
| if stat.st_size < 16: | |
| return False | |
| if expected_size is not None and stat.st_size != int(expected_size): | |
| return False | |
| with path.open("rb") as handle: | |
| return handle.read(16) == b"SQLite format 3\x00" | |
| except OSError: | |
| return False | |
| def _cache_lock(lock_path: Path): | |
| try: | |
| timeout = max(float(os.environ.get("PSI_OMNI_SQLITE_LOCAL_CACHE_LOCK_TIMEOUT", "1800")), 1.0) | |
| except ValueError: | |
| timeout = 1800.0 | |
| deadline = time.monotonic() + timeout | |
| lock_file = lock_path.open("a+b") | |
| try: | |
| while True: | |
| try: | |
| fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| break | |
| except BlockingIOError: | |
| if time.monotonic() >= deadline: | |
| raise TimeoutError(f"Timed out waiting for SQLite cache lock: {lock_path}") | |
| time.sleep(0.1) | |
| yield | |
| finally: | |
| try: | |
| fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) | |
| finally: | |
| lock_file.close() | |
| def local_sqlite_path(db_path: Path, *, expected_size: int | None = None) -> Path: | |
| """Return a verified read-only SQLite source, optionally staged in /tmp. | |
| The OSS/FUSE source must never be opened in SQLite read-write mode. A | |
| missing, empty, truncated, or non-SQLite source is rejected before it can | |
| replace a valid local cache. Task-level indexes may fall back to a | |
| separately preserved CPFS copy via ``PSI_OMNI_SQLITE_FALLBACK_ROOT``. | |
| """ | |
| db_path = Path(db_path) | |
| fallback_path = _fallback_path(db_path) | |
| if _is_valid_sqlite_file(db_path, expected_size=expected_size): | |
| source_path = db_path | |
| elif fallback_path is not None and _is_valid_sqlite_file( | |
| fallback_path, expected_size=expected_size | |
| ): | |
| source_path = fallback_path | |
| if int(os.environ.get("RANK", "0")) == 0: | |
| print( | |
| f"[OMNI SQLITE FALLBACK] invalid source={db_path}; using={fallback_path}", | |
| flush=True, | |
| ) | |
| else: | |
| source_path = None | |
| digest = hashlib.sha1(str(db_path.parent).encode("utf-8")).hexdigest()[:16] | |
| local_dir = _cache_root() / digest | |
| local_path = local_dir / db_path.name | |
| # An already verified local copy is preferable to returning a corrupt OSS | |
| # source, even when the preserved CPFS fallback is temporarily unavailable. | |
| if source_path is None: | |
| if _is_valid_sqlite_file(local_path, expected_size=expected_size): | |
| return local_path | |
| raise RuntimeError( | |
| "No valid SQLite index is available: " | |
| f"source={db_path}, fallback={fallback_path}, local={local_path}, " | |
| f"expected_size={expected_size}" | |
| ) | |
| if not _enabled(): | |
| return source_path | |
| source_stat = source_path.stat() | |
| local_dir.mkdir(parents=True, exist_ok=True) | |
| try: | |
| local_stat = local_path.stat() | |
| if ( | |
| local_stat.st_size == source_stat.st_size | |
| and local_stat.st_mtime >= source_stat.st_mtime | |
| and _is_valid_sqlite_file(local_path, expected_size=expected_size) | |
| ): | |
| return local_path | |
| except OSError: | |
| pass | |
| lock_path = local_path.with_suffix(local_path.suffix + ".lock") | |
| with _cache_lock(lock_path): | |
| try: | |
| local_stat = local_path.stat() | |
| if ( | |
| local_stat.st_size == source_stat.st_size | |
| and local_stat.st_mtime >= source_stat.st_mtime | |
| and _is_valid_sqlite_file(local_path, expected_size=expected_size) | |
| ): | |
| return local_path | |
| except OSError: | |
| pass | |
| tmp_path = local_path.with_suffix(local_path.suffix + f".tmp.{os.getpid()}") | |
| try: | |
| shutil.copy2(source_path, tmp_path) | |
| if not _is_valid_sqlite_file(tmp_path, expected_size=expected_size): | |
| raise RuntimeError( | |
| f"SQLite cache copy failed validation: {source_path} -> {tmp_path}" | |
| ) | |
| if tmp_path.stat().st_size != source_stat.st_size: | |
| raise RuntimeError( | |
| f"SQLite cache copy size mismatch: source={source_stat.st_size}, " | |
| f"copy={tmp_path.stat().st_size}, path={source_path}" | |
| ) | |
| os.replace(tmp_path, local_path) | |
| finally: | |
| try: | |
| tmp_path.unlink() | |
| except OSError: | |
| pass | |
| return local_path | |