code-HFD / src /psi /data /seed /startup_index_cache.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
4.04 kB
"""Local, content-keyed CPU indexes. Never cache augmented items or model KV.
Pickles are written by this process into a private cache, not downloaded data.
An advisory lock prevents all DDP ranks rebuilding the same index concurrently.
"""
from __future__ import annotations
import dataclasses
import fcntl
import hashlib
import json
import os
from pathlib import Path
import pickle
import sys
import tempfile
import time
import numpy as np
def _canonical(value):
if dataclasses.is_dataclass(value):
return [type(value).__qualname__, _canonical(dataclasses.asdict(value))]
if isinstance(value, Path):
return ["path", str(value)]
if isinstance(value, np.ndarray):
return ["array", value.dtype.str, list(value.shape),
hashlib.sha256(value.tobytes()).hexdigest()]
if isinstance(value, np.generic):
return _canonical(value.item())
if isinstance(value, dict):
return ["dict", [[_canonical(key), _canonical(value[key])]
for key in sorted(value, key=repr)]]
if isinstance(value, (set, frozenset)):
return ["set", sorted((_canonical(item) for item in value), key=repr)]
if isinstance(value, (list, tuple)):
return [type(value).__name__, [_canonical(item) for item in value]]
if value is None or isinstance(value, (str, bool, int, float)):
return value
raise TypeError(f"Unsupported startup index key: {type(value).__name__}")
def cache_key(inputs, code_paths=()):
code = {str(path): hashlib.sha256(Path(path).read_bytes()).hexdigest()
for path in (*code_paths, Path(__file__))}
content = json.dumps(_canonical((inputs, code, sys.version, np.__version__)), ensure_ascii=False,
separators=(",", ":"), allow_nan=False).encode()
return hashlib.sha256(content).hexdigest()
def cache_enabled():
return os.environ.get("HFD_STARTUP_INDEX_CACHE", "1").lower() not in {
"0", "false", "off", "no",
}
def cached_index(namespace, inputs, builder, *, code_paths=()):
"""Reuse deterministic outputs only; broken/partial indexes are rebuilt."""
if not cache_enabled():
return builder()
started = time.monotonic()
key = cache_key(inputs, code_paths)
root = Path(os.environ.get(
"HFD_STARTUP_INDEX_CACHE_DIR",
str(Path(__file__).resolve().parents[4] / "cache/full_duplex_startup"),
))
root.mkdir(parents=True, exist_ok=True, mode=0o700)
path = root / f"{namespace}-{key}.pkl"
# Never wait for an unavailable rank-0: any rank may build; kernel releases
# the lock if its owner crashes. Readers cannot observe half-written data.
with (root / f"{namespace}-{key}.lock").open("a+b") as lock:
if not path.exists():
print(f"[DATA INDEX] {namespace}: cache miss; one process builds, others wait", flush=True)
fcntl.flock(lock, fcntl.LOCK_EX)
try:
with path.open("rb") as handle:
payload = pickle.load(handle)
if payload["key"] != key:
raise ValueError("startup index key mismatch")
except (OSError, EOFError, ValueError, TypeError, KeyError,
pickle.UnpicklingError, AttributeError, ImportError):
value = builder()
fd, temporary = tempfile.mkstemp(prefix=f".{namespace}-", dir=root)
try:
with os.fdopen(fd, "wb") as handle:
pickle.dump({"key": key, "value": value}, handle,
protocol=pickle.HIGHEST_PROTOCOL)
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
status = "built"
else:
value = payload["value"]
status = "hit"
print(f"[DATA INDEX] {namespace}: {status} {time.monotonic()-started:.1f}s {path}", flush=True)
return value