"""Local, content-keyed CPU indexes. Never cache augmented items or model KV. Pickles are written by this process into a private cache, not downloaded data. An advisory lock prevents all DDP ranks rebuilding the same index concurrently. """ from __future__ import annotations import dataclasses import fcntl import hashlib import json import os from pathlib import Path import pickle import sys import tempfile import time import numpy as np def _canonical(value): if dataclasses.is_dataclass(value): return [type(value).__qualname__, _canonical(dataclasses.asdict(value))] if isinstance(value, Path): return ["path", str(value)] if isinstance(value, np.ndarray): return ["array", value.dtype.str, list(value.shape), hashlib.sha256(value.tobytes()).hexdigest()] if isinstance(value, np.generic): return _canonical(value.item()) if isinstance(value, dict): return ["dict", [[_canonical(key), _canonical(value[key])] for key in sorted(value, key=repr)]] if isinstance(value, (set, frozenset)): return ["set", sorted((_canonical(item) for item in value), key=repr)] if isinstance(value, (list, tuple)): return [type(value).__name__, [_canonical(item) for item in value]] if value is None or isinstance(value, (str, bool, int, float)): return value raise TypeError(f"Unsupported startup index key: {type(value).__name__}") def cache_key(inputs, code_paths=()): code = {str(path): hashlib.sha256(Path(path).read_bytes()).hexdigest() for path in (*code_paths, Path(__file__))} content = json.dumps(_canonical((inputs, code, sys.version, np.__version__)), ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode() return hashlib.sha256(content).hexdigest() def cache_enabled(): return os.environ.get("HFD_STARTUP_INDEX_CACHE", "1").lower() not in { "0", "false", "off", "no", } def cached_index(namespace, inputs, builder, *, code_paths=()): """Reuse deterministic outputs only; broken/partial indexes are rebuilt.""" if not cache_enabled(): return builder() started = time.monotonic() key = cache_key(inputs, code_paths) root = Path(os.environ.get( "HFD_STARTUP_INDEX_CACHE_DIR", str(Path(__file__).resolve().parents[4] / "cache/full_duplex_startup"), )) root.mkdir(parents=True, exist_ok=True, mode=0o700) path = root / f"{namespace}-{key}.pkl" # Never wait for an unavailable rank-0: any rank may build; kernel releases # the lock if its owner crashes. Readers cannot observe half-written data. with (root / f"{namespace}-{key}.lock").open("a+b") as lock: if not path.exists(): print(f"[DATA INDEX] {namespace}: cache miss; one process builds, others wait", flush=True) fcntl.flock(lock, fcntl.LOCK_EX) try: with path.open("rb") as handle: payload = pickle.load(handle) if payload["key"] != key: raise ValueError("startup index key mismatch") except (OSError, EOFError, ValueError, TypeError, KeyError, pickle.UnpicklingError, AttributeError, ImportError): value = builder() fd, temporary = tempfile.mkstemp(prefix=f".{namespace}-", dir=root) try: with os.fdopen(fd, "wb") as handle: pickle.dump({"key": key, "value": value}, handle, protocol=pickle.HIGHEST_PROTOCOL) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) finally: if os.path.exists(temporary): os.unlink(temporary) status = "built" else: value = payload["value"] status = "hit" print(f"[DATA INDEX] {namespace}: {status} {time.monotonic()-started:.1f}s {path}", flush=True) return value