Download src/psi/data/seed/startup_index_cache.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 4.04 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/startup_index_cache.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/seed/startup_index_cache.py
-
curl -L -o startup_index_cache.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/startup_index_cache.py
4.04 kB
| """Local, content-keyed CPU indexes. Never cache augmented items or model KV. | |
| Pickles are written by this process into a private cache, not downloaded data. | |
| An advisory lock prevents all DDP ranks rebuilding the same index concurrently. | |
| """ | |
| from __future__ import annotations | |
| import dataclasses | |
| import fcntl | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import pickle | |
| import sys | |
| import tempfile | |
| import time | |
| import numpy as np | |
| def _canonical(value): | |
| if dataclasses.is_dataclass(value): | |
| return [type(value).__qualname__, _canonical(dataclasses.asdict(value))] | |
| if isinstance(value, Path): | |
| return ["path", str(value)] | |
| if isinstance(value, np.ndarray): | |
| return ["array", value.dtype.str, list(value.shape), | |
| hashlib.sha256(value.tobytes()).hexdigest()] | |
| if isinstance(value, np.generic): | |
| return _canonical(value.item()) | |
| if isinstance(value, dict): | |
| return ["dict", [[_canonical(key), _canonical(value[key])] | |
| for key in sorted(value, key=repr)]] | |
| if isinstance(value, (set, frozenset)): | |
| return ["set", sorted((_canonical(item) for item in value), key=repr)] | |
| if isinstance(value, (list, tuple)): | |
| return [type(value).__name__, [_canonical(item) for item in value]] | |
| if value is None or isinstance(value, (str, bool, int, float)): | |
| return value | |
| raise TypeError(f"Unsupported startup index key: {type(value).__name__}") | |
| def cache_key(inputs, code_paths=()): | |
| code = {str(path): hashlib.sha256(Path(path).read_bytes()).hexdigest() | |
| for path in (*code_paths, Path(__file__))} | |
| content = json.dumps(_canonical((inputs, code, sys.version, np.__version__)), ensure_ascii=False, | |
| separators=(",", ":"), allow_nan=False).encode() | |
| return hashlib.sha256(content).hexdigest() | |
| def cache_enabled(): | |
| return os.environ.get("HFD_STARTUP_INDEX_CACHE", "1").lower() not in { | |
| "0", "false", "off", "no", | |
| } | |
| def cached_index(namespace, inputs, builder, *, code_paths=()): | |
| """Reuse deterministic outputs only; broken/partial indexes are rebuilt.""" | |
| if not cache_enabled(): | |
| return builder() | |
| started = time.monotonic() | |
| key = cache_key(inputs, code_paths) | |
| root = Path(os.environ.get( | |
| "HFD_STARTUP_INDEX_CACHE_DIR", | |
| str(Path(__file__).resolve().parents[4] / "cache/full_duplex_startup"), | |
| )) | |
| root.mkdir(parents=True, exist_ok=True, mode=0o700) | |
| path = root / f"{namespace}-{key}.pkl" | |
| # Never wait for an unavailable rank-0: any rank may build; kernel releases | |
| # the lock if its owner crashes. Readers cannot observe half-written data. | |
| with (root / f"{namespace}-{key}.lock").open("a+b") as lock: | |
| if not path.exists(): | |
| print(f"[DATA INDEX] {namespace}: cache miss; one process builds, others wait", flush=True) | |
| fcntl.flock(lock, fcntl.LOCK_EX) | |
| try: | |
| with path.open("rb") as handle: | |
| payload = pickle.load(handle) | |
| if payload["key"] != key: | |
| raise ValueError("startup index key mismatch") | |
| except (OSError, EOFError, ValueError, TypeError, KeyError, | |
| pickle.UnpicklingError, AttributeError, ImportError): | |
| value = builder() | |
| fd, temporary = tempfile.mkstemp(prefix=f".{namespace}-", dir=root) | |
| try: | |
| with os.fdopen(fd, "wb") as handle: | |
| pickle.dump({"key": key, "value": value}, handle, | |
| protocol=pickle.HIGHEST_PROTOCOL) | |
| handle.flush() | |
| os.fsync(handle.fileno()) | |
| os.replace(temporary, path) | |
| finally: | |
| if os.path.exists(temporary): | |
| os.unlink(temporary) | |
| status = "built" | |
| else: | |
| value = payload["value"] | |
| status = "hit" | |
| print(f"[DATA INDEX] {namespace}: {status} {time.monotonic()-started:.1f}s {path}", flush=True) | |
| return value | |