File size: 4,037 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
"""Local, content-keyed CPU indexes. Never cache augmented items or model KV.

Pickles are written by this process into a private cache, not downloaded data.
An advisory lock prevents all DDP ranks rebuilding the same index concurrently.
"""
from __future__ import annotations

import dataclasses
import fcntl
import hashlib
import json
import os
from pathlib import Path
import pickle
import sys
import tempfile
import time

import numpy as np


def _canonical(value):
    if dataclasses.is_dataclass(value):
        return [type(value).__qualname__, _canonical(dataclasses.asdict(value))]
    if isinstance(value, Path):
        return ["path", str(value)]
    if isinstance(value, np.ndarray):
        return ["array", value.dtype.str, list(value.shape),
                hashlib.sha256(value.tobytes()).hexdigest()]
    if isinstance(value, np.generic):
        return _canonical(value.item())
    if isinstance(value, dict):
        return ["dict", [[_canonical(key), _canonical(value[key])]
                         for key in sorted(value, key=repr)]]
    if isinstance(value, (set, frozenset)):
        return ["set", sorted((_canonical(item) for item in value), key=repr)]
    if isinstance(value, (list, tuple)):
        return [type(value).__name__, [_canonical(item) for item in value]]
    if value is None or isinstance(value, (str, bool, int, float)):
        return value
    raise TypeError(f"Unsupported startup index key: {type(value).__name__}")


def cache_key(inputs, code_paths=()):
    code = {str(path): hashlib.sha256(Path(path).read_bytes()).hexdigest()
            for path in (*code_paths, Path(__file__))}
    content = json.dumps(_canonical((inputs, code, sys.version, np.__version__)), ensure_ascii=False,
                         separators=(",", ":"), allow_nan=False).encode()
    return hashlib.sha256(content).hexdigest()


def cache_enabled():
    return os.environ.get("HFD_STARTUP_INDEX_CACHE", "1").lower() not in {
        "0", "false", "off", "no",
    }


def cached_index(namespace, inputs, builder, *, code_paths=()):
    """Reuse deterministic outputs only; broken/partial indexes are rebuilt."""
    if not cache_enabled():
        return builder()
    started = time.monotonic()
    key = cache_key(inputs, code_paths)
    root = Path(os.environ.get(
        "HFD_STARTUP_INDEX_CACHE_DIR",
        str(Path(__file__).resolve().parents[4] / "cache/full_duplex_startup"),
    ))
    root.mkdir(parents=True, exist_ok=True, mode=0o700)
    path = root / f"{namespace}-{key}.pkl"
    # Never wait for an unavailable rank-0: any rank may build; kernel releases
    # the lock if its owner crashes. Readers cannot observe half-written data.
    with (root / f"{namespace}-{key}.lock").open("a+b") as lock:
        if not path.exists():
            print(f"[DATA INDEX] {namespace}: cache miss; one process builds, others wait", flush=True)
        fcntl.flock(lock, fcntl.LOCK_EX)
        try:
            with path.open("rb") as handle:
                payload = pickle.load(handle)
            if payload["key"] != key:
                raise ValueError("startup index key mismatch")
        except (OSError, EOFError, ValueError, TypeError, KeyError,
                pickle.UnpicklingError, AttributeError, ImportError):
            value = builder()
            fd, temporary = tempfile.mkstemp(prefix=f".{namespace}-", dir=root)
            try:
                with os.fdopen(fd, "wb") as handle:
                    pickle.dump({"key": key, "value": value}, handle,
                                protocol=pickle.HIGHEST_PROTOCOL)
                    handle.flush()
                    os.fsync(handle.fileno())
                os.replace(temporary, path)
            finally:
                if os.path.exists(temporary):
                    os.unlink(temporary)
            status = "built"
        else:
            value = payload["value"]
            status = "hit"
    print(f"[DATA INDEX] {namespace}: {status} {time.monotonic()-started:.1f}s {path}", flush=True)
    return value