Download src/psi/data/seed/omni_hidden_cache.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 25.9 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/omni_hidden_cache.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/seed/omni_hidden_cache.py
-
curl -L -o omni_hidden_cache.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/omni_hidden_cache.py
25.9 kB
| from __future__ import annotations | |
| import atexit | |
| import contextlib | |
| import fcntl | |
| import hashlib | |
| import math | |
| import os | |
| import subprocess | |
| import threading | |
| import time | |
| from collections import OrderedDict, defaultdict, deque | |
| from pathlib import Path | |
| import torch | |
| from safetensors import safe_open | |
| from psi.data.seed.full_duplex_audio_contract import ( | |
| AUDIO_COMMAND_STREAMING_PROMPT, | |
| HFD_AUDIO_SOURCE_FORMAT, | |
| HFD_ONLINE_ENCODER_ATTENTION, | |
| HFD_ONLINE_ENCODER_BATCH_SIZE, | |
| HFD_ONLINE_ENCODER_CONTRACT, | |
| HFD_ONLINE_ENCODER_DTYPE, | |
| ) | |
| _TRUE_VALUES = {"1", "true", "yes", "y", "on"} | |
| def _max_open_shards() -> int: | |
| raw_value = os.environ.get("PSI_OMNI_SAFE_OPEN_CACHE_SIZE", "0") | |
| try: | |
| return max(int(raw_value), 0) | |
| except ValueError: | |
| return 128 | |
| _LOCK = threading.Lock() | |
| _OPEN_SHARDS: OrderedDict[str, object] = OrderedDict() | |
| _HIDDEN_TENSORS: OrderedDict[tuple[str, int, int], torch.Tensor] = OrderedDict() | |
| # Process-local resident shard cache. In a DataLoader worker this keeps a | |
| # bounded number of complete shards in that worker's CPU RAM, so all samples | |
| # from a shard reuse one sequential materialization instead of repeatedly | |
| # faulting remote mmap pages. | |
| _FULL_SHARDS: OrderedDict[str, torch.Tensor] = OrderedDict() | |
| _FULL_SHARD_LOADS: dict[str, threading.Event] = {} | |
| _OSS_BATCH_HINTS = threading.local() | |
| _FULL_SHARD_PREFETCH_THREAD: threading.Thread | None = None | |
| _FULL_SHARD_PREFETCH_KEY: str | None = None | |
| _FULL_SHARD_PREFETCH_LAST_KEY: str | None = None | |
| _FULL_SHARD_PREFETCH_PENDING_KEY: str | None = None | |
| _FULL_SHARD_PREFETCH_PENDING_CALLS = 0 | |
| def install_omni_oss_batch_hints( | |
| shard_path: str | Path, | |
| hints: list[tuple[str, str, int, int, int]], | |
| *, | |
| materialize_full_shard: bool = True, | |
| ) -> None: | |
| canonical_path = _canonical_shard_path(shard_path) | |
| by_sample = defaultdict(deque) | |
| by_key = {} | |
| for source_group, source_key, local_index, offset, length in hints: | |
| entry = (str(source_key), Path(canonical_path), int(offset), int(length)) | |
| by_sample[(str(source_group), int(local_index))].append(entry) | |
| by_key[(str(source_group), str(source_key))] = entry | |
| _OSS_BATCH_HINTS.by_sample = by_sample | |
| _OSS_BATCH_HINTS.by_key = by_key | |
| _OSS_BATCH_HINTS.active = None | |
| _OSS_BATCH_HINTS.materialize_full_shard = bool(materialize_full_shard) | |
| # The shard-aware sampler groups this worker batch by shard. Prime its | |
| # resident copy once before the dataset fetches the individual slices. | |
| if materialize_full_shard and _max_cached_full_shards() > 0: | |
| _get_full_shard_tensor(Path(canonical_path)) | |
| def claim_omni_oss_sample_hint( | |
| source_group: str, | |
| local_index: int, | |
| ) -> tuple[str, Path, int, int] | None: | |
| by_sample = getattr(_OSS_BATCH_HINTS, "by_sample", None) | |
| if by_sample is None: | |
| return None | |
| queue = by_sample.get((str(source_group), int(local_index))) | |
| if not queue: | |
| return None | |
| entry = queue.popleft() | |
| _OSS_BATCH_HINTS.active = (str(source_group), *entry) | |
| return entry | |
| def prepared_omni_oss_shard_entry( | |
| source_group: str, | |
| source_key: str, | |
| ) -> tuple[Path, int, int] | None: | |
| source_group = str(source_group) | |
| source_key = str(source_key) | |
| active = getattr(_OSS_BATCH_HINTS, "active", None) | |
| if active is not None and active[0] == source_group and active[1] == source_key: | |
| return active[2], int(active[3]), int(active[4]) | |
| by_key = getattr(_OSS_BATCH_HINTS, "by_key", None) | |
| if by_key is None: | |
| return None | |
| entry = by_key.get((source_group, source_key)) | |
| if entry is None: | |
| return None | |
| return entry[1], int(entry[2]), int(entry[3]) | |
| def clear_omni_oss_batch_hints() -> None: | |
| _OSS_BATCH_HINTS.by_sample = None | |
| _OSS_BATCH_HINTS.by_key = None | |
| _OSS_BATCH_HINTS.active = None | |
| _OSS_BATCH_HINTS.materialize_full_shard = True | |
| def _canonical_shard_path(shard_path: str | Path) -> str: | |
| return os.path.realpath(os.fspath(shard_path)) | |
| def _max_cached_hidden_tensors() -> int: | |
| raw_value = os.environ.get("PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE", "0") | |
| try: | |
| return max(int(raw_value), 0) | |
| except ValueError: | |
| return 0 | |
| def _max_cached_full_shards() -> int: | |
| raw_value = os.environ.get("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "0") | |
| try: | |
| return max(int(raw_value), 0) | |
| except ValueError: | |
| return 0 | |
| def _max_full_shard_bytes() -> int: | |
| """Optional per-shard ceiling; zero disables the explicit size limit.""" | |
| raw_value = os.environ.get("PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB", "0") | |
| try: | |
| gib = float(raw_value) | |
| except ValueError: | |
| return 0 | |
| return max(int(gib * 1024**3), 0) | |
| def _full_shard_prefetch_enabled() -> bool: | |
| return os.environ.get("PSI_OMNI_FULL_SHARD_PREFETCH", "1").strip().lower() in { | |
| "1", "true", "yes", "on" | |
| } | |
| def _full_shard_prefetch_stagger_slots() -> int: | |
| try: | |
| return max(int(os.environ.get("PSI_OMNI_FULL_SHARD_PREFETCH_STAGGER", "32")), 1) | |
| except ValueError: | |
| return 32 | |
| def _full_shard_io_slot_count() -> int: | |
| try: | |
| return max(int(os.environ.get("PSI_OMNI_FULL_SHARD_IO_SLOTS", "0")), 0) | |
| except ValueError: | |
| return 0 | |
| def _direct_oss_stage_enabled() -> bool: | |
| return os.environ.get("PSI_OMNI_DIRECT_OSS_STAGE", "0").strip().lower() in _TRUE_VALUES | |
| def _direct_oss_uri(shard_path: Path) -> str: | |
| """Translate one mounted OSS shard path to its canonical object URI.""" | |
| mount_root_value = os.environ.get("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", "").strip() | |
| uri_root = os.environ.get("PSI_OMNI_DIRECT_OSS_URI_ROOT", "").strip().rstrip("/") | |
| if not mount_root_value or not uri_root: | |
| raise RuntimeError( | |
| "Direct OSS staging requires PSI_OMNI_DIRECT_OSS_MOUNT_ROOT and " | |
| "PSI_OMNI_DIRECT_OSS_URI_ROOT" | |
| ) | |
| mount_root = Path(mount_root_value).resolve() | |
| canonical = Path(_canonical_shard_path(shard_path)) | |
| try: | |
| relative = canonical.relative_to(mount_root) | |
| except ValueError as exc: | |
| raise RuntimeError( | |
| f"Shard is outside the configured direct-OSS mount root: " | |
| f"shard={canonical} root={mount_root}" | |
| ) from exc | |
| return f"{uri_root}/{relative.as_posix()}" | |
| def _direct_oss_staged_path(shard_path: Path): | |
| """Download one shard with parallel ranged GETs, then remove the scratch file.""" | |
| if not _direct_oss_stage_enabled(): | |
| yield shard_path | |
| return | |
| binary = os.environ.get("PSI_OMNI_OSSUTIL_BIN", "").strip() | |
| if not binary: | |
| raise RuntimeError("PSI_OMNI_DIRECT_OSS_STAGE=1 but PSI_OMNI_OSSUTIL_BIN is empty") | |
| config_path = os.environ.get("PSI_OMNI_OSSUTIL_CONFIG", "").strip() | |
| stage_root_value = os.environ.get("PSI_OMNI_DIRECT_OSS_STAGE_DIR", "").strip() | |
| if not stage_root_value: | |
| raise RuntimeError( | |
| "PSI_OMNI_DIRECT_OSS_STAGE=1 but PSI_OMNI_DIRECT_OSS_STAGE_DIR is empty" | |
| ) | |
| try: | |
| parallel = max(int(os.environ.get("PSI_OMNI_OSSUTIL_PARALLEL", "32")), 1) | |
| except ValueError: | |
| parallel = 32 | |
| stage_root = Path(stage_root_value) | |
| stage_root.mkdir(parents=True, exist_ok=True) | |
| source_uri = _direct_oss_uri(shard_path) | |
| digest = hashlib.sha1(source_uri.encode("utf-8")).hexdigest()[:16] | |
| destination = stage_root / ( | |
| f"{digest}.{os.getpid()}.{threading.get_ident()}.{time.time_ns()}.safetensors" | |
| ) | |
| temp_destination = Path(f"{destination}.temp") | |
| command = [binary] | |
| if config_path: | |
| command.extend(["-c", config_path]) | |
| endpoint = os.environ.get("PSI_OMNI_OSSUTIL_ENDPOINT", "").strip() | |
| if endpoint: | |
| command.extend(["-e", endpoint]) | |
| command.extend( | |
| [ | |
| "cp", | |
| "--force", | |
| "--quiet", | |
| "--no-progress", | |
| "--parallel", | |
| str(parallel), | |
| source_uri, | |
| str(destination), | |
| ] | |
| ) | |
| started = time.perf_counter() | |
| try: | |
| # Hold the node-wide network slot only for the ranged download. The | |
| # subsequent local-file -> RAM clone should not block another transfer. | |
| with _full_shard_io_slot(): | |
| completed = subprocess.run( | |
| command, | |
| check=False, | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT, | |
| text=True, | |
| ) | |
| if completed.returncode != 0 or not destination.is_file(): | |
| detail = (completed.stdout or "").strip().replace("\n", " ")[-1000:] | |
| raise RuntimeError( | |
| f"Direct OSS shard download failed with code={completed.returncode}: " | |
| f"source={source_uri} detail={detail}" | |
| ) | |
| if os.environ.get("PSI_OMNI_DIRECT_OSS_STAGE_LOG", "0").strip().lower() in _TRUE_VALUES: | |
| print( | |
| f"[OMNI DIRECT OSS] pid={os.getpid()} shard={shard_path.name} " | |
| f"bytes={destination.stat().st_size / 1024**3:.2f}GiB " | |
| f"elapsed={time.perf_counter() - started:.3f}s parallel={parallel}", | |
| flush=True, | |
| ) | |
| yield destination | |
| finally: | |
| destination.unlink(missing_ok=True) | |
| temp_destination.unlink(missing_ok=True) | |
| def _full_shard_io_slot(): | |
| """Bound full-shard materializations across every rank and worker. | |
| The OSS FUSE mount has node-wide aggregate bandwidth but gets much slower | |
| per request when dozens of DataLoader workers clone shards concurrently. | |
| Advisory lock files in node-local ``/tmp`` provide a lightweight semaphore | |
| shared by all independently launched torchrun processes. | |
| """ | |
| slot_count = _full_shard_io_slot_count() | |
| lock_dir_value = os.environ.get("PSI_OMNI_FULL_SHARD_IO_LOCK_DIR", "").strip() | |
| if slot_count <= 0 or not lock_dir_value: | |
| yield | |
| return | |
| lock_dir = Path(lock_dir_value) | |
| lock_dir.mkdir(parents=True, exist_ok=True) | |
| started = time.perf_counter() | |
| lock_file = None | |
| try: | |
| if slot_count == 1: | |
| lock_file = (lock_dir / "slot.000.lock").open("a+b") | |
| fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) | |
| else: | |
| first_slot = os.getpid() % slot_count | |
| while lock_file is None: | |
| for offset in range(slot_count): | |
| slot = (first_slot + offset) % slot_count | |
| candidate = (lock_dir / f"slot.{slot:03d}.lock").open("a+b") | |
| try: | |
| fcntl.flock( | |
| candidate.fileno(), | |
| fcntl.LOCK_EX | fcntl.LOCK_NB, | |
| ) | |
| except BlockingIOError: | |
| candidate.close() | |
| continue | |
| lock_file = candidate | |
| break | |
| if lock_file is None: | |
| time.sleep(0.01) | |
| if os.environ.get("PSI_OMNI_FULL_SHARD_IO_LOCK_LOG", "0").strip().lower() in { | |
| "1", "true", "yes", "on" | |
| }: | |
| print( | |
| f"[OMNI FULL SHARD IO SLOT] pid={os.getpid()} " | |
| f"waited={time.perf_counter() - started:.3f}s slots={slot_count}", | |
| flush=True, | |
| ) | |
| yield | |
| finally: | |
| if lock_file is not None: | |
| try: | |
| fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) | |
| finally: | |
| lock_file.close() | |
| def omni_oss_bulk_io_slot(): | |
| """Share the node-wide OSS bulk-I/O limit with non-training prefetchers.""" | |
| with _full_shard_io_slot(): | |
| yield | |
| def _full_shard_read_path(shard_path: Path): | |
| """Yield a readable shard while limiting only the actual remote I/O.""" | |
| if _direct_oss_stage_enabled(): | |
| with _direct_oss_staged_path(shard_path) as staged_path: | |
| yield staged_path | |
| return | |
| with _full_shard_io_slot(): | |
| yield shard_path | |
| def _close_reader(reader: object) -> None: | |
| exit_fn = getattr(reader, "__exit__", None) | |
| if exit_fn is not None: | |
| try: | |
| exit_fn(None, None, None) | |
| except Exception: | |
| pass | |
| def close_cached_omni_shards() -> None: | |
| with _LOCK: | |
| readers = list(_OPEN_SHARDS.values()) | |
| load_events = list(_FULL_SHARD_LOADS.values()) | |
| _OPEN_SHARDS.clear() | |
| _HIDDEN_TENSORS.clear() | |
| _FULL_SHARDS.clear() | |
| _FULL_SHARD_LOADS.clear() | |
| for reader in readers: | |
| _close_reader(reader) | |
| for event in load_events: | |
| event.set() | |
| atexit.register(close_cached_omni_shards) | |
| def _get_reader(shard_path: Path): | |
| max_open = _max_open_shards() | |
| if max_open <= 0: | |
| return safe_open(str(shard_path), framework="pt", device="cpu") | |
| key = os.fspath(shard_path) | |
| with _LOCK: | |
| reader = _OPEN_SHARDS.get(key) | |
| if reader is not None: | |
| _OPEN_SHARDS.move_to_end(key) | |
| return reader | |
| reader = safe_open(key, framework="pt", device="cpu") | |
| _OPEN_SHARDS[key] = reader | |
| while len(_OPEN_SHARDS) > max_open: | |
| _, old_reader = _OPEN_SHARDS.popitem(last=False) | |
| _close_reader(old_reader) | |
| return reader | |
| def _get_full_shard_tensor(shard_path: Path) -> torch.Tensor | None: | |
| """Return a materialized CPU shard when the bounded cache is enabled.""" | |
| max_cached = _max_cached_full_shards() | |
| if max_cached <= 0: | |
| return None | |
| key = _canonical_shard_path(shard_path) | |
| while True: | |
| with _LOCK: | |
| cached = _FULL_SHARDS.get(key) | |
| if cached is not None: | |
| _FULL_SHARDS.move_to_end(key) | |
| return cached | |
| load_event = _FULL_SHARD_LOADS.get(key) | |
| if load_event is None: | |
| load_event = threading.Event() | |
| _FULL_SHARD_LOADS[key] = load_event | |
| break | |
| # Foreground reading and this process's prefetch thread can request the | |
| # same shard together. Exactly one owner performs the download/clone. | |
| load_event.wait() | |
| started = None | |
| if os.environ.get("PSI_OMNI_FULL_SHARD_CACHE_LOG", "0").strip().lower() in { | |
| "1", "true", "yes", "on" | |
| }: | |
| import time | |
| started = time.perf_counter() | |
| try: | |
| with _full_shard_read_path(Path(key)) as read_path: | |
| with safe_open(str(read_path), framework="pt", device="cpu") as reader: | |
| mapped = reader.get_tensor("hidden_states") | |
| nbytes = int(mapped.numel() * mapped.element_size()) | |
| max_bytes = _max_full_shard_bytes() | |
| if max_bytes and nbytes > max_bytes: | |
| return None | |
| # Break the mmap backing with one sequential materialization. | |
| resident = mapped.clone(memory_format=torch.contiguous_format) | |
| with _LOCK: | |
| _FULL_SHARDS[key] = resident | |
| while len(_FULL_SHARDS) > max_cached: | |
| _FULL_SHARDS.popitem(last=False) | |
| if started is not None: | |
| elapsed = time.perf_counter() - started | |
| print( | |
| f"[OMNI FULL SHARD CACHE] pid={os.getpid()} loaded {Path(key).name} " | |
| f"bytes={nbytes / 1024**3:.2f}GiB elapsed={elapsed:.3f}s", | |
| flush=True, | |
| ) | |
| return resident | |
| finally: | |
| with _LOCK: | |
| completed_event = _FULL_SHARD_LOADS.pop(key, None) | |
| if completed_event is not None: | |
| completed_event.set() | |
| def _prefetch_full_shard_worker(key: str) -> None: | |
| global _FULL_SHARD_PREFETCH_THREAD, _FULL_SHARD_PREFETCH_KEY | |
| try: | |
| _get_full_shard_tensor(Path(key)) | |
| except Exception as exc: | |
| # Prefetch is an optimization. A foreground read will retry and | |
| # surface the original exception if this shard is actually needed. | |
| if os.environ.get("PSI_OMNI_FULL_SHARD_CACHE_LOG", "0").strip().lower() in { | |
| "1", "true", "yes", "on" | |
| }: | |
| print( | |
| f"[OMNI FULL SHARD PREFETCH] pid={os.getpid()} failed " | |
| f"{Path(key).name}: {exc}", | |
| flush=True, | |
| ) | |
| finally: | |
| with _LOCK: | |
| if _FULL_SHARD_PREFETCH_THREAD is threading.current_thread(): | |
| _FULL_SHARD_PREFETCH_THREAD = None | |
| _FULL_SHARD_PREFETCH_KEY = None | |
| def prefetch_omni_oss_shard(shard_path: str | Path | None) -> None: | |
| """Asynchronously materialize one worker lane's next OSS shard. | |
| A single daemon thread bounds I/O concurrency to one next shard per | |
| DataLoader worker. Requiring two cache slots keeps both current and next | |
| resident, so prefetch cannot evict data still used by the active batch. | |
| """ | |
| global _FULL_SHARD_PREFETCH_THREAD, _FULL_SHARD_PREFETCH_KEY | |
| global _FULL_SHARD_PREFETCH_LAST_KEY, _FULL_SHARD_PREFETCH_PENDING_KEY | |
| global _FULL_SHARD_PREFETCH_PENDING_CALLS | |
| if not getattr(_OSS_BATCH_HINTS, "materialize_full_shard", True): | |
| # Short validation runs touch only a few batches from a shard. Their | |
| # DataLoader already prefetches the requested slices; materializing a | |
| # multi-GiB next shard here would amplify cold I/O without reuse. | |
| return | |
| if shard_path is None or not _full_shard_prefetch_enabled(): | |
| return | |
| if _max_cached_full_shards() < 2: | |
| return | |
| key = _canonical_shard_path(shard_path) | |
| with _LOCK: | |
| cached = _FULL_SHARDS.get(key) | |
| if cached is not None: | |
| _FULL_SHARDS.move_to_end(key) | |
| return | |
| thread = _FULL_SHARD_PREFETCH_THREAD | |
| if thread is not None and thread.is_alive(): | |
| return | |
| if key != _FULL_SHARD_PREFETCH_PENDING_KEY: | |
| _FULL_SHARD_PREFETCH_PENDING_KEY = key | |
| if _FULL_SHARD_PREFETCH_LAST_KEY is None: | |
| # Prime the first next shard immediately during startup. | |
| _FULL_SHARD_PREFETCH_PENDING_CALLS = 0 | |
| else: | |
| worker_info = torch.utils.data.get_worker_info() | |
| worker_id = int(worker_info.id) if worker_info is not None else 0 | |
| worker_lanes = max(int(os.environ.get("PSI_OMNI_OSS_WORKER_LANES", "1")), 1) | |
| rank = int(os.environ.get("RANK", "0")) | |
| _FULL_SHARD_PREFETCH_PENDING_CALLS = ( | |
| rank * worker_lanes + worker_id | |
| ) % _full_shard_prefetch_stagger_slots() | |
| if _FULL_SHARD_PREFETCH_PENDING_CALLS > 0: | |
| _FULL_SHARD_PREFETCH_PENDING_CALLS -= 1 | |
| return | |
| thread = threading.Thread( | |
| target=_prefetch_full_shard_worker, | |
| args=(key,), | |
| name="omni-full-shard-prefetch", | |
| daemon=True, | |
| ) | |
| _FULL_SHARD_PREFETCH_THREAD = thread | |
| _FULL_SHARD_PREFETCH_KEY = key | |
| _FULL_SHARD_PREFETCH_LAST_KEY = key | |
| _FULL_SHARD_PREFETCH_PENDING_KEY = None | |
| _FULL_SHARD_PREFETCH_PENDING_CALLS = 0 | |
| thread.start() | |
| def load_hidden_states_slice(shard_path: Path, offset: int, length: int) -> torch.Tensor: | |
| tensor_cache_size = _max_cached_hidden_tensors() | |
| tensor_key = (os.fspath(shard_path), int(offset), int(length)) | |
| if tensor_cache_size > 0: | |
| with _LOCK: | |
| cached = _HIDDEN_TENSORS.get(tensor_key) | |
| if cached is not None: | |
| _HIDDEN_TENSORS.move_to_end(tensor_key) | |
| return cached | |
| materialize_full_shard = getattr(_OSS_BATCH_HINTS, "materialize_full_shard", True) | |
| full_shard = _get_full_shard_tensor(shard_path) if materialize_full_shard else None | |
| if full_shard is not None: | |
| hidden_states = full_shard[int(offset) : int(offset) + int(length)].clone( | |
| memory_format=torch.contiguous_format | |
| ) | |
| # A row slice of a contiguous tensor is itself considered contiguous, | |
| # so contiguous() would retain the complete multi-GiB parent storage. | |
| # clone() is required: only this small slice may cross DataLoader IPC. | |
| elif _max_open_shards() <= 0: | |
| with safe_open(str(shard_path), framework="pt", device="cpu") as reader: | |
| hidden_states = reader.get_slice("hidden_states")[ | |
| int(offset) : int(offset) + int(length) | |
| ].clone(memory_format=torch.contiguous_format) | |
| else: | |
| reader = _get_reader(shard_path) | |
| hidden_states = reader.get_slice("hidden_states")[ | |
| int(offset) : int(offset) + int(length) | |
| ].clone(memory_format=torch.contiguous_format) | |
| # Safetensors row slices can report ``is_contiguous() == True`` while | |
| # retaining the complete shard's mmap storage. Every branch above must | |
| # therefore clone rather than rely on contiguous(): DataLoader IPC should | |
| # transfer only the requested hidden-state rows. | |
| if tensor_cache_size > 0: | |
| with _LOCK: | |
| _HIDDEN_TENSORS[tensor_key] = hidden_states | |
| while len(_HIDDEN_TENSORS) > tensor_cache_size: | |
| _HIDDEN_TENSORS.popitem(last=False) | |
| return hidden_states | |
| def load_hidden_states_source( | |
| source_path: Path, | |
| *, | |
| expected_window_seconds: float = 0.6, | |
| expected_step_frames: int = 20, | |
| expected_motion_fps: int = 50, | |
| expected_sampling_rate: int = 16000, | |
| ) -> tuple[tuple[int, ...], tuple[torch.Tensor, ...]]: | |
| """Load all ordered audio windows from one compact HFD WAV file. | |
| The full tensor is cloned once while the file is open. Returned window | |
| views share that small source-local storage, so one dataset item performs | |
| one open/read rather than several random reads from unrelated GiB shards. | |
| """ | |
| source_path = Path(source_path) | |
| if not source_path.is_file(): | |
| raise FileNotFoundError(f"Missing source hidden states: {source_path}") | |
| with safe_open(str(source_path), framework="pt", device="cpu") as reader: | |
| metadata = reader.metadata() or {} | |
| expected_metadata = { | |
| "format": HFD_AUDIO_SOURCE_FORMAT, | |
| "encoder_contract": HFD_ONLINE_ENCODER_CONTRACT, | |
| "encoder_batch_size": str(HFD_ONLINE_ENCODER_BATCH_SIZE), | |
| "encoder_dtype": HFD_ONLINE_ENCODER_DTYPE, | |
| "attention_implementation": HFD_ONLINE_ENCODER_ATTENTION, | |
| "instruction_sha1": hashlib.sha1( | |
| AUDIO_COMMAND_STREAMING_PROMPT.encode("utf-8") | |
| ).hexdigest(), | |
| } | |
| mismatched = { | |
| key: (metadata.get(key), expected) | |
| for key, expected in expected_metadata.items() | |
| if metadata.get(key) != expected | |
| } | |
| if mismatched: | |
| raise RuntimeError( | |
| f"Unsupported HFD online encoder metadata at {source_path}: " | |
| f"{mismatched}" | |
| ) | |
| try: | |
| count = int(metadata["count"]) | |
| window_seconds = float(metadata["window_seconds"]) | |
| step_frames = int(metadata["step_frames"]) | |
| motion_fps = int(metadata["motion_fps"]) | |
| sampling_rate = int(metadata["sampling_rate"]) | |
| except (KeyError, TypeError, ValueError) as exc: | |
| raise RuntimeError( | |
| f"Incomplete HFD source metadata at {source_path}" | |
| ) from exc | |
| if ( | |
| count <= 0 | |
| or not math.isclose( | |
| window_seconds, | |
| float(expected_window_seconds), | |
| rel_tol=0.0, | |
| abs_tol=1e-9, | |
| ) | |
| or step_frames != int(expected_step_frames) | |
| or motion_fps != int(expected_motion_fps) | |
| or sampling_rate != int(expected_sampling_rate) | |
| ): | |
| raise RuntimeError( | |
| "HFD source timing mismatch at " | |
| f"{source_path}: count={count}, window={window_seconds}, " | |
| f"step_frames={step_frames}, fps={motion_fps}, " | |
| f"sampling_rate={sampling_rate}" | |
| ) | |
| required = {"hidden_states", "offsets", "frame_ids"} | |
| if not required.issubset(set(reader.keys())): | |
| raise RuntimeError( | |
| f"Missing HFD source tensors at {source_path}: " | |
| f"{sorted(required - set(reader.keys()))}" | |
| ) | |
| hidden_states = reader.get_tensor("hidden_states").clone( | |
| memory_format=torch.contiguous_format | |
| ) | |
| offsets = reader.get_tensor("offsets").to(dtype=torch.int64) | |
| frame_ids_tensor = reader.get_tensor("frame_ids").to(dtype=torch.int64) | |
| if hidden_states.ndim != 2: | |
| raise RuntimeError( | |
| f"HFD source hidden_states must be 2D at {source_path}, " | |
| f"got {tuple(hidden_states.shape)}" | |
| ) | |
| if offsets.shape != (count + 1,) or frame_ids_tensor.shape != (count,): | |
| raise RuntimeError( | |
| f"HFD source index shape mismatch at {source_path}: " | |
| f"offsets={tuple(offsets.shape)} frame_ids={tuple(frame_ids_tensor.shape)} " | |
| f"count={count}" | |
| ) | |
| if int(offsets[0]) != 0 or int(offsets[-1]) != int(hidden_states.shape[0]): | |
| raise RuntimeError(f"Invalid HFD source offsets at {source_path}") | |
| if bool(torch.any(offsets[1:] <= offsets[:-1])): | |
| raise RuntimeError(f"Non-increasing HFD source offsets at {source_path}") | |
| expected_frame_ids = ( | |
| torch.arange(count, dtype=torch.int64) * int(expected_step_frames) | |
| ) | |
| if not torch.equal(frame_ids_tensor, expected_frame_ids): | |
| raise RuntimeError( | |
| f"Non-contiguous HFD source frame IDs at {source_path}" | |
| ) | |
| frame_ids = tuple(int(value) for value in frame_ids_tensor.tolist()) | |
| windows = tuple( | |
| hidden_states[int(offsets[index]) : int(offsets[index + 1])] | |
| for index in range(count) | |
| ) | |
| return frame_ids, windows | |