from __future__ import annotations import fcntl import hashlib import os import sqlite3 import threading import time from collections import defaultdict, deque from pathlib import Path from typing import Any, Callable, Iterator import numpy as np from torch.utils.data import ConcatDataset from psi.data.seed.omni_sqlite_cache import local_sqlite_path _TRUE_VALUES = {"1", "true", "yes", "y", "on"} _CACHE_VERSION = "psi0-video-motion-36d-f32-v1" _PREFETCH_LOCK = threading.Lock() _PREFETCH_THREAD: threading.Thread | None = None _PREFETCH_CURRENT_KEY: str | None = None _PREFETCH_PENDING: tuple[Any, str] | None = None _PREFETCHED_KEYS: deque[str] = deque(maxlen=16) def video_motion_local_cache_enabled() -> bool: return os.environ.get("PSI_VIDEO_MOTION_LOCAL_CACHE", "0").strip().lower() in _TRUE_VALUES def _motion_cache_root() -> Path: value = os.environ.get("PSI_VIDEO_MOTION_LOCAL_CACHE_DIR", "").strip() return Path(value or "/tmp/psi0_video_motion_arrays") def _motion_cache_path(motion_dir: Path) -> Path: # The source motion trees are immutable training inputs. The job-local # cache root is cleared by the launcher, so hashing path + format avoids # three remote stat calls on every cache hit. canonical = os.path.realpath(os.fspath(motion_dir)) digest = hashlib.sha1(f"{_CACHE_VERSION}\0{canonical}".encode("utf-8")).hexdigest() return _motion_cache_root() / digest[:2] / f"{digest}.npy" def _load_array(path: Path) -> np.ndarray: array = np.load(path, allow_pickle=False) if array.dtype != np.float32 or array.ndim != 2 or array.shape[1] != 36: raise ValueError( f"Invalid staged video motion array at {path}: " f"shape={array.shape} dtype={array.dtype}" ) return array def load_or_build_video_motion_array( motion_dir: Path, builder: Callable[[Path], np.ndarray], ) -> tuple[np.ndarray, bool]: """Return an exact float32 array from node-local storage. Advisory per-file locks let ranks and DataLoader workers share one local copy without modifying any source data. The boolean reports whether this call built the local artifact. """ motion_dir = Path(motion_dir) if not video_motion_local_cache_enabled(): return builder(motion_dir), False cache_path = _motion_cache_path(motion_dir) if cache_path.exists(): try: return _load_array(cache_path), False except (OSError, ValueError): pass cache_path.parent.mkdir(parents=True, exist_ok=True) lock_path = cache_path.with_suffix(cache_path.suffix + ".lock") with lock_path.open("a+b") as lock_file: fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) try: if cache_path.exists(): try: return _load_array(cache_path), False except (OSError, ValueError): cache_path.unlink(missing_ok=True) array = np.asarray(builder(motion_dir), dtype=np.float32) if array.ndim != 2 or array.shape[1] != 36: raise ValueError( f"Unexpected video motion array at {motion_dir}: shape={array.shape}" ) tmp_path = cache_path.with_name( f".{cache_path.name}.{os.getpid()}.{threading.get_ident()}.tmp" ) try: with tmp_path.open("wb") as f: np.save(f, array, allow_pickle=False) os.replace(tmp_path, cache_path) finally: tmp_path.unlink(missing_ok=True) return array, True finally: fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) def _iter_datasets(dataset: Any, visited: set[int] | None = None) -> Iterator[Any]: if visited is None: visited = set() identity = id(dataset) if identity in visited: return visited.add(identity) if callable(getattr(dataset, "prefetch_omni_motion_source_keys", None)): yield dataset return if isinstance(dataset, ConcatDataset): for child in dataset.datasets: yield from _iter_datasets(child, visited) return children = getattr(dataset, "datasets", None) if isinstance(children, (list, tuple)): for child in children: yield from _iter_datasets(child, visited) wrapped = getattr(dataset, "dataset", None) if wrapped is not None: yield from _iter_datasets(wrapped, visited) raw_dataset = getattr(dataset, "raw_dataset", None) if raw_dataset is not None: yield from _iter_datasets(raw_dataset, visited) def _prefetch_slot_count() -> int: try: return max(int(os.environ.get("PSI_VIDEO_MOTION_PREFETCH_SLOTS", "0")), 0) except ValueError: return 0 class _PrefetchSlot: def __init__(self) -> None: self.file = None def __enter__(self): slot_count = _prefetch_slot_count() lock_dir_value = os.environ.get("PSI_VIDEO_MOTION_PREFETCH_LOCK_DIR", "").strip() if slot_count <= 0 or not lock_dir_value: return self lock_dir = Path(lock_dir_value) lock_dir.mkdir(parents=True, exist_ok=True) first_slot = os.getpid() % slot_count while self.file is None: for offset in range(slot_count): slot = (first_slot + offset) % slot_count candidate = (lock_dir / f"slot.{slot:03d}.lock").open("a+b") try: fcntl.flock(candidate.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: candidate.close() continue self.file = candidate break if self.file is None: time.sleep(0.02) return self def __exit__(self, exc_type, exc, traceback) -> None: del exc_type, exc, traceback if self.file is not None: try: fcntl.flock(self.file.fileno(), fcntl.LOCK_UN) finally: self.file.close() self.file = None def _stage_motion_shard(dataset: Any, shard_key: str) -> None: leaves = { str(leaf.omni_oss_source_group()): leaf for leaf in _iter_datasets(dataset) if callable(getattr(leaf, "omni_oss_source_group", None)) } if not leaves: return shard_path = Path(shard_key) index_path = shard_path.parent / "index.rank00.sqlite" sqlite_path = local_sqlite_path(index_path) connection = sqlite3.connect( f"file:{sqlite_path}?mode=ro&immutable=1", uri=True, timeout=120.0, ) try: row = connection.execute( "SELECT shard_id FROM shards WHERE shard=?", (shard_path.name,), ).fetchone() if row is None: raise KeyError(f"OSS shard is absent from its index: {shard_path}") grouped_keys: dict[str, list[str]] = defaultdict(list) for source_group, source_key in connection.execute( "SELECT source_group, source_key FROM samples " "WHERE shard_id=? ORDER BY sample_idx", (int(row[0]),), ): source_group = str(source_group) if source_group in leaves: grouped_keys[source_group].append(str(source_key)) finally: connection.close() started = time.perf_counter() source_keys = 0 source_motions = 0 built = 0 with _PrefetchSlot(): for source_group, keys in grouped_keys.items(): counts = leaves[source_group].prefetch_omni_motion_source_keys(keys) source_keys += int(counts[0]) source_motions += int(counts[1]) built += int(counts[2]) if os.environ.get("PSI_VIDEO_MOTION_PREFETCH_LOG", "0").strip().lower() in _TRUE_VALUES: print( f"[VIDEO MOTION PREFETCH] pid={os.getpid()} shard={shard_path.name} " f"keys={source_keys:,} motions={source_motions:,} built={built:,} " f"elapsed={time.perf_counter() - started:.3f}s", flush=True, ) def _prefetch_worker(dataset: Any, shard_key: str) -> None: global _PREFETCH_THREAD, _PREFETCH_CURRENT_KEY, _PREFETCH_PENDING current_dataset = dataset current_key: str | None = shard_key while current_key is not None: try: _stage_motion_shard(current_dataset, current_key) except Exception as exc: if os.environ.get("PSI_VIDEO_MOTION_PREFETCH_LOG", "0").strip().lower() in _TRUE_VALUES: print( f"[VIDEO MOTION PREFETCH] pid={os.getpid()} failed " f"{Path(current_key).name}: {exc}", flush=True, ) with _PREFETCH_LOCK: _PREFETCHED_KEYS.append(current_key) pending = _PREFETCH_PENDING _PREFETCH_PENDING = None if pending is None: _PREFETCH_THREAD = None _PREFETCH_CURRENT_KEY = None current_key = None else: current_dataset, current_key = pending _PREFETCH_CURRENT_KEY = current_key def prefetch_video_motion_for_omni_shard(dataset: Any, shard_path: str | Path | None) -> None: """Stage only the next video shard's motion arrays, not full batches.""" global _PREFETCH_THREAD, _PREFETCH_CURRENT_KEY, _PREFETCH_PENDING if not video_motion_local_cache_enabled(): return if os.environ.get("PSI_VIDEO_MOTION_PREFETCH", "0").strip().lower() not in _TRUE_VALUES: return if shard_path is None: return if not any(True for _ in _iter_datasets(dataset)): return shard_key = os.path.realpath(os.fspath(shard_path)) with _PREFETCH_LOCK: if ( shard_key == _PREFETCH_CURRENT_KEY or shard_key in _PREFETCHED_KEYS or (_PREFETCH_PENDING is not None and shard_key == _PREFETCH_PENDING[1]) ): return thread = _PREFETCH_THREAD if thread is not None and thread.is_alive(): _PREFETCH_PENDING = (dataset, shard_key) return thread = threading.Thread( target=_prefetch_worker, args=(dataset, shard_key), name="video-motion-shard-prefetch", daemon=True, ) _PREFETCH_THREAD = thread _PREFETCH_CURRENT_KEY = shard_key thread.start()