Download src/psi/data/seed/video_motion_local_cache.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/video_motion_local_cache.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/seed/video_motion_local_cache.py
-
curl -L -o video_motion_local_cache.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/video_motion_local_cache.py
10.5 kB
| from __future__ import annotations | |
| import fcntl | |
| import hashlib | |
| import os | |
| import sqlite3 | |
| import threading | |
| import time | |
| from collections import defaultdict, deque | |
| from pathlib import Path | |
| from typing import Any, Callable, Iterator | |
| import numpy as np | |
| from torch.utils.data import ConcatDataset | |
| from psi.data.seed.omni_sqlite_cache import local_sqlite_path | |
| _TRUE_VALUES = {"1", "true", "yes", "y", "on"} | |
| _CACHE_VERSION = "psi0-video-motion-36d-f32-v1" | |
| _PREFETCH_LOCK = threading.Lock() | |
| _PREFETCH_THREAD: threading.Thread | None = None | |
| _PREFETCH_CURRENT_KEY: str | None = None | |
| _PREFETCH_PENDING: tuple[Any, str] | None = None | |
| _PREFETCHED_KEYS: deque[str] = deque(maxlen=16) | |
| def video_motion_local_cache_enabled() -> bool: | |
| return os.environ.get("PSI_VIDEO_MOTION_LOCAL_CACHE", "0").strip().lower() in _TRUE_VALUES | |
| def _motion_cache_root() -> Path: | |
| value = os.environ.get("PSI_VIDEO_MOTION_LOCAL_CACHE_DIR", "").strip() | |
| return Path(value or "/tmp/psi0_video_motion_arrays") | |
| def _motion_cache_path(motion_dir: Path) -> Path: | |
| # The source motion trees are immutable training inputs. The job-local | |
| # cache root is cleared by the launcher, so hashing path + format avoids | |
| # three remote stat calls on every cache hit. | |
| canonical = os.path.realpath(os.fspath(motion_dir)) | |
| digest = hashlib.sha1(f"{_CACHE_VERSION}\0{canonical}".encode("utf-8")).hexdigest() | |
| return _motion_cache_root() / digest[:2] / f"{digest}.npy" | |
| def _load_array(path: Path) -> np.ndarray: | |
| array = np.load(path, allow_pickle=False) | |
| if array.dtype != np.float32 or array.ndim != 2 or array.shape[1] != 36: | |
| raise ValueError( | |
| f"Invalid staged video motion array at {path}: " | |
| f"shape={array.shape} dtype={array.dtype}" | |
| ) | |
| return array | |
| def load_or_build_video_motion_array( | |
| motion_dir: Path, | |
| builder: Callable[[Path], np.ndarray], | |
| ) -> tuple[np.ndarray, bool]: | |
| """Return an exact float32 array from node-local storage. | |
| Advisory per-file locks let ranks and DataLoader workers share one local | |
| copy without modifying any source data. The boolean reports whether this | |
| call built the local artifact. | |
| """ | |
| motion_dir = Path(motion_dir) | |
| if not video_motion_local_cache_enabled(): | |
| return builder(motion_dir), False | |
| cache_path = _motion_cache_path(motion_dir) | |
| if cache_path.exists(): | |
| try: | |
| return _load_array(cache_path), False | |
| except (OSError, ValueError): | |
| pass | |
| cache_path.parent.mkdir(parents=True, exist_ok=True) | |
| lock_path = cache_path.with_suffix(cache_path.suffix + ".lock") | |
| with lock_path.open("a+b") as lock_file: | |
| fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) | |
| try: | |
| if cache_path.exists(): | |
| try: | |
| return _load_array(cache_path), False | |
| except (OSError, ValueError): | |
| cache_path.unlink(missing_ok=True) | |
| array = np.asarray(builder(motion_dir), dtype=np.float32) | |
| if array.ndim != 2 or array.shape[1] != 36: | |
| raise ValueError( | |
| f"Unexpected video motion array at {motion_dir}: shape={array.shape}" | |
| ) | |
| tmp_path = cache_path.with_name( | |
| f".{cache_path.name}.{os.getpid()}.{threading.get_ident()}.tmp" | |
| ) | |
| try: | |
| with tmp_path.open("wb") as f: | |
| np.save(f, array, allow_pickle=False) | |
| os.replace(tmp_path, cache_path) | |
| finally: | |
| tmp_path.unlink(missing_ok=True) | |
| return array, True | |
| finally: | |
| fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) | |
| def _iter_datasets(dataset: Any, visited: set[int] | None = None) -> Iterator[Any]: | |
| if visited is None: | |
| visited = set() | |
| identity = id(dataset) | |
| if identity in visited: | |
| return | |
| visited.add(identity) | |
| if callable(getattr(dataset, "prefetch_omni_motion_source_keys", None)): | |
| yield dataset | |
| return | |
| if isinstance(dataset, ConcatDataset): | |
| for child in dataset.datasets: | |
| yield from _iter_datasets(child, visited) | |
| return | |
| children = getattr(dataset, "datasets", None) | |
| if isinstance(children, (list, tuple)): | |
| for child in children: | |
| yield from _iter_datasets(child, visited) | |
| wrapped = getattr(dataset, "dataset", None) | |
| if wrapped is not None: | |
| yield from _iter_datasets(wrapped, visited) | |
| raw_dataset = getattr(dataset, "raw_dataset", None) | |
| if raw_dataset is not None: | |
| yield from _iter_datasets(raw_dataset, visited) | |
| def _prefetch_slot_count() -> int: | |
| try: | |
| return max(int(os.environ.get("PSI_VIDEO_MOTION_PREFETCH_SLOTS", "0")), 0) | |
| except ValueError: | |
| return 0 | |
| class _PrefetchSlot: | |
| def __init__(self) -> None: | |
| self.file = None | |
| def __enter__(self): | |
| slot_count = _prefetch_slot_count() | |
| lock_dir_value = os.environ.get("PSI_VIDEO_MOTION_PREFETCH_LOCK_DIR", "").strip() | |
| if slot_count <= 0 or not lock_dir_value: | |
| return self | |
| lock_dir = Path(lock_dir_value) | |
| lock_dir.mkdir(parents=True, exist_ok=True) | |
| first_slot = os.getpid() % slot_count | |
| while self.file is None: | |
| for offset in range(slot_count): | |
| slot = (first_slot + offset) % slot_count | |
| candidate = (lock_dir / f"slot.{slot:03d}.lock").open("a+b") | |
| try: | |
| fcntl.flock(candidate.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| except BlockingIOError: | |
| candidate.close() | |
| continue | |
| self.file = candidate | |
| break | |
| if self.file is None: | |
| time.sleep(0.02) | |
| return self | |
| def __exit__(self, exc_type, exc, traceback) -> None: | |
| del exc_type, exc, traceback | |
| if self.file is not None: | |
| try: | |
| fcntl.flock(self.file.fileno(), fcntl.LOCK_UN) | |
| finally: | |
| self.file.close() | |
| self.file = None | |
| def _stage_motion_shard(dataset: Any, shard_key: str) -> None: | |
| leaves = { | |
| str(leaf.omni_oss_source_group()): leaf | |
| for leaf in _iter_datasets(dataset) | |
| if callable(getattr(leaf, "omni_oss_source_group", None)) | |
| } | |
| if not leaves: | |
| return | |
| shard_path = Path(shard_key) | |
| index_path = shard_path.parent / "index.rank00.sqlite" | |
| sqlite_path = local_sqlite_path(index_path) | |
| connection = sqlite3.connect( | |
| f"file:{sqlite_path}?mode=ro&immutable=1", | |
| uri=True, | |
| timeout=120.0, | |
| ) | |
| try: | |
| row = connection.execute( | |
| "SELECT shard_id FROM shards WHERE shard=?", | |
| (shard_path.name,), | |
| ).fetchone() | |
| if row is None: | |
| raise KeyError(f"OSS shard is absent from its index: {shard_path}") | |
| grouped_keys: dict[str, list[str]] = defaultdict(list) | |
| for source_group, source_key in connection.execute( | |
| "SELECT source_group, source_key FROM samples " | |
| "WHERE shard_id=? ORDER BY sample_idx", | |
| (int(row[0]),), | |
| ): | |
| source_group = str(source_group) | |
| if source_group in leaves: | |
| grouped_keys[source_group].append(str(source_key)) | |
| finally: | |
| connection.close() | |
| started = time.perf_counter() | |
| source_keys = 0 | |
| source_motions = 0 | |
| built = 0 | |
| with _PrefetchSlot(): | |
| for source_group, keys in grouped_keys.items(): | |
| counts = leaves[source_group].prefetch_omni_motion_source_keys(keys) | |
| source_keys += int(counts[0]) | |
| source_motions += int(counts[1]) | |
| built += int(counts[2]) | |
| if os.environ.get("PSI_VIDEO_MOTION_PREFETCH_LOG", "0").strip().lower() in _TRUE_VALUES: | |
| print( | |
| f"[VIDEO MOTION PREFETCH] pid={os.getpid()} shard={shard_path.name} " | |
| f"keys={source_keys:,} motions={source_motions:,} built={built:,} " | |
| f"elapsed={time.perf_counter() - started:.3f}s", | |
| flush=True, | |
| ) | |
| def _prefetch_worker(dataset: Any, shard_key: str) -> None: | |
| global _PREFETCH_THREAD, _PREFETCH_CURRENT_KEY, _PREFETCH_PENDING | |
| current_dataset = dataset | |
| current_key: str | None = shard_key | |
| while current_key is not None: | |
| try: | |
| _stage_motion_shard(current_dataset, current_key) | |
| except Exception as exc: | |
| if os.environ.get("PSI_VIDEO_MOTION_PREFETCH_LOG", "0").strip().lower() in _TRUE_VALUES: | |
| print( | |
| f"[VIDEO MOTION PREFETCH] pid={os.getpid()} failed " | |
| f"{Path(current_key).name}: {exc}", | |
| flush=True, | |
| ) | |
| with _PREFETCH_LOCK: | |
| _PREFETCHED_KEYS.append(current_key) | |
| pending = _PREFETCH_PENDING | |
| _PREFETCH_PENDING = None | |
| if pending is None: | |
| _PREFETCH_THREAD = None | |
| _PREFETCH_CURRENT_KEY = None | |
| current_key = None | |
| else: | |
| current_dataset, current_key = pending | |
| _PREFETCH_CURRENT_KEY = current_key | |
| def prefetch_video_motion_for_omni_shard(dataset: Any, shard_path: str | Path | None) -> None: | |
| """Stage only the next video shard's motion arrays, not full batches.""" | |
| global _PREFETCH_THREAD, _PREFETCH_CURRENT_KEY, _PREFETCH_PENDING | |
| if not video_motion_local_cache_enabled(): | |
| return | |
| if os.environ.get("PSI_VIDEO_MOTION_PREFETCH", "0").strip().lower() not in _TRUE_VALUES: | |
| return | |
| if shard_path is None: | |
| return | |
| if not any(True for _ in _iter_datasets(dataset)): | |
| return | |
| shard_key = os.path.realpath(os.fspath(shard_path)) | |
| with _PREFETCH_LOCK: | |
| if ( | |
| shard_key == _PREFETCH_CURRENT_KEY | |
| or shard_key in _PREFETCHED_KEYS | |
| or (_PREFETCH_PENDING is not None and shard_key == _PREFETCH_PENDING[1]) | |
| ): | |
| return | |
| thread = _PREFETCH_THREAD | |
| if thread is not None and thread.is_alive(): | |
| _PREFETCH_PENDING = (dataset, shard_key) | |
| return | |
| thread = threading.Thread( | |
| target=_prefetch_worker, | |
| args=(dataset, shard_key), | |
| name="video-motion-shard-prefetch", | |
| daemon=True, | |
| ) | |
| _PREFETCH_THREAD = thread | |
| _PREFETCH_CURRENT_KEY = shard_key | |
| thread.start() | |