from __future__ import annotations import hashlib import json from collections.abc import Iterable, Sequence from pathlib import Path from typing import Any import numpy as np SOURCE_SPLIT_VERSION = 1 VIDEO_MANIFEST_IDENTITY_FORMAT = "psi0_video_manifest_identity_v1" _HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}" _UINT64_RANGE = 1 << 64 def video_manifest_identity_path(manifest_path: str | Path) -> Path: path = Path(manifest_path) return path.with_name(path.name + ".identity.json") def load_video_manifest_identity(manifest_path: str | Path) -> dict[str, Any] | None: """Load a path-independent identity for a copied/uploaded video manifest. Historical manifests keep the legacy path+mtime signature. New very large manifests ship this sidecar so indexes and lazy record caches built before OSS upload remain reusable after the mount path and mtime change. """ manifest = Path(manifest_path) identity_path = video_manifest_identity_path(manifest) if not identity_path.is_file(): return None payload = json.loads(identity_path.read_text(encoding="utf-8")) if payload.get("format") != VIDEO_MANIFEST_IDENTITY_FORMAT: raise RuntimeError(f"Unexpected video manifest identity format: {identity_path}") expected_bytes = int(payload.get("bytes") or -1) rows = int(payload.get("rows") or -1) sha256 = str(payload.get("sha256") or "") if expected_bytes <= 0 or rows <= 0 or len(sha256) != 64: raise RuntimeError(f"Incomplete video manifest identity: {identity_path}") actual_bytes = manifest.stat().st_size if actual_bytes != expected_bytes: raise RuntimeError( f"Video manifest identity size mismatch: {manifest} " f"actual={actual_bytes} expected={expected_bytes}" ) result = { "format": VIDEO_MANIFEST_IDENTITY_FORMAT, "bytes": expected_bytes, "rows": rows, "sha256": sha256, } for key in ("source_name", "sample_definition"): value = str(payload.get(key) or "").strip() if value: result[key] = value excluded_source_bases = payload.get("excluded_source_bases", []) if not isinstance(excluded_source_bases, list) or any( not isinstance(value, str) or not value.strip() for value in excluded_source_bases ): raise RuntimeError( f"Invalid excluded_source_bases in video manifest identity: {identity_path}" ) if len(set(excluded_source_bases)) != len(excluded_source_bases): raise RuntimeError( f"Duplicate excluded_source_bases in video manifest identity: {identity_path}" ) if excluded_source_bases: result["excluded_source_bases"] = sorted(excluded_source_bases) return result def source_leaf_key(value: object) -> str: """Return the stable motion leaf used across paired modalities. The current seed, dance, and 2264 manifests use the same unique motion basename in every modality. Video-clip prefix indexes already expose the corresponding basename (or an unambiguous encoded relative path). """ text = str(value).strip().replace("\\", "/").rstrip("/") if not text: raise ValueError("Motion source is empty") leaf = text.rsplit("/", 1)[-1] if leaf.lower().endswith(".csv"): leaf = leaf[:-4] if not leaf: raise ValueError(f"Cannot derive motion source key from {value!r}") return leaf def motion_source_id(namespace: str, value: object) -> str: namespace_norm = str(namespace).strip().replace("\\", "/").strip("/") if not namespace_norm: raise ValueError("split_source_namespace is required for source-grouped splitting") return f"{namespace_norm}/{source_leaf_key(value)}" def source_hash_u64(source_id: str, split_seed: int) -> int: payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8") return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False) def validation_source_ids( source_ids: Iterable[str], *, val_ratio: float, split_seed: int, ) -> set[str]: """Assign complete motion sources to validation with a stable hash. Hash thresholding makes the assignment independent of manifest row order and keeps the same source on the same side across modalities. For tiny datasets, deterministic fallbacks preserve a non-empty train and val split whenever that is mathematically possible. """ ratio = float(val_ratio) if not 0.0 <= ratio <= 1.0: raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}") unique = sorted(set(str(source_id) for source_id in source_ids)) if any(not source_id for source_id in unique): raise ValueError("Motion source ID is empty") if not unique or ratio <= 0.0: return set() if ratio >= 1.0: return set(unique) scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique] threshold = int(ratio * _UINT64_RANGE) selected = {source_id for score, source_id in scored if score < threshold} if len(unique) > 1 and not selected: selected.add(min(scored)[1]) if len(unique) > 1 and len(selected) == len(unique): selected.remove(max(scored)[1]) return selected def select_source_row_ids( source_ids: Sequence[str], *, split: str, val_ratio: float, split_seed: int, ) -> np.ndarray: """Return selected row IDs in original manifest order.""" split_norm = str(split or "train").lower() if split_norm == "all": return np.arange(len(source_ids), dtype=np.int64) if split_norm not in {"train", "val", "valid", "validation", "test"}: raise ValueError(f"Unsupported split={split!r}, expected train/val/all") val_sources = validation_source_ids( source_ids, val_ratio=val_ratio, split_seed=split_seed, ) choose_val = split_norm in {"val", "valid", "validation", "test"} selected = [ row_id for row_id, source_id in enumerate(source_ids) if (str(source_id) in val_sources) == choose_val ] return np.asarray(selected, dtype=np.int64)