from __future__ import annotations import hashlib from collections.abc import Iterable, Sequence import numpy as np SOURCE_SPLIT_VERSION = 1 _HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}" _UINT64_RANGE = 1 << 64 def source_leaf_key(value: object) -> str: """Return the stable motion leaf used across paired modalities. The current seed, dance, and 2264 manifests use the same unique motion basename in every modality. Video-clip prefix indexes already expose the corresponding basename (or an unambiguous encoded relative path). """ text = str(value).strip().replace("\\", "/").rstrip("/") if not text: raise ValueError("Motion source is empty") leaf = text.rsplit("/", 1)[-1] if leaf.lower().endswith(".csv"): leaf = leaf[:-4] if not leaf: raise ValueError(f"Cannot derive motion source key from {value!r}") return leaf def motion_source_id(namespace: str, value: object) -> str: namespace_norm = str(namespace).strip().replace("\\", "/").strip("/") if not namespace_norm: raise ValueError("split_source_namespace is required for source-grouped splitting") return f"{namespace_norm}/{source_leaf_key(value)}" def source_hash_u64(source_id: str, split_seed: int) -> int: payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8") return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False) def validation_source_ids( source_ids: Iterable[str], *, val_ratio: float, split_seed: int, ) -> set[str]: """Assign complete motion sources to validation with a stable hash. Hash thresholding makes the assignment independent of manifest row order and keeps the same source on the same side across modalities. For tiny datasets, deterministic fallbacks preserve a non-empty train and val split whenever that is mathematically possible. """ ratio = float(val_ratio) if not 0.0 <= ratio <= 1.0: raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}") unique = sorted(set(str(source_id) for source_id in source_ids)) if any(not source_id for source_id in unique): raise ValueError("Motion source ID is empty") if not unique or ratio <= 0.0: return set() if ratio >= 1.0: return set(unique) scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique] threshold = int(ratio * _UINT64_RANGE) selected = {source_id for score, source_id in scored if score < threshold} if len(unique) > 1 and not selected: selected.add(min(scored)[1]) if len(unique) > 1 and len(selected) == len(unique): selected.remove(max(scored)[1]) return selected def select_source_row_ids( source_ids: Sequence[str], *, split: str, val_ratio: float, split_seed: int, ) -> np.ndarray: """Return selected row IDs in original manifest order.""" split_norm = str(split or "train").lower() if split_norm == "all": return np.arange(len(source_ids), dtype=np.int64) if split_norm not in {"train", "val", "valid", "validation", "test"}: raise ValueError(f"Unsupported split={split!r}, expected train/val/all") val_sources = validation_source_ids( source_ids, val_ratio=val_ratio, split_seed=split_seed, ) choose_val = split_norm in {"val", "valid", "validation", "test"} selected = [ row_id for row_id, source_id in enumerate(source_ids) if (str(source_id) in val_sources) == choose_val ] return np.asarray(selected, dtype=np.int64)