File size: 3,652 Bytes
02626a9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | from __future__ import annotations
import hashlib
from collections.abc import Iterable, Sequence
import numpy as np
SOURCE_SPLIT_VERSION = 1
_HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}"
_UINT64_RANGE = 1 << 64
def source_leaf_key(value: object) -> str:
"""Return the stable motion leaf used across paired modalities.
The current seed, dance, and 2264 manifests use the same unique motion
basename in every modality. Video-clip prefix indexes already expose the
corresponding basename (or an unambiguous encoded relative path).
"""
text = str(value).strip().replace("\\", "/").rstrip("/")
if not text:
raise ValueError("Motion source is empty")
leaf = text.rsplit("/", 1)[-1]
if leaf.lower().endswith(".csv"):
leaf = leaf[:-4]
if not leaf:
raise ValueError(f"Cannot derive motion source key from {value!r}")
return leaf
def motion_source_id(namespace: str, value: object) -> str:
namespace_norm = str(namespace).strip().replace("\\", "/").strip("/")
if not namespace_norm:
raise ValueError("split_source_namespace is required for source-grouped splitting")
return f"{namespace_norm}/{source_leaf_key(value)}"
def source_hash_u64(source_id: str, split_seed: int) -> int:
payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8")
return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False)
def validation_source_ids(
source_ids: Iterable[str],
*,
val_ratio: float,
split_seed: int,
) -> set[str]:
"""Assign complete motion sources to validation with a stable hash.
Hash thresholding makes the assignment independent of manifest row order
and keeps the same source on the same side across modalities. For tiny
datasets, deterministic fallbacks preserve a non-empty train and val split
whenever that is mathematically possible.
"""
ratio = float(val_ratio)
if not 0.0 <= ratio <= 1.0:
raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}")
unique = sorted(set(str(source_id) for source_id in source_ids))
if any(not source_id for source_id in unique):
raise ValueError("Motion source ID is empty")
if not unique or ratio <= 0.0:
return set()
if ratio >= 1.0:
return set(unique)
scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique]
threshold = int(ratio * _UINT64_RANGE)
selected = {source_id for score, source_id in scored if score < threshold}
if len(unique) > 1 and not selected:
selected.add(min(scored)[1])
if len(unique) > 1 and len(selected) == len(unique):
selected.remove(max(scored)[1])
return selected
def select_source_row_ids(
source_ids: Sequence[str],
*,
split: str,
val_ratio: float,
split_seed: int,
) -> np.ndarray:
"""Return selected row IDs in original manifest order."""
split_norm = str(split or "train").lower()
if split_norm == "all":
return np.arange(len(source_ids), dtype=np.int64)
if split_norm not in {"train", "val", "valid", "validation", "test"}:
raise ValueError(f"Unsupported split={split!r}, expected train/val/all")
val_sources = validation_source_ids(
source_ids,
val_ratio=val_ratio,
split_seed=split_seed,
)
choose_val = split_norm in {"val", "valid", "validation", "test"}
selected = [
row_id
for row_id, source_id in enumerate(source_ids)
if (str(source_id) in val_sources) == choose_val
]
return np.asarray(selected, dtype=np.int64)
|