File size: 3,652 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
from __future__ import annotations

import hashlib
from collections.abc import Iterable, Sequence

import numpy as np


SOURCE_SPLIT_VERSION = 1
_HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}"
_UINT64_RANGE = 1 << 64


def source_leaf_key(value: object) -> str:
    """Return the stable motion leaf used across paired modalities.

    The current seed, dance, and 2264 manifests use the same unique motion
    basename in every modality.  Video-clip prefix indexes already expose the
    corresponding basename (or an unambiguous encoded relative path).
    """

    text = str(value).strip().replace("\\", "/").rstrip("/")
    if not text:
        raise ValueError("Motion source is empty")
    leaf = text.rsplit("/", 1)[-1]
    if leaf.lower().endswith(".csv"):
        leaf = leaf[:-4]
    if not leaf:
        raise ValueError(f"Cannot derive motion source key from {value!r}")
    return leaf


def motion_source_id(namespace: str, value: object) -> str:
    namespace_norm = str(namespace).strip().replace("\\", "/").strip("/")
    if not namespace_norm:
        raise ValueError("split_source_namespace is required for source-grouped splitting")
    return f"{namespace_norm}/{source_leaf_key(value)}"


def source_hash_u64(source_id: str, split_seed: int) -> int:
    payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8")
    return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False)


def validation_source_ids(
    source_ids: Iterable[str],
    *,
    val_ratio: float,
    split_seed: int,
) -> set[str]:
    """Assign complete motion sources to validation with a stable hash.

    Hash thresholding makes the assignment independent of manifest row order
    and keeps the same source on the same side across modalities.  For tiny
    datasets, deterministic fallbacks preserve a non-empty train and val split
    whenever that is mathematically possible.
    """

    ratio = float(val_ratio)
    if not 0.0 <= ratio <= 1.0:
        raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}")
    unique = sorted(set(str(source_id) for source_id in source_ids))
    if any(not source_id for source_id in unique):
        raise ValueError("Motion source ID is empty")
    if not unique or ratio <= 0.0:
        return set()
    if ratio >= 1.0:
        return set(unique)

    scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique]
    threshold = int(ratio * _UINT64_RANGE)
    selected = {source_id for score, source_id in scored if score < threshold}
    if len(unique) > 1 and not selected:
        selected.add(min(scored)[1])
    if len(unique) > 1 and len(selected) == len(unique):
        selected.remove(max(scored)[1])
    return selected


def select_source_row_ids(
    source_ids: Sequence[str],
    *,
    split: str,
    val_ratio: float,
    split_seed: int,
) -> np.ndarray:
    """Return selected row IDs in original manifest order."""

    split_norm = str(split or "train").lower()
    if split_norm == "all":
        return np.arange(len(source_ids), dtype=np.int64)
    if split_norm not in {"train", "val", "valid", "validation", "test"}:
        raise ValueError(f"Unsupported split={split!r}, expected train/val/all")

    val_sources = validation_source_ids(
        source_ids,
        val_ratio=val_ratio,
        split_seed=split_seed,
    )
    choose_val = split_norm in {"val", "valid", "validation", "test"}
    selected = [
        row_id
        for row_id, source_id in enumerate(source_ids)
        if (str(source_id) in val_sources) == choose_val
    ]
    return np.asarray(selected, dtype=np.int64)