File size: 6,233 Bytes
980282b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
from __future__ import annotations

import hashlib
import json
from collections.abc import Iterable, Sequence
from pathlib import Path
from typing import Any

import numpy as np


SOURCE_SPLIT_VERSION = 1
VIDEO_MANIFEST_IDENTITY_FORMAT = "psi0_video_manifest_identity_v1"
_HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}"
_UINT64_RANGE = 1 << 64


def video_manifest_identity_path(manifest_path: str | Path) -> Path:
    path = Path(manifest_path)
    return path.with_name(path.name + ".identity.json")


def load_video_manifest_identity(manifest_path: str | Path) -> dict[str, Any] | None:
    """Load a path-independent identity for a copied/uploaded video manifest.

    Historical manifests keep the legacy path+mtime signature. New very large
    manifests ship this sidecar so indexes and lazy record caches built before
    OSS upload remain reusable after the mount path and mtime change.
    """
    manifest = Path(manifest_path)
    identity_path = video_manifest_identity_path(manifest)
    if not identity_path.is_file():
        return None
    payload = json.loads(identity_path.read_text(encoding="utf-8"))
    if payload.get("format") != VIDEO_MANIFEST_IDENTITY_FORMAT:
        raise RuntimeError(f"Unexpected video manifest identity format: {identity_path}")
    expected_bytes = int(payload.get("bytes") or -1)
    rows = int(payload.get("rows") or -1)
    sha256 = str(payload.get("sha256") or "")
    if expected_bytes <= 0 or rows <= 0 or len(sha256) != 64:
        raise RuntimeError(f"Incomplete video manifest identity: {identity_path}")
    actual_bytes = manifest.stat().st_size
    if actual_bytes != expected_bytes:
        raise RuntimeError(
            f"Video manifest identity size mismatch: {manifest} "
            f"actual={actual_bytes} expected={expected_bytes}"
        )
    result = {
        "format": VIDEO_MANIFEST_IDENTITY_FORMAT,
        "bytes": expected_bytes,
        "rows": rows,
        "sha256": sha256,
    }
    for key in ("source_name", "sample_definition"):
        value = str(payload.get(key) or "").strip()
        if value:
            result[key] = value
    excluded_source_bases = payload.get("excluded_source_bases", [])
    if not isinstance(excluded_source_bases, list) or any(
        not isinstance(value, str) or not value.strip()
        for value in excluded_source_bases
    ):
        raise RuntimeError(
            f"Invalid excluded_source_bases in video manifest identity: {identity_path}"
        )
    if len(set(excluded_source_bases)) != len(excluded_source_bases):
        raise RuntimeError(
            f"Duplicate excluded_source_bases in video manifest identity: {identity_path}"
        )
    if excluded_source_bases:
        result["excluded_source_bases"] = sorted(excluded_source_bases)
    return result


def source_leaf_key(value: object) -> str:
    """Return the stable motion leaf used across paired modalities.

    The current seed, dance, and 2264 manifests use the same unique motion
    basename in every modality.  Video-clip prefix indexes already expose the
    corresponding basename (or an unambiguous encoded relative path).
    """

    text = str(value).strip().replace("\\", "/").rstrip("/")
    if not text:
        raise ValueError("Motion source is empty")
    leaf = text.rsplit("/", 1)[-1]
    if leaf.lower().endswith(".csv"):
        leaf = leaf[:-4]
    if not leaf:
        raise ValueError(f"Cannot derive motion source key from {value!r}")
    return leaf


def motion_source_id(namespace: str, value: object) -> str:
    namespace_norm = str(namespace).strip().replace("\\", "/").strip("/")
    if not namespace_norm:
        raise ValueError("split_source_namespace is required for source-grouped splitting")
    return f"{namespace_norm}/{source_leaf_key(value)}"


def source_hash_u64(source_id: str, split_seed: int) -> int:
    payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8")
    return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False)


def validation_source_ids(
    source_ids: Iterable[str],
    *,
    val_ratio: float,
    split_seed: int,
) -> set[str]:
    """Assign complete motion sources to validation with a stable hash.

    Hash thresholding makes the assignment independent of manifest row order
    and keeps the same source on the same side across modalities.  For tiny
    datasets, deterministic fallbacks preserve a non-empty train and val split
    whenever that is mathematically possible.
    """

    ratio = float(val_ratio)
    if not 0.0 <= ratio <= 1.0:
        raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}")
    unique = sorted(set(str(source_id) for source_id in source_ids))
    if any(not source_id for source_id in unique):
        raise ValueError("Motion source ID is empty")
    if not unique or ratio <= 0.0:
        return set()
    if ratio >= 1.0:
        return set(unique)

    scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique]
    threshold = int(ratio * _UINT64_RANGE)
    selected = {source_id for score, source_id in scored if score < threshold}
    if len(unique) > 1 and not selected:
        selected.add(min(scored)[1])
    if len(unique) > 1 and len(selected) == len(unique):
        selected.remove(max(scored)[1])
    return selected


def select_source_row_ids(
    source_ids: Sequence[str],
    *,
    split: str,
    val_ratio: float,
    split_seed: int,
) -> np.ndarray:
    """Return selected row IDs in original manifest order."""

    split_norm = str(split or "train").lower()
    if split_norm == "all":
        return np.arange(len(source_ids), dtype=np.int64)
    if split_norm not in {"train", "val", "valid", "validation", "test"}:
        raise ValueError(f"Unsupported split={split!r}, expected train/val/all")

    val_sources = validation_source_ids(
        source_ids,
        val_ratio=val_ratio,
        split_seed=split_seed,
    )
    choose_val = split_norm in {"val", "valid", "validation", "test"}
    selected = [
        row_id
        for row_id, source_id in enumerate(source_ids)
        if (str(source_id) in val_sources) == choose_val
    ]
    return np.asarray(selected, dtype=np.int64)