File size: 5,345 Bytes
980282b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
from __future__ import annotations

import json

import numpy as np

from psi.data.seed.omni_oss_sampler import (
    _video_manifest_signature,
    ensure_video_split_inverse_index,
)
from psi.data.seed.source_split import (
    VIDEO_MANIFEST_IDENTITY_FORMAT,
    load_video_manifest_identity,
    motion_source_id,
    select_source_row_ids,
    source_leaf_key,
    video_manifest_identity_path,
    validation_source_ids,
)


def test_motion_source_id_matches_across_modalities():
    text = motion_source_id("seed", "g1/csv/240918/body_check_001__A548.csv")
    command = motion_source_id("seed", "body_check_001__A548")
    assert text == command == "seed/body_check_001__A548"
    assert source_leaf_key(r"converted_motion\00001\dance.foo") == "dance.foo"


def test_source_rows_are_atomic_complementary_and_manifest_ordered():
    sources = ["family/a", "family/a", "family/b", "family/c", "family/c", "family/d"]
    train = select_source_row_ids(sources, split="train", val_ratio=0.5, split_seed=42)
    val = select_source_row_ids(sources, split="val", val_ratio=0.5, split_seed=42)

    assert np.all(train[:-1] < train[1:])
    assert np.all(val[:-1] < val[1:])
    assert set(train).isdisjoint(set(val))
    assert set(train) | set(val) == set(range(len(sources)))
    for source in set(sources):
        memberships = {int(row_id in set(val)) for row_id, value in enumerate(sources) if value == source}
        assert len(memberships) == 1


def test_source_assignment_is_independent_of_input_order():
    sources = [f"source/{index:03d}" for index in range(100)]
    selected = validation_source_ids(sources, val_ratio=0.1, split_seed=7)
    reversed_selected = validation_source_ids(reversed(sources), val_ratio=0.1, split_seed=7)
    assert selected == reversed_selected


def test_tiny_dataset_keeps_train_and_val_nonempty():
    sources = ["source/a", "source/b"]
    val = validation_source_ids(sources, val_ratio=1e-12, split_seed=1)
    assert len(val) == 1
    val = validation_source_ids(sources, val_ratio=1.0 - 1e-12, split_seed=1)
    assert len(val) == 1


def test_manifest_identity_survives_copy_and_mtime_change(tmp_path):
    payload = b'{"video_path":"demo.chunk00.f000000.mp4"}\n'
    identity = {
        "format": VIDEO_MANIFEST_IDENTITY_FORMAT,
        "bytes": len(payload),
        "rows": 1,
        "sha256": "a" * 64,
    }
    manifests = []
    for index in range(2):
        root = tmp_path / f"mount-{index}"
        root.mkdir()
        manifest = root / "metadata.jsonl"
        manifest.write_bytes(payload)
        video_manifest_identity_path(manifest).write_text(
            json.dumps(identity),
            encoding="utf-8",
        )
        manifests.append(manifest)
    manifests[1].touch()

    assert load_video_manifest_identity(manifests[0]) == identity
    assert load_video_manifest_identity(manifests[1]) == identity
    assert _video_manifest_signature(manifests[0]) == _video_manifest_signature(manifests[1])


def test_manifest_identity_keeps_explicit_excluded_video_sources(tmp_path):
    payload = b'{"video_path":"kept.chunk00.f000000.mp4"}\n'
    manifest = tmp_path / "metadata.jsonl"
    manifest.write_bytes(payload)
    video_manifest_identity_path(manifest).write_text(
        json.dumps(
            {
                "format": VIDEO_MANIFEST_IDENTITY_FORMAT,
                "bytes": len(payload),
                "rows": 1,
                "sha256": "c" * 64,
                "excluded_source_bases": ["missing-b", "missing-a"],
            }
        ),
        encoding="utf-8",
    )

    identity = load_video_manifest_identity(manifest)

    assert identity is not None
    assert identity["excluded_source_bases"] == ["missing-a", "missing-b"]


def test_manifest_identity_rejects_truncated_copy(tmp_path):
    manifest = tmp_path / "metadata.jsonl"
    manifest.write_bytes(b"{}\n")
    video_manifest_identity_path(manifest).write_text(
        json.dumps(
            {
                "format": VIDEO_MANIFEST_IDENTITY_FORMAT,
                "bytes": 999,
                "rows": 1,
                "sha256": "b" * 64,
            }
        ),
        encoding="utf-8",
    )
    with np.testing.assert_raises_regex(RuntimeError, "size mismatch"):
        load_video_manifest_identity(manifest)


def test_video_source_inverse_has_no_gaps_overlap_or_partial_sources(tmp_path):
    prefixes = {
        "a": (0, 3, "numeric"),
        "b": (3, 2, "numeric"),
        "c": (5, 4, "numeric"),
        "d": (9, 1, "numeric"),
    }
    kwargs = {
        "total_rows": 10,
        "val_ratio": 0.5,
        "split_seed": 42,
        "cache_dir": tmp_path,
        "source_signature": "synthetic",
        "prefixes": prefixes,
        "split_source_namespace": "video",
    }
    train = np.load(ensure_video_split_inverse_index(split="train", **kwargs))
    val = np.load(ensure_video_split_inverse_index(split="val", **kwargs))
    train_mask = train >= 0
    val_mask = val >= 0

    assert np.all(np.logical_xor(train_mask, val_mask))
    assert np.array_equal(train[train_mask], np.arange(train_mask.sum(), dtype=np.int32))
    assert np.array_equal(val[val_mask], np.arange(val_mask.sum(), dtype=np.int32))
    for start, count, _ in prefixes.values():
        assert train_mask[start : start + count].all() or val_mask[start : start + count].all()