File size: 5,345 Bytes
980282b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | from __future__ import annotations
import json
import numpy as np
from psi.data.seed.omni_oss_sampler import (
_video_manifest_signature,
ensure_video_split_inverse_index,
)
from psi.data.seed.source_split import (
VIDEO_MANIFEST_IDENTITY_FORMAT,
load_video_manifest_identity,
motion_source_id,
select_source_row_ids,
source_leaf_key,
video_manifest_identity_path,
validation_source_ids,
)
def test_motion_source_id_matches_across_modalities():
text = motion_source_id("seed", "g1/csv/240918/body_check_001__A548.csv")
command = motion_source_id("seed", "body_check_001__A548")
assert text == command == "seed/body_check_001__A548"
assert source_leaf_key(r"converted_motion\00001\dance.foo") == "dance.foo"
def test_source_rows_are_atomic_complementary_and_manifest_ordered():
sources = ["family/a", "family/a", "family/b", "family/c", "family/c", "family/d"]
train = select_source_row_ids(sources, split="train", val_ratio=0.5, split_seed=42)
val = select_source_row_ids(sources, split="val", val_ratio=0.5, split_seed=42)
assert np.all(train[:-1] < train[1:])
assert np.all(val[:-1] < val[1:])
assert set(train).isdisjoint(set(val))
assert set(train) | set(val) == set(range(len(sources)))
for source in set(sources):
memberships = {int(row_id in set(val)) for row_id, value in enumerate(sources) if value == source}
assert len(memberships) == 1
def test_source_assignment_is_independent_of_input_order():
sources = [f"source/{index:03d}" for index in range(100)]
selected = validation_source_ids(sources, val_ratio=0.1, split_seed=7)
reversed_selected = validation_source_ids(reversed(sources), val_ratio=0.1, split_seed=7)
assert selected == reversed_selected
def test_tiny_dataset_keeps_train_and_val_nonempty():
sources = ["source/a", "source/b"]
val = validation_source_ids(sources, val_ratio=1e-12, split_seed=1)
assert len(val) == 1
val = validation_source_ids(sources, val_ratio=1.0 - 1e-12, split_seed=1)
assert len(val) == 1
def test_manifest_identity_survives_copy_and_mtime_change(tmp_path):
payload = b'{"video_path":"demo.chunk00.f000000.mp4"}\n'
identity = {
"format": VIDEO_MANIFEST_IDENTITY_FORMAT,
"bytes": len(payload),
"rows": 1,
"sha256": "a" * 64,
}
manifests = []
for index in range(2):
root = tmp_path / f"mount-{index}"
root.mkdir()
manifest = root / "metadata.jsonl"
manifest.write_bytes(payload)
video_manifest_identity_path(manifest).write_text(
json.dumps(identity),
encoding="utf-8",
)
manifests.append(manifest)
manifests[1].touch()
assert load_video_manifest_identity(manifests[0]) == identity
assert load_video_manifest_identity(manifests[1]) == identity
assert _video_manifest_signature(manifests[0]) == _video_manifest_signature(manifests[1])
def test_manifest_identity_keeps_explicit_excluded_video_sources(tmp_path):
payload = b'{"video_path":"kept.chunk00.f000000.mp4"}\n'
manifest = tmp_path / "metadata.jsonl"
manifest.write_bytes(payload)
video_manifest_identity_path(manifest).write_text(
json.dumps(
{
"format": VIDEO_MANIFEST_IDENTITY_FORMAT,
"bytes": len(payload),
"rows": 1,
"sha256": "c" * 64,
"excluded_source_bases": ["missing-b", "missing-a"],
}
),
encoding="utf-8",
)
identity = load_video_manifest_identity(manifest)
assert identity is not None
assert identity["excluded_source_bases"] == ["missing-a", "missing-b"]
def test_manifest_identity_rejects_truncated_copy(tmp_path):
manifest = tmp_path / "metadata.jsonl"
manifest.write_bytes(b"{}\n")
video_manifest_identity_path(manifest).write_text(
json.dumps(
{
"format": VIDEO_MANIFEST_IDENTITY_FORMAT,
"bytes": 999,
"rows": 1,
"sha256": "b" * 64,
}
),
encoding="utf-8",
)
with np.testing.assert_raises_regex(RuntimeError, "size mismatch"):
load_video_manifest_identity(manifest)
def test_video_source_inverse_has_no_gaps_overlap_or_partial_sources(tmp_path):
prefixes = {
"a": (0, 3, "numeric"),
"b": (3, 2, "numeric"),
"c": (5, 4, "numeric"),
"d": (9, 1, "numeric"),
}
kwargs = {
"total_rows": 10,
"val_ratio": 0.5,
"split_seed": 42,
"cache_dir": tmp_path,
"source_signature": "synthetic",
"prefixes": prefixes,
"split_source_namespace": "video",
}
train = np.load(ensure_video_split_inverse_index(split="train", **kwargs))
val = np.load(ensure_video_split_inverse_index(split="val", **kwargs))
train_mask = train >= 0
val_mask = val >= 0
assert np.all(np.logical_xor(train_mask, val_mask))
assert np.array_equal(train[train_mask], np.arange(train_mask.sum(), dtype=np.int32))
assert np.array_equal(val[val_mask], np.arange(val_mask.sum(), dtype=np.int32))
for start, count, _ in prefixes.values():
assert train_mask[start : start + count].all() or val_mask[start : start + count].all()
|