code-Movens-Zero_train / src /psi /data /seed /source_split_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
980282b verified
Raw History Blame Contribute Delete
5.35 kB
from __future__ import annotations
import json
import numpy as np
from psi.data.seed.omni_oss_sampler import (
_video_manifest_signature,
ensure_video_split_inverse_index,
)
from psi.data.seed.source_split import (
VIDEO_MANIFEST_IDENTITY_FORMAT,
load_video_manifest_identity,
motion_source_id,
select_source_row_ids,
source_leaf_key,
video_manifest_identity_path,
validation_source_ids,
)
def test_motion_source_id_matches_across_modalities():
text = motion_source_id("seed", "g1/csv/240918/body_check_001__A548.csv")
command = motion_source_id("seed", "body_check_001__A548")
assert text == command == "seed/body_check_001__A548"
assert source_leaf_key(r"converted_motion\00001\dance.foo") == "dance.foo"
def test_source_rows_are_atomic_complementary_and_manifest_ordered():
sources = ["family/a", "family/a", "family/b", "family/c", "family/c", "family/d"]
train = select_source_row_ids(sources, split="train", val_ratio=0.5, split_seed=42)
val = select_source_row_ids(sources, split="val", val_ratio=0.5, split_seed=42)
assert np.all(train[:-1] < train[1:])
assert np.all(val[:-1] < val[1:])
assert set(train).isdisjoint(set(val))
assert set(train) | set(val) == set(range(len(sources)))
for source in set(sources):
memberships = {int(row_id in set(val)) for row_id, value in enumerate(sources) if value == source}
assert len(memberships) == 1
def test_source_assignment_is_independent_of_input_order():
sources = [f"source/{index:03d}" for index in range(100)]
selected = validation_source_ids(sources, val_ratio=0.1, split_seed=7)
reversed_selected = validation_source_ids(reversed(sources), val_ratio=0.1, split_seed=7)
assert selected == reversed_selected
def test_tiny_dataset_keeps_train_and_val_nonempty():
sources = ["source/a", "source/b"]
val = validation_source_ids(sources, val_ratio=1e-12, split_seed=1)
assert len(val) == 1
val = validation_source_ids(sources, val_ratio=1.0 - 1e-12, split_seed=1)
assert len(val) == 1
def test_manifest_identity_survives_copy_and_mtime_change(tmp_path):
payload = b'{"video_path":"demo.chunk00.f000000.mp4"}\n'
identity = {
"format": VIDEO_MANIFEST_IDENTITY_FORMAT,
"bytes": len(payload),
"rows": 1,
"sha256": "a" * 64,
}
manifests = []
for index in range(2):
root = tmp_path / f"mount-{index}"
root.mkdir()
manifest = root / "metadata.jsonl"
manifest.write_bytes(payload)
video_manifest_identity_path(manifest).write_text(
json.dumps(identity),
encoding="utf-8",
)
manifests.append(manifest)
manifests[1].touch()
assert load_video_manifest_identity(manifests[0]) == identity
assert load_video_manifest_identity(manifests[1]) == identity
assert _video_manifest_signature(manifests[0]) == _video_manifest_signature(manifests[1])
def test_manifest_identity_keeps_explicit_excluded_video_sources(tmp_path):
payload = b'{"video_path":"kept.chunk00.f000000.mp4"}\n'
manifest = tmp_path / "metadata.jsonl"
manifest.write_bytes(payload)
video_manifest_identity_path(manifest).write_text(
json.dumps(
{
"format": VIDEO_MANIFEST_IDENTITY_FORMAT,
"bytes": len(payload),
"rows": 1,
"sha256": "c" * 64,
"excluded_source_bases": ["missing-b", "missing-a"],
}
),
encoding="utf-8",
)
identity = load_video_manifest_identity(manifest)
assert identity is not None
assert identity["excluded_source_bases"] == ["missing-a", "missing-b"]
def test_manifest_identity_rejects_truncated_copy(tmp_path):
manifest = tmp_path / "metadata.jsonl"
manifest.write_bytes(b"{}\n")
video_manifest_identity_path(manifest).write_text(
json.dumps(
{
"format": VIDEO_MANIFEST_IDENTITY_FORMAT,
"bytes": 999,
"rows": 1,
"sha256": "b" * 64,
}
),
encoding="utf-8",
)
with np.testing.assert_raises_regex(RuntimeError, "size mismatch"):
load_video_manifest_identity(manifest)
def test_video_source_inverse_has_no_gaps_overlap_or_partial_sources(tmp_path):
prefixes = {
"a": (0, 3, "numeric"),
"b": (3, 2, "numeric"),
"c": (5, 4, "numeric"),
"d": (9, 1, "numeric"),
}
kwargs = {
"total_rows": 10,
"val_ratio": 0.5,
"split_seed": 42,
"cache_dir": tmp_path,
"source_signature": "synthetic",
"prefixes": prefixes,
"split_source_namespace": "video",
}
train = np.load(ensure_video_split_inverse_index(split="train", **kwargs))
val = np.load(ensure_video_split_inverse_index(split="val", **kwargs))
train_mask = train >= 0
val_mask = val >= 0
assert np.all(np.logical_xor(train_mask, val_mask))
assert np.array_equal(train[train_mask], np.arange(train_mask.sum(), dtype=np.int32))
assert np.array_equal(val[val_mask], np.arange(val_mask.sum(), dtype=np.int32))
for start, count, _ in prefixes.values():
assert train_mask[start : start + count].all() or val_mask[start : start + count].all()