Download src/psi/data/seed/source_split_test.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 5.35 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/src/psi/data/seed/source_split_test.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/src/psi/data/seed/source_split_test.py
-
curl -L -o source_split_test.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/src/psi/data/seed/source_split_test.py
5.35 kB
| from __future__ import annotations | |
| import json | |
| import numpy as np | |
| from psi.data.seed.omni_oss_sampler import ( | |
| _video_manifest_signature, | |
| ensure_video_split_inverse_index, | |
| ) | |
| from psi.data.seed.source_split import ( | |
| VIDEO_MANIFEST_IDENTITY_FORMAT, | |
| load_video_manifest_identity, | |
| motion_source_id, | |
| select_source_row_ids, | |
| source_leaf_key, | |
| video_manifest_identity_path, | |
| validation_source_ids, | |
| ) | |
| def test_motion_source_id_matches_across_modalities(): | |
| text = motion_source_id("seed", "g1/csv/240918/body_check_001__A548.csv") | |
| command = motion_source_id("seed", "body_check_001__A548") | |
| assert text == command == "seed/body_check_001__A548" | |
| assert source_leaf_key(r"converted_motion\00001\dance.foo") == "dance.foo" | |
| def test_source_rows_are_atomic_complementary_and_manifest_ordered(): | |
| sources = ["family/a", "family/a", "family/b", "family/c", "family/c", "family/d"] | |
| train = select_source_row_ids(sources, split="train", val_ratio=0.5, split_seed=42) | |
| val = select_source_row_ids(sources, split="val", val_ratio=0.5, split_seed=42) | |
| assert np.all(train[:-1] < train[1:]) | |
| assert np.all(val[:-1] < val[1:]) | |
| assert set(train).isdisjoint(set(val)) | |
| assert set(train) | set(val) == set(range(len(sources))) | |
| for source in set(sources): | |
| memberships = {int(row_id in set(val)) for row_id, value in enumerate(sources) if value == source} | |
| assert len(memberships) == 1 | |
| def test_source_assignment_is_independent_of_input_order(): | |
| sources = [f"source/{index:03d}" for index in range(100)] | |
| selected = validation_source_ids(sources, val_ratio=0.1, split_seed=7) | |
| reversed_selected = validation_source_ids(reversed(sources), val_ratio=0.1, split_seed=7) | |
| assert selected == reversed_selected | |
| def test_tiny_dataset_keeps_train_and_val_nonempty(): | |
| sources = ["source/a", "source/b"] | |
| val = validation_source_ids(sources, val_ratio=1e-12, split_seed=1) | |
| assert len(val) == 1 | |
| val = validation_source_ids(sources, val_ratio=1.0 - 1e-12, split_seed=1) | |
| assert len(val) == 1 | |
| def test_manifest_identity_survives_copy_and_mtime_change(tmp_path): | |
| payload = b'{"video_path":"demo.chunk00.f000000.mp4"}\n' | |
| identity = { | |
| "format": VIDEO_MANIFEST_IDENTITY_FORMAT, | |
| "bytes": len(payload), | |
| "rows": 1, | |
| "sha256": "a" * 64, | |
| } | |
| manifests = [] | |
| for index in range(2): | |
| root = tmp_path / f"mount-{index}" | |
| root.mkdir() | |
| manifest = root / "metadata.jsonl" | |
| manifest.write_bytes(payload) | |
| video_manifest_identity_path(manifest).write_text( | |
| json.dumps(identity), | |
| encoding="utf-8", | |
| ) | |
| manifests.append(manifest) | |
| manifests[1].touch() | |
| assert load_video_manifest_identity(manifests[0]) == identity | |
| assert load_video_manifest_identity(manifests[1]) == identity | |
| assert _video_manifest_signature(manifests[0]) == _video_manifest_signature(manifests[1]) | |
| def test_manifest_identity_keeps_explicit_excluded_video_sources(tmp_path): | |
| payload = b'{"video_path":"kept.chunk00.f000000.mp4"}\n' | |
| manifest = tmp_path / "metadata.jsonl" | |
| manifest.write_bytes(payload) | |
| video_manifest_identity_path(manifest).write_text( | |
| json.dumps( | |
| { | |
| "format": VIDEO_MANIFEST_IDENTITY_FORMAT, | |
| "bytes": len(payload), | |
| "rows": 1, | |
| "sha256": "c" * 64, | |
| "excluded_source_bases": ["missing-b", "missing-a"], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| identity = load_video_manifest_identity(manifest) | |
| assert identity is not None | |
| assert identity["excluded_source_bases"] == ["missing-a", "missing-b"] | |
| def test_manifest_identity_rejects_truncated_copy(tmp_path): | |
| manifest = tmp_path / "metadata.jsonl" | |
| manifest.write_bytes(b"{}\n") | |
| video_manifest_identity_path(manifest).write_text( | |
| json.dumps( | |
| { | |
| "format": VIDEO_MANIFEST_IDENTITY_FORMAT, | |
| "bytes": 999, | |
| "rows": 1, | |
| "sha256": "b" * 64, | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| with np.testing.assert_raises_regex(RuntimeError, "size mismatch"): | |
| load_video_manifest_identity(manifest) | |
| def test_video_source_inverse_has_no_gaps_overlap_or_partial_sources(tmp_path): | |
| prefixes = { | |
| "a": (0, 3, "numeric"), | |
| "b": (3, 2, "numeric"), | |
| "c": (5, 4, "numeric"), | |
| "d": (9, 1, "numeric"), | |
| } | |
| kwargs = { | |
| "total_rows": 10, | |
| "val_ratio": 0.5, | |
| "split_seed": 42, | |
| "cache_dir": tmp_path, | |
| "source_signature": "synthetic", | |
| "prefixes": prefixes, | |
| "split_source_namespace": "video", | |
| } | |
| train = np.load(ensure_video_split_inverse_index(split="train", **kwargs)) | |
| val = np.load(ensure_video_split_inverse_index(split="val", **kwargs)) | |
| train_mask = train >= 0 | |
| val_mask = val >= 0 | |
| assert np.all(np.logical_xor(train_mask, val_mask)) | |
| assert np.array_equal(train[train_mask], np.arange(train_mask.sum(), dtype=np.int32)) | |
| assert np.array_equal(val[val_mask], np.arange(val_mask.sum(), dtype=np.int32)) | |
| for start, count, _ in prefixes.values(): | |
| assert train_mask[start : start + count].all() or val_mask[start : start + count].all() | |