code-Movens-Zero_train / src /psi /data /seed /source_split.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
980282b verified
Raw History Blame Contribute Delete
6.23 kB
from __future__ import annotations
import hashlib
import json
from collections.abc import Iterable, Sequence
from pathlib import Path
from typing import Any
import numpy as np
SOURCE_SPLIT_VERSION = 1
VIDEO_MANIFEST_IDENTITY_FORMAT = "psi0_video_manifest_identity_v1"
_HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}"
_UINT64_RANGE = 1 << 64
def video_manifest_identity_path(manifest_path: str | Path) -> Path:
path = Path(manifest_path)
return path.with_name(path.name + ".identity.json")
def load_video_manifest_identity(manifest_path: str | Path) -> dict[str, Any] | None:
"""Load a path-independent identity for a copied/uploaded video manifest.
Historical manifests keep the legacy path+mtime signature. New very large
manifests ship this sidecar so indexes and lazy record caches built before
OSS upload remain reusable after the mount path and mtime change.
"""
manifest = Path(manifest_path)
identity_path = video_manifest_identity_path(manifest)
if not identity_path.is_file():
return None
payload = json.loads(identity_path.read_text(encoding="utf-8"))
if payload.get("format") != VIDEO_MANIFEST_IDENTITY_FORMAT:
raise RuntimeError(f"Unexpected video manifest identity format: {identity_path}")
expected_bytes = int(payload.get("bytes") or -1)
rows = int(payload.get("rows") or -1)
sha256 = str(payload.get("sha256") or "")
if expected_bytes <= 0 or rows <= 0 or len(sha256) != 64:
raise RuntimeError(f"Incomplete video manifest identity: {identity_path}")
actual_bytes = manifest.stat().st_size
if actual_bytes != expected_bytes:
raise RuntimeError(
f"Video manifest identity size mismatch: {manifest} "
f"actual={actual_bytes} expected={expected_bytes}"
)
result = {
"format": VIDEO_MANIFEST_IDENTITY_FORMAT,
"bytes": expected_bytes,
"rows": rows,
"sha256": sha256,
}
for key in ("source_name", "sample_definition"):
value = str(payload.get(key) or "").strip()
if value:
result[key] = value
excluded_source_bases = payload.get("excluded_source_bases", [])
if not isinstance(excluded_source_bases, list) or any(
not isinstance(value, str) or not value.strip()
for value in excluded_source_bases
):
raise RuntimeError(
f"Invalid excluded_source_bases in video manifest identity: {identity_path}"
)
if len(set(excluded_source_bases)) != len(excluded_source_bases):
raise RuntimeError(
f"Duplicate excluded_source_bases in video manifest identity: {identity_path}"
)
if excluded_source_bases:
result["excluded_source_bases"] = sorted(excluded_source_bases)
return result
def source_leaf_key(value: object) -> str:
"""Return the stable motion leaf used across paired modalities.
The current seed, dance, and 2264 manifests use the same unique motion
basename in every modality. Video-clip prefix indexes already expose the
corresponding basename (or an unambiguous encoded relative path).
"""
text = str(value).strip().replace("\\", "/").rstrip("/")
if not text:
raise ValueError("Motion source is empty")
leaf = text.rsplit("/", 1)[-1]
if leaf.lower().endswith(".csv"):
leaf = leaf[:-4]
if not leaf:
raise ValueError(f"Cannot derive motion source key from {value!r}")
return leaf
def motion_source_id(namespace: str, value: object) -> str:
namespace_norm = str(namespace).strip().replace("\\", "/").strip("/")
if not namespace_norm:
raise ValueError("split_source_namespace is required for source-grouped splitting")
return f"{namespace_norm}/{source_leaf_key(value)}"
def source_hash_u64(source_id: str, split_seed: int) -> int:
payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8")
return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False)
def validation_source_ids(
source_ids: Iterable[str],
*,
val_ratio: float,
split_seed: int,
) -> set[str]:
"""Assign complete motion sources to validation with a stable hash.
Hash thresholding makes the assignment independent of manifest row order
and keeps the same source on the same side across modalities. For tiny
datasets, deterministic fallbacks preserve a non-empty train and val split
whenever that is mathematically possible.
"""
ratio = float(val_ratio)
if not 0.0 <= ratio <= 1.0:
raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}")
unique = sorted(set(str(source_id) for source_id in source_ids))
if any(not source_id for source_id in unique):
raise ValueError("Motion source ID is empty")
if not unique or ratio <= 0.0:
return set()
if ratio >= 1.0:
return set(unique)
scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique]
threshold = int(ratio * _UINT64_RANGE)
selected = {source_id for score, source_id in scored if score < threshold}
if len(unique) > 1 and not selected:
selected.add(min(scored)[1])
if len(unique) > 1 and len(selected) == len(unique):
selected.remove(max(scored)[1])
return selected
def select_source_row_ids(
source_ids: Sequence[str],
*,
split: str,
val_ratio: float,
split_seed: int,
) -> np.ndarray:
"""Return selected row IDs in original manifest order."""
split_norm = str(split or "train").lower()
if split_norm == "all":
return np.arange(len(source_ids), dtype=np.int64)
if split_norm not in {"train", "val", "valid", "validation", "test"}:
raise ValueError(f"Unsupported split={split!r}, expected train/val/all")
val_sources = validation_source_ids(
source_ids,
val_ratio=val_ratio,
split_seed=split_seed,
)
choose_val = split_norm in {"val", "valid", "validation", "test"}
selected = [
row_id
for row_id, source_id in enumerate(source_ids)
if (str(source_id) in val_sources) == choose_val
]
return np.asarray(selected, dtype=np.int64)