File size: 6,233 Bytes
980282b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | from __future__ import annotations
import hashlib
import json
from collections.abc import Iterable, Sequence
from pathlib import Path
from typing import Any
import numpy as np
SOURCE_SPLIT_VERSION = 1
VIDEO_MANIFEST_IDENTITY_FORMAT = "psi0_video_manifest_identity_v1"
_HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}"
_UINT64_RANGE = 1 << 64
def video_manifest_identity_path(manifest_path: str | Path) -> Path:
path = Path(manifest_path)
return path.with_name(path.name + ".identity.json")
def load_video_manifest_identity(manifest_path: str | Path) -> dict[str, Any] | None:
"""Load a path-independent identity for a copied/uploaded video manifest.
Historical manifests keep the legacy path+mtime signature. New very large
manifests ship this sidecar so indexes and lazy record caches built before
OSS upload remain reusable after the mount path and mtime change.
"""
manifest = Path(manifest_path)
identity_path = video_manifest_identity_path(manifest)
if not identity_path.is_file():
return None
payload = json.loads(identity_path.read_text(encoding="utf-8"))
if payload.get("format") != VIDEO_MANIFEST_IDENTITY_FORMAT:
raise RuntimeError(f"Unexpected video manifest identity format: {identity_path}")
expected_bytes = int(payload.get("bytes") or -1)
rows = int(payload.get("rows") or -1)
sha256 = str(payload.get("sha256") or "")
if expected_bytes <= 0 or rows <= 0 or len(sha256) != 64:
raise RuntimeError(f"Incomplete video manifest identity: {identity_path}")
actual_bytes = manifest.stat().st_size
if actual_bytes != expected_bytes:
raise RuntimeError(
f"Video manifest identity size mismatch: {manifest} "
f"actual={actual_bytes} expected={expected_bytes}"
)
result = {
"format": VIDEO_MANIFEST_IDENTITY_FORMAT,
"bytes": expected_bytes,
"rows": rows,
"sha256": sha256,
}
for key in ("source_name", "sample_definition"):
value = str(payload.get(key) or "").strip()
if value:
result[key] = value
excluded_source_bases = payload.get("excluded_source_bases", [])
if not isinstance(excluded_source_bases, list) or any(
not isinstance(value, str) or not value.strip()
for value in excluded_source_bases
):
raise RuntimeError(
f"Invalid excluded_source_bases in video manifest identity: {identity_path}"
)
if len(set(excluded_source_bases)) != len(excluded_source_bases):
raise RuntimeError(
f"Duplicate excluded_source_bases in video manifest identity: {identity_path}"
)
if excluded_source_bases:
result["excluded_source_bases"] = sorted(excluded_source_bases)
return result
def source_leaf_key(value: object) -> str:
"""Return the stable motion leaf used across paired modalities.
The current seed, dance, and 2264 manifests use the same unique motion
basename in every modality. Video-clip prefix indexes already expose the
corresponding basename (or an unambiguous encoded relative path).
"""
text = str(value).strip().replace("\\", "/").rstrip("/")
if not text:
raise ValueError("Motion source is empty")
leaf = text.rsplit("/", 1)[-1]
if leaf.lower().endswith(".csv"):
leaf = leaf[:-4]
if not leaf:
raise ValueError(f"Cannot derive motion source key from {value!r}")
return leaf
def motion_source_id(namespace: str, value: object) -> str:
namespace_norm = str(namespace).strip().replace("\\", "/").strip("/")
if not namespace_norm:
raise ValueError("split_source_namespace is required for source-grouped splitting")
return f"{namespace_norm}/{source_leaf_key(value)}"
def source_hash_u64(source_id: str, split_seed: int) -> int:
payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8")
return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False)
def validation_source_ids(
source_ids: Iterable[str],
*,
val_ratio: float,
split_seed: int,
) -> set[str]:
"""Assign complete motion sources to validation with a stable hash.
Hash thresholding makes the assignment independent of manifest row order
and keeps the same source on the same side across modalities. For tiny
datasets, deterministic fallbacks preserve a non-empty train and val split
whenever that is mathematically possible.
"""
ratio = float(val_ratio)
if not 0.0 <= ratio <= 1.0:
raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}")
unique = sorted(set(str(source_id) for source_id in source_ids))
if any(not source_id for source_id in unique):
raise ValueError("Motion source ID is empty")
if not unique or ratio <= 0.0:
return set()
if ratio >= 1.0:
return set(unique)
scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique]
threshold = int(ratio * _UINT64_RANGE)
selected = {source_id for score, source_id in scored if score < threshold}
if len(unique) > 1 and not selected:
selected.add(min(scored)[1])
if len(unique) > 1 and len(selected) == len(unique):
selected.remove(max(scored)[1])
return selected
def select_source_row_ids(
source_ids: Sequence[str],
*,
split: str,
val_ratio: float,
split_seed: int,
) -> np.ndarray:
"""Return selected row IDs in original manifest order."""
split_norm = str(split or "train").lower()
if split_norm == "all":
return np.arange(len(source_ids), dtype=np.int64)
if split_norm not in {"train", "val", "valid", "validation", "test"}:
raise ValueError(f"Unsupported split={split!r}, expected train/val/all")
val_sources = validation_source_ids(
source_ids,
val_ratio=val_ratio,
split_seed=split_seed,
)
choose_val = split_norm in {"val", "valid", "validation", "test"}
selected = [
row_id
for row_id, source_id in enumerate(source_ids)
if (str(source_id) in val_sources) == choose_val
]
return np.asarray(selected, dtype=np.int64)
|