Download src/psi/data/seed/source_split.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 6.23 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/src/psi/data/seed/source_split.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/src/psi/data/seed/source_split.py
-
curl -L -o source_split.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/src/psi/data/seed/source_split.py
6.23 kB
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| from collections.abc import Iterable, Sequence | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| SOURCE_SPLIT_VERSION = 1 | |
| VIDEO_MANIFEST_IDENTITY_FORMAT = "psi0_video_manifest_identity_v1" | |
| _HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}" | |
| _UINT64_RANGE = 1 << 64 | |
| def video_manifest_identity_path(manifest_path: str | Path) -> Path: | |
| path = Path(manifest_path) | |
| return path.with_name(path.name + ".identity.json") | |
| def load_video_manifest_identity(manifest_path: str | Path) -> dict[str, Any] | None: | |
| """Load a path-independent identity for a copied/uploaded video manifest. | |
| Historical manifests keep the legacy path+mtime signature. New very large | |
| manifests ship this sidecar so indexes and lazy record caches built before | |
| OSS upload remain reusable after the mount path and mtime change. | |
| """ | |
| manifest = Path(manifest_path) | |
| identity_path = video_manifest_identity_path(manifest) | |
| if not identity_path.is_file(): | |
| return None | |
| payload = json.loads(identity_path.read_text(encoding="utf-8")) | |
| if payload.get("format") != VIDEO_MANIFEST_IDENTITY_FORMAT: | |
| raise RuntimeError(f"Unexpected video manifest identity format: {identity_path}") | |
| expected_bytes = int(payload.get("bytes") or -1) | |
| rows = int(payload.get("rows") or -1) | |
| sha256 = str(payload.get("sha256") or "") | |
| if expected_bytes <= 0 or rows <= 0 or len(sha256) != 64: | |
| raise RuntimeError(f"Incomplete video manifest identity: {identity_path}") | |
| actual_bytes = manifest.stat().st_size | |
| if actual_bytes != expected_bytes: | |
| raise RuntimeError( | |
| f"Video manifest identity size mismatch: {manifest} " | |
| f"actual={actual_bytes} expected={expected_bytes}" | |
| ) | |
| result = { | |
| "format": VIDEO_MANIFEST_IDENTITY_FORMAT, | |
| "bytes": expected_bytes, | |
| "rows": rows, | |
| "sha256": sha256, | |
| } | |
| for key in ("source_name", "sample_definition"): | |
| value = str(payload.get(key) or "").strip() | |
| if value: | |
| result[key] = value | |
| excluded_source_bases = payload.get("excluded_source_bases", []) | |
| if not isinstance(excluded_source_bases, list) or any( | |
| not isinstance(value, str) or not value.strip() | |
| for value in excluded_source_bases | |
| ): | |
| raise RuntimeError( | |
| f"Invalid excluded_source_bases in video manifest identity: {identity_path}" | |
| ) | |
| if len(set(excluded_source_bases)) != len(excluded_source_bases): | |
| raise RuntimeError( | |
| f"Duplicate excluded_source_bases in video manifest identity: {identity_path}" | |
| ) | |
| if excluded_source_bases: | |
| result["excluded_source_bases"] = sorted(excluded_source_bases) | |
| return result | |
| def source_leaf_key(value: object) -> str: | |
| """Return the stable motion leaf used across paired modalities. | |
| The current seed, dance, and 2264 manifests use the same unique motion | |
| basename in every modality. Video-clip prefix indexes already expose the | |
| corresponding basename (or an unambiguous encoded relative path). | |
| """ | |
| text = str(value).strip().replace("\\", "/").rstrip("/") | |
| if not text: | |
| raise ValueError("Motion source is empty") | |
| leaf = text.rsplit("/", 1)[-1] | |
| if leaf.lower().endswith(".csv"): | |
| leaf = leaf[:-4] | |
| if not leaf: | |
| raise ValueError(f"Cannot derive motion source key from {value!r}") | |
| return leaf | |
| def motion_source_id(namespace: str, value: object) -> str: | |
| namespace_norm = str(namespace).strip().replace("\\", "/").strip("/") | |
| if not namespace_norm: | |
| raise ValueError("split_source_namespace is required for source-grouped splitting") | |
| return f"{namespace_norm}/{source_leaf_key(value)}" | |
| def source_hash_u64(source_id: str, split_seed: int) -> int: | |
| payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8") | |
| return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False) | |
| def validation_source_ids( | |
| source_ids: Iterable[str], | |
| *, | |
| val_ratio: float, | |
| split_seed: int, | |
| ) -> set[str]: | |
| """Assign complete motion sources to validation with a stable hash. | |
| Hash thresholding makes the assignment independent of manifest row order | |
| and keeps the same source on the same side across modalities. For tiny | |
| datasets, deterministic fallbacks preserve a non-empty train and val split | |
| whenever that is mathematically possible. | |
| """ | |
| ratio = float(val_ratio) | |
| if not 0.0 <= ratio <= 1.0: | |
| raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}") | |
| unique = sorted(set(str(source_id) for source_id in source_ids)) | |
| if any(not source_id for source_id in unique): | |
| raise ValueError("Motion source ID is empty") | |
| if not unique or ratio <= 0.0: | |
| return set() | |
| if ratio >= 1.0: | |
| return set(unique) | |
| scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique] | |
| threshold = int(ratio * _UINT64_RANGE) | |
| selected = {source_id for score, source_id in scored if score < threshold} | |
| if len(unique) > 1 and not selected: | |
| selected.add(min(scored)[1]) | |
| if len(unique) > 1 and len(selected) == len(unique): | |
| selected.remove(max(scored)[1]) | |
| return selected | |
| def select_source_row_ids( | |
| source_ids: Sequence[str], | |
| *, | |
| split: str, | |
| val_ratio: float, | |
| split_seed: int, | |
| ) -> np.ndarray: | |
| """Return selected row IDs in original manifest order.""" | |
| split_norm = str(split or "train").lower() | |
| if split_norm == "all": | |
| return np.arange(len(source_ids), dtype=np.int64) | |
| if split_norm not in {"train", "val", "valid", "validation", "test"}: | |
| raise ValueError(f"Unsupported split={split!r}, expected train/val/all") | |
| val_sources = validation_source_ids( | |
| source_ids, | |
| val_ratio=val_ratio, | |
| split_seed=split_seed, | |
| ) | |
| choose_val = split_norm in {"val", "valid", "validation", "test"} | |
| selected = [ | |
| row_id | |
| for row_id, source_id in enumerate(source_ids) | |
| if (str(source_id) in val_sources) == choose_val | |
| ] | |
| return np.asarray(selected, dtype=np.int64) | |