Download src/psi/data/seed/source_split.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 3.65 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/src/psi/data/seed/source_split.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/src/psi/data/seed/source_split.py
-
curl -L -o source_split.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/src/psi/data/seed/source_split.py
3.65 kB
| from __future__ import annotations | |
| import hashlib | |
| from collections.abc import Iterable, Sequence | |
| import numpy as np | |
| SOURCE_SPLIT_VERSION = 1 | |
| _HASH_DOMAIN = f"psi0-motion-source-split-v{SOURCE_SPLIT_VERSION}" | |
| _UINT64_RANGE = 1 << 64 | |
| def source_leaf_key(value: object) -> str: | |
| """Return the stable motion leaf used across paired modalities. | |
| The current seed, dance, and 2264 manifests use the same unique motion | |
| basename in every modality. Video-clip prefix indexes already expose the | |
| corresponding basename (or an unambiguous encoded relative path). | |
| """ | |
| text = str(value).strip().replace("\\", "/").rstrip("/") | |
| if not text: | |
| raise ValueError("Motion source is empty") | |
| leaf = text.rsplit("/", 1)[-1] | |
| if leaf.lower().endswith(".csv"): | |
| leaf = leaf[:-4] | |
| if not leaf: | |
| raise ValueError(f"Cannot derive motion source key from {value!r}") | |
| return leaf | |
| def motion_source_id(namespace: str, value: object) -> str: | |
| namespace_norm = str(namespace).strip().replace("\\", "/").strip("/") | |
| if not namespace_norm: | |
| raise ValueError("split_source_namespace is required for source-grouped splitting") | |
| return f"{namespace_norm}/{source_leaf_key(value)}" | |
| def source_hash_u64(source_id: str, split_seed: int) -> int: | |
| payload = f"{_HASH_DOMAIN}\0{int(split_seed)}\0{source_id}".encode("utf-8") | |
| return int.from_bytes(hashlib.sha1(payload).digest()[:8], "big", signed=False) | |
| def validation_source_ids( | |
| source_ids: Iterable[str], | |
| *, | |
| val_ratio: float, | |
| split_seed: int, | |
| ) -> set[str]: | |
| """Assign complete motion sources to validation with a stable hash. | |
| Hash thresholding makes the assignment independent of manifest row order | |
| and keeps the same source on the same side across modalities. For tiny | |
| datasets, deterministic fallbacks preserve a non-empty train and val split | |
| whenever that is mathematically possible. | |
| """ | |
| ratio = float(val_ratio) | |
| if not 0.0 <= ratio <= 1.0: | |
| raise ValueError(f"val_ratio must be in [0, 1], got {val_ratio}") | |
| unique = sorted(set(str(source_id) for source_id in source_ids)) | |
| if any(not source_id for source_id in unique): | |
| raise ValueError("Motion source ID is empty") | |
| if not unique or ratio <= 0.0: | |
| return set() | |
| if ratio >= 1.0: | |
| return set(unique) | |
| scored = [(source_hash_u64(source_id, split_seed), source_id) for source_id in unique] | |
| threshold = int(ratio * _UINT64_RANGE) | |
| selected = {source_id for score, source_id in scored if score < threshold} | |
| if len(unique) > 1 and not selected: | |
| selected.add(min(scored)[1]) | |
| if len(unique) > 1 and len(selected) == len(unique): | |
| selected.remove(max(scored)[1]) | |
| return selected | |
| def select_source_row_ids( | |
| source_ids: Sequence[str], | |
| *, | |
| split: str, | |
| val_ratio: float, | |
| split_seed: int, | |
| ) -> np.ndarray: | |
| """Return selected row IDs in original manifest order.""" | |
| split_norm = str(split or "train").lower() | |
| if split_norm == "all": | |
| return np.arange(len(source_ids), dtype=np.int64) | |
| if split_norm not in {"train", "val", "valid", "validation", "test"}: | |
| raise ValueError(f"Unsupported split={split!r}, expected train/val/all") | |
| val_sources = validation_source_ids( | |
| source_ids, | |
| val_ratio=val_ratio, | |
| split_seed=split_seed, | |
| ) | |
| choose_val = split_norm in {"val", "valid", "validation", "test"} | |
| selected = [ | |
| row_id | |
| for row_id, source_id in enumerate(source_ids) | |
| if (str(source_id) in val_sources) == choose_val | |
| ] | |
| return np.asarray(selected, dtype=np.int64) | |