Download scripts/data/audit_source_split_artifacts.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 16.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/audit_source_split_artifacts.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/audit_source_split_artifacts.py
-
curl -L -o audit_source_split_artifacts.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/audit_source_split_artifacts.py
16.5 kB
| from __future__ import annotations | |
| import argparse | |
| import gc | |
| import json | |
| import pickle | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| REPO_ROOT = Path(__file__).resolve().parents[2] | |
| SRC_ROOT = REPO_ROOT / "src" | |
| DEFAULT_SOURCE_SPLIT_ROOT = Path( | |
| "/mnt/data/data/omni_hidden_states_v23_oss/training_support/source_split_v1" | |
| ) | |
| if str(SRC_ROOT) not in sys.path: | |
| sys.path.insert(0, str(SRC_ROOT)) | |
| from psi.data.seed.omni_oss_sampler import ensure_video_training_indexes | |
| from psi.data.seed.source_split import motion_source_id, validation_source_ids | |
| _VIDEO_PATH_RE = re.compile(br'"video_path"\s*:\s*"[^"]*/([^"/]+)\.mp4"') | |
| _VIDEO_STEM_RE = re.compile(br"^(.*)\.chunk(\d+)\.f(\d+)$") | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Full audit of v26 motion-source split artifacts.") | |
| parser.add_argument( | |
| "--cache-root", | |
| type=Path, | |
| default=DEFAULT_SOURCE_SPLIT_ROOT, | |
| ) | |
| parser.add_argument( | |
| "--runtime-index-dir", | |
| type=Path, | |
| default=DEFAULT_SOURCE_SPLIT_ROOT / "runtime_indexes", | |
| ) | |
| parser.add_argument("--val-ratio", type=float, default=0.1) | |
| parser.add_argument("--split-seed", type=int, default=42) | |
| parser.add_argument( | |
| "--video-root", | |
| type=Path, | |
| default=Path( | |
| "/mnt/data/data/omni_hidden_states_v23_oss/training_support/" | |
| "video_clip_dense_stride1" | |
| ), | |
| ) | |
| parser.add_argument("--skip-full-record-scan", action="store_true") | |
| parser.add_argument("--report", type=Path, default=None) | |
| return parser.parse_args() | |
| def find_one(root: Path, pattern: str) -> Path: | |
| matches = sorted(root.glob(pattern)) | |
| if len(matches) != 1: | |
| raise RuntimeError(f"Expected exactly one {pattern!r} under {root}, got {matches}") | |
| return matches[0] | |
| def load_pickle_records(path: Path) -> list[dict[str, Any]]: | |
| with path.open("rb") as f: | |
| value = pickle.load(f) | |
| if isinstance(value, dict) and "data" in value: | |
| value = value["data"] | |
| if not isinstance(value, list): | |
| raise TypeError(f"Expected list cache at {path}") | |
| return value | |
| def source_sets_from_pickles( | |
| root: Path, | |
| pattern_prefix: str, | |
| namespace: str, | |
| *, | |
| val_ratio: float, | |
| split_seed: int, | |
| ) -> tuple[set[str], set[str], dict[str, int]]: | |
| train_path = find_one(root, f"{pattern_prefix}.train.*.pkl") | |
| val_path = find_one(root, f"{pattern_prefix}.val.*.pkl") | |
| train_records = load_pickle_records(train_path) | |
| val_records = load_pickle_records(val_path) | |
| train = {motion_source_id(namespace, record["motion_dir"]) for record in train_records} | |
| val = {motion_source_id(namespace, record["motion_dir"]) for record in val_records} | |
| if len(train) != len(train_records) or len(val) != len(val_records): | |
| raise RuntimeError(f"Duplicate motion-source records in {pattern_prefix}") | |
| if train & val: | |
| raise RuntimeError(f"Source overlap in {pattern_prefix}: {len(train & val)}") | |
| expected_val = validation_source_ids( | |
| train | val, | |
| val_ratio=val_ratio, | |
| split_seed=split_seed, | |
| ) | |
| if val != expected_val: | |
| raise RuntimeError(f"Stable-hash assignment mismatch in {pattern_prefix}") | |
| result = { | |
| "train_records": len(train_records), | |
| "val_records": len(val_records), | |
| "train_sources": len(train), | |
| "val_sources": len(val), | |
| } | |
| del train_records, val_records | |
| gc.collect() | |
| return train, val, result | |
| def lazy_paths_from_meta(meta_path: Path) -> tuple[Path, Path, Path]: | |
| suffix = ".lazy.meta.json" | |
| base = meta_path.name[: -len(suffix)] | |
| return ( | |
| meta_path.with_name(base + ".lazy.jsonl"), | |
| meta_path.with_name(base + ".lazy.offsets.npy"), | |
| meta_path.with_name(base + ".pkl"), | |
| ) | |
| def assert_dense_inverse(inverse: np.ndarray, label: str) -> int: | |
| mask = inverse >= 0 | |
| count = int(mask.sum()) | |
| selected = np.asarray(inverse[mask]) | |
| if count: | |
| if int(selected[0]) != 0 or int(selected[-1]) != count - 1: | |
| raise RuntimeError(f"Non-dense {label} inverse endpoints") | |
| if count > 1 and not np.all(np.diff(selected) == 1): | |
| raise RuntimeError(f"Non-dense {label} inverse ordering") | |
| return count | |
| def audit_video( | |
| *, | |
| manifest: Path, | |
| namespace: str, | |
| source_name: str, | |
| cache_dir: Path, | |
| runtime_index_dir: Path, | |
| val_ratio: float, | |
| split_seed: int, | |
| full_scan: bool, | |
| ) -> tuple[dict[str, Any], set[str], set[str]]: | |
| prefix_path, train_inverse_path = ensure_video_training_indexes( | |
| manifest, | |
| split="train", | |
| val_ratio=val_ratio, | |
| split_seed=split_seed, | |
| cache_dir=runtime_index_dir, | |
| split_source_namespace=namespace, | |
| ) | |
| _, val_inverse_path = ensure_video_training_indexes( | |
| manifest, | |
| split="val", | |
| val_ratio=val_ratio, | |
| split_seed=split_seed, | |
| cache_dir=runtime_index_dir, | |
| split_source_namespace=namespace, | |
| ) | |
| with prefix_path.open("rb") as f: | |
| prefix_payload = pickle.load(f) | |
| prefixes: dict[str, tuple[int, int, str]] = prefix_payload["prefixes"] | |
| rows = int(prefix_payload["rows"]) | |
| train_inverse = np.load(train_inverse_path, mmap_mode="r") | |
| val_inverse = np.load(val_inverse_path, mmap_mode="r") | |
| if len(train_inverse) != rows or len(val_inverse) != rows: | |
| raise RuntimeError(f"Inverse length mismatch for {source_name}") | |
| if not np.all(np.logical_xor(train_inverse >= 0, val_inverse >= 0)): | |
| raise RuntimeError(f"Train/val inverse is not an exact partition for {source_name}") | |
| train_count = assert_dense_inverse(train_inverse, f"{source_name}/train") | |
| val_count = assert_dense_inverse(val_inverse, f"{source_name}/val") | |
| all_sources = { | |
| base: motion_source_id(namespace, base) | |
| for base in prefixes | |
| } | |
| expected_val_sources = validation_source_ids( | |
| all_sources.values(), | |
| val_ratio=val_ratio, | |
| split_seed=split_seed, | |
| ) | |
| train_sources: set[str] = set() | |
| val_sources: set[str] = set() | |
| covered = 0 | |
| for base, (start, count, _) in prefixes.items(): | |
| if int(start) != covered: | |
| raise RuntimeError(f"Gap or overlap before {base}: start={start} covered={covered}") | |
| train_all = bool(np.all(train_inverse[start : start + count] >= 0)) | |
| val_all = bool(np.all(val_inverse[start : start + count] >= 0)) | |
| if train_all == val_all: | |
| raise RuntimeError(f"Partially split source {base}") | |
| source_id = all_sources[base] | |
| if val_all: | |
| val_sources.add(source_id) | |
| else: | |
| train_sources.add(source_id) | |
| if val_all != (source_id in expected_val_sources): | |
| raise RuntimeError(f"Stable-hash assignment mismatch for {base}") | |
| covered += int(count) | |
| if covered != rows or train_sources & val_sources: | |
| raise RuntimeError(f"Source range coverage failed for {source_name}") | |
| train_meta_path = find_one(cache_dir, f"records.{source_name}.train.*.lazy.meta.json") | |
| val_meta_path = find_one(cache_dir, f"records.{source_name}.val.*.lazy.meta.json") | |
| train_records_path, train_offsets_path, _ = lazy_paths_from_meta(train_meta_path) | |
| val_records_path, val_offsets_path, _ = lazy_paths_from_meta(val_meta_path) | |
| with train_meta_path.open("r", encoding="utf-8") as f: | |
| train_meta = json.load(f) | |
| with val_meta_path.open("r", encoding="utf-8") as f: | |
| val_meta = json.load(f) | |
| train_offsets = np.load(train_offsets_path, mmap_mode="r") | |
| val_offsets = np.load(val_offsets_path, mmap_mode="r") | |
| if int(train_meta["count"]) != train_count or len(train_offsets) != train_count: | |
| raise RuntimeError(f"Train record cache count mismatch for {source_name}") | |
| if int(val_meta["count"]) != val_count or len(val_offsets) != val_count: | |
| raise RuntimeError(f"Val record cache count mismatch for {source_name}") | |
| for meta in (train_meta, val_meta): | |
| if int(meta.get("source_split_version", 0)) != 1: | |
| raise RuntimeError(f"Missing source split version for {source_name}") | |
| if str(meta.get("split_source_namespace", "")) != namespace: | |
| raise RuntimeError(f"Source namespace mismatch for {source_name}") | |
| train_sample_path = find_one(cache_dir, f"sample_index.{source_name}.train.*.pkl") | |
| val_sample_path = find_one(cache_dir, f"sample_index.{source_name}.val.*.pkl") | |
| for split, sample_path, expected_count in ( | |
| ("train", train_sample_path, train_count), | |
| ("val", val_sample_path, val_count), | |
| ): | |
| with sample_path.open("rb") as f: | |
| sample_index = pickle.load(f) | |
| if isinstance(sample_index, dict) and "data" in sample_index: | |
| sample_index = sample_index["data"] | |
| if len(sample_index) != expected_count: | |
| raise RuntimeError(f"Sample index count mismatch for {source_name}/{split}") | |
| if expected_count and ( | |
| sample_index[0] != (0, 0) | |
| or sample_index[-1] != (expected_count - 1, 0) | |
| ): | |
| raise RuntimeError(f"Sample index is not identity for {source_name}/{split}") | |
| if full_scan: | |
| mismatch = next( | |
| ( | |
| index | |
| for index, value in enumerate(sample_index) | |
| if value != (index, 0) | |
| ), | |
| None, | |
| ) | |
| if mismatch is not None: | |
| raise RuntimeError( | |
| f"Sample index mismatch for {source_name}/{split} at {mismatch}" | |
| ) | |
| del sample_index | |
| gc.collect() | |
| scan_seconds = 0.0 | |
| scan_gib_per_second = 0.0 | |
| if full_scan: | |
| started = time.perf_counter() | |
| bytes_read = 0 | |
| original_row = 0 | |
| with train_records_path.open("rb") as train_file, val_records_path.open("rb") as val_file: | |
| for base, (_, count, order_mode) in prefixes.items(): | |
| chunk_order = list(range(count)) | |
| if order_mode == "lexical": | |
| chunk_order.sort(key=lambda chunk: f"{chunk:02d}") | |
| elif order_mode != "numeric": | |
| raise RuntimeError(f"Unknown order mode {order_mode}") | |
| base_bytes = base.encode("utf-8") | |
| for expected_chunk in chunk_order: | |
| train_local_index = int(train_inverse[original_row]) | |
| if train_local_index >= 0: | |
| selected_file = train_file | |
| selected_offsets = train_offsets | |
| local_index = train_local_index | |
| else: | |
| selected_file = val_file | |
| selected_offsets = val_offsets | |
| local_index = int(val_inverse[original_row]) | |
| actual_offset = selected_file.tell() | |
| expected_offset = int(selected_offsets[local_index]) | |
| if actual_offset != expected_offset: | |
| raise RuntimeError( | |
| f"Record offset mismatch for {source_name} row={original_row} " | |
| f"local={local_index} actual={actual_offset} expected={expected_offset}" | |
| ) | |
| line = selected_file.readline() | |
| if not line: | |
| raise RuntimeError(f"Early EOF for {source_name} row {original_row}") | |
| bytes_read += len(line) | |
| path_match = _VIDEO_PATH_RE.search(line) | |
| stem_match = _VIDEO_STEM_RE.match(path_match.group(1)) if path_match else None | |
| if ( | |
| stem_match is None | |
| or stem_match.group(1) != base_bytes | |
| or int(stem_match.group(2)) != expected_chunk | |
| ): | |
| raise RuntimeError( | |
| f"Record/original-row mismatch for {source_name} row={original_row} " | |
| f"base={base} chunk={expected_chunk}" | |
| ) | |
| original_row += 1 | |
| if train_file.readline() or val_file.readline(): | |
| raise RuntimeError(f"Trailing records after full scan for {source_name}") | |
| if original_row != rows: | |
| raise RuntimeError(f"Full scan row count mismatch for {source_name}") | |
| scan_seconds = time.perf_counter() - started | |
| scan_gib_per_second = bytes_read / 1024**3 / max(scan_seconds, 1e-9) | |
| return ( | |
| { | |
| "rows": rows, | |
| "train_rows": train_count, | |
| "val_rows": val_count, | |
| "sources": len(prefixes), | |
| "train_sources": len(train_sources), | |
| "val_sources": len(val_sources), | |
| "train_record_gib": train_records_path.stat().st_size / 1024**3, | |
| "val_record_gib": val_records_path.stat().st_size / 1024**3, | |
| "full_scan_seconds": scan_seconds, | |
| "full_scan_gib_per_second": scan_gib_per_second, | |
| "prefix_index": str(prefix_path), | |
| "train_inverse": str(train_inverse_path), | |
| "val_inverse": str(val_inverse_path), | |
| }, | |
| train_sources, | |
| val_sources, | |
| ) | |
| def main() -> None: | |
| args = parse_args() | |
| video_root = args.video_root | |
| specs = [ | |
| ("dance_1723_video_10s", "dance_1723_video_10s_clip50"), | |
| ("motion_example_0506_demo", "motion_example_0506_demo_clip50"), | |
| ("2264_with_audio", "2264_with_audio_clip50"), | |
| ] | |
| report: dict[str, Any] = {"status": "running", "video": {}, "records": {}} | |
| video_source_sets: dict[str, tuple[set[str], set[str]]] = {} | |
| for namespace, source_name in specs: | |
| result, train_sources, val_sources = audit_video( | |
| manifest=video_root / namespace / "metadata.jsonl", | |
| namespace=namespace, | |
| source_name=source_name, | |
| cache_dir=args.cache_root / "video_clip_records", | |
| runtime_index_dir=args.runtime_index_dir, | |
| val_ratio=float(args.val_ratio), | |
| split_seed=int(args.split_seed), | |
| full_scan=not args.skip_full_record_scan, | |
| ) | |
| report["video"][namespace] = result | |
| video_source_sets[namespace] = (train_sources, val_sources) | |
| print(f"[AUDIT OK] video {namespace}: {result}", flush=True) | |
| text_train, text_val, text_result = source_sets_from_pickles( | |
| args.cache_root / "text_records", | |
| "records.seed_text", | |
| "seed", | |
| val_ratio=float(args.val_ratio), | |
| split_seed=int(args.split_seed), | |
| ) | |
| command_train, command_val, command_result = source_sets_from_pickles( | |
| args.cache_root / "audio_command_records", | |
| "records.qwen3_tts_seed_audio_command", | |
| "seed", | |
| val_ratio=float(args.val_ratio), | |
| split_seed=int(args.split_seed), | |
| ) | |
| if text_train != command_train or text_val != command_val: | |
| raise RuntimeError("Text and audio-command motion split assignments differ") | |
| report["records"]["text"] = text_result | |
| report["records"]["audio_command"] = command_result | |
| for namespace, prefix in ( | |
| ("dance_1723_video_10s", "records.psi0_dance_audio"), | |
| ("2264_with_audio", "records.2264_with_audio"), | |
| ): | |
| audio_train, audio_val, result = source_sets_from_pickles( | |
| args.cache_root / "audio_records", | |
| prefix, | |
| namespace, | |
| val_ratio=float(args.val_ratio), | |
| split_seed=int(args.split_seed), | |
| ) | |
| video_train, video_val = video_source_sets[namespace] | |
| if not audio_train <= video_train or not audio_val <= video_val: | |
| raise RuntimeError(f"Audio/video source assignment differs for {namespace}") | |
| if namespace == "2264_with_audio" and ( | |
| audio_train != video_train or audio_val != video_val | |
| ): | |
| raise RuntimeError("2264 audio/video source sets should be identical") | |
| report["records"][f"audio/{namespace}"] = result | |
| report["status"] = "ok" | |
| report_path = args.report or args.cache_root / "audit_report.json" | |
| report_path.parent.mkdir(parents=True, exist_ok=True) | |
| report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") | |
| print(f"[AUDIT COMPLETE] status=ok report={report_path}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |