from __future__ import annotations import argparse import gc import json import pickle import re import sys import time from pathlib import Path from typing import Any import numpy as np REPO_ROOT = Path(__file__).resolve().parents[2] SRC_ROOT = REPO_ROOT / "src" DEFAULT_SOURCE_SPLIT_ROOT = Path( "/mnt/data/data/omni_hidden_states_v23_oss/training_support/source_split_v1" ) if str(SRC_ROOT) not in sys.path: sys.path.insert(0, str(SRC_ROOT)) from psi.data.seed.omni_oss_sampler import ensure_video_training_indexes from psi.data.seed.source_split import motion_source_id, validation_source_ids _VIDEO_PATH_RE = re.compile(br'"video_path"\s*:\s*"[^"]*/([^"/]+)\.mp4"') _VIDEO_STEM_RE = re.compile(br"^(.*)\.chunk(\d+)\.f(\d+)$") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Full audit of v26 motion-source split artifacts.") parser.add_argument( "--cache-root", type=Path, default=DEFAULT_SOURCE_SPLIT_ROOT, ) parser.add_argument( "--runtime-index-dir", type=Path, default=DEFAULT_SOURCE_SPLIT_ROOT / "runtime_indexes", ) parser.add_argument("--val-ratio", type=float, default=0.1) parser.add_argument("--split-seed", type=int, default=42) parser.add_argument( "--video-root", type=Path, default=Path( "/mnt/data/data/omni_hidden_states_v23_oss/training_support/" "video_clip_dense_stride1" ), ) parser.add_argument("--skip-full-record-scan", action="store_true") parser.add_argument("--report", type=Path, default=None) return parser.parse_args() def find_one(root: Path, pattern: str) -> Path: matches = sorted(root.glob(pattern)) if len(matches) != 1: raise RuntimeError(f"Expected exactly one {pattern!r} under {root}, got {matches}") return matches[0] def load_pickle_records(path: Path) -> list[dict[str, Any]]: with path.open("rb") as f: value = pickle.load(f) if isinstance(value, dict) and "data" in value: value = value["data"] if not isinstance(value, list): raise TypeError(f"Expected list cache at {path}") return value def source_sets_from_pickles( root: Path, pattern_prefix: str, namespace: str, *, val_ratio: float, split_seed: int, ) -> tuple[set[str], set[str], dict[str, int]]: train_path = find_one(root, f"{pattern_prefix}.train.*.pkl") val_path = find_one(root, f"{pattern_prefix}.val.*.pkl") train_records = load_pickle_records(train_path) val_records = load_pickle_records(val_path) train = {motion_source_id(namespace, record["motion_dir"]) for record in train_records} val = {motion_source_id(namespace, record["motion_dir"]) for record in val_records} if len(train) != len(train_records) or len(val) != len(val_records): raise RuntimeError(f"Duplicate motion-source records in {pattern_prefix}") if train & val: raise RuntimeError(f"Source overlap in {pattern_prefix}: {len(train & val)}") expected_val = validation_source_ids( train | val, val_ratio=val_ratio, split_seed=split_seed, ) if val != expected_val: raise RuntimeError(f"Stable-hash assignment mismatch in {pattern_prefix}") result = { "train_records": len(train_records), "val_records": len(val_records), "train_sources": len(train), "val_sources": len(val), } del train_records, val_records gc.collect() return train, val, result def lazy_paths_from_meta(meta_path: Path) -> tuple[Path, Path, Path]: suffix = ".lazy.meta.json" base = meta_path.name[: -len(suffix)] return ( meta_path.with_name(base + ".lazy.jsonl"), meta_path.with_name(base + ".lazy.offsets.npy"), meta_path.with_name(base + ".pkl"), ) def assert_dense_inverse(inverse: np.ndarray, label: str) -> int: mask = inverse >= 0 count = int(mask.sum()) selected = np.asarray(inverse[mask]) if count: if int(selected[0]) != 0 or int(selected[-1]) != count - 1: raise RuntimeError(f"Non-dense {label} inverse endpoints") if count > 1 and not np.all(np.diff(selected) == 1): raise RuntimeError(f"Non-dense {label} inverse ordering") return count def audit_video( *, manifest: Path, namespace: str, source_name: str, cache_dir: Path, runtime_index_dir: Path, val_ratio: float, split_seed: int, full_scan: bool, ) -> tuple[dict[str, Any], set[str], set[str]]: prefix_path, train_inverse_path = ensure_video_training_indexes( manifest, split="train", val_ratio=val_ratio, split_seed=split_seed, cache_dir=runtime_index_dir, split_source_namespace=namespace, ) _, val_inverse_path = ensure_video_training_indexes( manifest, split="val", val_ratio=val_ratio, split_seed=split_seed, cache_dir=runtime_index_dir, split_source_namespace=namespace, ) with prefix_path.open("rb") as f: prefix_payload = pickle.load(f) prefixes: dict[str, tuple[int, int, str]] = prefix_payload["prefixes"] rows = int(prefix_payload["rows"]) train_inverse = np.load(train_inverse_path, mmap_mode="r") val_inverse = np.load(val_inverse_path, mmap_mode="r") if len(train_inverse) != rows or len(val_inverse) != rows: raise RuntimeError(f"Inverse length mismatch for {source_name}") if not np.all(np.logical_xor(train_inverse >= 0, val_inverse >= 0)): raise RuntimeError(f"Train/val inverse is not an exact partition for {source_name}") train_count = assert_dense_inverse(train_inverse, f"{source_name}/train") val_count = assert_dense_inverse(val_inverse, f"{source_name}/val") all_sources = { base: motion_source_id(namespace, base) for base in prefixes } expected_val_sources = validation_source_ids( all_sources.values(), val_ratio=val_ratio, split_seed=split_seed, ) train_sources: set[str] = set() val_sources: set[str] = set() covered = 0 for base, (start, count, _) in prefixes.items(): if int(start) != covered: raise RuntimeError(f"Gap or overlap before {base}: start={start} covered={covered}") train_all = bool(np.all(train_inverse[start : start + count] >= 0)) val_all = bool(np.all(val_inverse[start : start + count] >= 0)) if train_all == val_all: raise RuntimeError(f"Partially split source {base}") source_id = all_sources[base] if val_all: val_sources.add(source_id) else: train_sources.add(source_id) if val_all != (source_id in expected_val_sources): raise RuntimeError(f"Stable-hash assignment mismatch for {base}") covered += int(count) if covered != rows or train_sources & val_sources: raise RuntimeError(f"Source range coverage failed for {source_name}") train_meta_path = find_one(cache_dir, f"records.{source_name}.train.*.lazy.meta.json") val_meta_path = find_one(cache_dir, f"records.{source_name}.val.*.lazy.meta.json") train_records_path, train_offsets_path, _ = lazy_paths_from_meta(train_meta_path) val_records_path, val_offsets_path, _ = lazy_paths_from_meta(val_meta_path) with train_meta_path.open("r", encoding="utf-8") as f: train_meta = json.load(f) with val_meta_path.open("r", encoding="utf-8") as f: val_meta = json.load(f) train_offsets = np.load(train_offsets_path, mmap_mode="r") val_offsets = np.load(val_offsets_path, mmap_mode="r") if int(train_meta["count"]) != train_count or len(train_offsets) != train_count: raise RuntimeError(f"Train record cache count mismatch for {source_name}") if int(val_meta["count"]) != val_count or len(val_offsets) != val_count: raise RuntimeError(f"Val record cache count mismatch for {source_name}") for meta in (train_meta, val_meta): if int(meta.get("source_split_version", 0)) != 1: raise RuntimeError(f"Missing source split version for {source_name}") if str(meta.get("split_source_namespace", "")) != namespace: raise RuntimeError(f"Source namespace mismatch for {source_name}") train_sample_path = find_one(cache_dir, f"sample_index.{source_name}.train.*.pkl") val_sample_path = find_one(cache_dir, f"sample_index.{source_name}.val.*.pkl") for split, sample_path, expected_count in ( ("train", train_sample_path, train_count), ("val", val_sample_path, val_count), ): with sample_path.open("rb") as f: sample_index = pickle.load(f) if isinstance(sample_index, dict) and "data" in sample_index: sample_index = sample_index["data"] if len(sample_index) != expected_count: raise RuntimeError(f"Sample index count mismatch for {source_name}/{split}") if expected_count and ( sample_index[0] != (0, 0) or sample_index[-1] != (expected_count - 1, 0) ): raise RuntimeError(f"Sample index is not identity for {source_name}/{split}") if full_scan: mismatch = next( ( index for index, value in enumerate(sample_index) if value != (index, 0) ), None, ) if mismatch is not None: raise RuntimeError( f"Sample index mismatch for {source_name}/{split} at {mismatch}" ) del sample_index gc.collect() scan_seconds = 0.0 scan_gib_per_second = 0.0 if full_scan: started = time.perf_counter() bytes_read = 0 original_row = 0 with train_records_path.open("rb") as train_file, val_records_path.open("rb") as val_file: for base, (_, count, order_mode) in prefixes.items(): chunk_order = list(range(count)) if order_mode == "lexical": chunk_order.sort(key=lambda chunk: f"{chunk:02d}") elif order_mode != "numeric": raise RuntimeError(f"Unknown order mode {order_mode}") base_bytes = base.encode("utf-8") for expected_chunk in chunk_order: train_local_index = int(train_inverse[original_row]) if train_local_index >= 0: selected_file = train_file selected_offsets = train_offsets local_index = train_local_index else: selected_file = val_file selected_offsets = val_offsets local_index = int(val_inverse[original_row]) actual_offset = selected_file.tell() expected_offset = int(selected_offsets[local_index]) if actual_offset != expected_offset: raise RuntimeError( f"Record offset mismatch for {source_name} row={original_row} " f"local={local_index} actual={actual_offset} expected={expected_offset}" ) line = selected_file.readline() if not line: raise RuntimeError(f"Early EOF for {source_name} row {original_row}") bytes_read += len(line) path_match = _VIDEO_PATH_RE.search(line) stem_match = _VIDEO_STEM_RE.match(path_match.group(1)) if path_match else None if ( stem_match is None or stem_match.group(1) != base_bytes or int(stem_match.group(2)) != expected_chunk ): raise RuntimeError( f"Record/original-row mismatch for {source_name} row={original_row} " f"base={base} chunk={expected_chunk}" ) original_row += 1 if train_file.readline() or val_file.readline(): raise RuntimeError(f"Trailing records after full scan for {source_name}") if original_row != rows: raise RuntimeError(f"Full scan row count mismatch for {source_name}") scan_seconds = time.perf_counter() - started scan_gib_per_second = bytes_read / 1024**3 / max(scan_seconds, 1e-9) return ( { "rows": rows, "train_rows": train_count, "val_rows": val_count, "sources": len(prefixes), "train_sources": len(train_sources), "val_sources": len(val_sources), "train_record_gib": train_records_path.stat().st_size / 1024**3, "val_record_gib": val_records_path.stat().st_size / 1024**3, "full_scan_seconds": scan_seconds, "full_scan_gib_per_second": scan_gib_per_second, "prefix_index": str(prefix_path), "train_inverse": str(train_inverse_path), "val_inverse": str(val_inverse_path), }, train_sources, val_sources, ) def main() -> None: args = parse_args() video_root = args.video_root specs = [ ("dance_1723_video_10s", "dance_1723_video_10s_clip50"), ("motion_example_0506_demo", "motion_example_0506_demo_clip50"), ("2264_with_audio", "2264_with_audio_clip50"), ] report: dict[str, Any] = {"status": "running", "video": {}, "records": {}} video_source_sets: dict[str, tuple[set[str], set[str]]] = {} for namespace, source_name in specs: result, train_sources, val_sources = audit_video( manifest=video_root / namespace / "metadata.jsonl", namespace=namespace, source_name=source_name, cache_dir=args.cache_root / "video_clip_records", runtime_index_dir=args.runtime_index_dir, val_ratio=float(args.val_ratio), split_seed=int(args.split_seed), full_scan=not args.skip_full_record_scan, ) report["video"][namespace] = result video_source_sets[namespace] = (train_sources, val_sources) print(f"[AUDIT OK] video {namespace}: {result}", flush=True) text_train, text_val, text_result = source_sets_from_pickles( args.cache_root / "text_records", "records.seed_text", "seed", val_ratio=float(args.val_ratio), split_seed=int(args.split_seed), ) command_train, command_val, command_result = source_sets_from_pickles( args.cache_root / "audio_command_records", "records.qwen3_tts_seed_audio_command", "seed", val_ratio=float(args.val_ratio), split_seed=int(args.split_seed), ) if text_train != command_train or text_val != command_val: raise RuntimeError("Text and audio-command motion split assignments differ") report["records"]["text"] = text_result report["records"]["audio_command"] = command_result for namespace, prefix in ( ("dance_1723_video_10s", "records.psi0_dance_audio"), ("2264_with_audio", "records.2264_with_audio"), ): audio_train, audio_val, result = source_sets_from_pickles( args.cache_root / "audio_records", prefix, namespace, val_ratio=float(args.val_ratio), split_seed=int(args.split_seed), ) video_train, video_val = video_source_sets[namespace] if not audio_train <= video_train or not audio_val <= video_val: raise RuntimeError(f"Audio/video source assignment differs for {namespace}") if namespace == "2264_with_audio" and ( audio_train != video_train or audio_val != video_val ): raise RuntimeError("2264 audio/video source sets should be identical") report["records"][f"audio/{namespace}"] = result report["status"] = "ok" report_path = args.report or args.cache_root / "audit_report.json" report_path.parent.mkdir(parents=True, exist_ok=True) report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") print(f"[AUDIT COMPLETE] status=ok report={report_path}", flush=True) if __name__ == "__main__": main()