code-HFD_lzk / scripts /data /audit_source_split_artifacts.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
16.5 kB
from __future__ import annotations
import argparse
import gc
import json
import pickle
import re
import sys
import time
from pathlib import Path
from typing import Any
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[2]
SRC_ROOT = REPO_ROOT / "src"
DEFAULT_SOURCE_SPLIT_ROOT = Path(
"/mnt/data/data/omni_hidden_states_v23_oss/training_support/source_split_v1"
)
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))
from psi.data.seed.omni_oss_sampler import ensure_video_training_indexes
from psi.data.seed.source_split import motion_source_id, validation_source_ids
_VIDEO_PATH_RE = re.compile(br'"video_path"\s*:\s*"[^"]*/([^"/]+)\.mp4"')
_VIDEO_STEM_RE = re.compile(br"^(.*)\.chunk(\d+)\.f(\d+)$")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Full audit of v26 motion-source split artifacts.")
parser.add_argument(
"--cache-root",
type=Path,
default=DEFAULT_SOURCE_SPLIT_ROOT,
)
parser.add_argument(
"--runtime-index-dir",
type=Path,
default=DEFAULT_SOURCE_SPLIT_ROOT / "runtime_indexes",
)
parser.add_argument("--val-ratio", type=float, default=0.1)
parser.add_argument("--split-seed", type=int, default=42)
parser.add_argument(
"--video-root",
type=Path,
default=Path(
"/mnt/data/data/omni_hidden_states_v23_oss/training_support/"
"video_clip_dense_stride1"
),
)
parser.add_argument("--skip-full-record-scan", action="store_true")
parser.add_argument("--report", type=Path, default=None)
return parser.parse_args()
def find_one(root: Path, pattern: str) -> Path:
matches = sorted(root.glob(pattern))
if len(matches) != 1:
raise RuntimeError(f"Expected exactly one {pattern!r} under {root}, got {matches}")
return matches[0]
def load_pickle_records(path: Path) -> list[dict[str, Any]]:
with path.open("rb") as f:
value = pickle.load(f)
if isinstance(value, dict) and "data" in value:
value = value["data"]
if not isinstance(value, list):
raise TypeError(f"Expected list cache at {path}")
return value
def source_sets_from_pickles(
root: Path,
pattern_prefix: str,
namespace: str,
*,
val_ratio: float,
split_seed: int,
) -> tuple[set[str], set[str], dict[str, int]]:
train_path = find_one(root, f"{pattern_prefix}.train.*.pkl")
val_path = find_one(root, f"{pattern_prefix}.val.*.pkl")
train_records = load_pickle_records(train_path)
val_records = load_pickle_records(val_path)
train = {motion_source_id(namespace, record["motion_dir"]) for record in train_records}
val = {motion_source_id(namespace, record["motion_dir"]) for record in val_records}
if len(train) != len(train_records) or len(val) != len(val_records):
raise RuntimeError(f"Duplicate motion-source records in {pattern_prefix}")
if train & val:
raise RuntimeError(f"Source overlap in {pattern_prefix}: {len(train & val)}")
expected_val = validation_source_ids(
train | val,
val_ratio=val_ratio,
split_seed=split_seed,
)
if val != expected_val:
raise RuntimeError(f"Stable-hash assignment mismatch in {pattern_prefix}")
result = {
"train_records": len(train_records),
"val_records": len(val_records),
"train_sources": len(train),
"val_sources": len(val),
}
del train_records, val_records
gc.collect()
return train, val, result
def lazy_paths_from_meta(meta_path: Path) -> tuple[Path, Path, Path]:
suffix = ".lazy.meta.json"
base = meta_path.name[: -len(suffix)]
return (
meta_path.with_name(base + ".lazy.jsonl"),
meta_path.with_name(base + ".lazy.offsets.npy"),
meta_path.with_name(base + ".pkl"),
)
def assert_dense_inverse(inverse: np.ndarray, label: str) -> int:
mask = inverse >= 0
count = int(mask.sum())
selected = np.asarray(inverse[mask])
if count:
if int(selected[0]) != 0 or int(selected[-1]) != count - 1:
raise RuntimeError(f"Non-dense {label} inverse endpoints")
if count > 1 and not np.all(np.diff(selected) == 1):
raise RuntimeError(f"Non-dense {label} inverse ordering")
return count
def audit_video(
*,
manifest: Path,
namespace: str,
source_name: str,
cache_dir: Path,
runtime_index_dir: Path,
val_ratio: float,
split_seed: int,
full_scan: bool,
) -> tuple[dict[str, Any], set[str], set[str]]:
prefix_path, train_inverse_path = ensure_video_training_indexes(
manifest,
split="train",
val_ratio=val_ratio,
split_seed=split_seed,
cache_dir=runtime_index_dir,
split_source_namespace=namespace,
)
_, val_inverse_path = ensure_video_training_indexes(
manifest,
split="val",
val_ratio=val_ratio,
split_seed=split_seed,
cache_dir=runtime_index_dir,
split_source_namespace=namespace,
)
with prefix_path.open("rb") as f:
prefix_payload = pickle.load(f)
prefixes: dict[str, tuple[int, int, str]] = prefix_payload["prefixes"]
rows = int(prefix_payload["rows"])
train_inverse = np.load(train_inverse_path, mmap_mode="r")
val_inverse = np.load(val_inverse_path, mmap_mode="r")
if len(train_inverse) != rows or len(val_inverse) != rows:
raise RuntimeError(f"Inverse length mismatch for {source_name}")
if not np.all(np.logical_xor(train_inverse >= 0, val_inverse >= 0)):
raise RuntimeError(f"Train/val inverse is not an exact partition for {source_name}")
train_count = assert_dense_inverse(train_inverse, f"{source_name}/train")
val_count = assert_dense_inverse(val_inverse, f"{source_name}/val")
all_sources = {
base: motion_source_id(namespace, base)
for base in prefixes
}
expected_val_sources = validation_source_ids(
all_sources.values(),
val_ratio=val_ratio,
split_seed=split_seed,
)
train_sources: set[str] = set()
val_sources: set[str] = set()
covered = 0
for base, (start, count, _) in prefixes.items():
if int(start) != covered:
raise RuntimeError(f"Gap or overlap before {base}: start={start} covered={covered}")
train_all = bool(np.all(train_inverse[start : start + count] >= 0))
val_all = bool(np.all(val_inverse[start : start + count] >= 0))
if train_all == val_all:
raise RuntimeError(f"Partially split source {base}")
source_id = all_sources[base]
if val_all:
val_sources.add(source_id)
else:
train_sources.add(source_id)
if val_all != (source_id in expected_val_sources):
raise RuntimeError(f"Stable-hash assignment mismatch for {base}")
covered += int(count)
if covered != rows or train_sources & val_sources:
raise RuntimeError(f"Source range coverage failed for {source_name}")
train_meta_path = find_one(cache_dir, f"records.{source_name}.train.*.lazy.meta.json")
val_meta_path = find_one(cache_dir, f"records.{source_name}.val.*.lazy.meta.json")
train_records_path, train_offsets_path, _ = lazy_paths_from_meta(train_meta_path)
val_records_path, val_offsets_path, _ = lazy_paths_from_meta(val_meta_path)
with train_meta_path.open("r", encoding="utf-8") as f:
train_meta = json.load(f)
with val_meta_path.open("r", encoding="utf-8") as f:
val_meta = json.load(f)
train_offsets = np.load(train_offsets_path, mmap_mode="r")
val_offsets = np.load(val_offsets_path, mmap_mode="r")
if int(train_meta["count"]) != train_count or len(train_offsets) != train_count:
raise RuntimeError(f"Train record cache count mismatch for {source_name}")
if int(val_meta["count"]) != val_count or len(val_offsets) != val_count:
raise RuntimeError(f"Val record cache count mismatch for {source_name}")
for meta in (train_meta, val_meta):
if int(meta.get("source_split_version", 0)) != 1:
raise RuntimeError(f"Missing source split version for {source_name}")
if str(meta.get("split_source_namespace", "")) != namespace:
raise RuntimeError(f"Source namespace mismatch for {source_name}")
train_sample_path = find_one(cache_dir, f"sample_index.{source_name}.train.*.pkl")
val_sample_path = find_one(cache_dir, f"sample_index.{source_name}.val.*.pkl")
for split, sample_path, expected_count in (
("train", train_sample_path, train_count),
("val", val_sample_path, val_count),
):
with sample_path.open("rb") as f:
sample_index = pickle.load(f)
if isinstance(sample_index, dict) and "data" in sample_index:
sample_index = sample_index["data"]
if len(sample_index) != expected_count:
raise RuntimeError(f"Sample index count mismatch for {source_name}/{split}")
if expected_count and (
sample_index[0] != (0, 0)
or sample_index[-1] != (expected_count - 1, 0)
):
raise RuntimeError(f"Sample index is not identity for {source_name}/{split}")
if full_scan:
mismatch = next(
(
index
for index, value in enumerate(sample_index)
if value != (index, 0)
),
None,
)
if mismatch is not None:
raise RuntimeError(
f"Sample index mismatch for {source_name}/{split} at {mismatch}"
)
del sample_index
gc.collect()
scan_seconds = 0.0
scan_gib_per_second = 0.0
if full_scan:
started = time.perf_counter()
bytes_read = 0
original_row = 0
with train_records_path.open("rb") as train_file, val_records_path.open("rb") as val_file:
for base, (_, count, order_mode) in prefixes.items():
chunk_order = list(range(count))
if order_mode == "lexical":
chunk_order.sort(key=lambda chunk: f"{chunk:02d}")
elif order_mode != "numeric":
raise RuntimeError(f"Unknown order mode {order_mode}")
base_bytes = base.encode("utf-8")
for expected_chunk in chunk_order:
train_local_index = int(train_inverse[original_row])
if train_local_index >= 0:
selected_file = train_file
selected_offsets = train_offsets
local_index = train_local_index
else:
selected_file = val_file
selected_offsets = val_offsets
local_index = int(val_inverse[original_row])
actual_offset = selected_file.tell()
expected_offset = int(selected_offsets[local_index])
if actual_offset != expected_offset:
raise RuntimeError(
f"Record offset mismatch for {source_name} row={original_row} "
f"local={local_index} actual={actual_offset} expected={expected_offset}"
)
line = selected_file.readline()
if not line:
raise RuntimeError(f"Early EOF for {source_name} row {original_row}")
bytes_read += len(line)
path_match = _VIDEO_PATH_RE.search(line)
stem_match = _VIDEO_STEM_RE.match(path_match.group(1)) if path_match else None
if (
stem_match is None
or stem_match.group(1) != base_bytes
or int(stem_match.group(2)) != expected_chunk
):
raise RuntimeError(
f"Record/original-row mismatch for {source_name} row={original_row} "
f"base={base} chunk={expected_chunk}"
)
original_row += 1
if train_file.readline() or val_file.readline():
raise RuntimeError(f"Trailing records after full scan for {source_name}")
if original_row != rows:
raise RuntimeError(f"Full scan row count mismatch for {source_name}")
scan_seconds = time.perf_counter() - started
scan_gib_per_second = bytes_read / 1024**3 / max(scan_seconds, 1e-9)
return (
{
"rows": rows,
"train_rows": train_count,
"val_rows": val_count,
"sources": len(prefixes),
"train_sources": len(train_sources),
"val_sources": len(val_sources),
"train_record_gib": train_records_path.stat().st_size / 1024**3,
"val_record_gib": val_records_path.stat().st_size / 1024**3,
"full_scan_seconds": scan_seconds,
"full_scan_gib_per_second": scan_gib_per_second,
"prefix_index": str(prefix_path),
"train_inverse": str(train_inverse_path),
"val_inverse": str(val_inverse_path),
},
train_sources,
val_sources,
)
def main() -> None:
args = parse_args()
video_root = args.video_root
specs = [
("dance_1723_video_10s", "dance_1723_video_10s_clip50"),
("motion_example_0506_demo", "motion_example_0506_demo_clip50"),
("2264_with_audio", "2264_with_audio_clip50"),
]
report: dict[str, Any] = {"status": "running", "video": {}, "records": {}}
video_source_sets: dict[str, tuple[set[str], set[str]]] = {}
for namespace, source_name in specs:
result, train_sources, val_sources = audit_video(
manifest=video_root / namespace / "metadata.jsonl",
namespace=namespace,
source_name=source_name,
cache_dir=args.cache_root / "video_clip_records",
runtime_index_dir=args.runtime_index_dir,
val_ratio=float(args.val_ratio),
split_seed=int(args.split_seed),
full_scan=not args.skip_full_record_scan,
)
report["video"][namespace] = result
video_source_sets[namespace] = (train_sources, val_sources)
print(f"[AUDIT OK] video {namespace}: {result}", flush=True)
text_train, text_val, text_result = source_sets_from_pickles(
args.cache_root / "text_records",
"records.seed_text",
"seed",
val_ratio=float(args.val_ratio),
split_seed=int(args.split_seed),
)
command_train, command_val, command_result = source_sets_from_pickles(
args.cache_root / "audio_command_records",
"records.qwen3_tts_seed_audio_command",
"seed",
val_ratio=float(args.val_ratio),
split_seed=int(args.split_seed),
)
if text_train != command_train or text_val != command_val:
raise RuntimeError("Text and audio-command motion split assignments differ")
report["records"]["text"] = text_result
report["records"]["audio_command"] = command_result
for namespace, prefix in (
("dance_1723_video_10s", "records.psi0_dance_audio"),
("2264_with_audio", "records.2264_with_audio"),
):
audio_train, audio_val, result = source_sets_from_pickles(
args.cache_root / "audio_records",
prefix,
namespace,
val_ratio=float(args.val_ratio),
split_seed=int(args.split_seed),
)
video_train, video_val = video_source_sets[namespace]
if not audio_train <= video_train or not audio_val <= video_val:
raise RuntimeError(f"Audio/video source assignment differs for {namespace}")
if namespace == "2264_with_audio" and (
audio_train != video_train or audio_val != video_val
):
raise RuntimeError("2264 audio/video source sets should be identical")
report["records"][f"audio/{namespace}"] = result
report["status"] = "ok"
report_path = args.report or args.cache_root / "audit_report.json"
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
print(f"[AUDIT COMPLETE] status=ok report={report_path}", flush=True)
if __name__ == "__main__":
main()