File size: 16,540 Bytes
8562e9b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
from __future__ import annotations

import argparse
import gc
import json
import pickle
import re
import sys
import time
from pathlib import Path
from typing import Any

import numpy as np

REPO_ROOT = Path(__file__).resolve().parents[2]
SRC_ROOT = REPO_ROOT / "src"
DEFAULT_SOURCE_SPLIT_ROOT = Path(
    "/mnt/data/data/omni_hidden_states_v23_oss/training_support/source_split_v1"
)
if str(SRC_ROOT) not in sys.path:
    sys.path.insert(0, str(SRC_ROOT))

from psi.data.seed.omni_oss_sampler import ensure_video_training_indexes
from psi.data.seed.source_split import motion_source_id, validation_source_ids


_VIDEO_PATH_RE = re.compile(br'"video_path"\s*:\s*"[^"]*/([^"/]+)\.mp4"')
_VIDEO_STEM_RE = re.compile(br"^(.*)\.chunk(\d+)\.f(\d+)$")


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Full audit of v26 motion-source split artifacts.")
    parser.add_argument(
        "--cache-root",
        type=Path,
        default=DEFAULT_SOURCE_SPLIT_ROOT,
    )
    parser.add_argument(
        "--runtime-index-dir",
        type=Path,
        default=DEFAULT_SOURCE_SPLIT_ROOT / "runtime_indexes",
    )
    parser.add_argument("--val-ratio", type=float, default=0.1)
    parser.add_argument("--split-seed", type=int, default=42)
    parser.add_argument(
        "--video-root",
        type=Path,
        default=Path(
            "/mnt/data/data/omni_hidden_states_v23_oss/training_support/"
            "video_clip_dense_stride1"
        ),
    )
    parser.add_argument("--skip-full-record-scan", action="store_true")
    parser.add_argument("--report", type=Path, default=None)
    return parser.parse_args()


def find_one(root: Path, pattern: str) -> Path:
    matches = sorted(root.glob(pattern))
    if len(matches) != 1:
        raise RuntimeError(f"Expected exactly one {pattern!r} under {root}, got {matches}")
    return matches[0]


def load_pickle_records(path: Path) -> list[dict[str, Any]]:
    with path.open("rb") as f:
        value = pickle.load(f)
    if isinstance(value, dict) and "data" in value:
        value = value["data"]
    if not isinstance(value, list):
        raise TypeError(f"Expected list cache at {path}")
    return value


def source_sets_from_pickles(
    root: Path,
    pattern_prefix: str,
    namespace: str,
    *,
    val_ratio: float,
    split_seed: int,
) -> tuple[set[str], set[str], dict[str, int]]:
    train_path = find_one(root, f"{pattern_prefix}.train.*.pkl")
    val_path = find_one(root, f"{pattern_prefix}.val.*.pkl")
    train_records = load_pickle_records(train_path)
    val_records = load_pickle_records(val_path)
    train = {motion_source_id(namespace, record["motion_dir"]) for record in train_records}
    val = {motion_source_id(namespace, record["motion_dir"]) for record in val_records}
    if len(train) != len(train_records) or len(val) != len(val_records):
        raise RuntimeError(f"Duplicate motion-source records in {pattern_prefix}")
    if train & val:
        raise RuntimeError(f"Source overlap in {pattern_prefix}: {len(train & val)}")
    expected_val = validation_source_ids(
        train | val,
        val_ratio=val_ratio,
        split_seed=split_seed,
    )
    if val != expected_val:
        raise RuntimeError(f"Stable-hash assignment mismatch in {pattern_prefix}")
    result = {
        "train_records": len(train_records),
        "val_records": len(val_records),
        "train_sources": len(train),
        "val_sources": len(val),
    }
    del train_records, val_records
    gc.collect()
    return train, val, result


def lazy_paths_from_meta(meta_path: Path) -> tuple[Path, Path, Path]:
    suffix = ".lazy.meta.json"
    base = meta_path.name[: -len(suffix)]
    return (
        meta_path.with_name(base + ".lazy.jsonl"),
        meta_path.with_name(base + ".lazy.offsets.npy"),
        meta_path.with_name(base + ".pkl"),
    )


def assert_dense_inverse(inverse: np.ndarray, label: str) -> int:
    mask = inverse >= 0
    count = int(mask.sum())
    selected = np.asarray(inverse[mask])
    if count:
        if int(selected[0]) != 0 or int(selected[-1]) != count - 1:
            raise RuntimeError(f"Non-dense {label} inverse endpoints")
        if count > 1 and not np.all(np.diff(selected) == 1):
            raise RuntimeError(f"Non-dense {label} inverse ordering")
    return count


def audit_video(
    *,
    manifest: Path,
    namespace: str,
    source_name: str,
    cache_dir: Path,
    runtime_index_dir: Path,
    val_ratio: float,
    split_seed: int,
    full_scan: bool,
) -> tuple[dict[str, Any], set[str], set[str]]:
    prefix_path, train_inverse_path = ensure_video_training_indexes(
        manifest,
        split="train",
        val_ratio=val_ratio,
        split_seed=split_seed,
        cache_dir=runtime_index_dir,
        split_source_namespace=namespace,
    )
    _, val_inverse_path = ensure_video_training_indexes(
        manifest,
        split="val",
        val_ratio=val_ratio,
        split_seed=split_seed,
        cache_dir=runtime_index_dir,
        split_source_namespace=namespace,
    )
    with prefix_path.open("rb") as f:
        prefix_payload = pickle.load(f)
    prefixes: dict[str, tuple[int, int, str]] = prefix_payload["prefixes"]
    rows = int(prefix_payload["rows"])
    train_inverse = np.load(train_inverse_path, mmap_mode="r")
    val_inverse = np.load(val_inverse_path, mmap_mode="r")
    if len(train_inverse) != rows or len(val_inverse) != rows:
        raise RuntimeError(f"Inverse length mismatch for {source_name}")
    if not np.all(np.logical_xor(train_inverse >= 0, val_inverse >= 0)):
        raise RuntimeError(f"Train/val inverse is not an exact partition for {source_name}")
    train_count = assert_dense_inverse(train_inverse, f"{source_name}/train")
    val_count = assert_dense_inverse(val_inverse, f"{source_name}/val")

    all_sources = {
        base: motion_source_id(namespace, base)
        for base in prefixes
    }
    expected_val_sources = validation_source_ids(
        all_sources.values(),
        val_ratio=val_ratio,
        split_seed=split_seed,
    )
    train_sources: set[str] = set()
    val_sources: set[str] = set()
    covered = 0
    for base, (start, count, _) in prefixes.items():
        if int(start) != covered:
            raise RuntimeError(f"Gap or overlap before {base}: start={start} covered={covered}")
        train_all = bool(np.all(train_inverse[start : start + count] >= 0))
        val_all = bool(np.all(val_inverse[start : start + count] >= 0))
        if train_all == val_all:
            raise RuntimeError(f"Partially split source {base}")
        source_id = all_sources[base]
        if val_all:
            val_sources.add(source_id)
        else:
            train_sources.add(source_id)
        if val_all != (source_id in expected_val_sources):
            raise RuntimeError(f"Stable-hash assignment mismatch for {base}")
        covered += int(count)
    if covered != rows or train_sources & val_sources:
        raise RuntimeError(f"Source range coverage failed for {source_name}")

    train_meta_path = find_one(cache_dir, f"records.{source_name}.train.*.lazy.meta.json")
    val_meta_path = find_one(cache_dir, f"records.{source_name}.val.*.lazy.meta.json")
    train_records_path, train_offsets_path, _ = lazy_paths_from_meta(train_meta_path)
    val_records_path, val_offsets_path, _ = lazy_paths_from_meta(val_meta_path)
    with train_meta_path.open("r", encoding="utf-8") as f:
        train_meta = json.load(f)
    with val_meta_path.open("r", encoding="utf-8") as f:
        val_meta = json.load(f)
    train_offsets = np.load(train_offsets_path, mmap_mode="r")
    val_offsets = np.load(val_offsets_path, mmap_mode="r")
    if int(train_meta["count"]) != train_count or len(train_offsets) != train_count:
        raise RuntimeError(f"Train record cache count mismatch for {source_name}")
    if int(val_meta["count"]) != val_count or len(val_offsets) != val_count:
        raise RuntimeError(f"Val record cache count mismatch for {source_name}")
    for meta in (train_meta, val_meta):
        if int(meta.get("source_split_version", 0)) != 1:
            raise RuntimeError(f"Missing source split version for {source_name}")
        if str(meta.get("split_source_namespace", "")) != namespace:
            raise RuntimeError(f"Source namespace mismatch for {source_name}")

    train_sample_path = find_one(cache_dir, f"sample_index.{source_name}.train.*.pkl")
    val_sample_path = find_one(cache_dir, f"sample_index.{source_name}.val.*.pkl")
    for split, sample_path, expected_count in (
        ("train", train_sample_path, train_count),
        ("val", val_sample_path, val_count),
    ):
        with sample_path.open("rb") as f:
            sample_index = pickle.load(f)
        if isinstance(sample_index, dict) and "data" in sample_index:
            sample_index = sample_index["data"]
        if len(sample_index) != expected_count:
            raise RuntimeError(f"Sample index count mismatch for {source_name}/{split}")
        if expected_count and (
            sample_index[0] != (0, 0)
            or sample_index[-1] != (expected_count - 1, 0)
        ):
            raise RuntimeError(f"Sample index is not identity for {source_name}/{split}")
        if full_scan:
            mismatch = next(
                (
                    index
                    for index, value in enumerate(sample_index)
                    if value != (index, 0)
                ),
                None,
            )
            if mismatch is not None:
                raise RuntimeError(
                    f"Sample index mismatch for {source_name}/{split} at {mismatch}"
                )
        del sample_index
        gc.collect()

    scan_seconds = 0.0
    scan_gib_per_second = 0.0
    if full_scan:
        started = time.perf_counter()
        bytes_read = 0
        original_row = 0
        with train_records_path.open("rb") as train_file, val_records_path.open("rb") as val_file:
            for base, (_, count, order_mode) in prefixes.items():
                chunk_order = list(range(count))
                if order_mode == "lexical":
                    chunk_order.sort(key=lambda chunk: f"{chunk:02d}")
                elif order_mode != "numeric":
                    raise RuntimeError(f"Unknown order mode {order_mode}")
                base_bytes = base.encode("utf-8")
                for expected_chunk in chunk_order:
                    train_local_index = int(train_inverse[original_row])
                    if train_local_index >= 0:
                        selected_file = train_file
                        selected_offsets = train_offsets
                        local_index = train_local_index
                    else:
                        selected_file = val_file
                        selected_offsets = val_offsets
                        local_index = int(val_inverse[original_row])
                    actual_offset = selected_file.tell()
                    expected_offset = int(selected_offsets[local_index])
                    if actual_offset != expected_offset:
                        raise RuntimeError(
                            f"Record offset mismatch for {source_name} row={original_row} "
                            f"local={local_index} actual={actual_offset} expected={expected_offset}"
                        )
                    line = selected_file.readline()
                    if not line:
                        raise RuntimeError(f"Early EOF for {source_name} row {original_row}")
                    bytes_read += len(line)
                    path_match = _VIDEO_PATH_RE.search(line)
                    stem_match = _VIDEO_STEM_RE.match(path_match.group(1)) if path_match else None
                    if (
                        stem_match is None
                        or stem_match.group(1) != base_bytes
                        or int(stem_match.group(2)) != expected_chunk
                    ):
                        raise RuntimeError(
                            f"Record/original-row mismatch for {source_name} row={original_row} "
                            f"base={base} chunk={expected_chunk}"
                        )
                    original_row += 1
            if train_file.readline() or val_file.readline():
                raise RuntimeError(f"Trailing records after full scan for {source_name}")
        if original_row != rows:
            raise RuntimeError(f"Full scan row count mismatch for {source_name}")
        scan_seconds = time.perf_counter() - started
        scan_gib_per_second = bytes_read / 1024**3 / max(scan_seconds, 1e-9)

    return (
        {
            "rows": rows,
            "train_rows": train_count,
            "val_rows": val_count,
            "sources": len(prefixes),
            "train_sources": len(train_sources),
            "val_sources": len(val_sources),
            "train_record_gib": train_records_path.stat().st_size / 1024**3,
            "val_record_gib": val_records_path.stat().st_size / 1024**3,
            "full_scan_seconds": scan_seconds,
            "full_scan_gib_per_second": scan_gib_per_second,
            "prefix_index": str(prefix_path),
            "train_inverse": str(train_inverse_path),
            "val_inverse": str(val_inverse_path),
        },
        train_sources,
        val_sources,
    )


def main() -> None:
    args = parse_args()
    video_root = args.video_root
    specs = [
        ("dance_1723_video_10s", "dance_1723_video_10s_clip50"),
        ("motion_example_0506_demo", "motion_example_0506_demo_clip50"),
        ("2264_with_audio", "2264_with_audio_clip50"),
    ]
    report: dict[str, Any] = {"status": "running", "video": {}, "records": {}}
    video_source_sets: dict[str, tuple[set[str], set[str]]] = {}
    for namespace, source_name in specs:
        result, train_sources, val_sources = audit_video(
            manifest=video_root / namespace / "metadata.jsonl",
            namespace=namespace,
            source_name=source_name,
            cache_dir=args.cache_root / "video_clip_records",
            runtime_index_dir=args.runtime_index_dir,
            val_ratio=float(args.val_ratio),
            split_seed=int(args.split_seed),
            full_scan=not args.skip_full_record_scan,
        )
        report["video"][namespace] = result
        video_source_sets[namespace] = (train_sources, val_sources)
        print(f"[AUDIT OK] video {namespace}: {result}", flush=True)

    text_train, text_val, text_result = source_sets_from_pickles(
        args.cache_root / "text_records",
        "records.seed_text",
        "seed",
        val_ratio=float(args.val_ratio),
        split_seed=int(args.split_seed),
    )
    command_train, command_val, command_result = source_sets_from_pickles(
        args.cache_root / "audio_command_records",
        "records.qwen3_tts_seed_audio_command",
        "seed",
        val_ratio=float(args.val_ratio),
        split_seed=int(args.split_seed),
    )
    if text_train != command_train or text_val != command_val:
        raise RuntimeError("Text and audio-command motion split assignments differ")
    report["records"]["text"] = text_result
    report["records"]["audio_command"] = command_result

    for namespace, prefix in (
        ("dance_1723_video_10s", "records.psi0_dance_audio"),
        ("2264_with_audio", "records.2264_with_audio"),
    ):
        audio_train, audio_val, result = source_sets_from_pickles(
            args.cache_root / "audio_records",
            prefix,
            namespace,
            val_ratio=float(args.val_ratio),
            split_seed=int(args.split_seed),
        )
        video_train, video_val = video_source_sets[namespace]
        if not audio_train <= video_train or not audio_val <= video_val:
            raise RuntimeError(f"Audio/video source assignment differs for {namespace}")
        if namespace == "2264_with_audio" and (
            audio_train != video_train or audio_val != video_val
        ):
            raise RuntimeError("2264 audio/video source sets should be identical")
        report["records"][f"audio/{namespace}"] = result

    report["status"] = "ok"
    report_path = args.report or args.cache_root / "audit_report.json"
    report_path.parent.mkdir(parents=True, exist_ok=True)
    report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
    print(f"[AUDIT COMPLETE] status=ok report={report_path}", flush=True)


if __name__ == "__main__":
    main()