#!/usr/bin/env python3 """CPU-only startup index warmup from an exact saved HFD run configuration. Usage: .venv-psi/bin/python scripts/data/prepare_full_duplex_indexes.py RUN_DIR Set HFD_STARTUP_INDEX_CACHE_DIR identically here and on the training machine to share indexes. Set HFD_STARTUP_INDEX_CACHE=0 to bypass caching for diagnostics. No model/processor, WAV mixing, augmented item, CUDA or WandB is created. """ from __future__ import annotations import argparse import json import os from pathlib import Path import sys import time sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("run_config", type=Path, help="run directory or run_config.json") parser.add_argument("--splits", nargs="+", choices=("train", "val"), default=("train", "val")) parser.add_argument("--verify", action="store_true", help="Rebuild original audio composition in memory and compare every row/length") parser.add_argument("--listen-hold", action="store_true", help="Warm the revised listen targets' indexes without rewriting the saved run; " "with --verify also check the historical source membership and lengths") args = parser.parse_args() os.environ["CUDA_VISIBLE_DEVICES"] = "" from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig from psi.config.transform import SeedPosttrainRepackTransform path = args.run_config if path.is_dir(): path /= "run_config.json" payload = json.loads(path.read_text()) config = FullDuplexAudioCommandDataConfig.model_validate(payload["data"]) # Pydantic's base DataTransform otherwise drops the concrete repack fields. # Only geometry is consulted during construction; transforms are NOT run. config.transform.repack = SeedPosttrainRepackTransform.model_validate( payload["data"]["transform"]["repack"] ) if args.listen_hold: config.stationary_listen_hold = True warm_indexes(config, args.splits, verify=args.verify, listen_hold=args.listen_hold) def warm_indexes(config, splits=("train", "val"), *, verify=False, listen_hold=False): """Populate/reuse deterministic metadata only; never materialize augmented items.""" for split in splits: start = time.monotonic() dataset = config(split=split) lengths = [dataset.estimated_stream_steps(i) for i in range(len(dataset))] print(json.dumps({"split": split, "sources": len(dataset), "estimated_windows": sum(lengths), "seconds": round(time.monotonic() - start, 3)}), flush=True) if verify: raw = dataset.raw_dataset # The unified overlay wraps the same audio/motion dataset. Verify # that owner, not shadow attributes on the overlay wrapper. while hasattr(raw, "raw_dataset"): raw = raw.raw_dataset sessions = raw.sessions original_lengths = raw._estimated_stream_lengths summaries = {key: value for key, value in vars(raw).items() if key.startswith(("task_", "control_task_", "interruption_task_")) or key in {"long_session_filter_summary", "voice_reuse_after_coverage", "new_task_record_selection_summary"}} raw._build_session_index() assert sessions == raw.sessions, "Cached session composition differs" assert original_lengths == tuple(raw._estimate_stream_steps(i) for i in range(len(raw))), "Cached lengths differ" assert all(getattr(raw, key) == value for key, value in summaries.items()), "Cached task summaries differ" print(f"[verified] {split}: every session, length and task summary matches uncached construction", flush=True) if listen_hold: historical_config = config.model_copy(deep=True) historical_config.stationary_listen_hold = False historical = historical_config(split=split).raw_dataset while hasattr(historical, "raw_dataset"): historical = historical.raw_dataset assert historical.sessions == raw.sessions, "Listen hold changed source membership/order" assert historical._estimated_stream_lengths == raw._estimated_stream_lengths, "Listen hold changed timeline lengths" assert all(getattr(historical, key) == value for key, value in summaries.items()), "Listen hold changed task mixture" print(f"[verified] {split}: listen hold preserves every legacy session, timeline length and task summary", flush=True) if __name__ == "__main__": main()