code-HFD / scripts /data /prepare_full_duplex_indexes.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
4.89 kB
#!/usr/bin/env python3
"""CPU-only startup index warmup from an exact saved HFD run configuration.
Usage: .venv-psi/bin/python scripts/data/prepare_full_duplex_indexes.py RUN_DIR
Set HFD_STARTUP_INDEX_CACHE_DIR identically here and on the training machine to
share indexes. Set HFD_STARTUP_INDEX_CACHE=0 to bypass caching for diagnostics.
No model/processor, WAV mixing, augmented item, CUDA or WandB is created.
"""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src"))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("run_config", type=Path, help="run directory or run_config.json")
parser.add_argument("--splits", nargs="+", choices=("train", "val"),
default=("train", "val"))
parser.add_argument("--verify", action="store_true",
help="Rebuild original audio composition in memory and compare every row/length")
parser.add_argument("--listen-hold", action="store_true",
help="Warm the revised listen targets' indexes without rewriting the saved run; "
"with --verify also check the historical source membership and lengths")
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = ""
from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig
from psi.config.transform import SeedPosttrainRepackTransform
path = args.run_config
if path.is_dir():
path /= "run_config.json"
payload = json.loads(path.read_text())
config = FullDuplexAudioCommandDataConfig.model_validate(payload["data"])
# Pydantic's base DataTransform otherwise drops the concrete repack fields.
# Only geometry is consulted during construction; transforms are NOT run.
config.transform.repack = SeedPosttrainRepackTransform.model_validate(
payload["data"]["transform"]["repack"]
)
if args.listen_hold:
config.stationary_listen_hold = True
warm_indexes(config, args.splits, verify=args.verify, listen_hold=args.listen_hold)
def warm_indexes(config, splits=("train", "val"), *, verify=False, listen_hold=False):
"""Populate/reuse deterministic metadata only; never materialize augmented items."""
for split in splits:
start = time.monotonic()
dataset = config(split=split)
lengths = [dataset.estimated_stream_steps(i) for i in range(len(dataset))]
print(json.dumps({"split": split, "sources": len(dataset),
"estimated_windows": sum(lengths),
"seconds": round(time.monotonic() - start, 3)}), flush=True)
if verify:
raw = dataset.raw_dataset
# The unified overlay wraps the same audio/motion dataset. Verify
# that owner, not shadow attributes on the overlay wrapper.
while hasattr(raw, "raw_dataset"):
raw = raw.raw_dataset
sessions = raw.sessions
original_lengths = raw._estimated_stream_lengths
summaries = {key: value for key, value in vars(raw).items()
if key.startswith(("task_", "control_task_", "interruption_task_"))
or key in {"long_session_filter_summary", "voice_reuse_after_coverage",
"new_task_record_selection_summary"}}
raw._build_session_index()
assert sessions == raw.sessions, "Cached session composition differs"
assert original_lengths == tuple(raw._estimate_stream_steps(i) for i in range(len(raw))), "Cached lengths differ"
assert all(getattr(raw, key) == value for key, value in summaries.items()), "Cached task summaries differ"
print(f"[verified] {split}: every session, length and task summary matches uncached construction", flush=True)
if listen_hold:
historical_config = config.model_copy(deep=True)
historical_config.stationary_listen_hold = False
historical = historical_config(split=split).raw_dataset
while hasattr(historical, "raw_dataset"):
historical = historical.raw_dataset
assert historical.sessions == raw.sessions, "Listen hold changed source membership/order"
assert historical._estimated_stream_lengths == raw._estimated_stream_lengths, "Listen hold changed timeline lengths"
assert all(getattr(historical, key) == value for key, value in summaries.items()), "Listen hold changed task mixture"
print(f"[verified] {split}: listen hold preserves every legacy session, timeline length and task summary", flush=True)
if __name__ == "__main__":
main()