Download scripts/data/prepare_full_duplex_indexes.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 4.89 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/prepare_full_duplex_indexes.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/prepare_full_duplex_indexes.py
-
curl -L -o prepare_full_duplex_indexes.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/prepare_full_duplex_indexes.py
4.89 kB
| #!/usr/bin/env python3 | |
| """CPU-only startup index warmup from an exact saved HFD run configuration. | |
| Usage: .venv-psi/bin/python scripts/data/prepare_full_duplex_indexes.py RUN_DIR | |
| Set HFD_STARTUP_INDEX_CACHE_DIR identically here and on the training machine to | |
| share indexes. Set HFD_STARTUP_INDEX_CACHE=0 to bypass caching for diagnostics. | |
| No model/processor, WAV mixing, augmented item, CUDA or WandB is created. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| from pathlib import Path | |
| import sys | |
| import time | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("run_config", type=Path, help="run directory or run_config.json") | |
| parser.add_argument("--splits", nargs="+", choices=("train", "val"), | |
| default=("train", "val")) | |
| parser.add_argument("--verify", action="store_true", | |
| help="Rebuild original audio composition in memory and compare every row/length") | |
| parser.add_argument("--listen-hold", action="store_true", | |
| help="Warm the revised listen targets' indexes without rewriting the saved run; " | |
| "with --verify also check the historical source membership and lengths") | |
| args = parser.parse_args() | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "" | |
| from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig | |
| from psi.config.transform import SeedPosttrainRepackTransform | |
| path = args.run_config | |
| if path.is_dir(): | |
| path /= "run_config.json" | |
| payload = json.loads(path.read_text()) | |
| config = FullDuplexAudioCommandDataConfig.model_validate(payload["data"]) | |
| # Pydantic's base DataTransform otherwise drops the concrete repack fields. | |
| # Only geometry is consulted during construction; transforms are NOT run. | |
| config.transform.repack = SeedPosttrainRepackTransform.model_validate( | |
| payload["data"]["transform"]["repack"] | |
| ) | |
| if args.listen_hold: | |
| config.stationary_listen_hold = True | |
| warm_indexes(config, args.splits, verify=args.verify, listen_hold=args.listen_hold) | |
| def warm_indexes(config, splits=("train", "val"), *, verify=False, listen_hold=False): | |
| """Populate/reuse deterministic metadata only; never materialize augmented items.""" | |
| for split in splits: | |
| start = time.monotonic() | |
| dataset = config(split=split) | |
| lengths = [dataset.estimated_stream_steps(i) for i in range(len(dataset))] | |
| print(json.dumps({"split": split, "sources": len(dataset), | |
| "estimated_windows": sum(lengths), | |
| "seconds": round(time.monotonic() - start, 3)}), flush=True) | |
| if verify: | |
| raw = dataset.raw_dataset | |
| # The unified overlay wraps the same audio/motion dataset. Verify | |
| # that owner, not shadow attributes on the overlay wrapper. | |
| while hasattr(raw, "raw_dataset"): | |
| raw = raw.raw_dataset | |
| sessions = raw.sessions | |
| original_lengths = raw._estimated_stream_lengths | |
| summaries = {key: value for key, value in vars(raw).items() | |
| if key.startswith(("task_", "control_task_", "interruption_task_")) | |
| or key in {"long_session_filter_summary", "voice_reuse_after_coverage", | |
| "new_task_record_selection_summary"}} | |
| raw._build_session_index() | |
| assert sessions == raw.sessions, "Cached session composition differs" | |
| assert original_lengths == tuple(raw._estimate_stream_steps(i) for i in range(len(raw))), "Cached lengths differ" | |
| assert all(getattr(raw, key) == value for key, value in summaries.items()), "Cached task summaries differ" | |
| print(f"[verified] {split}: every session, length and task summary matches uncached construction", flush=True) | |
| if listen_hold: | |
| historical_config = config.model_copy(deep=True) | |
| historical_config.stationary_listen_hold = False | |
| historical = historical_config(split=split).raw_dataset | |
| while hasattr(historical, "raw_dataset"): | |
| historical = historical.raw_dataset | |
| assert historical.sessions == raw.sessions, "Listen hold changed source membership/order" | |
| assert historical._estimated_stream_lengths == raw._estimated_stream_lengths, "Listen hold changed timeline lengths" | |
| assert all(getattr(historical, key) == value for key, value in summaries.items()), "Listen hold changed task mixture" | |
| print(f"[verified] {split}: listen hold preserves every legacy session, timeline length and task summary", flush=True) | |
| if __name__ == "__main__": | |
| main() | |