#!/usr/bin/env python3 """Formal causal camera+audio -> action+reply inference for SWA checkpoints.""" from __future__ import annotations import argparse import json import os import sys from pathlib import Path import numpy as np import torch ROOT = Path(__file__).resolve().parents[2] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from psi.config.config import LaunchConfig from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset, _load_body from psi.models.interaction_arch.loading import load_swa_checkpoint from psi.utils import parse_args_to_tyro_config from scripts.infer.full_duplex_audio_to_motion import export_motion from scripts.infer.seed_text_to_motion import resolve_ckpt_step @torch.inference_mode() def rollout(model, launch_config, source, output: Path, speech) -> dict: from psi.runtime.swa_rollout import rollout_source, body_metrics motion, events = rollout_source(model, launch_config, source, speech=speech) body = _load_body(str(source["swa_job"]["path"])) output.mkdir(parents=True, exist_ok=True) np.save(output / "generated_abs.npy", motion) export_motion(output, motion) answers = speech.finish() from psi.runtime.swa_speech import schedule_speech result = { "native_speech_version": 1, "speech_timeline": schedule_speech(answers), "speech_scope": "offline causal AV replay; measured per-reply PCM availability; FIFO software playback, not hardware realtime", "id": source["swa_job"]["id"], "question": source["swa_job"]["question"]["question"], "answer_gt": source["swa_job"]["question"]["answer"], "answers": answers, "events": events, "control_accuracy": float(np.mean([x["control"] == x["control_gt"] for x in events])), "causal_camera_audio": True, "action_rtc": "30/20/10, 10-step flow solver", "body_metrics": body_metrics(motion, body["gt_abs"]), "progress_geometry": "inplace_blend_v1", "reply_debounce_windows": 2, # Retain the exact input identity for checkpoint-independent rendering. # Only provenance, never extra model inputs or GT answer speech. "render_source": source["swa_job"], } (output / "metrics.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") return result def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("run_dir", type=Path) parser.add_argument("--ckpt-step", default="best") parser.add_argument("--split", choices=("train", "val", "test"), default="val") parser.add_argument("--index", type=int, default=0) parser.add_argument("--device", default="cuda:0") parser.add_argument("--output", type=Path) args = parser.parse_args() run_dir = args.run_dir.expanduser().resolve() template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore launch_config = template.model_validate(json.loads((run_dir / "run_config.json").read_text())) if not bool(getattr(launch_config.model, "swa_visual_qa_enabled", False)): raise RuntimeError("Checkpoint is audio-only; refusing fake SWA inference") checkpoint = resolve_ckpt_step(run_dir, args.ckpt_step) model = load_swa_checkpoint( run_dir, checkpoint, launch_config, torch.device(args.device) ) dataset = SwaVisualQaDataset(launch_config.data.swa_source_plan, args.split) source = dataset[args.index % len(dataset)] output = args.output or run_dir / "swa_visual_qa" / f"ckpt_{checkpoint}" / args.split / source["swa_job"]["id"] from psi.runtime.swa_speech import SwaSpeechWorker speech = SwaSpeechWorker(model, os.environ.get("PSI_PRETRAIN_BACKBONE_PATH", "") or launch_config.model.model_name_or_path, output) try: result = rollout(model, launch_config, source, output, speech) finally: speech.close() print(json.dumps({"output": str(output), "answer": result["answers"]}, ensure_ascii=False)) if __name__ == "__main__": main()