File size: 4,083 Bytes
06f9561 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | #!/usr/bin/env python3
"""Formal causal camera+audio -> action+reply inference for SWA checkpoints."""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
import numpy as np
import torch
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from psi.config.config import LaunchConfig
from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset, _load_body
from psi.models.interaction_arch.loading import load_swa_checkpoint
from psi.utils import parse_args_to_tyro_config
from scripts.infer.full_duplex_audio_to_motion import export_motion
from scripts.infer.seed_text_to_motion import resolve_ckpt_step
@torch.inference_mode()
def rollout(model, launch_config, source, output: Path, speech) -> dict:
from psi.runtime.swa_rollout import rollout_source, body_metrics
motion, events = rollout_source(model, launch_config, source, speech=speech)
body = _load_body(str(source["swa_job"]["path"]))
output.mkdir(parents=True, exist_ok=True)
np.save(output / "generated_abs.npy", motion)
export_motion(output, motion)
answers = speech.finish()
from psi.runtime.swa_speech import schedule_speech
result = {
"native_speech_version": 1,
"speech_timeline": schedule_speech(answers),
"speech_scope": "offline causal AV replay; measured per-reply PCM availability; FIFO software playback, not hardware realtime",
"id": source["swa_job"]["id"],
"question": source["swa_job"]["question"]["question"],
"answer_gt": source["swa_job"]["question"]["answer"],
"answers": answers, "events": events,
"control_accuracy": float(np.mean([x["control"] == x["control_gt"] for x in events])),
"causal_camera_audio": True,
"action_rtc": "30/20/10, 10-step flow solver",
"body_metrics": body_metrics(motion, body["gt_abs"]),
"progress_geometry": "inplace_blend_v1",
"reply_debounce_windows": 2,
# Retain the exact input identity for checkpoint-independent rendering.
# Only provenance, never extra model inputs or GT answer speech.
"render_source": source["swa_job"],
}
(output / "metrics.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
return result
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("run_dir", type=Path)
parser.add_argument("--ckpt-step", default="best")
parser.add_argument("--split", choices=("train", "val", "test"), default="val")
parser.add_argument("--index", type=int, default=0)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--output", type=Path)
args = parser.parse_args()
run_dir = args.run_dir.expanduser().resolve()
template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
launch_config = template.model_validate(json.loads((run_dir / "run_config.json").read_text()))
if not bool(getattr(launch_config.model, "swa_visual_qa_enabled", False)):
raise RuntimeError("Checkpoint is audio-only; refusing fake SWA inference")
checkpoint = resolve_ckpt_step(run_dir, args.ckpt_step)
model = load_swa_checkpoint(
run_dir, checkpoint, launch_config, torch.device(args.device)
)
dataset = SwaVisualQaDataset(launch_config.data.swa_source_plan, args.split)
source = dataset[args.index % len(dataset)]
output = args.output or run_dir / "swa_visual_qa" / f"ckpt_{checkpoint}" / args.split / source["swa_job"]["id"]
from psi.runtime.swa_speech import SwaSpeechWorker
speech = SwaSpeechWorker(model, os.environ.get("PSI_PRETRAIN_BACKBONE_PATH", "") or
launch_config.model.model_name_or_path, output)
try:
result = rollout(model, launch_config, source, output, speech)
finally:
speech.close()
print(json.dumps({"output": str(output), "answer": result["answers"]}, ensure_ascii=False))
if __name__ == "__main__":
main()
|