File size: 4,083 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#!/usr/bin/env python3
"""Formal causal camera+audio -> action+reply inference for SWA checkpoints."""
from __future__ import annotations

import argparse
import json
import os
import sys
from pathlib import Path

import numpy as np
import torch

ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
    sys.path.insert(0, str(ROOT))

from psi.config.config import LaunchConfig
from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset, _load_body
from psi.models.interaction_arch.loading import load_swa_checkpoint
from psi.utils import parse_args_to_tyro_config
from scripts.infer.full_duplex_audio_to_motion import export_motion
from scripts.infer.seed_text_to_motion import resolve_ckpt_step


@torch.inference_mode()
def rollout(model, launch_config, source, output: Path, speech) -> dict:
    from psi.runtime.swa_rollout import rollout_source, body_metrics
    motion, events = rollout_source(model, launch_config, source, speech=speech)
    body = _load_body(str(source["swa_job"]["path"]))
    output.mkdir(parents=True, exist_ok=True)
    np.save(output / "generated_abs.npy", motion)
    export_motion(output, motion)
    answers = speech.finish()
    from psi.runtime.swa_speech import schedule_speech
    result = {
        "native_speech_version": 1,
        "speech_timeline": schedule_speech(answers),
        "speech_scope": "offline causal AV replay; measured per-reply PCM availability; FIFO software playback, not hardware realtime",
        "id": source["swa_job"]["id"],
        "question": source["swa_job"]["question"]["question"],
        "answer_gt": source["swa_job"]["question"]["answer"],
        "answers": answers, "events": events,
        "control_accuracy": float(np.mean([x["control"] == x["control_gt"] for x in events])),
        "causal_camera_audio": True,
        "action_rtc": "30/20/10, 10-step flow solver",
        "body_metrics": body_metrics(motion, body["gt_abs"]),
        "progress_geometry": "inplace_blend_v1",
        "reply_debounce_windows": 2,
        # Retain the exact input identity for checkpoint-independent rendering.
        # Only provenance, never extra model inputs or GT answer speech.
        "render_source": source["swa_job"],
    }
    (output / "metrics.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
    return result


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("run_dir", type=Path)
    parser.add_argument("--ckpt-step", default="best")
    parser.add_argument("--split", choices=("train", "val", "test"), default="val")
    parser.add_argument("--index", type=int, default=0)
    parser.add_argument("--device", default="cuda:0")
    parser.add_argument("--output", type=Path)
    args = parser.parse_args()
    run_dir = args.run_dir.expanduser().resolve()
    template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt")  # type: ignore
    launch_config = template.model_validate(json.loads((run_dir / "run_config.json").read_text()))
    if not bool(getattr(launch_config.model, "swa_visual_qa_enabled", False)):
        raise RuntimeError("Checkpoint is audio-only; refusing fake SWA inference")
    checkpoint = resolve_ckpt_step(run_dir, args.ckpt_step)
    model = load_swa_checkpoint(
        run_dir, checkpoint, launch_config, torch.device(args.device)
    )
    dataset = SwaVisualQaDataset(launch_config.data.swa_source_plan, args.split)
    source = dataset[args.index % len(dataset)]
    output = args.output or run_dir / "swa_visual_qa" / f"ckpt_{checkpoint}" / args.split / source["swa_job"]["id"]
    from psi.runtime.swa_speech import SwaSpeechWorker
    speech = SwaSpeechWorker(model, os.environ.get("PSI_PRETRAIN_BACKBONE_PATH", "") or
                             launch_config.model.model_name_or_path, output)
    try:
        result = rollout(model, launch_config, source, output, speech)
    finally:
        speech.close()
    print(json.dumps({"output": str(output), "answer": result["answers"]}, ensure_ascii=False))


if __name__ == "__main__":
    main()