Download scripts/infer/swa_visual_qa.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 4.08 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/swa_visual_qa.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/infer/swa_visual_qa.py
-
curl -L -o swa_visual_qa.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/swa_visual_qa.py
4.08 kB
| #!/usr/bin/env python3 | |
| """Formal causal camera+audio -> action+reply inference for SWA checkpoints.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| ROOT = Path(__file__).resolve().parents[2] | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| from psi.config.config import LaunchConfig | |
| from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset, _load_body | |
| from psi.models.interaction_arch.loading import load_swa_checkpoint | |
| from psi.utils import parse_args_to_tyro_config | |
| from scripts.infer.full_duplex_audio_to_motion import export_motion | |
| from scripts.infer.seed_text_to_motion import resolve_ckpt_step | |
| def rollout(model, launch_config, source, output: Path, speech) -> dict: | |
| from psi.runtime.swa_rollout import rollout_source, body_metrics | |
| motion, events = rollout_source(model, launch_config, source, speech=speech) | |
| body = _load_body(str(source["swa_job"]["path"])) | |
| output.mkdir(parents=True, exist_ok=True) | |
| np.save(output / "generated_abs.npy", motion) | |
| export_motion(output, motion) | |
| answers = speech.finish() | |
| from psi.runtime.swa_speech import schedule_speech | |
| result = { | |
| "native_speech_version": 1, | |
| "speech_timeline": schedule_speech(answers), | |
| "speech_scope": "offline causal AV replay; measured per-reply PCM availability; FIFO software playback, not hardware realtime", | |
| "id": source["swa_job"]["id"], | |
| "question": source["swa_job"]["question"]["question"], | |
| "answer_gt": source["swa_job"]["question"]["answer"], | |
| "answers": answers, "events": events, | |
| "control_accuracy": float(np.mean([x["control"] == x["control_gt"] for x in events])), | |
| "causal_camera_audio": True, | |
| "action_rtc": "30/20/10, 10-step flow solver", | |
| "body_metrics": body_metrics(motion, body["gt_abs"]), | |
| "progress_geometry": "inplace_blend_v1", | |
| "reply_debounce_windows": 2, | |
| # Retain the exact input identity for checkpoint-independent rendering. | |
| # Only provenance, never extra model inputs or GT answer speech. | |
| "render_source": source["swa_job"], | |
| } | |
| (output / "metrics.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") | |
| return result | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("run_dir", type=Path) | |
| parser.add_argument("--ckpt-step", default="best") | |
| parser.add_argument("--split", choices=("train", "val", "test"), default="val") | |
| parser.add_argument("--index", type=int, default=0) | |
| parser.add_argument("--device", default="cuda:0") | |
| parser.add_argument("--output", type=Path) | |
| args = parser.parse_args() | |
| run_dir = args.run_dir.expanduser().resolve() | |
| template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore | |
| launch_config = template.model_validate(json.loads((run_dir / "run_config.json").read_text())) | |
| if not bool(getattr(launch_config.model, "swa_visual_qa_enabled", False)): | |
| raise RuntimeError("Checkpoint is audio-only; refusing fake SWA inference") | |
| checkpoint = resolve_ckpt_step(run_dir, args.ckpt_step) | |
| model = load_swa_checkpoint( | |
| run_dir, checkpoint, launch_config, torch.device(args.device) | |
| ) | |
| dataset = SwaVisualQaDataset(launch_config.data.swa_source_plan, args.split) | |
| source = dataset[args.index % len(dataset)] | |
| output = args.output or run_dir / "swa_visual_qa" / f"ckpt_{checkpoint}" / args.split / source["swa_job"]["id"] | |
| from psi.runtime.swa_speech import SwaSpeechWorker | |
| speech = SwaSpeechWorker(model, os.environ.get("PSI_PRETRAIN_BACKBONE_PATH", "") or | |
| launch_config.model.model_name_or_path, output) | |
| try: | |
| result = rollout(model, launch_config, source, output, speech) | |
| finally: | |
| speech.close() | |
| print(json.dumps({"output": str(output), "answer": result["answers"]}, ensure_ascii=False)) | |
| if __name__ == "__main__": | |
| main() | |