"""CPU-only SWA rendering: no model, GPU, dataset scan or reference-answer TTS.""" import hashlib import importlib.util import json from pathlib import Path import subprocess import numpy as np import pytest import torch from scipy.io import wavfile ROOT = Path(__file__).resolve().parents[1] spec = importlib.util.spec_from_file_location("swa_render_test", ROOT / "scripts/infer/render_swa_result.py") render = importlib.util.module_from_spec(spec) spec.loader.exec_module(render) def fixture_source(tmp_path): row = {"id": "fixture", "split": "val", "camera_domain": "static_image", "frontend_root": str(tmp_path / "frontend"), "swa_has_question": True, "path": str(tmp_path / "body.pt"), "question": {"question": "What color?", "answer": "GT MUST NOT BE SPOKEN", "image_path": str(tmp_path / "image.png")}} from PIL import Image Image.new("RGB", (32, 24), "red").save(row["question"]["image_path"]) voice = tmp_path / "voice.wav" wavfile.write(voice, 16000, np.full(3200, 1024, np.int16)) timeline = {"question_start": .2, "question_end": .4, "question_gain": .5, "question_voice": str(voice), "camera_domain": "static_image"} frontend = Path(row["frontend_root"]) / "data/frontend" frontend.mkdir(parents=True) torch.save({"format": "static_factorized_v1", "timeline": timeline}, frontend / "fixture.pt") initial = np.zeros(36, np.float32) initial[29:33] = [1, 2, .8, 1] body = {"base_audio": np.zeros(9600, np.float32), "initial_abs_state": initial} torch.save(body, row["path"]) return row, body, timeline def test_display_holds_initial_until_audio_available_without_changing_predictions(): initial = np.zeros(36, np.float32) generated = np.ones((40, 36), np.float32) generated[20:] = 2 original = generated.copy() poses, starts = render.display_motion(initial, generated, [{"seconds": .6}, {"seconds": 1.}]) assert starts == [30, 50] and len(poses) == 70 np.testing.assert_array_equal(poses[:30], np.zeros((30, 36))) np.testing.assert_array_equal(poses[30:], generated) np.testing.assert_array_equal(generated, original) with pytest.raises(ValueError, match="cadence"): render.display_motion(initial, generated, [{"seconds": .6}, {"seconds": 1.1}]) def test_static_microphone_reconstructs_encoder_gain_fades_and_timing(tmp_path): row, body, timeline = fixture_source(tmp_path) actual, _ = render.microphone(row, body) expected = body["base_audio"].copy() question = np.full(3200, 1024 / 32768, np.float32) * .5 question[:160] *= np.linspace(0, 1, 160) question[-160:] *= np.linspace(1, 0, 160) expected[3200:6400] += question np.testing.assert_array_equal(actual, expected) row["swa_has_question"] = False ordinary, _ = render.microphone(row, body) np.testing.assert_array_equal(ordinary, body["base_audio"]) def test_recorded_microphone_is_verified_not_replaced_with_answer(tmp_path): row, body, _ = fixture_source(tmp_path) directory = Path(row["frontend_root"]) / "data/timeline" directory.mkdir() mic = np.linspace(-.1, .1, 9600, dtype=np.float32) timeline = {"audio_sha256": hashlib.sha256(mic.tobytes()).hexdigest()} path = directory / "fixture.pt" torch.save({"mixed_audio": mic, "timeline": timeline}, path) actual, _ = render.microphone(row, body) np.testing.assert_array_equal(actual, mic) torch.save({"mixed_audio": mic * 2, "timeline": timeline}, path) with pytest.raises(ValueError, match="hash mismatch"): render.microphone(row, body) def test_legacy_prepare_resolves_id_and_migrates_only_marker_then_composes(tmp_path): row, body, _ = fixture_source(tmp_path) plan = tmp_path / "plan.json" plan.write_text(json.dumps([{"id": "other"}, row])) (tmp_path / "run_config.json").write_text(json.dumps({"data": {"swa_source_plan": str(plan)}})) output = tmp_path / "swa_visual_qa/ckpt_1/val/sample_0000" output.mkdir(parents=True) generated = np.repeat(body["initial_abs_state"][None], 20, axis=0) np.save(output / "generated_abs.npy", generated) before = (output / "generated_abs.npy").read_bytes() metrics = {"id": "fixture", "question": "What color?", "events": [{"seconds": .6}], "answers": [{"seconds": .6, "text": "red"}]} (output / "metrics.json").write_text(json.dumps(metrics)) (output / ".complete").touch() render.prepare(tmp_path, output) assert not (output / ".complete").exists() assert (output / ".inferred").exists() assert (output / "generated_abs.npy").read_bytes() == before meta = render.read(output / "infer_meta.json") assert meta["total_frames"] == 50 and meta["initial_hold_frames"] == 30 sr, audio = wavfile.read(output / "assembled_input_audio_16k.wav") assert sr == 16000 and len(audio) == 16000 assert not (tmp_path / "checkpoints").exists() # Replace only the expensive robot renderer with a real 50fps color video. subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "color=c=blue:s=64x64:r=50:d=1", "-c:v", "libx264", "-threads", "1", str(output / "render_motion/reference_motion.mp4")], check=True) render.compose(output) assert (output / "reference_motion_with_audio.mp4").stat().st_size > 0 meta = render.read(output / "infer_meta.json") assert meta["swa_review"]["generated_speech"] is False assert meta["swa_review"]["frames"] == 50 def test_camera_decode_is_causal_and_keeps_recorded_pts(tmp_path): path = tmp_path / "camera.mp4" subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "color=c=green:s=32x32:r=10:d=1", "-c:v", "libx264", "-threads", "1", str(path)], check=True) images, pts = render.camera_frames(path, np.array([0., .15, .26, .41])) np.testing.assert_allclose(pts, [0., .1, .2, .4], atol=1e-8) assert len(images) == 4 def test_native_answer_is_audible_and_tail_is_not_truncated(tmp_path): from psi.runtime.swa_speech import schedule_speech row, body, _ = fixture_source(tmp_path) output = tmp_path / 'native' output.mkdir() np.save(output / 'generated_abs.npy', np.repeat(body['initial_abs_state'][None], 20, axis=0)) pcm = (.2 * np.sin(2 * np.pi * 440 * np.arange(24000) / 24000)).astype(np.float32) wavfile.write(output / 'answer_000_24k.wav', 24000, pcm) answer = dict(text='red', seconds=.6, audio_path='answer_000_24k.wav', samples=len(pcm), playback_buffer_seconds=.2, pcm_releases=[dict(sample_start=0, sample_end=len(pcm), available_seconds=.8)]) metrics = dict(id='fixture', question='What color?', render_source=row, events=[dict(seconds=.6)], answers=[answer], native_speech_version=1, speech_timeline=schedule_speech([answer])) (output / 'metrics.json').write_text(json.dumps(metrics)) render.prepare(tmp_path, output) meta = render.read(output / 'infer_meta.json') assert meta['total_frames'] == 100 and meta['speech_tail_hold_frames'] == 50 sr, speaker = wavfile.read(output / 'assembled_answer_audio_24k.wav') assert sr == 24000 and len(speaker) == 48000 np.testing.assert_array_equal(speaker[24000:], pcm) _, mixed = wavfile.read(output / 'assembled_review_audio_24k.wav') np.testing.assert_allclose(mixed[24000:], pcm, atol=1e-7) subprocess.run(['ffmpeg', '-v', 'error', '-y', '-f', 'lavfi', '-i', 'color=c=blue:s=64x64:r=50:d=2', '-c:v', 'libx264', '-threads', '1', str(output / 'render_motion/reference_motion.mp4')], check=True) render.compose(output) decoded = subprocess.check_output(['ffmpeg', '-v', 'error', '-i', str(output / 'reference_motion_with_audio.mp4'), '-f', 'f32le', '-ac', '1', '-ar', '24000', 'pipe:1']) audio = np.frombuffer(decoded, np.float32) assert np.sqrt(np.mean(audio[25000:47000] ** 2)) > .1 assert render.read(output / 'infer_meta.json')['swa_review']['generated_speech'] is True def test_legacy_complete_swa_is_recovered_by_async_queue_without_checkpoint(tmp_path): from psi.async_render_queue import AsyncRenderQueue, render_complete, swa_outputs_ready output = tmp_path / "swa_visual_qa/ckpt_1/val/sample_0000" output.mkdir(parents=True) for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv"): (output / name).write_text("fixture") (output / "metrics.json").write_text('{"id":"fixture","events":[{"seconds":0.6}]}') (output / ".complete").touch() assert swa_outputs_ready(output) and not render_complete(output) with AsyncRenderQueue(ROOT, tmp_path) as queue: queue.recover([output]) assert list(queue.pending) == [output] assert (output / ".inferred").exists() assert not (tmp_path / "checkpoints").exists() def test_native_swa_cannot_mark_missing_speech_or_old_silent_video_complete(tmp_path): from psi.async_render_queue import swa_outputs_ready, render_complete for name in ('generated_abs.npy', 'body_pos.csv', 'body_quat.csv', 'joint_pos.csv'): (tmp_path / name).write_text('fixture') metrics = dict(id='fixture', events=[dict(seconds=.6)], native_speech_version=1, answers=[dict(native_speech=True, samples=24000, audio_path='answer.wav')]) (tmp_path / 'metrics.json').write_text(json.dumps(metrics)) assert not swa_outputs_ready(tmp_path) wavfile.write(tmp_path / 'answer.wav', 24000, np.zeros(24000, np.float32)) assert swa_outputs_ready(tmp_path) (tmp_path / '.complete').touch() (tmp_path / 'reference_motion_with_audio.mp4').write_bytes(b'video') (tmp_path / 'infer_meta.json').write_text('{"swa_render_version":1}') assert not render_complete(tmp_path) (tmp_path / 'infer_meta.json').write_text('{"swa_render_version":2}') assert render_complete(tmp_path)