code-HFD / tests /test_swa_render.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
5d9f970 verified
Raw History Blame Contribute Delete
10 kB
"""CPU-only SWA rendering: no model, GPU, dataset scan or reference-answer TTS."""
import hashlib
import importlib.util
import json
from pathlib import Path
import subprocess
import numpy as np
import pytest
import torch
from scipy.io import wavfile
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("swa_render_test", ROOT / "scripts/infer/render_swa_result.py")
render = importlib.util.module_from_spec(spec)
spec.loader.exec_module(render)
def fixture_source(tmp_path):
row = {"id": "fixture", "split": "val", "camera_domain": "static_image",
"frontend_root": str(tmp_path / "frontend"), "swa_has_question": True,
"path": str(tmp_path / "body.pt"),
"question": {"question": "What color?", "answer": "GT MUST NOT BE SPOKEN",
"image_path": str(tmp_path / "image.png")}}
from PIL import Image
Image.new("RGB", (32, 24), "red").save(row["question"]["image_path"])
voice = tmp_path / "voice.wav"
wavfile.write(voice, 16000, np.full(3200, 1024, np.int16))
timeline = {"question_start": .2, "question_end": .4, "question_gain": .5,
"question_voice": str(voice), "camera_domain": "static_image"}
frontend = Path(row["frontend_root"]) / "data/frontend"
frontend.mkdir(parents=True)
torch.save({"format": "static_factorized_v1", "timeline": timeline}, frontend / "fixture.pt")
initial = np.zeros(36, np.float32)
initial[29:33] = [1, 2, .8, 1]
body = {"base_audio": np.zeros(9600, np.float32), "initial_abs_state": initial}
torch.save(body, row["path"])
return row, body, timeline
def test_display_holds_initial_until_audio_available_without_changing_predictions():
initial = np.zeros(36, np.float32)
generated = np.ones((40, 36), np.float32)
generated[20:] = 2
original = generated.copy()
poses, starts = render.display_motion(initial, generated, [{"seconds": .6}, {"seconds": 1.}])
assert starts == [30, 50] and len(poses) == 70
np.testing.assert_array_equal(poses[:30], np.zeros((30, 36)))
np.testing.assert_array_equal(poses[30:], generated)
np.testing.assert_array_equal(generated, original)
with pytest.raises(ValueError, match="cadence"):
render.display_motion(initial, generated, [{"seconds": .6}, {"seconds": 1.1}])
def test_static_microphone_reconstructs_encoder_gain_fades_and_timing(tmp_path):
row, body, timeline = fixture_source(tmp_path)
actual, _ = render.microphone(row, body)
expected = body["base_audio"].copy()
question = np.full(3200, 1024 / 32768, np.float32) * .5
question[:160] *= np.linspace(0, 1, 160)
question[-160:] *= np.linspace(1, 0, 160)
expected[3200:6400] += question
np.testing.assert_array_equal(actual, expected)
row["swa_has_question"] = False
ordinary, _ = render.microphone(row, body)
np.testing.assert_array_equal(ordinary, body["base_audio"])
def test_recorded_microphone_is_verified_not_replaced_with_answer(tmp_path):
row, body, _ = fixture_source(tmp_path)
directory = Path(row["frontend_root"]) / "data/timeline"
directory.mkdir()
mic = np.linspace(-.1, .1, 9600, dtype=np.float32)
timeline = {"audio_sha256": hashlib.sha256(mic.tobytes()).hexdigest()}
path = directory / "fixture.pt"
torch.save({"mixed_audio": mic, "timeline": timeline}, path)
actual, _ = render.microphone(row, body)
np.testing.assert_array_equal(actual, mic)
torch.save({"mixed_audio": mic * 2, "timeline": timeline}, path)
with pytest.raises(ValueError, match="hash mismatch"):
render.microphone(row, body)
def test_legacy_prepare_resolves_id_and_migrates_only_marker_then_composes(tmp_path):
row, body, _ = fixture_source(tmp_path)
plan = tmp_path / "plan.json"
plan.write_text(json.dumps([{"id": "other"}, row]))
(tmp_path / "run_config.json").write_text(json.dumps({"data": {"swa_source_plan": str(plan)}}))
output = tmp_path / "swa_visual_qa/ckpt_1/val/sample_0000"
output.mkdir(parents=True)
generated = np.repeat(body["initial_abs_state"][None], 20, axis=0)
np.save(output / "generated_abs.npy", generated)
before = (output / "generated_abs.npy").read_bytes()
metrics = {"id": "fixture", "question": "What color?", "events": [{"seconds": .6}],
"answers": [{"seconds": .6, "text": "red"}]}
(output / "metrics.json").write_text(json.dumps(metrics))
(output / ".complete").touch()
render.prepare(tmp_path, output)
assert not (output / ".complete").exists()
assert (output / ".inferred").exists()
assert (output / "generated_abs.npy").read_bytes() == before
meta = render.read(output / "infer_meta.json")
assert meta["total_frames"] == 50 and meta["initial_hold_frames"] == 30
sr, audio = wavfile.read(output / "assembled_input_audio_16k.wav")
assert sr == 16000 and len(audio) == 16000
assert not (tmp_path / "checkpoints").exists()
# Replace only the expensive robot renderer with a real 50fps color video.
subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i",
"color=c=blue:s=64x64:r=50:d=1", "-c:v", "libx264", "-threads", "1",
str(output / "render_motion/reference_motion.mp4")], check=True)
render.compose(output)
assert (output / "reference_motion_with_audio.mp4").stat().st_size > 0
meta = render.read(output / "infer_meta.json")
assert meta["swa_review"]["generated_speech"] is False
assert meta["swa_review"]["frames"] == 50
def test_camera_decode_is_causal_and_keeps_recorded_pts(tmp_path):
path = tmp_path / "camera.mp4"
subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i",
"color=c=green:s=32x32:r=10:d=1", "-c:v", "libx264", "-threads", "1",
str(path)], check=True)
images, pts = render.camera_frames(path, np.array([0., .15, .26, .41]))
np.testing.assert_allclose(pts, [0., .1, .2, .4], atol=1e-8)
assert len(images) == 4
def test_native_answer_is_audible_and_tail_is_not_truncated(tmp_path):
from psi.runtime.swa_speech import schedule_speech
row, body, _ = fixture_source(tmp_path)
output = tmp_path / 'native'
output.mkdir()
np.save(output / 'generated_abs.npy', np.repeat(body['initial_abs_state'][None], 20, axis=0))
pcm = (.2 * np.sin(2 * np.pi * 440 * np.arange(24000) / 24000)).astype(np.float32)
wavfile.write(output / 'answer_000_24k.wav', 24000, pcm)
answer = dict(text='red', seconds=.6, audio_path='answer_000_24k.wav', samples=len(pcm),
playback_buffer_seconds=.2,
pcm_releases=[dict(sample_start=0, sample_end=len(pcm), available_seconds=.8)])
metrics = dict(id='fixture', question='What color?', render_source=row,
events=[dict(seconds=.6)], answers=[answer], native_speech_version=1,
speech_timeline=schedule_speech([answer]))
(output / 'metrics.json').write_text(json.dumps(metrics))
render.prepare(tmp_path, output)
meta = render.read(output / 'infer_meta.json')
assert meta['total_frames'] == 100 and meta['speech_tail_hold_frames'] == 50
sr, speaker = wavfile.read(output / 'assembled_answer_audio_24k.wav')
assert sr == 24000 and len(speaker) == 48000
np.testing.assert_array_equal(speaker[24000:], pcm)
_, mixed = wavfile.read(output / 'assembled_review_audio_24k.wav')
np.testing.assert_allclose(mixed[24000:], pcm, atol=1e-7)
subprocess.run(['ffmpeg', '-v', 'error', '-y', '-f', 'lavfi', '-i',
'color=c=blue:s=64x64:r=50:d=2', '-c:v', 'libx264', '-threads', '1',
str(output / 'render_motion/reference_motion.mp4')], check=True)
render.compose(output)
decoded = subprocess.check_output(['ffmpeg', '-v', 'error', '-i',
str(output / 'reference_motion_with_audio.mp4'), '-f', 'f32le', '-ac', '1', '-ar', '24000', 'pipe:1'])
audio = np.frombuffer(decoded, np.float32)
assert np.sqrt(np.mean(audio[25000:47000] ** 2)) > .1
assert render.read(output / 'infer_meta.json')['swa_review']['generated_speech'] is True
def test_legacy_complete_swa_is_recovered_by_async_queue_without_checkpoint(tmp_path):
from psi.async_render_queue import AsyncRenderQueue, render_complete, swa_outputs_ready
output = tmp_path / "swa_visual_qa/ckpt_1/val/sample_0000"
output.mkdir(parents=True)
for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv"):
(output / name).write_text("fixture")
(output / "metrics.json").write_text('{"id":"fixture","events":[{"seconds":0.6}]}')
(output / ".complete").touch()
assert swa_outputs_ready(output) and not render_complete(output)
with AsyncRenderQueue(ROOT, tmp_path) as queue:
queue.recover([output])
assert list(queue.pending) == [output]
assert (output / ".inferred").exists()
assert not (tmp_path / "checkpoints").exists()
def test_native_swa_cannot_mark_missing_speech_or_old_silent_video_complete(tmp_path):
from psi.async_render_queue import swa_outputs_ready, render_complete
for name in ('generated_abs.npy', 'body_pos.csv', 'body_quat.csv', 'joint_pos.csv'):
(tmp_path / name).write_text('fixture')
metrics = dict(id='fixture', events=[dict(seconds=.6)], native_speech_version=1,
answers=[dict(native_speech=True, samples=24000, audio_path='answer.wav')])
(tmp_path / 'metrics.json').write_text(json.dumps(metrics))
assert not swa_outputs_ready(tmp_path)
wavfile.write(tmp_path / 'answer.wav', 24000, np.zeros(24000, np.float32))
assert swa_outputs_ready(tmp_path)
(tmp_path / '.complete').touch()
(tmp_path / 'reference_motion_with_audio.mp4').write_bytes(b'video')
(tmp_path / 'infer_meta.json').write_text('{"swa_render_version":1}')
assert not render_complete(tmp_path)
(tmp_path / 'infer_meta.json').write_text('{"swa_render_version":2}')
assert render_complete(tmp_path)