Download scripts/infer/render_swa_result.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 17.3 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/render_swa_result.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/infer/render_swa_result.py
-
curl -L -o render_swa_result.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/render_swa_result.py
17.3 kB
| #!/usr/bin/env python3 | |
| """CPU-only SWA review assets: exact microphone, causal camera, generated body. | |
| No model/checkpoint and no answer TTS. Original generated poses are immutable; | |
| the separate display CSVs hold the initial state until the first packet arrives. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import subprocess | |
| import textwrap | |
| import numpy as np | |
| FPS, SR = 50, 16000 | |
| def read(path): | |
| return json.loads(Path(path).read_text(encoding="utf-8")) | |
| def write(path, data): | |
| path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| def load_cpu(path): | |
| import torch | |
| torch.set_num_threads(1) | |
| return torch.load(path, map_location="cpu", weights_only=False, mmap=True) | |
| def read_wave(path): | |
| from scipy.io import wavfile | |
| sr, wave = wavfile.read(path) | |
| if sr != SR or wave.ndim != 1: | |
| raise ValueError(f"SWA microphone must be mono 16 kHz: {path}") | |
| if wave.dtype == np.int16: | |
| wave = wave.astype(np.float32) / 32768 | |
| elif wave.dtype.kind != "f": | |
| raise ValueError(f"Unsupported SWA waveform encoding: {wave.dtype}") | |
| return np.asarray(wave, np.float32) | |
| def source_row(run, metrics): | |
| row = metrics.get("render_source") | |
| if row is None: | |
| # Old reports lack provenance. Resolve by immutable ID, not today's | |
| # dataset index, which may move when a source plan grows. | |
| plan = read(run / "run_config.json")["data"]["swa_source_plan"] | |
| matches = [r for r in read(plan) if r["id"] == metrics["id"]] | |
| if len(matches) != 1: | |
| raise ValueError(f"Cannot uniquely recover SWA source {metrics['id']}") | |
| row = matches[0] | |
| if row["id"] != metrics["id"] or row["question"]["question"] != metrics["question"]: | |
| raise ValueError("Saved SWA identity/question differs from render source") | |
| return row | |
| def microphone(row, body): | |
| root = Path(row["frontend_root"]) | |
| timeline_path = root / "data/timeline" / f"{row['id']}.pt" | |
| if timeline_path.is_file(): | |
| data = load_cpu(timeline_path) | |
| timeline = data["timeline"] | |
| audio = np.asarray(data["mixed_audio"], np.float32).copy() | |
| expected = timeline.get("audio_sha256") | |
| if expected and hashlib.sha256(audio.tobytes()).hexdigest() != expected: | |
| raise ValueError("SWA stored microphone hash mismatch") | |
| else: | |
| frontend = load_cpu(root / "data/frontend" / f"{row['id']}.pt") | |
| if frontend.get("format") != "static_factorized_v1": | |
| raise ValueError("Missing original SWA microphone; refusing guessed audio") | |
| timeline = frontend["timeline"] | |
| audio = np.asarray(body["base_audio"], np.float32).copy() | |
| if row.get("swa_has_question", True): | |
| voice = Path(timeline["question_voice"]) | |
| expected = row.get("asr_review", {}).get("wave_sha256") | |
| if expected and hashlib.sha256(voice.read_bytes()).hexdigest() != expected: | |
| raise ValueError("SWA question WAV differs from screened input") | |
| question = read_wave(voice).copy() | |
| question *= float(timeline["question_gain"]) | |
| fade = min(160, len(question) // 4) | |
| if fade: | |
| question[:fade] *= np.linspace(0, 1, fade) | |
| question[-fade:] *= np.linspace(1, 0, fade) | |
| start = round(float(timeline["question_start"]) * SR) | |
| end = start + len(question) | |
| if start < 0 or end > len(audio): | |
| raise ValueError("SWA question is outside source audio") | |
| if abs(end / SR - float(timeline["question_end"])) > 1 / SR: | |
| raise ValueError("SWA question timeline differs from input encoding") | |
| audio[start:end] += question | |
| if not row.get("swa_has_question", True): | |
| audio = np.asarray(body["base_audio"], np.float32).copy() | |
| if audio.ndim != 1 or not np.isfinite(audio).all() or np.max(np.abs(audio)) >= 1: | |
| raise ValueError("Invalid/clipped SWA microphone") | |
| return audio, timeline | |
| def display_motion(initial, generated, events): | |
| if (generated.ndim != 2 or generated.shape[1] != 36 | |
| or len(generated) != 20 * len(events) or not len(events) | |
| or not np.isfinite(generated).all()): | |
| raise ValueError("SWA generated poses/events do not match 20-frame execution") | |
| starts = np.rint(np.array([e["seconds"] for e in events]) * FPS).astype(int) | |
| if starts[0] < 0 or np.any(np.diff(starts) != 20): | |
| raise ValueError("SWA event arrivals do not follow the recorded 0.4s cadence") | |
| initial = np.asarray(initial, np.float32) | |
| if initial.shape != (36,) or not np.isfinite(initial).all(): | |
| raise ValueError("Invalid initial robot state") | |
| poses = np.repeat(initial[None], starts[-1] + 20, axis=0) | |
| for i, start in enumerate(starts): | |
| poses[start:start + 20] = generated[i * 20:(i + 1) * 20] | |
| return poses, starts.tolist() | |
| def answer_audio(output, metrics): | |
| """Read only actual generated PCM and its measured FIFO schedule, never GT.""" | |
| from scipy.io import wavfile | |
| from psi.runtime.swa_speech import schedule_speech | |
| segments = schedule_speech(metrics['answers']) | |
| if segments != metrics['speech_timeline']: | |
| raise ValueError('Saved speech schedule differs from measured PCM releases') | |
| wave = np.zeros(max([0] + [s['end'] for s in segments]), np.float32) | |
| loaded = {} | |
| for segment in segments: | |
| name = segment['path'] | |
| path = (output / name).resolve() | |
| if not path.is_relative_to(output.resolve()): | |
| raise ValueError('Answer PCM must be inside its result directory') | |
| if name not in loaded: | |
| sr, audio = wavfile.read(path) | |
| if sr != 24000 or audio.ndim != 1 or audio.dtype.kind != 'f' or not np.isfinite(audio).all(): | |
| raise ValueError('Invalid native answer WAV') | |
| loaded[name] = audio | |
| audio = loaded[name][segment['source_start']:segment['source_end']] | |
| if len(audio) != segment['end'] - segment['start']: | |
| raise ValueError('Answer WAV is incomplete') | |
| wave[segment['start']:segment['end']] = audio | |
| return wave, segments | |
| def prepare(run, output): | |
| from scipy.io import wavfile | |
| metrics = read(output / "metrics.json") | |
| row = source_row(run, metrics) | |
| body = load_cpu(row["path"]) | |
| generated = np.load(output / "generated_abs.npy", allow_pickle=False) | |
| poses, starts = display_motion(body["initial_abs_state"], generated, metrics["events"]) | |
| motion_frames = len(poses) | |
| audio, timeline = microphone(row, body) | |
| native = metrics.get('native_speech_version') == 1 | |
| speaker, segments = answer_audio(output, metrics) if native else (np.zeros(0, np.float32), []) | |
| required_frames = max(len(poses), int(np.ceil(len(speaker) / 480))) | |
| if required_frames > len(poses): | |
| poses = np.concatenate((poses, np.repeat(poses[-1:], required_frames - len(poses), axis=0))) | |
| samples = len(poses) * (SR // FPS) | |
| if len(audio) > samples: | |
| raise ValueError("Review would truncate the SWA microphone") | |
| audio = np.pad(audio, (0, samples - len(audio))) | |
| motion_dir = output / "render_motion" | |
| motion_dir.mkdir(exist_ok=True) | |
| for name, values in (("joint_pos", poses[:, :29]), ("body_pos", poses[:, 29:32]), | |
| ("body_quat", poses[:, 32:36])): | |
| np.savetxt(motion_dir / f"{name}.csv", values, delimiter=",", fmt="%.9g") | |
| # IEEE float WAV preserves the exact model-input samples. | |
| wavfile.write(output / "assembled_input_audio_16k.wav", SR, audio) | |
| mix_gain = 1. | |
| if native: | |
| from scipy.signal import resample_poly | |
| mic24 = resample_poly(audio, 3, 2).astype(np.float32) | |
| speaker = np.pad(speaker, (0, len(mic24) - len(speaker))) | |
| mixed = mic24 + speaker | |
| mix_gain = min(1., .98 / max(float(np.max(np.abs(mixed))), 1e-8)) | |
| wavfile.write(output / 'assembled_answer_audio_24k.wav', 24000, speaker) | |
| wavfile.write(output / 'assembled_review_audio_24k.wav', 24000, mixed * mix_gain) | |
| write(output / "infer_meta.json", { | |
| "swa_render_version": 2 if native else 1, "source_id": row["id"], "fps": FPS, | |
| "total_frames": len(poses), "generated_frames": len(generated), | |
| "action_display_start_frames": starts, "initial_hold_frames": starts[0], | |
| "timeline": timeline, "render_source": row, | |
| "audio_mode": "original_microphone_plus_native_generated_speech" if native else "original_microphone_only_no_generated_speech", | |
| "speech_segments": segments, "audio_mix_gain": mix_gain, | |
| "motion_end_frame": motion_frames, "speech_tail_hold_frames": len(poses) - motion_frames, | |
| "audio_file": "assembled_review_audio_24k.wav" if native else "assembled_input_audio_16k.wav", | |
| "answer_mode": "model_text_at_native_speech_playback" if native else "model_text_at_logical_packet_time_not_wall_clock_latency", | |
| "original_generated_motion_unchanged": True, | |
| }) | |
| # Migrate only the old misleading stage marker, not any result tensors. | |
| if (output / ".complete").exists(): | |
| (output / ".complete").unlink() | |
| (output / ".inferred").touch() | |
| print(f"[swa-render] prepared {row['id']}: {len(poses)} frames, native_answers={len(segments)} PCM chunks", flush=True) | |
| def camera_frames(path, deadlines): | |
| """Same PTS<=deadline rule as the frozen frontend camera protocol.""" | |
| import av | |
| from PIL import ImageOps | |
| av.logging.restore_default_callback() | |
| images, pts, index = [], [], 0 | |
| previous = previous_pts = None | |
| with av.open(str(path)) as container: | |
| stream = container.streams.video[0] | |
| stream.thread_type = "AUTO" | |
| stream.codec_context.thread_count = 2 | |
| container.seek(max(0, int((deadlines[0] - 2) / float(stream.time_base))), | |
| stream=stream, backward=True) | |
| for frame in container.decode(stream): | |
| if frame.pts is None: | |
| continue | |
| now = float(frame.pts * stream.time_base) | |
| while index < len(deadlines) and now > deadlines[index] + 1e-8: | |
| if previous is None: | |
| raise ValueError("No camera frame before recorded arrival") | |
| images.append(previous) | |
| pts.append(previous_pts) | |
| index += 1 | |
| if index == len(deadlines): | |
| break | |
| previous = ImageOps.pad(frame.to_image(), (224, 224), color=(0, 0, 0)) | |
| previous_pts = now | |
| while index < len(deadlines): | |
| if previous is None: | |
| raise ValueError("Empty source camera") | |
| images.append(previous) | |
| pts.append(previous_pts) | |
| index += 1 | |
| if np.max(np.asarray(deadlines) - pts) >= .2: | |
| raise ValueError("Camera recording ended before the SWA timeline") | |
| return images, np.asarray(pts) | |
| def compose(output): | |
| import cv2 | |
| from PIL import Image, ImageDraw, ImageFont, ImageOps | |
| cv2.setNumThreads(1) | |
| metadata = read(output / "infer_meta.json") | |
| metrics = read(output / "metrics.json") | |
| row, timeline = metadata["render_source"], metadata["timeline"] | |
| total = int(metadata["total_frames"]) | |
| if row.get("camera_domain") == "static_image": | |
| captures = np.array([0.]) | |
| with Image.open(row["question"]["image_path"]) as image: | |
| cameras = [ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0))] | |
| else: | |
| captures = np.asarray(timeline["camera_capture_seconds"]) | |
| cameras, pts = camera_frames(row["question"]["video_path"], | |
| captures + float(timeline["video_offset"])) | |
| if not np.allclose(pts, timeline["camera_pts"], atol=1e-7, rtol=0): | |
| raise ValueError("Review camera PTS differ from actual encoded frames") | |
| cameras = [image.resize((384, 384)) for image in cameras] | |
| font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 18) | |
| robot = cv2.VideoCapture(str(output / "render_motion/reference_motion.mp4")) | |
| if (not robot.isOpened() or abs(robot.get(cv2.CAP_PROP_FPS) - FPS) > .01 | |
| or int(robot.get(cv2.CAP_PROP_FRAME_COUNT)) != total): | |
| robot.release() | |
| raise ValueError("SWA robot render frame count/FPS mismatch") | |
| target = output / "reference_motion_with_audio.mp4" | |
| temporary = output / ".swa_review.partial.mp4" | |
| command = ["ffmpeg", "-v", "error", "-y", "-f", "rawvideo", "-pixel_format", "rgb24", | |
| "-video_size", "960x560", "-framerate", str(FPS), "-i", "pipe:0", | |
| "-i", str(output / metadata.get('audio_file', 'assembled_input_audio_16k.wav')), | |
| "-map", "0:v:0", "-map", "1:a:0", "-c:v", "libx264", "-preset", "veryfast", | |
| "-threads", "2", "-crf", "21", "-pix_fmt", "yuv420p", "-c:a", "aac", | |
| "-t", str(total / FPS), "-movflags", "+faststart", str(temporary)] | |
| child = subprocess.Popen(command, stdin=subprocess.PIPE) | |
| try: | |
| for frame in range(total): | |
| ok, bgr = robot.read() | |
| if not ok: | |
| raise ValueError("SWA robot video ended early") | |
| t = frame / FPS | |
| canvas = Image.new("RGB", (960, 560), (20, 20, 20)) | |
| camera_index = np.searchsorted(captures, t + 1e-8, side="right") - 1 | |
| if camera_index >= 0: | |
| canvas.paste(cameras[min(camera_index, len(cameras) - 1)], (0, 36)) | |
| body = Image.fromarray(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)) | |
| canvas.paste(ImageOps.pad(body, (576, 384), color=(0, 0, 0)), (384, 36)) | |
| draw = ImageDraw.Draw(canvas) | |
| draw.text((8, 7), f"SWA camera input | Generated robot {t:.2f}s", | |
| font=font, fill="white") | |
| native = metadata['swa_render_version'] >= 2 | |
| notice = "Audio: microphone + native model voice (offline causal replay)." if native else "Audio: original microphone. Model answer: text only (no generated voice)." | |
| if frame >= metadata.get('motion_end_frame', total): | |
| notice = "Answer audio tail: holding last generated robot pose." | |
| draw.text((8, 424), notice, | |
| font=font, fill="white") | |
| question = metrics["question"] if t >= float(timeline["question_start"]) and row.get("swa_has_question", True) else "(not spoken yet)" | |
| def answer_time(answer): | |
| matching = [s['start'] / 24000 for s in metadata.get('speech_segments', []) if s['path'] == answer.get('audio_path')] | |
| return min(matching) if matching else float(answer['seconds']) | |
| available = [a for a in metrics["answers"] if answer_time(a) <= t + 1e-8] | |
| answer = available[-1]["text"] if available else "(no answer yet)" | |
| lines = (textwrap.wrap("Question: " + question, 97)[:2] | |
| + textwrap.wrap("Model: " + answer, 97)[:3]) | |
| for index, line in enumerate(lines): | |
| draw.text((8, 447 + index * 21), line, font=font, fill="white") | |
| child.stdin.write(np.asarray(canvas, np.uint8).tobytes()) | |
| child.stdin.close() | |
| if child.wait() != 0: | |
| raise RuntimeError("SWA video/audio encoding failed") | |
| finally: | |
| robot.release() | |
| if not child.stdin.closed: | |
| try: | |
| child.stdin.close() | |
| except BrokenPipeError: | |
| pass | |
| if child.poll() is None: | |
| child.kill() | |
| child.wait() | |
| probe = json.loads(subprocess.check_output( | |
| ["ffprobe", "-v", "error", "-show_streams", "-of", "json", str(temporary)])) | |
| video = next(s for s in probe["streams"] if s["codec_type"] == "video") | |
| audio = next(s for s in probe["streams"] if s["codec_type"] == "audio") | |
| if (int(video["nb_frames"]) != total or video["avg_frame_rate"] != "50/1" | |
| or abs(float(audio.get("start_time", 0))) > .021): | |
| raise ValueError("SWA composed video lost frames or shifted microphone") | |
| temporary.replace(target) | |
| if metadata['swa_render_version'] >= 2: | |
| metadata['answer_mode'] = 'model_text_at_native_speech_playback' | |
| metadata["swa_review"] = {"video": str(target), "camera_pts_verified": True, | |
| "frames": total, "fps": FPS, | |
| "generated_speech": metadata['swa_render_version'] >= 2 and bool(metrics['answers'])} | |
| write(output / "infer_meta.json", metadata) | |
| print(f"[done] SWA synchronized review: {target}", flush=True) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("mode", choices=("prepare", "compose")) | |
| parser.add_argument("--run-dir", type=Path, required=True) | |
| parser.add_argument("--output-dir", type=Path, required=True) | |
| args = parser.parse_args() | |
| if not args.output_dir.resolve().is_relative_to(args.run_dir.resolve()): | |
| parser.error("SWA output must be inside the selected run") | |
| if args.mode == "prepare": | |
| prepare(args.run_dir, args.output_dir) | |
| else: | |
| compose(args.output_dir) | |
| if __name__ == "__main__": | |
| main() | |