"""Render saved benchmark output using the formal camera/body/audio renderer. No extra model inference, answer TTS, camera cropping, or pose smoothing. Playback uses packet arrival times; motion scoring uses target frame order. """ import argparse import json from pathlib import Path import sys import subprocess import numpy as np from common import ROOT,WORK,write_json sys.path[:0]=[str(ROOT),str(ROOT/'src')] def review_panel(folder, metadata): """Benchmark-only CJK font fallback; never changes model inputs.""" from PIL import Image, ImageDraw, ImageFont fonts = ('/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc', '/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc') path = next((p for p in fonts if Path(p).is_file()), None) if path is None: raise FileNotFoundError('A CJK font is required for review captions') font = ImageFont.truetype(path, 17) canvas = Image.new('RGB', (640, 480), '#f5f6f8') draw = ImageDraw.Draw(canvas) lines = [metadata['review_kind']+' | '+metadata['split'], 'Task: '+metadata['task'], 'Input: '+metadata['instruction_text'], 'Question: '+metadata['question'], 'Model answer: '+metadata['answer'], f"Question time: {metadata['question_start']:.2f} - {metadata['question_end']:.2f} s", 'Camera: continuous real-time video', 'Captions are review-only, NOT model inputs.'] y = 12 for text in lines: line = '' for char in text: if draw.textlength(line+char, font=font)>610: draw.text((14,y), line, font=font, fill='#151b26'); y+=21; line='' line+=char draw.text((14,y), line, font=font, fill='#151b26'); y+=25 if y>480: raise ValueError('Review captions overflow') output=folder/'review_panel.png'; canvas.save(output) return output def remux(folder, metadata): panel=review_panel(folder, metadata) target=folder/'model_session_with_audio.mp4' subprocess.run(['ffmpeg','-y','-i',str(folder/'reference_motion.mp4'), '-i',str(folder/'camera_overlay.mp4'),'-i',str(folder/'review_audio_with_answer.wav'), '-loop','1','-i',str(panel),'-filter_complex', '[0:v]scale=640:360[r];[1:v][r]hstack=inputs=2[scene];' '[scene]pad=1280:480:0:60:color=black[sc];[sc][3:v]hstack=inputs=2[v]', '-map','[v]','-map','2:a:0','-shortest','-t',str(metadata['motion_seconds']), '-r','50','-c:v','libx264','-pix_fmt','yuv420p','-c:a','aac','-b:a','128k',str(target)], check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) return target def main(): p=argparse.ArgumentParser(description=__doc__) p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048') p.add_argument('--generated',type=Path,required=True) p.add_argument('--sample',required=True) p.add_argument('--remux-only',action='store_true',help='Reuse existing rendered video and native answer audio') a=p.parse_args() if a.remux_only: folder=a.generated/a.sample/'preview' print('PREVIEW',remux(folder,json.loads((folder/'preview_metadata.json').read_text())),flush=True) return import torch from scipy.io import wavfile from scipy.signal import resample_poly from scripts.infer.render_swa_result import display_motion,answer_audio from scripts.infer.full_duplex_audio_to_motion import export_motion from scripts.data import render_hfd_unified_multimodal_gt as renderer renderer.write_review_panel=review_panel torch.set_num_threads(1) row=json.loads((a.cohort/(a.sample+'.json')).read_text()) result=json.loads((a.generated/(a.sample+'.json')).read_text()) if row['source_id']!=result['source_id']:raise ValueError('Preview source mismatch') source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True) with np.load(a.generated/(a.sample+'.npz')) as data:motion=data['absolute'] poses,_=display_motion(source['initial_abs_state'],motion,result['events']) folder=a.generated/a.sample/'preview';folder.mkdir(parents=True,exist_ok=True) speaker,_=answer_audio(a.generated/a.sample,result) rate,mic=wavfile.read(a.cohort/row['microphone_file']) if rate!=16000 or mic.ndim!=1 or mic.dtype!=np.int16:raise ValueError('Unexpected microphone format') mic=resample_poly(mic.astype(np.float32)/32768,3,2) n=max(len(poses),int(np.ceil(max(len(mic),len(speaker))/480))) poses=np.concatenate([poses,np.repeat(poses[-1:],n-len(poses),axis=0)]) wave=np.zeros(n*480,np.float32);wave[:len(mic)]+=mic;wave[:len(speaker)]+=speaker wave*=min(1.,.98/max(float(np.abs(wave).max()),1e-8)) export_motion(folder,poses) wavfile.write(folder/'review_audio_with_answer.wav',24000,wave) overlay=row['overlay'] metadata=dict(camera_video_path=overlay['video_path'],camera_video_offset=0.,motion_seconds=n/50, input_duration_seconds=len(mic)/24000,review_kind='MODEL OUTPUT - not GT', task=row['task'],split=row['split'], instruction_text=' / '.join(s['final_instruction'] for s in row['segments'])[:170], question=overlay['question'],answer='; '.join(x.get('text','') for x in result['answers'])[:180], question_start=overlay['question_start'],question_end=overlay['question_end']) write_json(folder/'preview_metadata.json',dict(**metadata, note='Actual generated speech only; no GT answer waveform. Offline scheduling is not hardware latency evaluation.')) video=renderer.render_video(folder,metadata) target=folder/'model_session_with_audio.mp4' video.replace(target) print('PREVIEW',target,flush=True) if __name__=='__main__':main()