code-HFD / benchmark /preview.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
fb0074a verified
Raw History Blame Contribute Delete
5.73 kB
"""Render saved benchmark output using the formal camera/body/audio renderer.
No extra model inference, answer TTS, camera cropping, or pose smoothing.
Playback uses packet arrival times; motion scoring uses target frame order.
"""
import argparse
import json
from pathlib import Path
import sys
import subprocess
import numpy as np
from common import ROOT,WORK,write_json
sys.path[:0]=[str(ROOT),str(ROOT/'src')]
def review_panel(folder, metadata):
"""Benchmark-only CJK font fallback; never changes model inputs."""
from PIL import Image, ImageDraw, ImageFont
fonts = ('/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc',
'/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc')
path = next((p for p in fonts if Path(p).is_file()), None)
if path is None: raise FileNotFoundError('A CJK font is required for review captions')
font = ImageFont.truetype(path, 17)
canvas = Image.new('RGB', (640, 480), '#f5f6f8')
draw = ImageDraw.Draw(canvas)
lines = [metadata['review_kind']+' | '+metadata['split'], 'Task: '+metadata['task'],
'Input: '+metadata['instruction_text'], 'Question: '+metadata['question'],
'Model answer: '+metadata['answer'],
f"Question time: {metadata['question_start']:.2f} - {metadata['question_end']:.2f} s",
'Camera: continuous real-time video', 'Captions are review-only, NOT model inputs.']
y = 12
for text in lines:
line = ''
for char in text:
if draw.textlength(line+char, font=font)>610:
draw.text((14,y), line, font=font, fill='#151b26'); y+=21; line=''
line+=char
draw.text((14,y), line, font=font, fill='#151b26'); y+=25
if y>480: raise ValueError('Review captions overflow')
output=folder/'review_panel.png'; canvas.save(output)
return output
def remux(folder, metadata):
panel=review_panel(folder, metadata)
target=folder/'model_session_with_audio.mp4'
subprocess.run(['ffmpeg','-y','-i',str(folder/'reference_motion.mp4'),
'-i',str(folder/'camera_overlay.mp4'),'-i',str(folder/'review_audio_with_answer.wav'),
'-loop','1','-i',str(panel),'-filter_complex',
'[0:v]scale=640:360[r];[1:v][r]hstack=inputs=2[scene];'
'[scene]pad=1280:480:0:60:color=black[sc];[sc][3:v]hstack=inputs=2[v]',
'-map','[v]','-map','2:a:0','-shortest','-t',str(metadata['motion_seconds']),
'-r','50','-c:v','libx264','-pix_fmt','yuv420p','-c:a','aac','-b:a','128k',str(target)],
check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
return target
def main():
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048')
p.add_argument('--generated',type=Path,required=True)
p.add_argument('--sample',required=True)
p.add_argument('--remux-only',action='store_true',help='Reuse existing rendered video and native answer audio')
a=p.parse_args()
if a.remux_only:
folder=a.generated/a.sample/'preview'
print('PREVIEW',remux(folder,json.loads((folder/'preview_metadata.json').read_text())),flush=True)
return
import torch
from scipy.io import wavfile
from scipy.signal import resample_poly
from scripts.infer.render_swa_result import display_motion,answer_audio
from scripts.infer.full_duplex_audio_to_motion import export_motion
from scripts.data import render_hfd_unified_multimodal_gt as renderer
renderer.write_review_panel=review_panel
torch.set_num_threads(1)
row=json.loads((a.cohort/(a.sample+'.json')).read_text())
result=json.loads((a.generated/(a.sample+'.json')).read_text())
if row['source_id']!=result['source_id']:raise ValueError('Preview source mismatch')
source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True)
with np.load(a.generated/(a.sample+'.npz')) as data:motion=data['absolute']
poses,_=display_motion(source['initial_abs_state'],motion,result['events'])
folder=a.generated/a.sample/'preview';folder.mkdir(parents=True,exist_ok=True)
speaker,_=answer_audio(a.generated/a.sample,result)
rate,mic=wavfile.read(a.cohort/row['microphone_file'])
if rate!=16000 or mic.ndim!=1 or mic.dtype!=np.int16:raise ValueError('Unexpected microphone format')
mic=resample_poly(mic.astype(np.float32)/32768,3,2)
n=max(len(poses),int(np.ceil(max(len(mic),len(speaker))/480)))
poses=np.concatenate([poses,np.repeat(poses[-1:],n-len(poses),axis=0)])
wave=np.zeros(n*480,np.float32);wave[:len(mic)]+=mic;wave[:len(speaker)]+=speaker
wave*=min(1.,.98/max(float(np.abs(wave).max()),1e-8))
export_motion(folder,poses)
wavfile.write(folder/'review_audio_with_answer.wav',24000,wave)
overlay=row['overlay']
metadata=dict(camera_video_path=overlay['video_path'],camera_video_offset=0.,motion_seconds=n/50,
input_duration_seconds=len(mic)/24000,review_kind='MODEL OUTPUT - not GT',
task=row['task'],split=row['split'],
instruction_text=' / '.join(s['final_instruction'] for s in row['segments'])[:170],
question=overlay['question'],answer='; '.join(x.get('text','') for x in result['answers'])[:180],
question_start=overlay['question_start'],question_end=overlay['question_end'])
write_json(folder/'preview_metadata.json',dict(**metadata,
note='Actual generated speech only; no GT answer waveform. Offline scheduling is not hardware latency evaluation.'))
video=renderer.render_video(folder,metadata)
target=folder/'model_session_with_audio.mp4'
video.replace(target)
print('PREVIEW',target,flush=True)
if __name__=='__main__':main()