Download benchmark/preview.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.73 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/preview.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/preview.py
-
curl -L -o preview.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/preview.py
5.73 kB
| """Render saved benchmark output using the formal camera/body/audio renderer. | |
| No extra model inference, answer TTS, camera cropping, or pose smoothing. | |
| Playback uses packet arrival times; motion scoring uses target frame order. | |
| """ | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import sys | |
| import subprocess | |
| import numpy as np | |
| from common import ROOT,WORK,write_json | |
| sys.path[:0]=[str(ROOT),str(ROOT/'src')] | |
| def review_panel(folder, metadata): | |
| """Benchmark-only CJK font fallback; never changes model inputs.""" | |
| from PIL import Image, ImageDraw, ImageFont | |
| fonts = ('/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc', | |
| '/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc') | |
| path = next((p for p in fonts if Path(p).is_file()), None) | |
| if path is None: raise FileNotFoundError('A CJK font is required for review captions') | |
| font = ImageFont.truetype(path, 17) | |
| canvas = Image.new('RGB', (640, 480), '#f5f6f8') | |
| draw = ImageDraw.Draw(canvas) | |
| lines = [metadata['review_kind']+' | '+metadata['split'], 'Task: '+metadata['task'], | |
| 'Input: '+metadata['instruction_text'], 'Question: '+metadata['question'], | |
| 'Model answer: '+metadata['answer'], | |
| f"Question time: {metadata['question_start']:.2f} - {metadata['question_end']:.2f} s", | |
| 'Camera: continuous real-time video', 'Captions are review-only, NOT model inputs.'] | |
| y = 12 | |
| for text in lines: | |
| line = '' | |
| for char in text: | |
| if draw.textlength(line+char, font=font)>610: | |
| draw.text((14,y), line, font=font, fill='#151b26'); y+=21; line='' | |
| line+=char | |
| draw.text((14,y), line, font=font, fill='#151b26'); y+=25 | |
| if y>480: raise ValueError('Review captions overflow') | |
| output=folder/'review_panel.png'; canvas.save(output) | |
| return output | |
| def remux(folder, metadata): | |
| panel=review_panel(folder, metadata) | |
| target=folder/'model_session_with_audio.mp4' | |
| subprocess.run(['ffmpeg','-y','-i',str(folder/'reference_motion.mp4'), | |
| '-i',str(folder/'camera_overlay.mp4'),'-i',str(folder/'review_audio_with_answer.wav'), | |
| '-loop','1','-i',str(panel),'-filter_complex', | |
| '[0:v]scale=640:360[r];[1:v][r]hstack=inputs=2[scene];' | |
| '[scene]pad=1280:480:0:60:color=black[sc];[sc][3:v]hstack=inputs=2[v]', | |
| '-map','[v]','-map','2:a:0','-shortest','-t',str(metadata['motion_seconds']), | |
| '-r','50','-c:v','libx264','-pix_fmt','yuv420p','-c:a','aac','-b:a','128k',str(target)], | |
| check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) | |
| return target | |
| def main(): | |
| p=argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048') | |
| p.add_argument('--generated',type=Path,required=True) | |
| p.add_argument('--sample',required=True) | |
| p.add_argument('--remux-only',action='store_true',help='Reuse existing rendered video and native answer audio') | |
| a=p.parse_args() | |
| if a.remux_only: | |
| folder=a.generated/a.sample/'preview' | |
| print('PREVIEW',remux(folder,json.loads((folder/'preview_metadata.json').read_text())),flush=True) | |
| return | |
| import torch | |
| from scipy.io import wavfile | |
| from scipy.signal import resample_poly | |
| from scripts.infer.render_swa_result import display_motion,answer_audio | |
| from scripts.infer.full_duplex_audio_to_motion import export_motion | |
| from scripts.data import render_hfd_unified_multimodal_gt as renderer | |
| renderer.write_review_panel=review_panel | |
| torch.set_num_threads(1) | |
| row=json.loads((a.cohort/(a.sample+'.json')).read_text()) | |
| result=json.loads((a.generated/(a.sample+'.json')).read_text()) | |
| if row['source_id']!=result['source_id']:raise ValueError('Preview source mismatch') | |
| source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True) | |
| with np.load(a.generated/(a.sample+'.npz')) as data:motion=data['absolute'] | |
| poses,_=display_motion(source['initial_abs_state'],motion,result['events']) | |
| folder=a.generated/a.sample/'preview';folder.mkdir(parents=True,exist_ok=True) | |
| speaker,_=answer_audio(a.generated/a.sample,result) | |
| rate,mic=wavfile.read(a.cohort/row['microphone_file']) | |
| if rate!=16000 or mic.ndim!=1 or mic.dtype!=np.int16:raise ValueError('Unexpected microphone format') | |
| mic=resample_poly(mic.astype(np.float32)/32768,3,2) | |
| n=max(len(poses),int(np.ceil(max(len(mic),len(speaker))/480))) | |
| poses=np.concatenate([poses,np.repeat(poses[-1:],n-len(poses),axis=0)]) | |
| wave=np.zeros(n*480,np.float32);wave[:len(mic)]+=mic;wave[:len(speaker)]+=speaker | |
| wave*=min(1.,.98/max(float(np.abs(wave).max()),1e-8)) | |
| export_motion(folder,poses) | |
| wavfile.write(folder/'review_audio_with_answer.wav',24000,wave) | |
| overlay=row['overlay'] | |
| metadata=dict(camera_video_path=overlay['video_path'],camera_video_offset=0.,motion_seconds=n/50, | |
| input_duration_seconds=len(mic)/24000,review_kind='MODEL OUTPUT - not GT', | |
| task=row['task'],split=row['split'], | |
| instruction_text=' / '.join(s['final_instruction'] for s in row['segments'])[:170], | |
| question=overlay['question'],answer='; '.join(x.get('text','') for x in result['answers'])[:180], | |
| question_start=overlay['question_start'],question_end=overlay['question_end']) | |
| write_json(folder/'preview_metadata.json',dict(**metadata, | |
| note='Actual generated speech only; no GT answer waveform. Offline scheduling is not hardware latency evaluation.')) | |
| video=renderer.render_video(folder,metadata) | |
| target=folder/'model_session_with_audio.mp4' | |
| video.replace(target) | |
| print('PREVIEW',target,flush=True) | |
| if __name__=='__main__':main() | |