File size: 5,728 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
"""Render saved benchmark output using the formal camera/body/audio renderer.

No extra model inference, answer TTS, camera cropping, or pose smoothing.
Playback uses packet arrival times; motion scoring uses target frame order.
"""
import argparse
import json
from pathlib import Path
import sys
import subprocess
import numpy as np
from common import ROOT,WORK,write_json
sys.path[:0]=[str(ROOT),str(ROOT/'src')]


def review_panel(folder, metadata):
    """Benchmark-only CJK font fallback; never changes model inputs."""
    from PIL import Image, ImageDraw, ImageFont
    fonts = ('/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc',
             '/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc')
    path = next((p for p in fonts if Path(p).is_file()), None)
    if path is None: raise FileNotFoundError('A CJK font is required for review captions')
    font = ImageFont.truetype(path, 17)
    canvas = Image.new('RGB', (640, 480), '#f5f6f8')
    draw = ImageDraw.Draw(canvas)
    lines = [metadata['review_kind']+' | '+metadata['split'], 'Task: '+metadata['task'],
             'Input: '+metadata['instruction_text'], 'Question: '+metadata['question'],
             'Model answer: '+metadata['answer'],
             f"Question time: {metadata['question_start']:.2f} - {metadata['question_end']:.2f} s",
             'Camera: continuous real-time video', 'Captions are review-only, NOT model inputs.']
    y = 12
    for text in lines:
        line = ''
        for char in text:
            if draw.textlength(line+char, font=font)>610:
                draw.text((14,y), line, font=font, fill='#151b26'); y+=21; line=''
            line+=char
        draw.text((14,y), line, font=font, fill='#151b26'); y+=25
    if y>480: raise ValueError('Review captions overflow')
    output=folder/'review_panel.png'; canvas.save(output)
    return output


def remux(folder, metadata):
    panel=review_panel(folder, metadata)
    target=folder/'model_session_with_audio.mp4'
    subprocess.run(['ffmpeg','-y','-i',str(folder/'reference_motion.mp4'),
        '-i',str(folder/'camera_overlay.mp4'),'-i',str(folder/'review_audio_with_answer.wav'),
        '-loop','1','-i',str(panel),'-filter_complex',
        '[0:v]scale=640:360[r];[1:v][r]hstack=inputs=2[scene];'
        '[scene]pad=1280:480:0:60:color=black[sc];[sc][3:v]hstack=inputs=2[v]',
        '-map','[v]','-map','2:a:0','-shortest','-t',str(metadata['motion_seconds']),
        '-r','50','-c:v','libx264','-pix_fmt','yuv420p','-c:a','aac','-b:a','128k',str(target)],
        check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
    return target


def main():
    p=argparse.ArgumentParser(description=__doc__)
    p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048')
    p.add_argument('--generated',type=Path,required=True)
    p.add_argument('--sample',required=True)
    p.add_argument('--remux-only',action='store_true',help='Reuse existing rendered video and native answer audio')
    a=p.parse_args()
    if a.remux_only:
        folder=a.generated/a.sample/'preview'
        print('PREVIEW',remux(folder,json.loads((folder/'preview_metadata.json').read_text())),flush=True)
        return
    import torch
    from scipy.io import wavfile
    from scipy.signal import resample_poly
    from scripts.infer.render_swa_result import display_motion,answer_audio
    from scripts.infer.full_duplex_audio_to_motion import export_motion
    from scripts.data import render_hfd_unified_multimodal_gt as renderer
    renderer.write_review_panel=review_panel
    torch.set_num_threads(1)
    row=json.loads((a.cohort/(a.sample+'.json')).read_text())
    result=json.loads((a.generated/(a.sample+'.json')).read_text())
    if row['source_id']!=result['source_id']:raise ValueError('Preview source mismatch')
    source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True)
    with np.load(a.generated/(a.sample+'.npz')) as data:motion=data['absolute']
    poses,_=display_motion(source['initial_abs_state'],motion,result['events'])
    folder=a.generated/a.sample/'preview';folder.mkdir(parents=True,exist_ok=True)
    speaker,_=answer_audio(a.generated/a.sample,result)
    rate,mic=wavfile.read(a.cohort/row['microphone_file'])
    if rate!=16000 or mic.ndim!=1 or mic.dtype!=np.int16:raise ValueError('Unexpected microphone format')
    mic=resample_poly(mic.astype(np.float32)/32768,3,2)
    n=max(len(poses),int(np.ceil(max(len(mic),len(speaker))/480)))
    poses=np.concatenate([poses,np.repeat(poses[-1:],n-len(poses),axis=0)])
    wave=np.zeros(n*480,np.float32);wave[:len(mic)]+=mic;wave[:len(speaker)]+=speaker
    wave*=min(1.,.98/max(float(np.abs(wave).max()),1e-8))
    export_motion(folder,poses)
    wavfile.write(folder/'review_audio_with_answer.wav',24000,wave)
    overlay=row['overlay']
    metadata=dict(camera_video_path=overlay['video_path'],camera_video_offset=0.,motion_seconds=n/50,
        input_duration_seconds=len(mic)/24000,review_kind='MODEL OUTPUT - not GT',
        task=row['task'],split=row['split'],
        instruction_text=' / '.join(s['final_instruction'] for s in row['segments'])[:170],
        question=overlay['question'],answer='; '.join(x.get('text','') for x in result['answers'])[:180],
        question_start=overlay['question_start'],question_end=overlay['question_end'])
    write_json(folder/'preview_metadata.json',dict(**metadata,
        note='Actual generated speech only; no GT answer waveform. Offline scheduling is not hardware latency evaluation.'))
    video=renderer.render_video(folder,metadata)
    target=folder/'model_session_with_audio.mp4'
    video.replace(target)
    print('PREVIEW',target,flush=True)


if __name__=='__main__':main()