code-HFD / scripts /infer /unified_overlay.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
10.1 kB
"""Fixed real-overlay diagnostics; reuse formal dataset, rollout and native speech."""
import argparse, json, os, random, subprocess, sys
from pathlib import Path
import numpy as np
import torch
ROOT=Path(__file__).resolve().parents[2]
WORK=Path(os.environ.get('HFD_OVERLAY_FIXTURE_DIR', str(ROOT.parents[1]/'hfd_experiments/unified_overlay_fixtures'))).expanduser().resolve()
sys.path[:0]=[str(ROOT),str(ROOT/'src')]
from psi.runtime.overlay_cases import CASE_PATHS
def cfg(run):
from psi.utils import parse_args_to_tyro_config
cls=parse_args_to_tyro_config(run/'argv.txt')
return cls.model_validate(json.loads((run/'run_config.json').read_text()))
def prepare(run):
from transformers import Qwen2_5OmniProcessor
from scripts.data.render_hfd_unified_multimodal_gt import session_task
from scripts.data.render_hfd_hesitation_pause_gt import reconstruct_motion
from psi.data.seed.live_multimodal_overlay import continuous_audio
from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexTransformedDataset
config=cfg(run)
processor=Qwen2_5OmniProcessor.from_pretrained(config.model.model_name_or_path,local_files_only=True)
saved=[];evidence=[]
WORK.mkdir(parents=True,exist_ok=True)
overrides=os.environ.get('HFD_OVERLAY_MANIFESTS','')
if overrides:config.data.multimodal_overlay_manifests=tuple(overrides.split(':'))
for split in ('val','train'):
ds=config.data(split=split,transform_kwargs={'vlm_processor':processor})
overlay=ds.raw_dataset
raw=overlay.raw_dataset
wanted=['turn-taking','interruption','stop','backchannel','hesitation','pause']
if getattr(config.data, 'third_party_root', ''):
wanted.append('third-party')
for case_index,task in enumerate(wanted):
base_task=task
original_pools=overlay.by_voice
# Visual kind is an input property, never an additional task.
# Both splits cover all tasks, with opposite image/video choices.
video=(case_index+(split=='val'))%2==0
overlay.by_voice={k:[a for a in v if bool(a.get('video_path'))==video] for k,v in original_pools.items()}
indices=[]
for i,s in enumerate(raw.sessions):
kind=('third-party' if s[0].get('_third_party') else
'stop' if s[0].get('_session_interaction_mode')=='emergency-stop' else session_task(s))
if kind==base_task and s[0].get('_session_voice_name')=='female_01_en':
n=raw.estimated_stream_steps(i)
if 300<=n<=(1400 if video else 1900):indices.append((n,i))
choices=sorted(indices,key=lambda pair:abs(pair[0]-950))[:160]
# A short Stop may have no gap for a complete question. Keep the
# task as an explicit visual-only negative, never overwrite Stop.
choices=[(mode,i) for mode in ('qa','visual-only') for _,i in choices]
for mode,i in choices:
random.seed(260927+i);np.random.seed(260927+i);torch.manual_seed(260927+i)
source=overlay.compose(raw[i], mode)
if source['multimodal_overlay']['mode']!=mode:continue
motion,_=reconstruct_motion(source)
item=FullDuplexTransformedDataset([source],config.data.transform,{'vlm_processor':processor})[0]
item['probe_gt_abs']=motion;item['probe_audio']=continuous_audio(source)
item['probe_task']=task;item['probe_split']=split
item['probe_instruction_text']='; '.join(dict.fromkeys(s.get('latest_instruction_text','') for s in source['steps'] if s.get('latest_instruction_text')))
item['probe_controls']=np.asarray([s['control_label'] for s in source['steps']])
saved.append(item)
evidence.append(dict(index=len(saved)-1,split=split,task=task,source_id=item['source_id'],
dataset_index=i,overlay=item['multimodal_overlay']))
print('FIXTURE',split,task,i,item['multimodal_overlay']['data_domain'],flush=True)
break
else:raise RuntimeError(f'No QA fixture for {split}/{task}')
overlay.by_voice=original_pools
torch.save(saved,WORK/'sources.pt')
(WORK/'sources.json').write_text(json.dumps(evidence,indent=2,ensure_ascii=False))
def infer(run,step,output,limit,only_index=None):
from psi.models.interaction_arch.loading import load_swa_checkpoint
from psi.runtime.swa_rollout import rollout_source,body_metrics
from psi.runtime.swa_speech import SwaSpeechWorker,schedule_speech
config=cfg(run)
model=load_swa_checkpoint(run,step,config,torch.device('cuda:0'))
model.eval()
sources=torch.load(WORK/'sources.pt',map_location='cpu',weights_only=False,mmap=True)
for i,source in enumerate(sources[:limit] if limit else sources):
if only_index is not None and i!=only_index:continue
dest=output/source['probe_split']/CASE_PATHS[source['probe_task']]
if (dest/'metrics.json').exists():continue
dest.mkdir(parents=True,exist_ok=True)
torch.manual_seed(42);np.random.seed(42);random.seed(42)
speech=SwaSpeechWorker(model,config.model.model_name_or_path,dest)
try:
motion,events=rollout_source(model,config,source,speech=speech)
answers=speech.finish(allow_incomplete=True)
finally:speech.close()
np.save(dest/'generated_abs.npy',motion)
result=dict(source_index=i,source_id=source['source_id'],overlay=source['multimodal_overlay'],
events=events,answers=answers,speech_timeline=schedule_speech(answers),native_speech_version=1,
speech_failures=speech.failures, speech_complete=not speech.failures,
control_accuracy=float(np.mean([e['control']==e['control_gt'] for e in events])),
body_metrics=body_metrics(motion,source['probe_gt_abs'][1:]),
ground_truth_answer_audio_used=False,checkpoint_step=step,
overlay_fixture_file=str(WORK/'sources.pt'),probe_task=source['probe_task'],probe_split=source['probe_split'])
(dest/'metrics.json').write_text(json.dumps(result,indent=2,ensure_ascii=False))
print('INFERRED',dest,flush=True)
if os.environ.get('HFD_INFER_SKIP_RENDER')=='1':continue
with (dest/'render.log').open('a') as f:
subprocess.Popen([sys.executable,str(Path(__file__)), 'render',str(dest)],
cwd=ROOT,stdout=f,stderr=subprocess.STDOUT,start_new_session=True)
def render_content(output):
from scipy.io import wavfile
from scipy.signal import resample_poly
from scripts.infer.render_swa_result import display_motion,answer_audio
from scripts.infer.full_duplex_audio_to_motion import export_motion
from scripts.data.render_hfd_unified_multimodal_gt import render_video
result=json.loads((output/'metrics.json').read_text())
source=torch.load(result['overlay_fixture_file'],map_location='cpu',weights_only=False,mmap=True)[result['source_index']]
assert source['source_id']==result['source_id']
poses,_=display_motion(source['initial_abs_state'],np.load(output/'generated_abs.npy'),result['events'])
speaker,segments=answer_audio(output,result)
mic=resample_poly(source['probe_audio'],3,2)
n=max(len(poses),int(np.ceil(max(len(mic),len(speaker))/480)))
poses=np.concatenate([poses,np.repeat(poses[-1:],n-len(poses),axis=0)])
audio=np.zeros(n*480,np.float32);audio[:len(mic)]+=mic;audio[:len(speaker)]+=speaker
audio*=min(1.,.98/max(float(np.abs(audio).max()),1e-8))
folder=output/'render_motion';folder.mkdir(exist_ok=True);export_motion(folder,poses)
wavfile.write(folder/'review_audio_with_answer.wav',24000,audio)
relation=result['overlay'];camera=relation.get('video_path') or relation.get('image_path','')
video=render_video(folder,dict(camera_video_path=camera,camera_video_offset=0.,motion_seconds=n/50,
input_duration_seconds=len(mic)/24000,
review_kind='MODEL INFERENCE',split=source['probe_split'],task=source['probe_task'],
instruction_text=source.get('probe_instruction_text','See command audio'),
question=relation['question'] if relation['mode']=='qa' else '(visual-only; no question)',
answer='; '.join(a.get('text','') for a in result['answers']),
question_start=relation['question_start'],question_end=relation['question_end']))
video.replace(output/'reference_motion_with_audio.mp4')
(output/'infer_meta.json').write_text(json.dumps(dict(swa_render_version=2,total_frames=n)))
(output/'.rendered').touch()
(output/'.complete').touch()
(output/'.render_failed').unlink(missing_ok=True)
print('RENDERED',output,flush=True)
def render(output):
import fcntl,time
WORK.mkdir(parents=True,exist_ok=True)
with (output/'.render.lock').open('a') as own:
try:fcntl.flock(own,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError:return
if (output/'.rendered').exists():return
while True:
slot=None
for i in range(2):
candidate=(WORK/f'render-slot-{i}.lock').open('a')
try:fcntl.flock(candidate,fcntl.LOCK_EX|fcntl.LOCK_NB);slot=candidate;break
except BlockingIOError:candidate.close()
if slot is not None:break
time.sleep(5)
try:render_content(output)
except Exception as e:
(output/'.render_failed').write_text(repr(e));raise
finally:slot.close()
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','infer','render'])
p.add_argument('path',type=Path);p.add_argument('--step',type=int);p.add_argument('--output',type=Path)
p.add_argument('--limit',type=int,default=0);p.add_argument('--only-index',type=int);a=p.parse_args()
if a.mode=='prepare':prepare(a.path)
elif a.mode=='render':render(a.path)
else:infer(a.path,a.step,a.output,a.limit,a.only_index)