"""Fixed real-overlay diagnostics; reuse formal dataset, rollout and native speech.""" import argparse, json, os, random, subprocess, sys from pathlib import Path import numpy as np import torch ROOT=Path(__file__).resolve().parents[2] WORK=Path(os.environ.get('HFD_OVERLAY_FIXTURE_DIR', str(ROOT.parents[1]/'hfd_experiments/unified_overlay_fixtures'))).expanduser().resolve() sys.path[:0]=[str(ROOT),str(ROOT/'src')] from psi.runtime.overlay_cases import CASE_PATHS def cfg(run): from psi.utils import parse_args_to_tyro_config cls=parse_args_to_tyro_config(run/'argv.txt') return cls.model_validate(json.loads((run/'run_config.json').read_text())) def prepare(run): from transformers import Qwen2_5OmniProcessor from scripts.data.render_hfd_unified_multimodal_gt import session_task from scripts.data.render_hfd_hesitation_pause_gt import reconstruct_motion from psi.data.seed.live_multimodal_overlay import continuous_audio from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexTransformedDataset config=cfg(run) processor=Qwen2_5OmniProcessor.from_pretrained(config.model.model_name_or_path,local_files_only=True) saved=[];evidence=[] WORK.mkdir(parents=True,exist_ok=True) overrides=os.environ.get('HFD_OVERLAY_MANIFESTS','') if overrides:config.data.multimodal_overlay_manifests=tuple(overrides.split(':')) for split in ('val','train'): ds=config.data(split=split,transform_kwargs={'vlm_processor':processor}) overlay=ds.raw_dataset raw=overlay.raw_dataset wanted=['turn-taking','interruption','stop','backchannel','hesitation','pause'] if getattr(config.data, 'third_party_root', ''): wanted.append('third-party') for case_index,task in enumerate(wanted): base_task=task original_pools=overlay.by_voice # Visual kind is an input property, never an additional task. # Both splits cover all tasks, with opposite image/video choices. video=(case_index+(split=='val'))%2==0 overlay.by_voice={k:[a for a in v if bool(a.get('video_path'))==video] for k,v in original_pools.items()} indices=[] for i,s in enumerate(raw.sessions): kind=('third-party' if s[0].get('_third_party') else 'stop' if s[0].get('_session_interaction_mode')=='emergency-stop' else session_task(s)) if kind==base_task and s[0].get('_session_voice_name')=='female_01_en': n=raw.estimated_stream_steps(i) if 300<=n<=(1400 if video else 1900):indices.append((n,i)) choices=sorted(indices,key=lambda pair:abs(pair[0]-950))[:160] # A short Stop may have no gap for a complete question. Keep the # task as an explicit visual-only negative, never overwrite Stop. choices=[(mode,i) for mode in ('qa','visual-only') for _,i in choices] for mode,i in choices: random.seed(260927+i);np.random.seed(260927+i);torch.manual_seed(260927+i) source=overlay.compose(raw[i], mode) if source['multimodal_overlay']['mode']!=mode:continue motion,_=reconstruct_motion(source) item=FullDuplexTransformedDataset([source],config.data.transform,{'vlm_processor':processor})[0] item['probe_gt_abs']=motion;item['probe_audio']=continuous_audio(source) item['probe_task']=task;item['probe_split']=split item['probe_instruction_text']='; '.join(dict.fromkeys(s.get('latest_instruction_text','') for s in source['steps'] if s.get('latest_instruction_text'))) item['probe_controls']=np.asarray([s['control_label'] for s in source['steps']]) saved.append(item) evidence.append(dict(index=len(saved)-1,split=split,task=task,source_id=item['source_id'], dataset_index=i,overlay=item['multimodal_overlay'])) print('FIXTURE',split,task,i,item['multimodal_overlay']['data_domain'],flush=True) break else:raise RuntimeError(f'No QA fixture for {split}/{task}') overlay.by_voice=original_pools torch.save(saved,WORK/'sources.pt') (WORK/'sources.json').write_text(json.dumps(evidence,indent=2,ensure_ascii=False)) def infer(run,step,output,limit,only_index=None): from psi.models.interaction_arch.loading import load_swa_checkpoint from psi.runtime.swa_rollout import rollout_source,body_metrics from psi.runtime.swa_speech import SwaSpeechWorker,schedule_speech config=cfg(run) model=load_swa_checkpoint(run,step,config,torch.device('cuda:0')) model.eval() sources=torch.load(WORK/'sources.pt',map_location='cpu',weights_only=False,mmap=True) for i,source in enumerate(sources[:limit] if limit else sources): if only_index is not None and i!=only_index:continue dest=output/source['probe_split']/CASE_PATHS[source['probe_task']] if (dest/'metrics.json').exists():continue dest.mkdir(parents=True,exist_ok=True) torch.manual_seed(42);np.random.seed(42);random.seed(42) speech=SwaSpeechWorker(model,config.model.model_name_or_path,dest) try: motion,events=rollout_source(model,config,source,speech=speech) answers=speech.finish(allow_incomplete=True) finally:speech.close() np.save(dest/'generated_abs.npy',motion) result=dict(source_index=i,source_id=source['source_id'],overlay=source['multimodal_overlay'], events=events,answers=answers,speech_timeline=schedule_speech(answers),native_speech_version=1, speech_failures=speech.failures, speech_complete=not speech.failures, control_accuracy=float(np.mean([e['control']==e['control_gt'] for e in events])), body_metrics=body_metrics(motion,source['probe_gt_abs'][1:]), ground_truth_answer_audio_used=False,checkpoint_step=step, overlay_fixture_file=str(WORK/'sources.pt'),probe_task=source['probe_task'],probe_split=source['probe_split']) (dest/'metrics.json').write_text(json.dumps(result,indent=2,ensure_ascii=False)) print('INFERRED',dest,flush=True) if os.environ.get('HFD_INFER_SKIP_RENDER')=='1':continue with (dest/'render.log').open('a') as f: subprocess.Popen([sys.executable,str(Path(__file__)), 'render',str(dest)], cwd=ROOT,stdout=f,stderr=subprocess.STDOUT,start_new_session=True) def render_content(output): from scipy.io import wavfile from scipy.signal import resample_poly from scripts.infer.render_swa_result import display_motion,answer_audio from scripts.infer.full_duplex_audio_to_motion import export_motion from scripts.data.render_hfd_unified_multimodal_gt import render_video result=json.loads((output/'metrics.json').read_text()) source=torch.load(result['overlay_fixture_file'],map_location='cpu',weights_only=False,mmap=True)[result['source_index']] assert source['source_id']==result['source_id'] poses,_=display_motion(source['initial_abs_state'],np.load(output/'generated_abs.npy'),result['events']) speaker,segments=answer_audio(output,result) mic=resample_poly(source['probe_audio'],3,2) n=max(len(poses),int(np.ceil(max(len(mic),len(speaker))/480))) poses=np.concatenate([poses,np.repeat(poses[-1:],n-len(poses),axis=0)]) audio=np.zeros(n*480,np.float32);audio[:len(mic)]+=mic;audio[:len(speaker)]+=speaker audio*=min(1.,.98/max(float(np.abs(audio).max()),1e-8)) folder=output/'render_motion';folder.mkdir(exist_ok=True);export_motion(folder,poses) wavfile.write(folder/'review_audio_with_answer.wav',24000,audio) relation=result['overlay'];camera=relation.get('video_path') or relation.get('image_path','') video=render_video(folder,dict(camera_video_path=camera,camera_video_offset=0.,motion_seconds=n/50, input_duration_seconds=len(mic)/24000, review_kind='MODEL INFERENCE',split=source['probe_split'],task=source['probe_task'], instruction_text=source.get('probe_instruction_text','See command audio'), question=relation['question'] if relation['mode']=='qa' else '(visual-only; no question)', answer='; '.join(a.get('text','') for a in result['answers']), question_start=relation['question_start'],question_end=relation['question_end'])) video.replace(output/'reference_motion_with_audio.mp4') (output/'infer_meta.json').write_text(json.dumps(dict(swa_render_version=2,total_frames=n))) (output/'.rendered').touch() (output/'.complete').touch() (output/'.render_failed').unlink(missing_ok=True) print('RENDERED',output,flush=True) def render(output): import fcntl,time WORK.mkdir(parents=True,exist_ok=True) with (output/'.render.lock').open('a') as own: try:fcntl.flock(own,fcntl.LOCK_EX|fcntl.LOCK_NB) except BlockingIOError:return if (output/'.rendered').exists():return while True: slot=None for i in range(2): candidate=(WORK/f'render-slot-{i}.lock').open('a') try:fcntl.flock(candidate,fcntl.LOCK_EX|fcntl.LOCK_NB);slot=candidate;break except BlockingIOError:candidate.close() if slot is not None:break time.sleep(5) try:render_content(output) except Exception as e: (output/'.render_failed').write_text(repr(e));raise finally:slot.close() if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','infer','render']) p.add_argument('path',type=Path);p.add_argument('--step',type=int);p.add_argument('--output',type=Path) p.add_argument('--limit',type=int,default=0);p.add_argument('--only-index',type=int);a=p.parse_args() if a.mode=='prepare':prepare(a.path) elif a.mode=='render':render(a.path) else:infer(a.path,a.step,a.output,a.limit,a.only_index)