Download scripts/infer/unified_overlay.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/unified_overlay.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/infer/unified_overlay.py
-
curl -L -o unified_overlay.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/unified_overlay.py
10.1 kB
| """Fixed real-overlay diagnostics; reuse formal dataset, rollout and native speech.""" | |
| import argparse, json, os, random, subprocess, sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| ROOT=Path(__file__).resolve().parents[2] | |
| WORK=Path(os.environ.get('HFD_OVERLAY_FIXTURE_DIR', str(ROOT.parents[1]/'hfd_experiments/unified_overlay_fixtures'))).expanduser().resolve() | |
| sys.path[:0]=[str(ROOT),str(ROOT/'src')] | |
| from psi.runtime.overlay_cases import CASE_PATHS | |
| def cfg(run): | |
| from psi.utils import parse_args_to_tyro_config | |
| cls=parse_args_to_tyro_config(run/'argv.txt') | |
| return cls.model_validate(json.loads((run/'run_config.json').read_text())) | |
| def prepare(run): | |
| from transformers import Qwen2_5OmniProcessor | |
| from scripts.data.render_hfd_unified_multimodal_gt import session_task | |
| from scripts.data.render_hfd_hesitation_pause_gt import reconstruct_motion | |
| from psi.data.seed.live_multimodal_overlay import continuous_audio | |
| from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexTransformedDataset | |
| config=cfg(run) | |
| processor=Qwen2_5OmniProcessor.from_pretrained(config.model.model_name_or_path,local_files_only=True) | |
| saved=[];evidence=[] | |
| WORK.mkdir(parents=True,exist_ok=True) | |
| overrides=os.environ.get('HFD_OVERLAY_MANIFESTS','') | |
| if overrides:config.data.multimodal_overlay_manifests=tuple(overrides.split(':')) | |
| for split in ('val','train'): | |
| ds=config.data(split=split,transform_kwargs={'vlm_processor':processor}) | |
| overlay=ds.raw_dataset | |
| raw=overlay.raw_dataset | |
| wanted=['turn-taking','interruption','stop','backchannel','hesitation','pause'] | |
| if getattr(config.data, 'third_party_root', ''): | |
| wanted.append('third-party') | |
| for case_index,task in enumerate(wanted): | |
| base_task=task | |
| original_pools=overlay.by_voice | |
| # Visual kind is an input property, never an additional task. | |
| # Both splits cover all tasks, with opposite image/video choices. | |
| video=(case_index+(split=='val'))%2==0 | |
| overlay.by_voice={k:[a for a in v if bool(a.get('video_path'))==video] for k,v in original_pools.items()} | |
| indices=[] | |
| for i,s in enumerate(raw.sessions): | |
| kind=('third-party' if s[0].get('_third_party') else | |
| 'stop' if s[0].get('_session_interaction_mode')=='emergency-stop' else session_task(s)) | |
| if kind==base_task and s[0].get('_session_voice_name')=='female_01_en': | |
| n=raw.estimated_stream_steps(i) | |
| if 300<=n<=(1400 if video else 1900):indices.append((n,i)) | |
| choices=sorted(indices,key=lambda pair:abs(pair[0]-950))[:160] | |
| # A short Stop may have no gap for a complete question. Keep the | |
| # task as an explicit visual-only negative, never overwrite Stop. | |
| choices=[(mode,i) for mode in ('qa','visual-only') for _,i in choices] | |
| for mode,i in choices: | |
| random.seed(260927+i);np.random.seed(260927+i);torch.manual_seed(260927+i) | |
| source=overlay.compose(raw[i], mode) | |
| if source['multimodal_overlay']['mode']!=mode:continue | |
| motion,_=reconstruct_motion(source) | |
| item=FullDuplexTransformedDataset([source],config.data.transform,{'vlm_processor':processor})[0] | |
| item['probe_gt_abs']=motion;item['probe_audio']=continuous_audio(source) | |
| item['probe_task']=task;item['probe_split']=split | |
| item['probe_instruction_text']='; '.join(dict.fromkeys(s.get('latest_instruction_text','') for s in source['steps'] if s.get('latest_instruction_text'))) | |
| item['probe_controls']=np.asarray([s['control_label'] for s in source['steps']]) | |
| saved.append(item) | |
| evidence.append(dict(index=len(saved)-1,split=split,task=task,source_id=item['source_id'], | |
| dataset_index=i,overlay=item['multimodal_overlay'])) | |
| print('FIXTURE',split,task,i,item['multimodal_overlay']['data_domain'],flush=True) | |
| break | |
| else:raise RuntimeError(f'No QA fixture for {split}/{task}') | |
| overlay.by_voice=original_pools | |
| torch.save(saved,WORK/'sources.pt') | |
| (WORK/'sources.json').write_text(json.dumps(evidence,indent=2,ensure_ascii=False)) | |
| def infer(run,step,output,limit,only_index=None): | |
| from psi.models.interaction_arch.loading import load_swa_checkpoint | |
| from psi.runtime.swa_rollout import rollout_source,body_metrics | |
| from psi.runtime.swa_speech import SwaSpeechWorker,schedule_speech | |
| config=cfg(run) | |
| model=load_swa_checkpoint(run,step,config,torch.device('cuda:0')) | |
| model.eval() | |
| sources=torch.load(WORK/'sources.pt',map_location='cpu',weights_only=False,mmap=True) | |
| for i,source in enumerate(sources[:limit] if limit else sources): | |
| if only_index is not None and i!=only_index:continue | |
| dest=output/source['probe_split']/CASE_PATHS[source['probe_task']] | |
| if (dest/'metrics.json').exists():continue | |
| dest.mkdir(parents=True,exist_ok=True) | |
| torch.manual_seed(42);np.random.seed(42);random.seed(42) | |
| speech=SwaSpeechWorker(model,config.model.model_name_or_path,dest) | |
| try: | |
| motion,events=rollout_source(model,config,source,speech=speech) | |
| answers=speech.finish(allow_incomplete=True) | |
| finally:speech.close() | |
| np.save(dest/'generated_abs.npy',motion) | |
| result=dict(source_index=i,source_id=source['source_id'],overlay=source['multimodal_overlay'], | |
| events=events,answers=answers,speech_timeline=schedule_speech(answers),native_speech_version=1, | |
| speech_failures=speech.failures, speech_complete=not speech.failures, | |
| control_accuracy=float(np.mean([e['control']==e['control_gt'] for e in events])), | |
| body_metrics=body_metrics(motion,source['probe_gt_abs'][1:]), | |
| ground_truth_answer_audio_used=False,checkpoint_step=step, | |
| overlay_fixture_file=str(WORK/'sources.pt'),probe_task=source['probe_task'],probe_split=source['probe_split']) | |
| (dest/'metrics.json').write_text(json.dumps(result,indent=2,ensure_ascii=False)) | |
| print('INFERRED',dest,flush=True) | |
| if os.environ.get('HFD_INFER_SKIP_RENDER')=='1':continue | |
| with (dest/'render.log').open('a') as f: | |
| subprocess.Popen([sys.executable,str(Path(__file__)), 'render',str(dest)], | |
| cwd=ROOT,stdout=f,stderr=subprocess.STDOUT,start_new_session=True) | |
| def render_content(output): | |
| from scipy.io import wavfile | |
| from scipy.signal import resample_poly | |
| from scripts.infer.render_swa_result import display_motion,answer_audio | |
| from scripts.infer.full_duplex_audio_to_motion import export_motion | |
| from scripts.data.render_hfd_unified_multimodal_gt import render_video | |
| result=json.loads((output/'metrics.json').read_text()) | |
| source=torch.load(result['overlay_fixture_file'],map_location='cpu',weights_only=False,mmap=True)[result['source_index']] | |
| assert source['source_id']==result['source_id'] | |
| poses,_=display_motion(source['initial_abs_state'],np.load(output/'generated_abs.npy'),result['events']) | |
| speaker,segments=answer_audio(output,result) | |
| mic=resample_poly(source['probe_audio'],3,2) | |
| n=max(len(poses),int(np.ceil(max(len(mic),len(speaker))/480))) | |
| poses=np.concatenate([poses,np.repeat(poses[-1:],n-len(poses),axis=0)]) | |
| audio=np.zeros(n*480,np.float32);audio[:len(mic)]+=mic;audio[:len(speaker)]+=speaker | |
| audio*=min(1.,.98/max(float(np.abs(audio).max()),1e-8)) | |
| folder=output/'render_motion';folder.mkdir(exist_ok=True);export_motion(folder,poses) | |
| wavfile.write(folder/'review_audio_with_answer.wav',24000,audio) | |
| relation=result['overlay'];camera=relation.get('video_path') or relation.get('image_path','') | |
| video=render_video(folder,dict(camera_video_path=camera,camera_video_offset=0.,motion_seconds=n/50, | |
| input_duration_seconds=len(mic)/24000, | |
| review_kind='MODEL INFERENCE',split=source['probe_split'],task=source['probe_task'], | |
| instruction_text=source.get('probe_instruction_text','See command audio'), | |
| question=relation['question'] if relation['mode']=='qa' else '(visual-only; no question)', | |
| answer='; '.join(a.get('text','') for a in result['answers']), | |
| question_start=relation['question_start'],question_end=relation['question_end'])) | |
| video.replace(output/'reference_motion_with_audio.mp4') | |
| (output/'infer_meta.json').write_text(json.dumps(dict(swa_render_version=2,total_frames=n))) | |
| (output/'.rendered').touch() | |
| (output/'.complete').touch() | |
| (output/'.render_failed').unlink(missing_ok=True) | |
| print('RENDERED',output,flush=True) | |
| def render(output): | |
| import fcntl,time | |
| WORK.mkdir(parents=True,exist_ok=True) | |
| with (output/'.render.lock').open('a') as own: | |
| try:fcntl.flock(own,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| except BlockingIOError:return | |
| if (output/'.rendered').exists():return | |
| while True: | |
| slot=None | |
| for i in range(2): | |
| candidate=(WORK/f'render-slot-{i}.lock').open('a') | |
| try:fcntl.flock(candidate,fcntl.LOCK_EX|fcntl.LOCK_NB);slot=candidate;break | |
| except BlockingIOError:candidate.close() | |
| if slot is not None:break | |
| time.sleep(5) | |
| try:render_content(output) | |
| except Exception as e: | |
| (output/'.render_failed').write_text(repr(e));raise | |
| finally:slot.close() | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','infer','render']) | |
| p.add_argument('path',type=Path);p.add_argument('--step',type=int);p.add_argument('--output',type=Path) | |
| p.add_argument('--limit',type=int,default=0);p.add_argument('--only-index',type=int);a=p.parse_args() | |
| if a.mode=='prepare':prepare(a.path) | |
| elif a.mode=='render':render(a.path) | |
| else:infer(a.path,a.step,a.output,a.limit,a.only_index) | |