"""Freeze complete multimodal validation sessions with the formal training loader.""" import argparse from collections import Counter from concurrent.futures import ProcessPoolExecutor import multiprocessing import json import os import random import sys from pathlib import Path from common import ROOT, WORK, DEFAULT_RUN, sha256, write_json from session_data import task_of, quotas, candidate_order, quality_asset, motion_segments sys.path[:0] = [str(ROOT), str(ROOT/'src')] os.environ.setdefault('HFD_STARTUP_INDEX_CACHE_DIR', str(WORK/'outputs/startup_cache')) import numpy as np import torch def build(job): index, kind, blocked_assets = job from scripts.data.render_hfd_hesitation_pause_gt import reconstruct_motion from psi.data.seed.live_multimodal_overlay import continuous_audio from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexTransformedDataset from scipy.io import wavfile torch.set_num_threads(1) sid = f'hfd_session_{index:05d}' dest = ARGS.output done = dest/f'{sid}.json' if done.exists(): row = json.loads(done.read_text()) if row['visual_kind'] != kind: return dict(rejected='previous_other_modality', dataset_index=index) if row['overlay']['asset_id'] not in blocked_assets: return row random.seed(ARGS.seed+index); np.random.seed(ARGS.seed+index); torch.manual_seed(ARGS.seed+index) voice = RAW.sessions[index][0]['_session_voice_name'] OVERLAY.by_voice = {voice: [r for r in POOLS[(voice, kind)] if r['asset_id'] not in blocked_assets]} source = OVERLAY.compose(RAW[index], 'qa') relation = source['multimodal_overlay'] if relation['mode'] != 'qa': return dict(rejected=relation.get('fallback_reason', relation['mode']), dataset_index=index, kind=kind) gt, controls = reconstruct_motion(source) segments, turns = motion_segments(source, RAW.sessions[index], controls) audio = continuous_audio(source) emitted = 20*sum(bool(s['update_semantic_cache']) for s in source['steps']) if emitted < len(gt)-1: raise ValueError('Rollout fails to cover entire reference') if not np.isfinite(gt).all() or not np.isfinite(audio).all(): raise ValueError('Nonfinite GT or microphone') wavfile.write(dest/f'{sid}_microphone.wav', 16000, (np.clip(audio,-1,1)*32767).astype(np.int16)) np.savez_compressed(dest/f'{sid}_reference.npz', absolute=gt[1:], controls=controls[1:], turn_indices=turns) event_keys = ('microphone_speech_intervals','turn_listen_start_frames','turn_speech_start_frames', 'turn_listen_reaction_frames','control_command_events','backchannel_events','disfluency_events', 'interruption_start_frames','interruption_remaining_frames','third_party_interference_intervals') events = {k:source[k] for k in event_keys if k in source} metadata = dict(sample_id=sid, source_id=source['source_id'], dataset_index=index, task=task_of(RAW.sessions[index]), task_family=relation['task'], turn_count=len(segments), third_party_profile=source.get('third_party_profile'), disfluency_subtype=source.get('disfluency_subtype'), voice=voice, split='val', visual_kind=kind, overlay=relation, events=events, segments=segments, source_file=f'{sid}.pt', reference_file=f'{sid}_reference.npz', microphone_file=f'{sid}_microphone.wav', reference_frames=len(gt)-1, score_end=len(gt)-1, clocks=dict(motion_fps=50, microphone_sample_rate=16000, motion_origin_in_microphone_seconds=.2), reference_answer=dict(text=relation['answer'], earliest_seconds=relation['question_end'], reference_waveform=None, note='Text GT; generated speech is an output, never an input.'), input_provenance=dict(run_config_sha256=RUN_HASH, seed=ARGS.seed+index)) # Deployment uses semantic packets. Keep full GT in a separate reference. source['steps'] = [s for s in source['steps'] if s['update_semantic_cache']] item = FullDuplexTransformedDataset([source], CONFIG.data.transform, {'vlm_processor':PROCESSOR})[0] if len(item['swa_packets'])*20 < metadata['reference_frames']: raise ValueError('Transformed packet coverage mismatch') tmp = dest/f'{sid}.pt.tmp'; torch.save(item,tmp); tmp.replace(dest/f'{sid}.pt') for key in ('source_file','reference_file','microphone_file'): metadata[key.replace('_file','_sha256')] = sha256(dest/metadata[key]) write_json(done,metadata) print('PREPARED', sid, metadata['task'],kind,len(segments),round(len(gt)/50,2),flush=True) return metadata def discard_unselected(row, destination): """Only delete this preparer's completed, unselected, reproducible outputs.""" sid=f"hfd_session_{int(row['dataset_index']):05d}" if row['sample_id']!=sid:raise ValueError('Unsafe unselected sample identity') for suffix in ('.json','.pt','_microphone.wav','_reference.npz'): path=destination/(sid+suffix) if path.is_symlink():raise ValueError('Refusing to delete symlinked sample') path.unlink(missing_ok=True) def main(): global ARGS, RAW, OVERLAY, CONFIG, PROCESSOR, POOLS, RUN_HASH p=argparse.ArgumentParser() p.add_argument('--run',type=Path,default=DEFAULT_RUN) p.add_argument('--output',type=Path,default=WORK/'outputs/sessions2048') p.add_argument('--count',type=int,default=2048) p.add_argument('--seed',type=int,default=42) p.add_argument('--workers',type=int,default=8) p.add_argument('--max-qa-reuse',type=int,default=64,help='Maximum sessions per original visual QA across all voices') p.add_argument('--stop-after',type=int,default=0,help='Pause for human review without freezing manifest') p.add_argument('--external-assets',type=Path,help='Reviewed external QA manifest; replaces all training-derived visuals') p.add_argument('--reuse-cohort',type=Path,help='Keep hash-identical samples admitted by the revised visual pool') ARGS=p.parse_args(); a=ARGS if a.max_qa_reuse<1:raise ValueError('max-qa-reuse must be positive') a.output.mkdir(parents=True,exist_ok=True) if (a.output/'manifest.json').exists(): raise FileExistsError('Frozen cohort already exists') torch.set_num_threads(1) from scripts.infer.unified_overlay import cfg from transformers import Qwen2_5OmniProcessor CONFIG=cfg(a.run); RUN_HASH=sha256(a.run/'run_config.json') PROCESSOR=Qwen2_5OmniProcessor.from_pretrained(CONFIG.model.model_name_or_path,local_files_only=True) OVERLAY=CONFIG.data(split='val',transform_kwargs={'vlm_processor':PROCESSOR}).raw_dataset RAW=OVERLAY.raw_dataset; POOLS={}; quality_counts={} if a.external_assets: from session_overlay import ExternalOverlay, reviewed payload=json.loads(a.external_assets.read_text()) if payload.get('format')!='hfd_bench_external_assets_v1':raise ValueError('Wrong external QA format') replacement=ExternalOverlay(OVERLAY.policy,seed=a.seed) for row in payload['assets']: if not reviewed(row):raise ValueError('External QA lacks verified review/ASR: '+row['asset_id']) replacement.by_voice.setdefault(row['voice'],[]).append(row) OVERLAY=replacement for voice, rows in OVERLAY.by_voice.items(): for kind in ('video','image'): eligible=[r for r in rows if bool(r.get('video_path')) == (kind=='video')] POOLS[(voice,kind)]=eligible if a.external_assets else [r for r in eligible if quality_asset(r)] quality_counts[f'{voice}/{kind}']=dict(before=len(eligible),after=len(POOLS[(voice,kind)])) print('QUALITY_POOLS',json.dumps(quality_counts),flush=True) wanted=quotas(a.count) image_ratio=0. if a.external_assets else .05 target={(task,kind):(round(n*image_ratio) if kind=='image' else n-round(n*image_ratio)) for task,n in wanted.items() for kind in ('video','image')} identity=dict(run=str(a.run.resolve()),run_config_sha256=RUN_HASH,count=a.count,seed=a.seed, schema='hfd_bench_session_v3',max_qa_reuse=a.max_qa_reuse) if a.external_assets: identity['external_assets_sha256']=sha256(a.external_assets) identity['external_assets_path']=str(a.external_assets.resolve()) progress=a.output/'selection_progress.json' if a.reuse_cohort and not progress.exists(): if not a.external_assets:raise ValueError('Cohort revision requires explicit external assets') from revision import seed_cohort write_json(progress,seed_cohort(a.reuse_cohort,a.output,identity,payload['assets'])) prior=json.loads(progress.read_text()) if progress.exists() else dict(identity=identity,rows=[],rejections=[],attempted=[]) if prior['identity']!=identity: raise ValueError('Selection identity changed') rows=prior['rows']; rejected=prior['rejections']; attempted={tuple(x) for x in prior['attempted']} counts=Counter((r['task'],r['visual_kind']) for r in rows) used={r['dataset_index'] for r in rows}; assets=Counter(r['overlay']['asset_id'] for r in rows) order=candidate_order(RAW,a.seed) with ProcessPoolExecutor(max_workers=a.workers,mp_context=multiprocessing.get_context('fork')) as pool: for kind in ('video','image'): pending=[i for i in order if i not in used and (i,kind) not in attempted] for offset in range(0,len(pending),a.workers*2): jobs=[(i,kind) for i in pending[offset:offset+a.workers*2] if counts[(task_of(RAW.sessions[i]),kind)]=a.max_qa_reuse) for job,row in zip(jobs,pool.map(build,[(i,k,blocked) for i,k in jobs])): attempted.add(job) if 'rejected' in row: rejected.append(row); continue key=(row['task'],kind) if counts[key]>=target[key]: discard_unselected(row,a.output) continue if assets[row['overlay']['asset_id']]>=a.max_qa_reuse: rejected.append(dict(dataset_index=job[0],kind=kind,rejected='qa_reuse_cap')) discard_unselected(row,a.output);continue rows.append(row);used.add(job[0]);counts[key]+=1;assets[row['overlay']['asset_id']]+=1 write_json(progress,dict(identity=identity,rows=rows,rejections=rejected,attempted=sorted(attempted))) print('SELECTED',len(rows),'/',a.count,dict(counts),flush=True) if a.stop_after and len(rows)>=a.stop_after: return if all(counts[(t,kind)]==target[(t,kind)] for t in wanted): break if len(rows)!=a.count: raise RuntimeError(f'Insufficient valid sessions: {dict(counts)} targets={target}; no silent fallback') random.Random(a.seed).shuffle(rows) summary=dict(candidate_sessions=len(RAW),tasks=dict(Counter(r['task'] for r in rows)), task_families=dict(Counter(r['task_family'] for r in rows)),modalities=dict(Counter(r['visual_kind'] for r in rows)), turns=dict(Counter(r['turn_count'] for r in rows)),voices=dict(Counter(r['voice'] for r in rows)), quality_pools=quality_counts,unique_visual_qa=len(assets),max_visual_qa_reuse=max(assets.values()), total_source_occurrences=sum(r['turn_count'] for r in rows), unique_motion_sources=len({s['motion_name'] for r in rows for s in r['segments']}), unique_final_commands=len({s['caption'] for r in rows for s in r['segments']}), semantic_category_occurrences=dict(Counter(c for r in rows for s in r['segments'] for c in s['semantic_categories'])), session_seconds_percentiles=dict(zip(('min','p25','median','p75','p95','max'), np.percentile([r['reference_frames']/50 for r in rows],[0,25,50,75,95,100]).tolist())), eligible_motion_segments=sum(s['eligible'] for r in rows for s in r['segments']), excluded_motion_segments=dict(Counter(s['exclusion_reason'] for r in rows for s in r['segments'] if not s['eligible'])), rejections=dict(Counter(r['rejected'] for r in rejected))) write_json(a.output/'selection_summary.json',summary) write_json(a.output/'manifest.json',dict(**identity,split='val',selection_summary=summary,rows=rows, protocol=f'Natural 1-4-turn held-out sessions; six task families balanced; {(1-image_ratio)*100:g}% video QA; GT-only selection')) if __name__=='__main__': main()