File size: 12,686 Bytes
fb0074a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 | """Freeze complete multimodal validation sessions with the formal training loader."""
import argparse
from collections import Counter
from concurrent.futures import ProcessPoolExecutor
import multiprocessing
import json
import os
import random
import sys
from pathlib import Path
from common import ROOT, WORK, DEFAULT_RUN, sha256, write_json
from session_data import task_of, quotas, candidate_order, quality_asset, motion_segments
sys.path[:0] = [str(ROOT), str(ROOT/'src')]
os.environ.setdefault('HFD_STARTUP_INDEX_CACHE_DIR', str(WORK/'outputs/startup_cache'))
import numpy as np
import torch
def build(job):
index, kind, blocked_assets = job
from scripts.data.render_hfd_hesitation_pause_gt import reconstruct_motion
from psi.data.seed.live_multimodal_overlay import continuous_audio
from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexTransformedDataset
from scipy.io import wavfile
torch.set_num_threads(1)
sid = f'hfd_session_{index:05d}'
dest = ARGS.output
done = dest/f'{sid}.json'
if done.exists():
row = json.loads(done.read_text())
if row['visual_kind'] != kind:
return dict(rejected='previous_other_modality', dataset_index=index)
if row['overlay']['asset_id'] not in blocked_assets:
return row
random.seed(ARGS.seed+index); np.random.seed(ARGS.seed+index); torch.manual_seed(ARGS.seed+index)
voice = RAW.sessions[index][0]['_session_voice_name']
OVERLAY.by_voice = {voice: [r for r in POOLS[(voice, kind)] if r['asset_id'] not in blocked_assets]}
source = OVERLAY.compose(RAW[index], 'qa')
relation = source['multimodal_overlay']
if relation['mode'] != 'qa':
return dict(rejected=relation.get('fallback_reason', relation['mode']), dataset_index=index, kind=kind)
gt, controls = reconstruct_motion(source)
segments, turns = motion_segments(source, RAW.sessions[index], controls)
audio = continuous_audio(source)
emitted = 20*sum(bool(s['update_semantic_cache']) for s in source['steps'])
if emitted < len(gt)-1:
raise ValueError('Rollout fails to cover entire reference')
if not np.isfinite(gt).all() or not np.isfinite(audio).all():
raise ValueError('Nonfinite GT or microphone')
wavfile.write(dest/f'{sid}_microphone.wav', 16000, (np.clip(audio,-1,1)*32767).astype(np.int16))
np.savez_compressed(dest/f'{sid}_reference.npz', absolute=gt[1:], controls=controls[1:], turn_indices=turns)
event_keys = ('microphone_speech_intervals','turn_listen_start_frames','turn_speech_start_frames',
'turn_listen_reaction_frames','control_command_events','backchannel_events','disfluency_events',
'interruption_start_frames','interruption_remaining_frames','third_party_interference_intervals')
events = {k:source[k] for k in event_keys if k in source}
metadata = dict(sample_id=sid, source_id=source['source_id'], dataset_index=index,
task=task_of(RAW.sessions[index]), task_family=relation['task'], turn_count=len(segments),
third_party_profile=source.get('third_party_profile'), disfluency_subtype=source.get('disfluency_subtype'),
voice=voice, split='val', visual_kind=kind, overlay=relation, events=events, segments=segments,
source_file=f'{sid}.pt', reference_file=f'{sid}_reference.npz', microphone_file=f'{sid}_microphone.wav',
reference_frames=len(gt)-1, score_end=len(gt)-1,
clocks=dict(motion_fps=50, microphone_sample_rate=16000, motion_origin_in_microphone_seconds=.2),
reference_answer=dict(text=relation['answer'], earliest_seconds=relation['question_end'],
reference_waveform=None, note='Text GT; generated speech is an output, never an input.'),
input_provenance=dict(run_config_sha256=RUN_HASH, seed=ARGS.seed+index))
# Deployment uses semantic packets. Keep full GT in a separate reference.
source['steps'] = [s for s in source['steps'] if s['update_semantic_cache']]
item = FullDuplexTransformedDataset([source], CONFIG.data.transform, {'vlm_processor':PROCESSOR})[0]
if len(item['swa_packets'])*20 < metadata['reference_frames']:
raise ValueError('Transformed packet coverage mismatch')
tmp = dest/f'{sid}.pt.tmp'; torch.save(item,tmp); tmp.replace(dest/f'{sid}.pt')
for key in ('source_file','reference_file','microphone_file'):
metadata[key.replace('_file','_sha256')] = sha256(dest/metadata[key])
write_json(done,metadata)
print('PREPARED', sid, metadata['task'],kind,len(segments),round(len(gt)/50,2),flush=True)
return metadata
def discard_unselected(row, destination):
"""Only delete this preparer's completed, unselected, reproducible outputs."""
sid=f"hfd_session_{int(row['dataset_index']):05d}"
if row['sample_id']!=sid:raise ValueError('Unsafe unselected sample identity')
for suffix in ('.json','.pt','_microphone.wav','_reference.npz'):
path=destination/(sid+suffix)
if path.is_symlink():raise ValueError('Refusing to delete symlinked sample')
path.unlink(missing_ok=True)
def main():
global ARGS, RAW, OVERLAY, CONFIG, PROCESSOR, POOLS, RUN_HASH
p=argparse.ArgumentParser()
p.add_argument('--run',type=Path,default=DEFAULT_RUN)
p.add_argument('--output',type=Path,default=WORK/'outputs/sessions2048')
p.add_argument('--count',type=int,default=2048)
p.add_argument('--seed',type=int,default=42)
p.add_argument('--workers',type=int,default=8)
p.add_argument('--max-qa-reuse',type=int,default=64,help='Maximum sessions per original visual QA across all voices')
p.add_argument('--stop-after',type=int,default=0,help='Pause for human review without freezing manifest')
p.add_argument('--external-assets',type=Path,help='Reviewed external QA manifest; replaces all training-derived visuals')
p.add_argument('--reuse-cohort',type=Path,help='Keep hash-identical samples admitted by the revised visual pool')
ARGS=p.parse_args(); a=ARGS
if a.max_qa_reuse<1:raise ValueError('max-qa-reuse must be positive')
a.output.mkdir(parents=True,exist_ok=True)
if (a.output/'manifest.json').exists():
raise FileExistsError('Frozen cohort already exists')
torch.set_num_threads(1)
from scripts.infer.unified_overlay import cfg
from transformers import Qwen2_5OmniProcessor
CONFIG=cfg(a.run); RUN_HASH=sha256(a.run/'run_config.json')
PROCESSOR=Qwen2_5OmniProcessor.from_pretrained(CONFIG.model.model_name_or_path,local_files_only=True)
OVERLAY=CONFIG.data(split='val',transform_kwargs={'vlm_processor':PROCESSOR}).raw_dataset
RAW=OVERLAY.raw_dataset; POOLS={}; quality_counts={}
if a.external_assets:
from session_overlay import ExternalOverlay, reviewed
payload=json.loads(a.external_assets.read_text())
if payload.get('format')!='hfd_bench_external_assets_v1':raise ValueError('Wrong external QA format')
replacement=ExternalOverlay(OVERLAY.policy,seed=a.seed)
for row in payload['assets']:
if not reviewed(row):raise ValueError('External QA lacks verified review/ASR: '+row['asset_id'])
replacement.by_voice.setdefault(row['voice'],[]).append(row)
OVERLAY=replacement
for voice, rows in OVERLAY.by_voice.items():
for kind in ('video','image'):
eligible=[r for r in rows if bool(r.get('video_path')) == (kind=='video')]
POOLS[(voice,kind)]=eligible if a.external_assets else [r for r in eligible if quality_asset(r)]
quality_counts[f'{voice}/{kind}']=dict(before=len(eligible),after=len(POOLS[(voice,kind)]))
print('QUALITY_POOLS',json.dumps(quality_counts),flush=True)
wanted=quotas(a.count)
image_ratio=0. if a.external_assets else .05
target={(task,kind):(round(n*image_ratio) if kind=='image' else n-round(n*image_ratio))
for task,n in wanted.items() for kind in ('video','image')}
identity=dict(run=str(a.run.resolve()),run_config_sha256=RUN_HASH,count=a.count,seed=a.seed,
schema='hfd_bench_session_v3',max_qa_reuse=a.max_qa_reuse)
if a.external_assets:
identity['external_assets_sha256']=sha256(a.external_assets)
identity['external_assets_path']=str(a.external_assets.resolve())
progress=a.output/'selection_progress.json'
if a.reuse_cohort and not progress.exists():
if not a.external_assets:raise ValueError('Cohort revision requires explicit external assets')
from revision import seed_cohort
write_json(progress,seed_cohort(a.reuse_cohort,a.output,identity,payload['assets']))
prior=json.loads(progress.read_text()) if progress.exists() else dict(identity=identity,rows=[],rejections=[],attempted=[])
if prior['identity']!=identity:
raise ValueError('Selection identity changed')
rows=prior['rows']; rejected=prior['rejections']; attempted={tuple(x) for x in prior['attempted']}
counts=Counter((r['task'],r['visual_kind']) for r in rows)
used={r['dataset_index'] for r in rows}; assets=Counter(r['overlay']['asset_id'] for r in rows)
order=candidate_order(RAW,a.seed)
with ProcessPoolExecutor(max_workers=a.workers,mp_context=multiprocessing.get_context('fork')) as pool:
for kind in ('video','image'):
pending=[i for i in order if i not in used and (i,kind) not in attempted]
for offset in range(0,len(pending),a.workers*2):
jobs=[(i,kind) for i in pending[offset:offset+a.workers*2]
if counts[(task_of(RAW.sessions[i]),kind)]<target[(task_of(RAW.sessions[i]),kind)]]
blocked=tuple(asset for asset,n in assets.items() if n>=a.max_qa_reuse)
for job,row in zip(jobs,pool.map(build,[(i,k,blocked) for i,k in jobs])):
attempted.add(job)
if 'rejected' in row:
rejected.append(row); continue
key=(row['task'],kind)
if counts[key]>=target[key]:
discard_unselected(row,a.output)
continue
if assets[row['overlay']['asset_id']]>=a.max_qa_reuse:
rejected.append(dict(dataset_index=job[0],kind=kind,rejected='qa_reuse_cap'))
discard_unselected(row,a.output);continue
rows.append(row);used.add(job[0]);counts[key]+=1;assets[row['overlay']['asset_id']]+=1
write_json(progress,dict(identity=identity,rows=rows,rejections=rejected,attempted=sorted(attempted)))
print('SELECTED',len(rows),'/',a.count,dict(counts),flush=True)
if a.stop_after and len(rows)>=a.stop_after:
return
if all(counts[(t,kind)]==target[(t,kind)] for t in wanted):
break
if len(rows)!=a.count:
raise RuntimeError(f'Insufficient valid sessions: {dict(counts)} targets={target}; no silent fallback')
random.Random(a.seed).shuffle(rows)
summary=dict(candidate_sessions=len(RAW),tasks=dict(Counter(r['task'] for r in rows)),
task_families=dict(Counter(r['task_family'] for r in rows)),modalities=dict(Counter(r['visual_kind'] for r in rows)),
turns=dict(Counter(r['turn_count'] for r in rows)),voices=dict(Counter(r['voice'] for r in rows)),
quality_pools=quality_counts,unique_visual_qa=len(assets),max_visual_qa_reuse=max(assets.values()),
total_source_occurrences=sum(r['turn_count'] for r in rows),
unique_motion_sources=len({s['motion_name'] for r in rows for s in r['segments']}),
unique_final_commands=len({s['caption'] for r in rows for s in r['segments']}),
semantic_category_occurrences=dict(Counter(c for r in rows for s in r['segments'] for c in s['semantic_categories'])),
session_seconds_percentiles=dict(zip(('min','p25','median','p75','p95','max'),
np.percentile([r['reference_frames']/50 for r in rows],[0,25,50,75,95,100]).tolist())),
eligible_motion_segments=sum(s['eligible'] for r in rows for s in r['segments']),
excluded_motion_segments=dict(Counter(s['exclusion_reason'] for r in rows for s in r['segments'] if not s['eligible'])),
rejections=dict(Counter(r['rejected'] for r in rejected)))
write_json(a.output/'selection_summary.json',summary)
write_json(a.output/'manifest.json',dict(**identity,split='val',selection_summary=summary,rows=rows,
protocol=f'Natural 1-4-turn held-out sessions; six task families balanced; {(1-image_ratio)*100:g}% video QA; GT-only selection'))
if __name__=='__main__':
main()
|