code-HFD / benchmark /prepare.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
fb0074a verified
Raw History Blame Contribute Delete
12.7 kB
"""Freeze complete multimodal validation sessions with the formal training loader."""
import argparse
from collections import Counter
from concurrent.futures import ProcessPoolExecutor
import multiprocessing
import json
import os
import random
import sys
from pathlib import Path
from common import ROOT, WORK, DEFAULT_RUN, sha256, write_json
from session_data import task_of, quotas, candidate_order, quality_asset, motion_segments
sys.path[:0] = [str(ROOT), str(ROOT/'src')]
os.environ.setdefault('HFD_STARTUP_INDEX_CACHE_DIR', str(WORK/'outputs/startup_cache'))
import numpy as np
import torch
def build(job):
index, kind, blocked_assets = job
from scripts.data.render_hfd_hesitation_pause_gt import reconstruct_motion
from psi.data.seed.live_multimodal_overlay import continuous_audio
from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexTransformedDataset
from scipy.io import wavfile
torch.set_num_threads(1)
sid = f'hfd_session_{index:05d}'
dest = ARGS.output
done = dest/f'{sid}.json'
if done.exists():
row = json.loads(done.read_text())
if row['visual_kind'] != kind:
return dict(rejected='previous_other_modality', dataset_index=index)
if row['overlay']['asset_id'] not in blocked_assets:
return row
random.seed(ARGS.seed+index); np.random.seed(ARGS.seed+index); torch.manual_seed(ARGS.seed+index)
voice = RAW.sessions[index][0]['_session_voice_name']
OVERLAY.by_voice = {voice: [r for r in POOLS[(voice, kind)] if r['asset_id'] not in blocked_assets]}
source = OVERLAY.compose(RAW[index], 'qa')
relation = source['multimodal_overlay']
if relation['mode'] != 'qa':
return dict(rejected=relation.get('fallback_reason', relation['mode']), dataset_index=index, kind=kind)
gt, controls = reconstruct_motion(source)
segments, turns = motion_segments(source, RAW.sessions[index], controls)
audio = continuous_audio(source)
emitted = 20*sum(bool(s['update_semantic_cache']) for s in source['steps'])
if emitted < len(gt)-1:
raise ValueError('Rollout fails to cover entire reference')
if not np.isfinite(gt).all() or not np.isfinite(audio).all():
raise ValueError('Nonfinite GT or microphone')
wavfile.write(dest/f'{sid}_microphone.wav', 16000, (np.clip(audio,-1,1)*32767).astype(np.int16))
np.savez_compressed(dest/f'{sid}_reference.npz', absolute=gt[1:], controls=controls[1:], turn_indices=turns)
event_keys = ('microphone_speech_intervals','turn_listen_start_frames','turn_speech_start_frames',
'turn_listen_reaction_frames','control_command_events','backchannel_events','disfluency_events',
'interruption_start_frames','interruption_remaining_frames','third_party_interference_intervals')
events = {k:source[k] for k in event_keys if k in source}
metadata = dict(sample_id=sid, source_id=source['source_id'], dataset_index=index,
task=task_of(RAW.sessions[index]), task_family=relation['task'], turn_count=len(segments),
third_party_profile=source.get('third_party_profile'), disfluency_subtype=source.get('disfluency_subtype'),
voice=voice, split='val', visual_kind=kind, overlay=relation, events=events, segments=segments,
source_file=f'{sid}.pt', reference_file=f'{sid}_reference.npz', microphone_file=f'{sid}_microphone.wav',
reference_frames=len(gt)-1, score_end=len(gt)-1,
clocks=dict(motion_fps=50, microphone_sample_rate=16000, motion_origin_in_microphone_seconds=.2),
reference_answer=dict(text=relation['answer'], earliest_seconds=relation['question_end'],
reference_waveform=None, note='Text GT; generated speech is an output, never an input.'),
input_provenance=dict(run_config_sha256=RUN_HASH, seed=ARGS.seed+index))
# Deployment uses semantic packets. Keep full GT in a separate reference.
source['steps'] = [s for s in source['steps'] if s['update_semantic_cache']]
item = FullDuplexTransformedDataset([source], CONFIG.data.transform, {'vlm_processor':PROCESSOR})[0]
if len(item['swa_packets'])*20 < metadata['reference_frames']:
raise ValueError('Transformed packet coverage mismatch')
tmp = dest/f'{sid}.pt.tmp'; torch.save(item,tmp); tmp.replace(dest/f'{sid}.pt')
for key in ('source_file','reference_file','microphone_file'):
metadata[key.replace('_file','_sha256')] = sha256(dest/metadata[key])
write_json(done,metadata)
print('PREPARED', sid, metadata['task'],kind,len(segments),round(len(gt)/50,2),flush=True)
return metadata
def discard_unselected(row, destination):
"""Only delete this preparer's completed, unselected, reproducible outputs."""
sid=f"hfd_session_{int(row['dataset_index']):05d}"
if row['sample_id']!=sid:raise ValueError('Unsafe unselected sample identity')
for suffix in ('.json','.pt','_microphone.wav','_reference.npz'):
path=destination/(sid+suffix)
if path.is_symlink():raise ValueError('Refusing to delete symlinked sample')
path.unlink(missing_ok=True)
def main():
global ARGS, RAW, OVERLAY, CONFIG, PROCESSOR, POOLS, RUN_HASH
p=argparse.ArgumentParser()
p.add_argument('--run',type=Path,default=DEFAULT_RUN)
p.add_argument('--output',type=Path,default=WORK/'outputs/sessions2048')
p.add_argument('--count',type=int,default=2048)
p.add_argument('--seed',type=int,default=42)
p.add_argument('--workers',type=int,default=8)
p.add_argument('--max-qa-reuse',type=int,default=64,help='Maximum sessions per original visual QA across all voices')
p.add_argument('--stop-after',type=int,default=0,help='Pause for human review without freezing manifest')
p.add_argument('--external-assets',type=Path,help='Reviewed external QA manifest; replaces all training-derived visuals')
p.add_argument('--reuse-cohort',type=Path,help='Keep hash-identical samples admitted by the revised visual pool')
ARGS=p.parse_args(); a=ARGS
if a.max_qa_reuse<1:raise ValueError('max-qa-reuse must be positive')
a.output.mkdir(parents=True,exist_ok=True)
if (a.output/'manifest.json').exists():
raise FileExistsError('Frozen cohort already exists')
torch.set_num_threads(1)
from scripts.infer.unified_overlay import cfg
from transformers import Qwen2_5OmniProcessor
CONFIG=cfg(a.run); RUN_HASH=sha256(a.run/'run_config.json')
PROCESSOR=Qwen2_5OmniProcessor.from_pretrained(CONFIG.model.model_name_or_path,local_files_only=True)
OVERLAY=CONFIG.data(split='val',transform_kwargs={'vlm_processor':PROCESSOR}).raw_dataset
RAW=OVERLAY.raw_dataset; POOLS={}; quality_counts={}
if a.external_assets:
from session_overlay import ExternalOverlay, reviewed
payload=json.loads(a.external_assets.read_text())
if payload.get('format')!='hfd_bench_external_assets_v1':raise ValueError('Wrong external QA format')
replacement=ExternalOverlay(OVERLAY.policy,seed=a.seed)
for row in payload['assets']:
if not reviewed(row):raise ValueError('External QA lacks verified review/ASR: '+row['asset_id'])
replacement.by_voice.setdefault(row['voice'],[]).append(row)
OVERLAY=replacement
for voice, rows in OVERLAY.by_voice.items():
for kind in ('video','image'):
eligible=[r for r in rows if bool(r.get('video_path')) == (kind=='video')]
POOLS[(voice,kind)]=eligible if a.external_assets else [r for r in eligible if quality_asset(r)]
quality_counts[f'{voice}/{kind}']=dict(before=len(eligible),after=len(POOLS[(voice,kind)]))
print('QUALITY_POOLS',json.dumps(quality_counts),flush=True)
wanted=quotas(a.count)
image_ratio=0. if a.external_assets else .05
target={(task,kind):(round(n*image_ratio) if kind=='image' else n-round(n*image_ratio))
for task,n in wanted.items() for kind in ('video','image')}
identity=dict(run=str(a.run.resolve()),run_config_sha256=RUN_HASH,count=a.count,seed=a.seed,
schema='hfd_bench_session_v3',max_qa_reuse=a.max_qa_reuse)
if a.external_assets:
identity['external_assets_sha256']=sha256(a.external_assets)
identity['external_assets_path']=str(a.external_assets.resolve())
progress=a.output/'selection_progress.json'
if a.reuse_cohort and not progress.exists():
if not a.external_assets:raise ValueError('Cohort revision requires explicit external assets')
from revision import seed_cohort
write_json(progress,seed_cohort(a.reuse_cohort,a.output,identity,payload['assets']))
prior=json.loads(progress.read_text()) if progress.exists() else dict(identity=identity,rows=[],rejections=[],attempted=[])
if prior['identity']!=identity:
raise ValueError('Selection identity changed')
rows=prior['rows']; rejected=prior['rejections']; attempted={tuple(x) for x in prior['attempted']}
counts=Counter((r['task'],r['visual_kind']) for r in rows)
used={r['dataset_index'] for r in rows}; assets=Counter(r['overlay']['asset_id'] for r in rows)
order=candidate_order(RAW,a.seed)
with ProcessPoolExecutor(max_workers=a.workers,mp_context=multiprocessing.get_context('fork')) as pool:
for kind in ('video','image'):
pending=[i for i in order if i not in used and (i,kind) not in attempted]
for offset in range(0,len(pending),a.workers*2):
jobs=[(i,kind) for i in pending[offset:offset+a.workers*2]
if counts[(task_of(RAW.sessions[i]),kind)]<target[(task_of(RAW.sessions[i]),kind)]]
blocked=tuple(asset for asset,n in assets.items() if n>=a.max_qa_reuse)
for job,row in zip(jobs,pool.map(build,[(i,k,blocked) for i,k in jobs])):
attempted.add(job)
if 'rejected' in row:
rejected.append(row); continue
key=(row['task'],kind)
if counts[key]>=target[key]:
discard_unselected(row,a.output)
continue
if assets[row['overlay']['asset_id']]>=a.max_qa_reuse:
rejected.append(dict(dataset_index=job[0],kind=kind,rejected='qa_reuse_cap'))
discard_unselected(row,a.output);continue
rows.append(row);used.add(job[0]);counts[key]+=1;assets[row['overlay']['asset_id']]+=1
write_json(progress,dict(identity=identity,rows=rows,rejections=rejected,attempted=sorted(attempted)))
print('SELECTED',len(rows),'/',a.count,dict(counts),flush=True)
if a.stop_after and len(rows)>=a.stop_after:
return
if all(counts[(t,kind)]==target[(t,kind)] for t in wanted):
break
if len(rows)!=a.count:
raise RuntimeError(f'Insufficient valid sessions: {dict(counts)} targets={target}; no silent fallback')
random.Random(a.seed).shuffle(rows)
summary=dict(candidate_sessions=len(RAW),tasks=dict(Counter(r['task'] for r in rows)),
task_families=dict(Counter(r['task_family'] for r in rows)),modalities=dict(Counter(r['visual_kind'] for r in rows)),
turns=dict(Counter(r['turn_count'] for r in rows)),voices=dict(Counter(r['voice'] for r in rows)),
quality_pools=quality_counts,unique_visual_qa=len(assets),max_visual_qa_reuse=max(assets.values()),
total_source_occurrences=sum(r['turn_count'] for r in rows),
unique_motion_sources=len({s['motion_name'] for r in rows for s in r['segments']}),
unique_final_commands=len({s['caption'] for r in rows for s in r['segments']}),
semantic_category_occurrences=dict(Counter(c for r in rows for s in r['segments'] for c in s['semantic_categories'])),
session_seconds_percentiles=dict(zip(('min','p25','median','p75','p95','max'),
np.percentile([r['reference_frames']/50 for r in rows],[0,25,50,75,95,100]).tolist())),
eligible_motion_segments=sum(s['eligible'] for r in rows for s in r['segments']),
excluded_motion_segments=dict(Counter(s['exclusion_reason'] for r in rows for s in r['segments'] if not s['eligible'])),
rejections=dict(Counter(r['rejected'] for r in rejected)))
write_json(a.output/'selection_summary.json',summary)
write_json(a.output/'manifest.json',dict(**identity,split='val',selection_summary=summary,rows=rows,
protocol=f'Natural 1-4-turn held-out sessions; six task families balanced; {(1-image_ratio)*100:g}% video QA; GT-only selection'))
if __name__=='__main__':
main()