Download benchmark/prepare.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 12.7 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/prepare.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/prepare.py
-
curl -L -o prepare.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/prepare.py
12.7 kB
| """Freeze complete multimodal validation sessions with the formal training loader.""" | |
| import argparse | |
| from collections import Counter | |
| from concurrent.futures import ProcessPoolExecutor | |
| import multiprocessing | |
| import json | |
| import os | |
| import random | |
| import sys | |
| from pathlib import Path | |
| from common import ROOT, WORK, DEFAULT_RUN, sha256, write_json | |
| from session_data import task_of, quotas, candidate_order, quality_asset, motion_segments | |
| sys.path[:0] = [str(ROOT), str(ROOT/'src')] | |
| os.environ.setdefault('HFD_STARTUP_INDEX_CACHE_DIR', str(WORK/'outputs/startup_cache')) | |
| import numpy as np | |
| import torch | |
| def build(job): | |
| index, kind, blocked_assets = job | |
| from scripts.data.render_hfd_hesitation_pause_gt import reconstruct_motion | |
| from psi.data.seed.live_multimodal_overlay import continuous_audio | |
| from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexTransformedDataset | |
| from scipy.io import wavfile | |
| torch.set_num_threads(1) | |
| sid = f'hfd_session_{index:05d}' | |
| dest = ARGS.output | |
| done = dest/f'{sid}.json' | |
| if done.exists(): | |
| row = json.loads(done.read_text()) | |
| if row['visual_kind'] != kind: | |
| return dict(rejected='previous_other_modality', dataset_index=index) | |
| if row['overlay']['asset_id'] not in blocked_assets: | |
| return row | |
| random.seed(ARGS.seed+index); np.random.seed(ARGS.seed+index); torch.manual_seed(ARGS.seed+index) | |
| voice = RAW.sessions[index][0]['_session_voice_name'] | |
| OVERLAY.by_voice = {voice: [r for r in POOLS[(voice, kind)] if r['asset_id'] not in blocked_assets]} | |
| source = OVERLAY.compose(RAW[index], 'qa') | |
| relation = source['multimodal_overlay'] | |
| if relation['mode'] != 'qa': | |
| return dict(rejected=relation.get('fallback_reason', relation['mode']), dataset_index=index, kind=kind) | |
| gt, controls = reconstruct_motion(source) | |
| segments, turns = motion_segments(source, RAW.sessions[index], controls) | |
| audio = continuous_audio(source) | |
| emitted = 20*sum(bool(s['update_semantic_cache']) for s in source['steps']) | |
| if emitted < len(gt)-1: | |
| raise ValueError('Rollout fails to cover entire reference') | |
| if not np.isfinite(gt).all() or not np.isfinite(audio).all(): | |
| raise ValueError('Nonfinite GT or microphone') | |
| wavfile.write(dest/f'{sid}_microphone.wav', 16000, (np.clip(audio,-1,1)*32767).astype(np.int16)) | |
| np.savez_compressed(dest/f'{sid}_reference.npz', absolute=gt[1:], controls=controls[1:], turn_indices=turns) | |
| event_keys = ('microphone_speech_intervals','turn_listen_start_frames','turn_speech_start_frames', | |
| 'turn_listen_reaction_frames','control_command_events','backchannel_events','disfluency_events', | |
| 'interruption_start_frames','interruption_remaining_frames','third_party_interference_intervals') | |
| events = {k:source[k] for k in event_keys if k in source} | |
| metadata = dict(sample_id=sid, source_id=source['source_id'], dataset_index=index, | |
| task=task_of(RAW.sessions[index]), task_family=relation['task'], turn_count=len(segments), | |
| third_party_profile=source.get('third_party_profile'), disfluency_subtype=source.get('disfluency_subtype'), | |
| voice=voice, split='val', visual_kind=kind, overlay=relation, events=events, segments=segments, | |
| source_file=f'{sid}.pt', reference_file=f'{sid}_reference.npz', microphone_file=f'{sid}_microphone.wav', | |
| reference_frames=len(gt)-1, score_end=len(gt)-1, | |
| clocks=dict(motion_fps=50, microphone_sample_rate=16000, motion_origin_in_microphone_seconds=.2), | |
| reference_answer=dict(text=relation['answer'], earliest_seconds=relation['question_end'], | |
| reference_waveform=None, note='Text GT; generated speech is an output, never an input.'), | |
| input_provenance=dict(run_config_sha256=RUN_HASH, seed=ARGS.seed+index)) | |
| # Deployment uses semantic packets. Keep full GT in a separate reference. | |
| source['steps'] = [s for s in source['steps'] if s['update_semantic_cache']] | |
| item = FullDuplexTransformedDataset([source], CONFIG.data.transform, {'vlm_processor':PROCESSOR})[0] | |
| if len(item['swa_packets'])*20 < metadata['reference_frames']: | |
| raise ValueError('Transformed packet coverage mismatch') | |
| tmp = dest/f'{sid}.pt.tmp'; torch.save(item,tmp); tmp.replace(dest/f'{sid}.pt') | |
| for key in ('source_file','reference_file','microphone_file'): | |
| metadata[key.replace('_file','_sha256')] = sha256(dest/metadata[key]) | |
| write_json(done,metadata) | |
| print('PREPARED', sid, metadata['task'],kind,len(segments),round(len(gt)/50,2),flush=True) | |
| return metadata | |
| def discard_unselected(row, destination): | |
| """Only delete this preparer's completed, unselected, reproducible outputs.""" | |
| sid=f"hfd_session_{int(row['dataset_index']):05d}" | |
| if row['sample_id']!=sid:raise ValueError('Unsafe unselected sample identity') | |
| for suffix in ('.json','.pt','_microphone.wav','_reference.npz'): | |
| path=destination/(sid+suffix) | |
| if path.is_symlink():raise ValueError('Refusing to delete symlinked sample') | |
| path.unlink(missing_ok=True) | |
| def main(): | |
| global ARGS, RAW, OVERLAY, CONFIG, PROCESSOR, POOLS, RUN_HASH | |
| p=argparse.ArgumentParser() | |
| p.add_argument('--run',type=Path,default=DEFAULT_RUN) | |
| p.add_argument('--output',type=Path,default=WORK/'outputs/sessions2048') | |
| p.add_argument('--count',type=int,default=2048) | |
| p.add_argument('--seed',type=int,default=42) | |
| p.add_argument('--workers',type=int,default=8) | |
| p.add_argument('--max-qa-reuse',type=int,default=64,help='Maximum sessions per original visual QA across all voices') | |
| p.add_argument('--stop-after',type=int,default=0,help='Pause for human review without freezing manifest') | |
| p.add_argument('--external-assets',type=Path,help='Reviewed external QA manifest; replaces all training-derived visuals') | |
| p.add_argument('--reuse-cohort',type=Path,help='Keep hash-identical samples admitted by the revised visual pool') | |
| ARGS=p.parse_args(); a=ARGS | |
| if a.max_qa_reuse<1:raise ValueError('max-qa-reuse must be positive') | |
| a.output.mkdir(parents=True,exist_ok=True) | |
| if (a.output/'manifest.json').exists(): | |
| raise FileExistsError('Frozen cohort already exists') | |
| torch.set_num_threads(1) | |
| from scripts.infer.unified_overlay import cfg | |
| from transformers import Qwen2_5OmniProcessor | |
| CONFIG=cfg(a.run); RUN_HASH=sha256(a.run/'run_config.json') | |
| PROCESSOR=Qwen2_5OmniProcessor.from_pretrained(CONFIG.model.model_name_or_path,local_files_only=True) | |
| OVERLAY=CONFIG.data(split='val',transform_kwargs={'vlm_processor':PROCESSOR}).raw_dataset | |
| RAW=OVERLAY.raw_dataset; POOLS={}; quality_counts={} | |
| if a.external_assets: | |
| from session_overlay import ExternalOverlay, reviewed | |
| payload=json.loads(a.external_assets.read_text()) | |
| if payload.get('format')!='hfd_bench_external_assets_v1':raise ValueError('Wrong external QA format') | |
| replacement=ExternalOverlay(OVERLAY.policy,seed=a.seed) | |
| for row in payload['assets']: | |
| if not reviewed(row):raise ValueError('External QA lacks verified review/ASR: '+row['asset_id']) | |
| replacement.by_voice.setdefault(row['voice'],[]).append(row) | |
| OVERLAY=replacement | |
| for voice, rows in OVERLAY.by_voice.items(): | |
| for kind in ('video','image'): | |
| eligible=[r for r in rows if bool(r.get('video_path')) == (kind=='video')] | |
| POOLS[(voice,kind)]=eligible if a.external_assets else [r for r in eligible if quality_asset(r)] | |
| quality_counts[f'{voice}/{kind}']=dict(before=len(eligible),after=len(POOLS[(voice,kind)])) | |
| print('QUALITY_POOLS',json.dumps(quality_counts),flush=True) | |
| wanted=quotas(a.count) | |
| image_ratio=0. if a.external_assets else .05 | |
| target={(task,kind):(round(n*image_ratio) if kind=='image' else n-round(n*image_ratio)) | |
| for task,n in wanted.items() for kind in ('video','image')} | |
| identity=dict(run=str(a.run.resolve()),run_config_sha256=RUN_HASH,count=a.count,seed=a.seed, | |
| schema='hfd_bench_session_v3',max_qa_reuse=a.max_qa_reuse) | |
| if a.external_assets: | |
| identity['external_assets_sha256']=sha256(a.external_assets) | |
| identity['external_assets_path']=str(a.external_assets.resolve()) | |
| progress=a.output/'selection_progress.json' | |
| if a.reuse_cohort and not progress.exists(): | |
| if not a.external_assets:raise ValueError('Cohort revision requires explicit external assets') | |
| from revision import seed_cohort | |
| write_json(progress,seed_cohort(a.reuse_cohort,a.output,identity,payload['assets'])) | |
| prior=json.loads(progress.read_text()) if progress.exists() else dict(identity=identity,rows=[],rejections=[],attempted=[]) | |
| if prior['identity']!=identity: | |
| raise ValueError('Selection identity changed') | |
| rows=prior['rows']; rejected=prior['rejections']; attempted={tuple(x) for x in prior['attempted']} | |
| counts=Counter((r['task'],r['visual_kind']) for r in rows) | |
| used={r['dataset_index'] for r in rows}; assets=Counter(r['overlay']['asset_id'] for r in rows) | |
| order=candidate_order(RAW,a.seed) | |
| with ProcessPoolExecutor(max_workers=a.workers,mp_context=multiprocessing.get_context('fork')) as pool: | |
| for kind in ('video','image'): | |
| pending=[i for i in order if i not in used and (i,kind) not in attempted] | |
| for offset in range(0,len(pending),a.workers*2): | |
| jobs=[(i,kind) for i in pending[offset:offset+a.workers*2] | |
| if counts[(task_of(RAW.sessions[i]),kind)]<target[(task_of(RAW.sessions[i]),kind)]] | |
| blocked=tuple(asset for asset,n in assets.items() if n>=a.max_qa_reuse) | |
| for job,row in zip(jobs,pool.map(build,[(i,k,blocked) for i,k in jobs])): | |
| attempted.add(job) | |
| if 'rejected' in row: | |
| rejected.append(row); continue | |
| key=(row['task'],kind) | |
| if counts[key]>=target[key]: | |
| discard_unselected(row,a.output) | |
| continue | |
| if assets[row['overlay']['asset_id']]>=a.max_qa_reuse: | |
| rejected.append(dict(dataset_index=job[0],kind=kind,rejected='qa_reuse_cap')) | |
| discard_unselected(row,a.output);continue | |
| rows.append(row);used.add(job[0]);counts[key]+=1;assets[row['overlay']['asset_id']]+=1 | |
| write_json(progress,dict(identity=identity,rows=rows,rejections=rejected,attempted=sorted(attempted))) | |
| print('SELECTED',len(rows),'/',a.count,dict(counts),flush=True) | |
| if a.stop_after and len(rows)>=a.stop_after: | |
| return | |
| if all(counts[(t,kind)]==target[(t,kind)] for t in wanted): | |
| break | |
| if len(rows)!=a.count: | |
| raise RuntimeError(f'Insufficient valid sessions: {dict(counts)} targets={target}; no silent fallback') | |
| random.Random(a.seed).shuffle(rows) | |
| summary=dict(candidate_sessions=len(RAW),tasks=dict(Counter(r['task'] for r in rows)), | |
| task_families=dict(Counter(r['task_family'] for r in rows)),modalities=dict(Counter(r['visual_kind'] for r in rows)), | |
| turns=dict(Counter(r['turn_count'] for r in rows)),voices=dict(Counter(r['voice'] for r in rows)), | |
| quality_pools=quality_counts,unique_visual_qa=len(assets),max_visual_qa_reuse=max(assets.values()), | |
| total_source_occurrences=sum(r['turn_count'] for r in rows), | |
| unique_motion_sources=len({s['motion_name'] for r in rows for s in r['segments']}), | |
| unique_final_commands=len({s['caption'] for r in rows for s in r['segments']}), | |
| semantic_category_occurrences=dict(Counter(c for r in rows for s in r['segments'] for c in s['semantic_categories'])), | |
| session_seconds_percentiles=dict(zip(('min','p25','median','p75','p95','max'), | |
| np.percentile([r['reference_frames']/50 for r in rows],[0,25,50,75,95,100]).tolist())), | |
| eligible_motion_segments=sum(s['eligible'] for r in rows for s in r['segments']), | |
| excluded_motion_segments=dict(Counter(s['exclusion_reason'] for r in rows for s in r['segments'] if not s['eligible'])), | |
| rejections=dict(Counter(r['rejected'] for r in rejected))) | |
| write_json(a.output/'selection_summary.json',summary) | |
| write_json(a.output/'manifest.json',dict(**identity,split='val',selection_summary=summary,rows=rows, | |
| protocol=f'Natural 1-4-turn held-out sessions; six task families balanced; {(1-image_ratio)*100:g}% video QA; GT-only selection')) | |
| if __name__=='__main__': | |
| main() | |