File size: 12,686 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
"""Freeze complete multimodal validation sessions with the formal training loader."""
import argparse
from collections import Counter
from concurrent.futures import ProcessPoolExecutor
import multiprocessing
import json
import os
import random
import sys
from pathlib import Path
from common import ROOT, WORK, DEFAULT_RUN, sha256, write_json
from session_data import task_of, quotas, candidate_order, quality_asset, motion_segments
sys.path[:0] = [str(ROOT), str(ROOT/'src')]
os.environ.setdefault('HFD_STARTUP_INDEX_CACHE_DIR', str(WORK/'outputs/startup_cache'))
import numpy as np
import torch


def build(job):
    index, kind, blocked_assets = job
    from scripts.data.render_hfd_hesitation_pause_gt import reconstruct_motion
    from psi.data.seed.live_multimodal_overlay import continuous_audio
    from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexTransformedDataset
    from scipy.io import wavfile
    torch.set_num_threads(1)
    sid = f'hfd_session_{index:05d}'
    dest = ARGS.output
    done = dest/f'{sid}.json'
    if done.exists():
        row = json.loads(done.read_text())
        if row['visual_kind'] != kind:
            return dict(rejected='previous_other_modality', dataset_index=index)
        if row['overlay']['asset_id'] not in blocked_assets:
            return row
    random.seed(ARGS.seed+index); np.random.seed(ARGS.seed+index); torch.manual_seed(ARGS.seed+index)
    voice = RAW.sessions[index][0]['_session_voice_name']
    OVERLAY.by_voice = {voice: [r for r in POOLS[(voice, kind)] if r['asset_id'] not in blocked_assets]}
    source = OVERLAY.compose(RAW[index], 'qa')
    relation = source['multimodal_overlay']
    if relation['mode'] != 'qa':
        return dict(rejected=relation.get('fallback_reason', relation['mode']), dataset_index=index, kind=kind)
    gt, controls = reconstruct_motion(source)
    segments, turns = motion_segments(source, RAW.sessions[index], controls)
    audio = continuous_audio(source)
    emitted = 20*sum(bool(s['update_semantic_cache']) for s in source['steps'])
    if emitted < len(gt)-1:
        raise ValueError('Rollout fails to cover entire reference')
    if not np.isfinite(gt).all() or not np.isfinite(audio).all():
        raise ValueError('Nonfinite GT or microphone')
    wavfile.write(dest/f'{sid}_microphone.wav', 16000, (np.clip(audio,-1,1)*32767).astype(np.int16))
    np.savez_compressed(dest/f'{sid}_reference.npz', absolute=gt[1:], controls=controls[1:], turn_indices=turns)
    event_keys = ('microphone_speech_intervals','turn_listen_start_frames','turn_speech_start_frames',
                  'turn_listen_reaction_frames','control_command_events','backchannel_events','disfluency_events',
                  'interruption_start_frames','interruption_remaining_frames','third_party_interference_intervals')
    events = {k:source[k] for k in event_keys if k in source}
    metadata = dict(sample_id=sid, source_id=source['source_id'], dataset_index=index,
        task=task_of(RAW.sessions[index]), task_family=relation['task'], turn_count=len(segments),
        third_party_profile=source.get('third_party_profile'), disfluency_subtype=source.get('disfluency_subtype'),
        voice=voice, split='val', visual_kind=kind, overlay=relation, events=events, segments=segments,
        source_file=f'{sid}.pt', reference_file=f'{sid}_reference.npz', microphone_file=f'{sid}_microphone.wav',
        reference_frames=len(gt)-1, score_end=len(gt)-1,
        clocks=dict(motion_fps=50, microphone_sample_rate=16000, motion_origin_in_microphone_seconds=.2),
        reference_answer=dict(text=relation['answer'], earliest_seconds=relation['question_end'],
                              reference_waveform=None, note='Text GT; generated speech is an output, never an input.'),
        input_provenance=dict(run_config_sha256=RUN_HASH, seed=ARGS.seed+index))
    # Deployment uses semantic packets. Keep full GT in a separate reference.
    source['steps'] = [s for s in source['steps'] if s['update_semantic_cache']]
    item = FullDuplexTransformedDataset([source], CONFIG.data.transform, {'vlm_processor':PROCESSOR})[0]
    if len(item['swa_packets'])*20 < metadata['reference_frames']:
        raise ValueError('Transformed packet coverage mismatch')
    tmp = dest/f'{sid}.pt.tmp'; torch.save(item,tmp); tmp.replace(dest/f'{sid}.pt')
    for key in ('source_file','reference_file','microphone_file'):
        metadata[key.replace('_file','_sha256')] = sha256(dest/metadata[key])
    write_json(done,metadata)
    print('PREPARED', sid, metadata['task'],kind,len(segments),round(len(gt)/50,2),flush=True)
    return metadata


def discard_unselected(row, destination):
    """Only delete this preparer's completed, unselected, reproducible outputs."""
    sid=f"hfd_session_{int(row['dataset_index']):05d}"
    if row['sample_id']!=sid:raise ValueError('Unsafe unselected sample identity')
    for suffix in ('.json','.pt','_microphone.wav','_reference.npz'):
        path=destination/(sid+suffix)
        if path.is_symlink():raise ValueError('Refusing to delete symlinked sample')
        path.unlink(missing_ok=True)


def main():
    global ARGS, RAW, OVERLAY, CONFIG, PROCESSOR, POOLS, RUN_HASH
    p=argparse.ArgumentParser()
    p.add_argument('--run',type=Path,default=DEFAULT_RUN)
    p.add_argument('--output',type=Path,default=WORK/'outputs/sessions2048')
    p.add_argument('--count',type=int,default=2048)
    p.add_argument('--seed',type=int,default=42)
    p.add_argument('--workers',type=int,default=8)
    p.add_argument('--max-qa-reuse',type=int,default=64,help='Maximum sessions per original visual QA across all voices')
    p.add_argument('--stop-after',type=int,default=0,help='Pause for human review without freezing manifest')
    p.add_argument('--external-assets',type=Path,help='Reviewed external QA manifest; replaces all training-derived visuals')
    p.add_argument('--reuse-cohort',type=Path,help='Keep hash-identical samples admitted by the revised visual pool')
    ARGS=p.parse_args(); a=ARGS
    if a.max_qa_reuse<1:raise ValueError('max-qa-reuse must be positive')
    a.output.mkdir(parents=True,exist_ok=True)
    if (a.output/'manifest.json').exists():
        raise FileExistsError('Frozen cohort already exists')
    torch.set_num_threads(1)
    from scripts.infer.unified_overlay import cfg
    from transformers import Qwen2_5OmniProcessor
    CONFIG=cfg(a.run); RUN_HASH=sha256(a.run/'run_config.json')
    PROCESSOR=Qwen2_5OmniProcessor.from_pretrained(CONFIG.model.model_name_or_path,local_files_only=True)
    OVERLAY=CONFIG.data(split='val',transform_kwargs={'vlm_processor':PROCESSOR}).raw_dataset
    RAW=OVERLAY.raw_dataset; POOLS={}; quality_counts={}
    if a.external_assets:
        from session_overlay import ExternalOverlay, reviewed
        payload=json.loads(a.external_assets.read_text())
        if payload.get('format')!='hfd_bench_external_assets_v1':raise ValueError('Wrong external QA format')
        replacement=ExternalOverlay(OVERLAY.policy,seed=a.seed)
        for row in payload['assets']:
            if not reviewed(row):raise ValueError('External QA lacks verified review/ASR: '+row['asset_id'])
            replacement.by_voice.setdefault(row['voice'],[]).append(row)
        OVERLAY=replacement
    for voice, rows in OVERLAY.by_voice.items():
        for kind in ('video','image'):
            eligible=[r for r in rows if bool(r.get('video_path')) == (kind=='video')]
            POOLS[(voice,kind)]=eligible if a.external_assets else [r for r in eligible if quality_asset(r)]
            quality_counts[f'{voice}/{kind}']=dict(before=len(eligible),after=len(POOLS[(voice,kind)]))
    print('QUALITY_POOLS',json.dumps(quality_counts),flush=True)
    wanted=quotas(a.count)
    image_ratio=0. if a.external_assets else .05
    target={(task,kind):(round(n*image_ratio) if kind=='image' else n-round(n*image_ratio))
            for task,n in wanted.items() for kind in ('video','image')}
    identity=dict(run=str(a.run.resolve()),run_config_sha256=RUN_HASH,count=a.count,seed=a.seed,
                  schema='hfd_bench_session_v3',max_qa_reuse=a.max_qa_reuse)
    if a.external_assets:
        identity['external_assets_sha256']=sha256(a.external_assets)
        identity['external_assets_path']=str(a.external_assets.resolve())
    progress=a.output/'selection_progress.json'
    if a.reuse_cohort and not progress.exists():
        if not a.external_assets:raise ValueError('Cohort revision requires explicit external assets')
        from revision import seed_cohort
        write_json(progress,seed_cohort(a.reuse_cohort,a.output,identity,payload['assets']))
    prior=json.loads(progress.read_text()) if progress.exists() else dict(identity=identity,rows=[],rejections=[],attempted=[])
    if prior['identity']!=identity:
        raise ValueError('Selection identity changed')
    rows=prior['rows']; rejected=prior['rejections']; attempted={tuple(x) for x in prior['attempted']}
    counts=Counter((r['task'],r['visual_kind']) for r in rows)
    used={r['dataset_index'] for r in rows}; assets=Counter(r['overlay']['asset_id'] for r in rows)
    order=candidate_order(RAW,a.seed)
    with ProcessPoolExecutor(max_workers=a.workers,mp_context=multiprocessing.get_context('fork')) as pool:
        for kind in ('video','image'):
            pending=[i for i in order if i not in used and (i,kind) not in attempted]
            for offset in range(0,len(pending),a.workers*2):
                jobs=[(i,kind) for i in pending[offset:offset+a.workers*2]
                      if counts[(task_of(RAW.sessions[i]),kind)]<target[(task_of(RAW.sessions[i]),kind)]]
                blocked=tuple(asset for asset,n in assets.items() if n>=a.max_qa_reuse)
                for job,row in zip(jobs,pool.map(build,[(i,k,blocked) for i,k in jobs])):
                    attempted.add(job)
                    if 'rejected' in row:
                        rejected.append(row); continue
                    key=(row['task'],kind)
                    if counts[key]>=target[key]:
                        discard_unselected(row,a.output)
                        continue
                    if assets[row['overlay']['asset_id']]>=a.max_qa_reuse:
                        rejected.append(dict(dataset_index=job[0],kind=kind,rejected='qa_reuse_cap'))
                        discard_unselected(row,a.output);continue
                    rows.append(row);used.add(job[0]);counts[key]+=1;assets[row['overlay']['asset_id']]+=1
                write_json(progress,dict(identity=identity,rows=rows,rejections=rejected,attempted=sorted(attempted)))
                print('SELECTED',len(rows),'/',a.count,dict(counts),flush=True)
                if a.stop_after and len(rows)>=a.stop_after:
                    return
                if all(counts[(t,kind)]==target[(t,kind)] for t in wanted):
                    break
    if len(rows)!=a.count:
        raise RuntimeError(f'Insufficient valid sessions: {dict(counts)} targets={target}; no silent fallback')
    random.Random(a.seed).shuffle(rows)
    summary=dict(candidate_sessions=len(RAW),tasks=dict(Counter(r['task'] for r in rows)),
        task_families=dict(Counter(r['task_family'] for r in rows)),modalities=dict(Counter(r['visual_kind'] for r in rows)),
        turns=dict(Counter(r['turn_count'] for r in rows)),voices=dict(Counter(r['voice'] for r in rows)),
        quality_pools=quality_counts,unique_visual_qa=len(assets),max_visual_qa_reuse=max(assets.values()),
        total_source_occurrences=sum(r['turn_count'] for r in rows),
        unique_motion_sources=len({s['motion_name'] for r in rows for s in r['segments']}),
        unique_final_commands=len({s['caption'] for r in rows for s in r['segments']}),
        semantic_category_occurrences=dict(Counter(c for r in rows for s in r['segments'] for c in s['semantic_categories'])),
        session_seconds_percentiles=dict(zip(('min','p25','median','p75','p95','max'),
            np.percentile([r['reference_frames']/50 for r in rows],[0,25,50,75,95,100]).tolist())),
        eligible_motion_segments=sum(s['eligible'] for r in rows for s in r['segments']),
        excluded_motion_segments=dict(Counter(s['exclusion_reason'] for r in rows for s in r['segments'] if not s['eligible'])),
        rejections=dict(Counter(r['rejected'] for r in rejected)))
    write_json(a.output/'selection_summary.json',summary)
    write_json(a.output/'manifest.json',dict(**identity,split='val',selection_summary=summary,rows=rows,
        protocol=f'Natural 1-4-turn held-out sessions; six task families balanced; {(1-image_ratio)*100:g}% video QA; GT-only selection'))


if __name__=='__main__':
    main()