File size: 7,259 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"""Use the unchanged formal B1 visual encoder; publish only complete QA assets."""
import argparse
import json
from pathlib import Path
import sys
from types import SimpleNamespace
from common import ROOT,DEFAULT_RUN,sha256,write_json
from external_visual import DEFAULT
sys.path[:0]=[str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')]


def features(root,rank,world):
    from prepare_hfd_video_assets import encode
    plan=json.loads((root/'speech/plan.json').read_text())
    media={r['video_sha256']:r for r in plan['rows']}
    rows=[dict(asset_id=sha,video_path=r['video_path'],video_sha256=sha,
               duration_seconds=min(r['duration_seconds'],180.)) for sha,r in sorted(media.items())]
    # Only stop encoding unneeded suffix; never move the query, speed up video,
    # or shorten the observed prefix. Sessions must fit the encoded duration.
    output=root/'features'
    write_json(output/'admission/split.json',dict(rows=rows))
    config=json.loads((DEFAULT_RUN/'run_config.json').read_text())
    encode(SimpleNamespace(root=output,base_model=Path(config['model']['model_name_or_path']),
        selected_before_split=False,world=world,rank=rank,limit=0,packet_batch=1))


def publish(root,output,allowlist=None):
    from expand_hfd_video_voices import VOICES,translation_path,audio_path,canonical
    plan=json.loads((root/'speech/plan.json').read_text())
    ids={r['question']:r['id'] for r in plan['questions']}
    manual_path=root/'manual_review.json'
    manual=json.loads(manual_path.read_text())['rows'] if manual_path.exists() else []
    veto={r['id'] for r in manual if not r['accepted']}
    inspected=None
    if allowlist:
        inspection=json.loads(allowlist.read_text())
        if inspection.get('complete') is not True:
            raise ValueError('Direct visual inspection unfinished')
        inspected={r['id']:r for r in inspection['rows']}
        if len(inspected)!=len(inspection['rows']):raise ValueError('Duplicate direct inspection IDs')
    rows=[];rejections=[]
    for row in plan['rows']:
        if inspected is not None and not inspected.get(row['id'],{}).get('accepted'):
            rejections.append(dict(id=row['id'],reason='direct_inspection_not_approved'));continue
        if row['id'] in veto:
            rejections.append(dict(id=row['id'],reason='manual_visual_veto'));continue
        qtranslation=translation_path(root/'speech','question',row['question'])
        atranslation=translation_path(root/'speech','answer:'+row['question'],row['answer'])
        if not qtranslation.exists() or not atranslation.exists():
            raise ValueError('Translation review unfinished')
        if not all(json.loads(p.read_text())['accepted'] for p in (qtranslation,atranslation)):
            rejections.append(dict(id=row['id'],reason='translation_not_verified'));continue
        fpath=root/'features/admission/feature_receipts'/(row['video_sha256']+'.json')
        if not fpath.exists():raise ValueError('Feature extraction not finished: '+str(fpath))
        feature=json.loads(fpath.read_text())
        if feature.get('rejected'):
            rejections.append(dict(id=row['id'],reason='feature_decode_rejected'));continue
        if sha256(feature['path'])!=feature['sha256']:raise ValueError('Feature file changed')
        qid=ids[row['question']];variants=[]
        for voice in VOICES:
            wave=audio_path(root/'speech',qid,voice);receipt=wave.with_suffix('.json')
            if not receipt.exists():raise ValueError('TTS/ASR unfinished: '+str(receipt))
            ar=json.loads(receipt.read_text())
            if not ar.get('accepted'):break
            if (ar['wave_sha256']!=sha256(wave) or not ar['asr'].get('independent_asr')
                    or canonical(ar['asr']['heard'],voice[-2:])!=canonical(ar['text'],voice[-2:])):
                raise ValueError('Independent ASR receipt mismatch')
            question,answer=row['question'],row['answer']
            if voice.endswith('_zh'):
                q=json.loads(translation_path(root/'speech','question',question).read_text())
                ans=json.loads(translation_path(root/'speech','answer:'+question,answer).read_text())
                if not q['accepted'] or not ans['accepted']:break
                question,answer=q['translation'],ans['translation']
            if ar['text']!=question:raise ValueError('Question text changed after TTS')
            variants.append(dict(asset_id=row['id'],external_id=row['id'],voice=voice,split='val',
                data_domain='OVO-Bench',question=question,answer=answer,
                original_question=row['question'],original_answer=row['answer'],
                question_time=row['question_time'],options=row['options'],correct_option=row['correct_option'],
                video_path=row['video_path'],video_sha256=row['video_sha256'],
                duration_seconds=min(row['duration_seconds'],180.),
                original_video_duration_seconds=row['duration_seconds'],
                visual_features_path=feature['path'],visual_features_sha256=feature['sha256'],
                visual_receipt=row['visual_receipt'],visual_receipt_sha256=sha256(row['visual_receipt']),
                question_audio_path=str(wave),question_audio_sha256=ar['wave_sha256'],asr_receipt=str(receipt)))
        if len(variants)!=8:
            rejections.append(dict(id=row['id'],reason='incomplete_verified_eight_voices'));continue
        rows.extend(variants)
        if allowlist:
            for variant in variants:
                variant.update(direct_inspection_path=str(allowlist.resolve()),
                               direct_inspection_sha256=sha256(allowlist))
    if not rows:raise ValueError('No complete reviewed external QA')
    payload=dict(format='hfd_bench_external_assets_v1',assets=rows,rejections=rejections,
        provenance=dict(source='OVO-Bench',speech_plan_sha256=sha256(root/'speech/plan.json'),
            split_audit_sha256=sha256(root/'split_audit.json'),original_options_preserved=True,
            manual_review_sha256=sha256(manual_path) if manual_path.exists() else None,
            direct_inspection_sha256=sha256(allowlist) if allowlist else None,
            protocol='HFD spoken open QA adaptation; question END at original timestamp; no history crop',
            training_data_modified=False))
    if output.exists() and json.loads(output.read_text())!=payload:raise ValueError('Refusing to overwrite a changed published manifest')
    write_json(output,payload)
    print('PUBLISHED',len(rows)//8,'questions',len(rows),'voices',output,flush=True)


if __name__=='__main__':
    p=argparse.ArgumentParser(description=__doc__)
    p.add_argument('command',choices=['features','publish'])
    p.add_argument('--root',type=Path,default=DEFAULT)
    p.add_argument('--output',type=Path,default=ROOT/'benchmark/outputs/external_visual/overlay_assets.json')
    p.add_argument('--manual-allowlist',type=Path,help='Completed direct inspection; only explicit accepted IDs admitted')
    p.add_argument('--rank',type=int,default=0);p.add_argument('--world',type=int,default=1)
    a=p.parse_args()
    if a.command=='features':features(a.root,a.rank,a.world)
    else:publish(a.root,a.output,a.manual_allowlist)