File size: 7,259 Bytes
fb0074a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """Use the unchanged formal B1 visual encoder; publish only complete QA assets."""
import argparse
import json
from pathlib import Path
import sys
from types import SimpleNamespace
from common import ROOT,DEFAULT_RUN,sha256,write_json
from external_visual import DEFAULT
sys.path[:0]=[str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')]
def features(root,rank,world):
from prepare_hfd_video_assets import encode
plan=json.loads((root/'speech/plan.json').read_text())
media={r['video_sha256']:r for r in plan['rows']}
rows=[dict(asset_id=sha,video_path=r['video_path'],video_sha256=sha,
duration_seconds=min(r['duration_seconds'],180.)) for sha,r in sorted(media.items())]
# Only stop encoding unneeded suffix; never move the query, speed up video,
# or shorten the observed prefix. Sessions must fit the encoded duration.
output=root/'features'
write_json(output/'admission/split.json',dict(rows=rows))
config=json.loads((DEFAULT_RUN/'run_config.json').read_text())
encode(SimpleNamespace(root=output,base_model=Path(config['model']['model_name_or_path']),
selected_before_split=False,world=world,rank=rank,limit=0,packet_batch=1))
def publish(root,output,allowlist=None):
from expand_hfd_video_voices import VOICES,translation_path,audio_path,canonical
plan=json.loads((root/'speech/plan.json').read_text())
ids={r['question']:r['id'] for r in plan['questions']}
manual_path=root/'manual_review.json'
manual=json.loads(manual_path.read_text())['rows'] if manual_path.exists() else []
veto={r['id'] for r in manual if not r['accepted']}
inspected=None
if allowlist:
inspection=json.loads(allowlist.read_text())
if inspection.get('complete') is not True:
raise ValueError('Direct visual inspection unfinished')
inspected={r['id']:r for r in inspection['rows']}
if len(inspected)!=len(inspection['rows']):raise ValueError('Duplicate direct inspection IDs')
rows=[];rejections=[]
for row in plan['rows']:
if inspected is not None and not inspected.get(row['id'],{}).get('accepted'):
rejections.append(dict(id=row['id'],reason='direct_inspection_not_approved'));continue
if row['id'] in veto:
rejections.append(dict(id=row['id'],reason='manual_visual_veto'));continue
qtranslation=translation_path(root/'speech','question',row['question'])
atranslation=translation_path(root/'speech','answer:'+row['question'],row['answer'])
if not qtranslation.exists() or not atranslation.exists():
raise ValueError('Translation review unfinished')
if not all(json.loads(p.read_text())['accepted'] for p in (qtranslation,atranslation)):
rejections.append(dict(id=row['id'],reason='translation_not_verified'));continue
fpath=root/'features/admission/feature_receipts'/(row['video_sha256']+'.json')
if not fpath.exists():raise ValueError('Feature extraction not finished: '+str(fpath))
feature=json.loads(fpath.read_text())
if feature.get('rejected'):
rejections.append(dict(id=row['id'],reason='feature_decode_rejected'));continue
if sha256(feature['path'])!=feature['sha256']:raise ValueError('Feature file changed')
qid=ids[row['question']];variants=[]
for voice in VOICES:
wave=audio_path(root/'speech',qid,voice);receipt=wave.with_suffix('.json')
if not receipt.exists():raise ValueError('TTS/ASR unfinished: '+str(receipt))
ar=json.loads(receipt.read_text())
if not ar.get('accepted'):break
if (ar['wave_sha256']!=sha256(wave) or not ar['asr'].get('independent_asr')
or canonical(ar['asr']['heard'],voice[-2:])!=canonical(ar['text'],voice[-2:])):
raise ValueError('Independent ASR receipt mismatch')
question,answer=row['question'],row['answer']
if voice.endswith('_zh'):
q=json.loads(translation_path(root/'speech','question',question).read_text())
ans=json.loads(translation_path(root/'speech','answer:'+question,answer).read_text())
if not q['accepted'] or not ans['accepted']:break
question,answer=q['translation'],ans['translation']
if ar['text']!=question:raise ValueError('Question text changed after TTS')
variants.append(dict(asset_id=row['id'],external_id=row['id'],voice=voice,split='val',
data_domain='OVO-Bench',question=question,answer=answer,
original_question=row['question'],original_answer=row['answer'],
question_time=row['question_time'],options=row['options'],correct_option=row['correct_option'],
video_path=row['video_path'],video_sha256=row['video_sha256'],
duration_seconds=min(row['duration_seconds'],180.),
original_video_duration_seconds=row['duration_seconds'],
visual_features_path=feature['path'],visual_features_sha256=feature['sha256'],
visual_receipt=row['visual_receipt'],visual_receipt_sha256=sha256(row['visual_receipt']),
question_audio_path=str(wave),question_audio_sha256=ar['wave_sha256'],asr_receipt=str(receipt)))
if len(variants)!=8:
rejections.append(dict(id=row['id'],reason='incomplete_verified_eight_voices'));continue
rows.extend(variants)
if allowlist:
for variant in variants:
variant.update(direct_inspection_path=str(allowlist.resolve()),
direct_inspection_sha256=sha256(allowlist))
if not rows:raise ValueError('No complete reviewed external QA')
payload=dict(format='hfd_bench_external_assets_v1',assets=rows,rejections=rejections,
provenance=dict(source='OVO-Bench',speech_plan_sha256=sha256(root/'speech/plan.json'),
split_audit_sha256=sha256(root/'split_audit.json'),original_options_preserved=True,
manual_review_sha256=sha256(manual_path) if manual_path.exists() else None,
direct_inspection_sha256=sha256(allowlist) if allowlist else None,
protocol='HFD spoken open QA adaptation; question END at original timestamp; no history crop',
training_data_modified=False))
if output.exists() and json.loads(output.read_text())!=payload:raise ValueError('Refusing to overwrite a changed published manifest')
write_json(output,payload)
print('PUBLISHED',len(rows)//8,'questions',len(rows),'voices',output,flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('command',choices=['features','publish'])
p.add_argument('--root',type=Path,default=DEFAULT)
p.add_argument('--output',type=Path,default=ROOT/'benchmark/outputs/external_visual/overlay_assets.json')
p.add_argument('--manual-allowlist',type=Path,help='Completed direct inspection; only explicit accepted IDs admitted')
p.add_argument('--rank',type=int,default=0);p.add_argument('--world',type=int,default=1)
a=p.parse_args()
if a.command=='features':features(a.root,a.rank,a.world)
else:publish(a.root,a.output,a.manual_allowlist)
|