Download benchmark/external_features.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 7.26 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/external_features.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/external_features.py
-
curl -L -o external_features.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/external_features.py
7.26 kB
| """Use the unchanged formal B1 visual encoder; publish only complete QA assets.""" | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import sys | |
| from types import SimpleNamespace | |
| from common import ROOT,DEFAULT_RUN,sha256,write_json | |
| from external_visual import DEFAULT | |
| sys.path[:0]=[str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')] | |
| def features(root,rank,world): | |
| from prepare_hfd_video_assets import encode | |
| plan=json.loads((root/'speech/plan.json').read_text()) | |
| media={r['video_sha256']:r for r in plan['rows']} | |
| rows=[dict(asset_id=sha,video_path=r['video_path'],video_sha256=sha, | |
| duration_seconds=min(r['duration_seconds'],180.)) for sha,r in sorted(media.items())] | |
| # Only stop encoding unneeded suffix; never move the query, speed up video, | |
| # or shorten the observed prefix. Sessions must fit the encoded duration. | |
| output=root/'features' | |
| write_json(output/'admission/split.json',dict(rows=rows)) | |
| config=json.loads((DEFAULT_RUN/'run_config.json').read_text()) | |
| encode(SimpleNamespace(root=output,base_model=Path(config['model']['model_name_or_path']), | |
| selected_before_split=False,world=world,rank=rank,limit=0,packet_batch=1)) | |
| def publish(root,output,allowlist=None): | |
| from expand_hfd_video_voices import VOICES,translation_path,audio_path,canonical | |
| plan=json.loads((root/'speech/plan.json').read_text()) | |
| ids={r['question']:r['id'] for r in plan['questions']} | |
| manual_path=root/'manual_review.json' | |
| manual=json.loads(manual_path.read_text())['rows'] if manual_path.exists() else [] | |
| veto={r['id'] for r in manual if not r['accepted']} | |
| inspected=None | |
| if allowlist: | |
| inspection=json.loads(allowlist.read_text()) | |
| if inspection.get('complete') is not True: | |
| raise ValueError('Direct visual inspection unfinished') | |
| inspected={r['id']:r for r in inspection['rows']} | |
| if len(inspected)!=len(inspection['rows']):raise ValueError('Duplicate direct inspection IDs') | |
| rows=[];rejections=[] | |
| for row in plan['rows']: | |
| if inspected is not None and not inspected.get(row['id'],{}).get('accepted'): | |
| rejections.append(dict(id=row['id'],reason='direct_inspection_not_approved'));continue | |
| if row['id'] in veto: | |
| rejections.append(dict(id=row['id'],reason='manual_visual_veto'));continue | |
| qtranslation=translation_path(root/'speech','question',row['question']) | |
| atranslation=translation_path(root/'speech','answer:'+row['question'],row['answer']) | |
| if not qtranslation.exists() or not atranslation.exists(): | |
| raise ValueError('Translation review unfinished') | |
| if not all(json.loads(p.read_text())['accepted'] for p in (qtranslation,atranslation)): | |
| rejections.append(dict(id=row['id'],reason='translation_not_verified'));continue | |
| fpath=root/'features/admission/feature_receipts'/(row['video_sha256']+'.json') | |
| if not fpath.exists():raise ValueError('Feature extraction not finished: '+str(fpath)) | |
| feature=json.loads(fpath.read_text()) | |
| if feature.get('rejected'): | |
| rejections.append(dict(id=row['id'],reason='feature_decode_rejected'));continue | |
| if sha256(feature['path'])!=feature['sha256']:raise ValueError('Feature file changed') | |
| qid=ids[row['question']];variants=[] | |
| for voice in VOICES: | |
| wave=audio_path(root/'speech',qid,voice);receipt=wave.with_suffix('.json') | |
| if not receipt.exists():raise ValueError('TTS/ASR unfinished: '+str(receipt)) | |
| ar=json.loads(receipt.read_text()) | |
| if not ar.get('accepted'):break | |
| if (ar['wave_sha256']!=sha256(wave) or not ar['asr'].get('independent_asr') | |
| or canonical(ar['asr']['heard'],voice[-2:])!=canonical(ar['text'],voice[-2:])): | |
| raise ValueError('Independent ASR receipt mismatch') | |
| question,answer=row['question'],row['answer'] | |
| if voice.endswith('_zh'): | |
| q=json.loads(translation_path(root/'speech','question',question).read_text()) | |
| ans=json.loads(translation_path(root/'speech','answer:'+question,answer).read_text()) | |
| if not q['accepted'] or not ans['accepted']:break | |
| question,answer=q['translation'],ans['translation'] | |
| if ar['text']!=question:raise ValueError('Question text changed after TTS') | |
| variants.append(dict(asset_id=row['id'],external_id=row['id'],voice=voice,split='val', | |
| data_domain='OVO-Bench',question=question,answer=answer, | |
| original_question=row['question'],original_answer=row['answer'], | |
| question_time=row['question_time'],options=row['options'],correct_option=row['correct_option'], | |
| video_path=row['video_path'],video_sha256=row['video_sha256'], | |
| duration_seconds=min(row['duration_seconds'],180.), | |
| original_video_duration_seconds=row['duration_seconds'], | |
| visual_features_path=feature['path'],visual_features_sha256=feature['sha256'], | |
| visual_receipt=row['visual_receipt'],visual_receipt_sha256=sha256(row['visual_receipt']), | |
| question_audio_path=str(wave),question_audio_sha256=ar['wave_sha256'],asr_receipt=str(receipt))) | |
| if len(variants)!=8: | |
| rejections.append(dict(id=row['id'],reason='incomplete_verified_eight_voices'));continue | |
| rows.extend(variants) | |
| if allowlist: | |
| for variant in variants: | |
| variant.update(direct_inspection_path=str(allowlist.resolve()), | |
| direct_inspection_sha256=sha256(allowlist)) | |
| if not rows:raise ValueError('No complete reviewed external QA') | |
| payload=dict(format='hfd_bench_external_assets_v1',assets=rows,rejections=rejections, | |
| provenance=dict(source='OVO-Bench',speech_plan_sha256=sha256(root/'speech/plan.json'), | |
| split_audit_sha256=sha256(root/'split_audit.json'),original_options_preserved=True, | |
| manual_review_sha256=sha256(manual_path) if manual_path.exists() else None, | |
| direct_inspection_sha256=sha256(allowlist) if allowlist else None, | |
| protocol='HFD spoken open QA adaptation; question END at original timestamp; no history crop', | |
| training_data_modified=False)) | |
| if output.exists() and json.loads(output.read_text())!=payload:raise ValueError('Refusing to overwrite a changed published manifest') | |
| write_json(output,payload) | |
| print('PUBLISHED',len(rows)//8,'questions',len(rows),'voices',output,flush=True) | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('command',choices=['features','publish']) | |
| p.add_argument('--root',type=Path,default=DEFAULT) | |
| p.add_argument('--output',type=Path,default=ROOT/'benchmark/outputs/external_visual/overlay_assets.json') | |
| p.add_argument('--manual-allowlist',type=Path,help='Completed direct inspection; only explicit accepted IDs admitted') | |
| p.add_argument('--rank',type=int,default=0);p.add_argument('--world',type=int,default=1) | |
| a=p.parse_args() | |
| if a.command=='features':features(a.root,a.rank,a.world) | |
| else:publish(a.root,a.output,a.manual_allowlist) | |