"""Reuse the established eight-voice translation/TTS/independent-ASR pipeline. Only benchmark assets are written. `plan` requires finished visual/split review; failed or pending QA cannot leak into the published overlay manifest. """ import argparse import json from pathlib import Path import sys from common import ROOT,sha256,write_json from external_visual import DEFAULT CAM=ROOT.parent/'HFD_swa_stream_2609111628_hzoGRo' sys.path[:0]=[str(CAM/'tts_deps'),str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')] from expand_hfd_video_voices import ( references,VOICES,TextModel,translate,SpeechModels,translation_path,audio_path, ) PUBLIC=Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen') def plan(root): rows=json.loads((root/'media_inventory.json').read_text())['rows'] split=json.loads((root/'split_audit.json').read_text()) if split['fingerprinted_training_videos']!=split['training_videos']: raise ValueError('Training-video overlap screening is not complete') disallowed={r['video_sha256'] for r in split['rows'] if not r['passed']} manual_path=root/'manual_review.json' manual=json.loads(manual_path.read_text())['rows'] if manual_path.exists() else [] veto={r['id'] for r in manual if not r['accepted']} selected=[] for row in rows: path=root/'reviews'/(row['id']+'.json') if not path.exists():raise ValueError('Visual review is not complete') review=json.loads(path.read_text()) if review['accepted'] and row['video_sha256'] not in disallowed and row['id'] not in veto: selected.append({**row,'visual_receipt':str(path)}) jobs=[] for i,question in enumerate(sorted({r['question'] for r in selected})): jobs.append(dict(id=f'{i:06d}',question=question, answers=sorted({r['answer'] for r in selected if r['question']==question}))) out=dict(rows=selected,questions=jobs,voices=list(VOICES),references=references(), tts_model=str(CAM/'data/tts_base'), asr_model=str(ROOT.parents[1]/'hfd_experiments/swa_260910/ego_selection/scaleup/asr_large_v3_turbo'), translator=str(PUBLIC/'Qwen2.5-14B-Instruct'),judge=str(PUBLIC/'Qwen2.5-7B-Instruct'), review_contract_sha256=sha256(root/'review_contract.json'),split_audit_sha256=sha256(root/'split_audit.json')) path=root/'speech/plan.json' if path.exists() and json.loads(path.read_text())!=out:raise ValueError('Frozen speech plan changed') write_json(path,out);print('SPEECH_PLAN',len(selected),'QA',len(jobs)*8,'waves',flush=True) def text(root): p=json.loads((root/'speech/plan.json').read_text()) translator=TextModel(p['translator']);judge=TextModel(p['judge']) pairs=[];contexts={} for job in p['questions']: pairs.append(('question',job['question'])) for answer in job['answers']: kind='answer:'+job['question'];pairs.append((kind,answer));contexts[kind]=job['question'] translate(root/'speech',pairs,translator,judge, dict(translator=p['translator'],judge=p['judge'],version='benchmark_translation_v1'),contexts) def audio(root,rank,world,batch): p=json.loads((root/'speech/plan.json').read_text()) model=SpeechModels(p,codec_graph=True) jobs=p['questions'][rank::world] for voice in VOICES: for i in range(0,len(jobs),batch): model.generate(root/'speech',jobs[i:i+batch],voice) print('SPEECH',rank,voice,min(i+batch,len(jobs)),'/',len(jobs),flush=True) if __name__=='__main__': p=argparse.ArgumentParser(description=__doc__) p.add_argument('command',choices=['plan','text','audio']) p.add_argument('--root',type=Path,default=DEFAULT) p.add_argument('--rank',type=int,default=0);p.add_argument('--world',type=int,default=1) p.add_argument('--batch-size',type=int,default=8) a=p.parse_args() if a.command=='plan':plan(a.root) elif a.command=='text':text(a.root) else:audio(a.root,a.rank,a.world,a.batch_size)