voihf200 / evaluate.py
amir0907's picture
Upload evaluate.py with huggingface_hub
a67a92f verified
Raw History Blame Contribute Delete
3.73 kB
#!/usr/bin/env python3
"""Single-device read-only evaluation of canonical weights; never a training fallback."""
from __future__ import annotations
import argparse,json,os,sys,time
from collections import defaultdict
from pathlib import Path
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);p.add_argument('--model',type=Path);p.add_argument('--tag',required=True);p.add_argument('--device',default='npu:7');p.add_argument('--samples',type=int,default=12);p.add_argument('--tts',action='store_true');a=p.parse_args();r=a.root.resolve()
sys.path[:0]=[str(r/'trusted/src'),str(r/'env')]
os.environ.update(HF_HUB_OFFLINE='1',TRANSFORMERS_OFFLINE='1',TOKENIZERS_PARALLELISM='false')
import torch,torch_npu
from dots_s2st_npu.tasks import load_task_modules,generate_text_en_fa,synthesize_no_reference
from dots_s2st_npu.checkpoint import load_portable_weights
from dots_s2st_npu.prepare import Pins,load_pretrained_codec
from dots_s2st_npu.data import normalize_persian
from sacrebleu.metrics import CHRF
import soundfile as sf
initial=r/'downloads/initial'
qwen,acoustic,cfg,tok=load_task_modules(a.model or initial,device='cpu',dtype=torch.float32)
qwen=qwen.to(a.device,dtype=torch.bfloat16).eval()
results=[];rows=[json.loads(x) for x in (r/'data/mt_validation.jsonl').read_text().splitlines() if x.strip()]
groups=defaultdict(list)
for x in rows:groups[x['category']].append(x)
chosen=[]
for k in sorted(groups):chosen+=groups[k][:max(1,a.samples//len(groups))]
chosen=chosen[:a.samples]
metric=CHRF(word_order=2)
with torch.inference_mode(),torch.autocast(device_type='npu',dtype=torch.bfloat16):
for i,row in enumerate(chosen):
start=time.monotonic();out=generate_text_en_fa(qwen,tok,cfg,row['source'],device=a.device,max_new_tokens=128,max_context_tokens=512)
score=max(metric.sentence_score(normalize_persian(out['text']),[normalize_persian(t)]).score for t in row['targets'])
rec={'id':row['id'],'category':row['category'],'source':row['source'],'targets':row['targets'],'prediction':out['text'],'termination':out['termination'],'chrf_plus_plus_best_reference':score,'seconds':time.monotonic()-start}
results.append(rec);print('MT_EVAL_SAMPLE '+json.dumps(rec,ensure_ascii=False),flush=True)
folder=r/'evaluation'/a.tag;folder.mkdir(parents=True,exist_ok=False)
(folder/'mt.json').write_text(json.dumps({'tag':a.tag,'samples':results,'macro_sentence_chrf':sum(x['chrf_plus_plus_best_reference'] for x in results)/max(1,len(results)),'human_eval':False,'single_device_evaluation_not_training':True},ensure_ascii=False,indent=2),encoding='utf-8')
if a.tts:
acoustic=acoustic.to(a.device,dtype=torch.bfloat16).eval()
codec,prov=load_pretrained_codec(r,Pins(r,r/'downloads/pins.json'))
texts=['سلام. امروز هوا آفتابی است و من برای دیدن دوستم به پارک می‌روم.','این کتاب بیست و پنج صفحه دارد، اما داستان آن هنوز تمام نشده است.']
for i,text in enumerate(texts):
with torch.inference_mode(),torch.autocast(device_type='npu',dtype=torch.bfloat16):out=synthesize_no_reference(qwen,acoustic,tok,cfg,text,device=a.device,max_patches=80,num_steps=8,max_context_tokens=384)
latent=out['latents'].float().cpu().T[None]
with torch.inference_mode():wav=codec.inference_from_latents(latent,do_sample=False).reshape(-1).numpy()
sf.write(str(folder/f'fa-{i}.wav'),wav,48000)
(folder/f'fa-{i}.json').write_text(json.dumps({'text':text,'termination':out['termination'],'samples':len(wav),'codec':prov,'human_intelligibility_verified':False},ensure_ascii=False,indent=2),encoding='utf-8')
print('NPU_EVAL_TERMINAL '+json.dumps({'state':'succeeded','tag':a.tag,'path':str(folder)}),flush=True)