Download evaluate.py from amir0907/voihf200: direct link, hf CLI and curl.
- Browser
- Download file 3.73 kB
-
https://huggingface.co/amir0907/voihf200/resolve/main/evaluate.py
- Command line
-
hf download hf://amir0907/voihf200/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/amir0907/voihf200/resolve/main/evaluate.py
3.73 kB
| #!/usr/bin/env python3 | |
| """Single-device read-only evaluation of canonical weights; never a training fallback.""" | |
| from __future__ import annotations | |
| import argparse,json,os,sys,time | |
| from collections import defaultdict | |
| from pathlib import Path | |
| p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);p.add_argument('--model',type=Path);p.add_argument('--tag',required=True);p.add_argument('--device',default='npu:7');p.add_argument('--samples',type=int,default=12);p.add_argument('--tts',action='store_true');a=p.parse_args();r=a.root.resolve() | |
| sys.path[:0]=[str(r/'trusted/src'),str(r/'env')] | |
| os.environ.update(HF_HUB_OFFLINE='1',TRANSFORMERS_OFFLINE='1',TOKENIZERS_PARALLELISM='false') | |
| import torch,torch_npu | |
| from dots_s2st_npu.tasks import load_task_modules,generate_text_en_fa,synthesize_no_reference | |
| from dots_s2st_npu.checkpoint import load_portable_weights | |
| from dots_s2st_npu.prepare import Pins,load_pretrained_codec | |
| from dots_s2st_npu.data import normalize_persian | |
| from sacrebleu.metrics import CHRF | |
| import soundfile as sf | |
| initial=r/'downloads/initial' | |
| qwen,acoustic,cfg,tok=load_task_modules(a.model or initial,device='cpu',dtype=torch.float32) | |
| qwen=qwen.to(a.device,dtype=torch.bfloat16).eval() | |
| results=[];rows=[json.loads(x) for x in (r/'data/mt_validation.jsonl').read_text().splitlines() if x.strip()] | |
| groups=defaultdict(list) | |
| for x in rows:groups[x['category']].append(x) | |
| chosen=[] | |
| for k in sorted(groups):chosen+=groups[k][:max(1,a.samples//len(groups))] | |
| chosen=chosen[:a.samples] | |
| metric=CHRF(word_order=2) | |
| with torch.inference_mode(),torch.autocast(device_type='npu',dtype=torch.bfloat16): | |
| for i,row in enumerate(chosen): | |
| start=time.monotonic();out=generate_text_en_fa(qwen,tok,cfg,row['source'],device=a.device,max_new_tokens=128,max_context_tokens=512) | |
| score=max(metric.sentence_score(normalize_persian(out['text']),[normalize_persian(t)]).score for t in row['targets']) | |
| rec={'id':row['id'],'category':row['category'],'source':row['source'],'targets':row['targets'],'prediction':out['text'],'termination':out['termination'],'chrf_plus_plus_best_reference':score,'seconds':time.monotonic()-start} | |
| results.append(rec);print('MT_EVAL_SAMPLE '+json.dumps(rec,ensure_ascii=False),flush=True) | |
| folder=r/'evaluation'/a.tag;folder.mkdir(parents=True,exist_ok=False) | |
| (folder/'mt.json').write_text(json.dumps({'tag':a.tag,'samples':results,'macro_sentence_chrf':sum(x['chrf_plus_plus_best_reference'] for x in results)/max(1,len(results)),'human_eval':False,'single_device_evaluation_not_training':True},ensure_ascii=False,indent=2),encoding='utf-8') | |
| if a.tts: | |
| acoustic=acoustic.to(a.device,dtype=torch.bfloat16).eval() | |
| codec,prov=load_pretrained_codec(r,Pins(r,r/'downloads/pins.json')) | |
| texts=['سلام. امروز هوا آفتابی است و من برای دیدن دوستم به پارک میروم.','این کتاب بیست و پنج صفحه دارد، اما داستان آن هنوز تمام نشده است.'] | |
| for i,text in enumerate(texts): | |
| with torch.inference_mode(),torch.autocast(device_type='npu',dtype=torch.bfloat16):out=synthesize_no_reference(qwen,acoustic,tok,cfg,text,device=a.device,max_patches=80,num_steps=8,max_context_tokens=384) | |
| latent=out['latents'].float().cpu().T[None] | |
| with torch.inference_mode():wav=codec.inference_from_latents(latent,do_sample=False).reshape(-1).numpy() | |
| sf.write(str(folder/f'fa-{i}.wav'),wav,48000) | |
| (folder/f'fa-{i}.json').write_text(json.dumps({'text':text,'termination':out['termination'],'samples':len(wav),'codec':prov,'human_intelligibility_verified':False},ensure_ascii=False,indent=2),encoding='utf-8') | |
| print('NPU_EVAL_TERMINAL '+json.dumps({'state':'succeeded','tag':a.tag,'path':str(folder)}),flush=True) | |