#!/usr/bin/env python3 """Single-device read-only evaluation of canonical weights; never a training fallback.""" from __future__ import annotations import argparse,json,os,sys,time from collections import defaultdict from pathlib import Path p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);p.add_argument('--model',type=Path);p.add_argument('--tag',required=True);p.add_argument('--device',default='npu:7');p.add_argument('--samples',type=int,default=12);p.add_argument('--tts',action='store_true');a=p.parse_args();r=a.root.resolve() sys.path[:0]=[str(r/'trusted/src'),str(r/'env')] os.environ.update(HF_HUB_OFFLINE='1',TRANSFORMERS_OFFLINE='1',TOKENIZERS_PARALLELISM='false') import torch,torch_npu from dots_s2st_npu.tasks import load_task_modules,generate_text_en_fa,synthesize_no_reference from dots_s2st_npu.checkpoint import load_portable_weights from dots_s2st_npu.prepare import Pins,load_pretrained_codec from dots_s2st_npu.data import normalize_persian from sacrebleu.metrics import CHRF import soundfile as sf initial=r/'downloads/initial' qwen,acoustic,cfg,tok=load_task_modules(a.model or initial,device='cpu',dtype=torch.float32) qwen=qwen.to(a.device,dtype=torch.bfloat16).eval() results=[];rows=[json.loads(x) for x in (r/'data/mt_validation.jsonl').read_text().splitlines() if x.strip()] groups=defaultdict(list) for x in rows:groups[x['category']].append(x) chosen=[] for k in sorted(groups):chosen+=groups[k][:max(1,a.samples//len(groups))] chosen=chosen[:a.samples] metric=CHRF(word_order=2) with torch.inference_mode(),torch.autocast(device_type='npu',dtype=torch.bfloat16): for i,row in enumerate(chosen): start=time.monotonic();out=generate_text_en_fa(qwen,tok,cfg,row['source'],device=a.device,max_new_tokens=128,max_context_tokens=512) score=max(metric.sentence_score(normalize_persian(out['text']),[normalize_persian(t)]).score for t in row['targets']) rec={'id':row['id'],'category':row['category'],'source':row['source'],'targets':row['targets'],'prediction':out['text'],'termination':out['termination'],'chrf_plus_plus_best_reference':score,'seconds':time.monotonic()-start} results.append(rec);print('MT_EVAL_SAMPLE '+json.dumps(rec,ensure_ascii=False),flush=True) folder=r/'evaluation'/a.tag;folder.mkdir(parents=True,exist_ok=False) (folder/'mt.json').write_text(json.dumps({'tag':a.tag,'samples':results,'macro_sentence_chrf':sum(x['chrf_plus_plus_best_reference'] for x in results)/max(1,len(results)),'human_eval':False,'single_device_evaluation_not_training':True},ensure_ascii=False,indent=2),encoding='utf-8') if a.tts: acoustic=acoustic.to(a.device,dtype=torch.bfloat16).eval() codec,prov=load_pretrained_codec(r,Pins(r,r/'downloads/pins.json')) texts=['سلام. امروز هوا آفتابی است و من برای دیدن دوستم به پارک می‌روم.','این کتاب بیست و پنج صفحه دارد، اما داستان آن هنوز تمام نشده است.'] for i,text in enumerate(texts): with torch.inference_mode(),torch.autocast(device_type='npu',dtype=torch.bfloat16):out=synthesize_no_reference(qwen,acoustic,tok,cfg,text,device=a.device,max_patches=80,num_steps=8,max_context_tokens=384) latent=out['latents'].float().cpu().T[None] with torch.inference_mode():wav=codec.inference_from_latents(latent,do_sample=False).reshape(-1).numpy() sf.write(str(folder/f'fa-{i}.wav'),wav,48000) (folder/f'fa-{i}.json').write_text(json.dumps({'text':text,'termination':out['termination'],'samples':len(wav),'codec':prov,'human_intelligibility_verified':False},ensure_ascii=False,indent=2),encoding='utf-8') print('NPU_EVAL_TERMINAL '+json.dumps({'state':'succeeded','tag':a.tag,'path':str(folder)}),flush=True)