File size: 4,548 Bytes
9a0cc30 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | import json
from pathlib import Path
import numpy as np
import torch
from transformers import AutoModel, AutoModelForSequenceClassification, AutoTokenizer
ROOT=Path(__file__).resolve().parent
NAMES=['roberta-base-go_emotions','chronos-bolt-small','contriever','mdeberta-v3-base']
torch.set_num_threads(4)
def load(name):
path=ROOT/'sources'/name
if name=='chronos-bolt-small':
from chronos.chronos_bolt import ChronosBoltModelForForecasting
model=ChronosBoltModelForForecasting.from_pretrained(path).eval()
model.config.use_cache=False
model.decoder.config.use_cache=False
else:
cls=AutoModelForSequenceClassification if name=='roberta-base-go_emotions' else AutoModel
model=cls.from_pretrained(path,attn_implementation='eager').eval()
return model
class Wrapper(torch.nn.Module):
def __init__(self,model,name): super().__init__(); self.model=model; self.name=name
def forward(self,input_ids,attention_mask=None):
if self.name=='chronos-bolt-small': return self.model(context=input_ids).quantile_preds
out=self.model(input_ids=input_ids,attention_mask=attention_mask,return_dict=False)[0]
if self.name=='contriever':
m=attention_mask.unsqueeze(-1).to(out.dtype)
return (out*m).sum(1)/m.sum(1).clamp(min=1)
return out
class ExportWrapper(Wrapper):
def forward(self,input_ids,attention_mask=None):
if self.name!='chronos-bolt-small': return super().forward(input_ids,attention_mask)
m=self.model; context=input_ids
valid=context==context
count=valid.sum(-1,keepdim=True).clamp(min=1).float()
loc=torch.where(valid,context,0.0).sum(-1,keepdim=True)/count
scale=(torch.where(valid,(context-loc).square(),0.0).sum(-1,keepdim=True)/count).sqrt()
scale=torch.where(valid.any(-1,keepdim=True),scale,1.0)
scale=torch.where(scale==0,1e-5,scale)
x=torch.where(valid,(context-loc)/scale,0.0)
size=m.chronos_config.input_patch_size
patches=x.reshape(context.shape[0],-1,size)
mask=valid.float().reshape(context.shape[0],-1,size)
am=mask.sum(-1)>0
embeds=m.input_patch_embedding(torch.cat([patches,mask],-1))
if m.chronos_config.use_reg_token:
reg=m.shared.weight[1].reshape(1,1,-1).expand(context.shape[0],1,-1)
embeds=torch.cat([embeds,reg],1)
am=torch.cat([am,torch.ones((context.shape[0],1),dtype=torch.bool)],1)
hidden=m.encoder(inputs_embeds=embeds,attention_mask=am,return_dict=False)[0]
dec=m.shared.weight[0].reshape(1,1,-1).expand(context.shape[0],1,-1)
output=m.decoder(inputs_embeds=dec,encoder_hidden_states=hidden,encoder_attention_mask=am,use_cache=False,return_dict=False)[0]
return m.output_patch_embedding(output).reshape(context.shape[0],m.num_quantiles,m.chronos_config.prediction_length)*scale.unsqueeze(-1)+loc.unsqueeze(-1)
def cases(name,length=128):
if name=='chronos-bolt-small':
length=2048
t=np.arange(length,dtype=np.float32)
return [dict(context=np.array([np.sin(t/9)+t/100],dtype=np.float32)),
dict(context=np.full((1,length),7,dtype=np.float32)),
dict(context=np.where(np.arange(length)[None]%11==0,np.nan,np.cos(t/7)[None]).astype(np.float32))]
tok=AutoTokenizer.from_pretrained(ROOT/'sources'/name,use_fast=name!='mdeberta-v3-base')
texts=['Thank you so much! I love this wonderful surprise.', 'I am angry and disappointed. This is terrible.',
'The document describes information retrieval and machine learning.', '今日は良い天気です。検索モデルを検証します。',
'', 'A short sentence. '*80]
return [dict(tok(t,padding='max_length',truncation=True,max_length=length,return_tensors='np')) for t in texts]
def args_for(name,case):
if name=='chronos-bolt-small': return (torch.tensor(case['context']),)
return tuple(torch.tensor(case[k],dtype=torch.int32) for k in ['input_ids','attention_mask'])
def reference(name,case,wrapper):
with torch.no_grad(): return wrapper(*args_for(name,case)).float().numpy()
def metrics(ref,out):
ref=ref.astype(np.float32); out=out.astype(np.float32)
assert ref.shape==out.shape and np.isfinite(out).all(), (ref.shape,out.shape)
return dict(max_abs=float(np.max(np.abs(ref-out))),mean_abs=float(np.mean(np.abs(ref-out))),
cosine=float(np.dot(ref.ravel(),out.ravel())/(np.linalg.norm(ref)*np.linalg.norm(out)+1e-30)))
|