File size: 4,548 Bytes
9a0cc30
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
import json
from pathlib import Path
import numpy as np
import torch
from transformers import AutoModel, AutoModelForSequenceClassification, AutoTokenizer
ROOT=Path(__file__).resolve().parent
NAMES=['roberta-base-go_emotions','chronos-bolt-small','contriever','mdeberta-v3-base']
torch.set_num_threads(4)

def load(name):
    path=ROOT/'sources'/name
    if name=='chronos-bolt-small':
        from chronos.chronos_bolt import ChronosBoltModelForForecasting
        model=ChronosBoltModelForForecasting.from_pretrained(path).eval()
        model.config.use_cache=False
        model.decoder.config.use_cache=False
    else:
        cls=AutoModelForSequenceClassification if name=='roberta-base-go_emotions' else AutoModel
        model=cls.from_pretrained(path,attn_implementation='eager').eval()
    return model

class Wrapper(torch.nn.Module):
    def __init__(self,model,name): super().__init__(); self.model=model; self.name=name
    def forward(self,input_ids,attention_mask=None):
        if self.name=='chronos-bolt-small': return self.model(context=input_ids).quantile_preds
        out=self.model(input_ids=input_ids,attention_mask=attention_mask,return_dict=False)[0]
        if self.name=='contriever':
            m=attention_mask.unsqueeze(-1).to(out.dtype)
            return (out*m).sum(1)/m.sum(1).clamp(min=1)
        return out

class ExportWrapper(Wrapper):
    def forward(self,input_ids,attention_mask=None):
        if self.name!='chronos-bolt-small': return super().forward(input_ids,attention_mask)
        m=self.model; context=input_ids
        valid=context==context
        count=valid.sum(-1,keepdim=True).clamp(min=1).float()
        loc=torch.where(valid,context,0.0).sum(-1,keepdim=True)/count
        scale=(torch.where(valid,(context-loc).square(),0.0).sum(-1,keepdim=True)/count).sqrt()
        scale=torch.where(valid.any(-1,keepdim=True),scale,1.0)
        scale=torch.where(scale==0,1e-5,scale)
        x=torch.where(valid,(context-loc)/scale,0.0)
        size=m.chronos_config.input_patch_size
        patches=x.reshape(context.shape[0],-1,size)
        mask=valid.float().reshape(context.shape[0],-1,size)
        am=mask.sum(-1)>0
        embeds=m.input_patch_embedding(torch.cat([patches,mask],-1))
        if m.chronos_config.use_reg_token:
            reg=m.shared.weight[1].reshape(1,1,-1).expand(context.shape[0],1,-1)
            embeds=torch.cat([embeds,reg],1)
            am=torch.cat([am,torch.ones((context.shape[0],1),dtype=torch.bool)],1)
        hidden=m.encoder(inputs_embeds=embeds,attention_mask=am,return_dict=False)[0]
        dec=m.shared.weight[0].reshape(1,1,-1).expand(context.shape[0],1,-1)
        output=m.decoder(inputs_embeds=dec,encoder_hidden_states=hidden,encoder_attention_mask=am,use_cache=False,return_dict=False)[0]
        return m.output_patch_embedding(output).reshape(context.shape[0],m.num_quantiles,m.chronos_config.prediction_length)*scale.unsqueeze(-1)+loc.unsqueeze(-1)

def cases(name,length=128):
    if name=='chronos-bolt-small':
        length=2048
        t=np.arange(length,dtype=np.float32)
        return [dict(context=np.array([np.sin(t/9)+t/100],dtype=np.float32)),
                dict(context=np.full((1,length),7,dtype=np.float32)),
                dict(context=np.where(np.arange(length)[None]%11==0,np.nan,np.cos(t/7)[None]).astype(np.float32))]
    tok=AutoTokenizer.from_pretrained(ROOT/'sources'/name,use_fast=name!='mdeberta-v3-base')
    texts=['Thank you so much! I love this wonderful surprise.', 'I am angry and disappointed. This is terrible.',
           'The document describes information retrieval and machine learning.', '今日は良い天気です。検索モデルを検証します。',
           '', 'A short sentence. '*80]
    return [dict(tok(t,padding='max_length',truncation=True,max_length=length,return_tensors='np')) for t in texts]

def args_for(name,case):
    if name=='chronos-bolt-small': return (torch.tensor(case['context']),)
    return tuple(torch.tensor(case[k],dtype=torch.int32) for k in ['input_ids','attention_mask'])

def reference(name,case,wrapper):
    with torch.no_grad(): return wrapper(*args_for(name,case)).float().numpy()

def metrics(ref,out):
    ref=ref.astype(np.float32); out=out.astype(np.float32)
    assert ref.shape==out.shape and np.isfinite(out).all(), (ref.shape,out.shape)
    return dict(max_abs=float(np.max(np.abs(ref-out))),mean_abs=float(np.mean(np.abs(ref-out))),
                cosine=float(np.dot(ref.ravel(),out.ravel())/(np.linalg.norm(ref)*np.linalg.norm(out)+1e-30)))