File size: 7,920 Bytes
14e3ecd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
"""Native MLX inference; original Hugging Face tensor names are retained."""
import json
from pathlib import Path
import numpy as np
import mlx.core as mx
import mlx.nn as nn

class Model:
    def __init__(self, path, gguf=False):
        p=Path(path)
        self.c=json.loads((p/'config.json').read_text())
        if gguf:
            from gguf import GGUFReader
            self.w={t.name: mx.array(np.array(t.data)) for t in GGUFReader(str(p/'model-f32.gguf')).tensors}
        else:
            self.w=mx.load(str(p/'model.safetensors'))
    def linear(self,x,p):
        y=x@self.w[p+'.weight'].T
        return y+self.w[p+'.bias'] if p+'.bias' in self.w else y
    def ln(self,x,p,eps=None):
        return mx.fast.layer_norm(x,self.w[p+'.weight'],self.w.get(p+'.bias'),eps or self.c.get('layer_norm_eps',1e-12))
    def heads(self,x,n):
        return x.reshape(*x.shape[:-1],n,-1).transpose(0,2,1,3)
    def __call__(self,input_ids=None,attention_mask=None,context=None):
        if 'chronos_config' in self.c: return self.chronos(mx.array(context))
        ids=mx.array(input_ids,dtype=mx.int32); mask=mx.array(attention_mask)
        if self.c['model_type']=='deberta-v2': return self.deberta(ids,mask)
        roberta=self.c['model_type']=='roberta'; root='roberta.' if roberta else ''
        e=root+'embeddings.'
        pos=mx.cumsum((ids!=1).astype(mx.int32),axis=1)*(ids!=1)+1 if roberta else mx.arange(ids.shape[1])[None,:]
        x=self.w[e+'word_embeddings.weight'][ids]+self.w[e+'position_embeddings.weight'][pos]+self.w[e+'token_type_embeddings.weight'][0]
        x=self.ln(x,e+'LayerNorm')
        n=self.c['num_attention_heads']
        for i in range(self.c['num_hidden_layers']):
            p=root+f'encoder.layer.{i}.'
            q,k,v=[self.heads(self.linear(x,p+'attention.self.'+t),n) for t in ['query','key','value']]
            a=mx.softmax(mx.where(mask[:,None,None,:]>0,(q@k.transpose(0,1,3,2))/(q.shape[-1]**.5),-1e30),axis=-1)
            y=(a@v).transpose(0,2,1,3).reshape(x.shape)
            x=self.ln(x+self.linear(y,p+'attention.output.dense'),p+'attention.output.LayerNorm')
            x=self.ln(x+self.linear(nn.gelu(self.linear(x,p+'intermediate.dense')),p+'output.dense'),p+'output.LayerNorm')
        if roberta: return self.linear(mx.tanh(self.linear(x[:,0], 'classifier.dense')),'classifier.out_proj')
        return (x*mask[...,None]).sum(1)/mx.maximum(mask.sum(1,keepdims=True),1)
    def rms(self,x,p):
        return x*mx.rsqrt(mx.mean(x*x,axis=-1,keepdims=True)+self.c['layer_norm_epsilon'])*self.w[p+'.weight']
    def residual(self,x,p):
        return self.linear(mx.maximum(self.linear(x,p+'.hidden_layer'),0),p+'.output_layer')+self.linear(x,p+'.residual_layer')
    def t5att(self,x,p,kv=None,bias=None,mask=None):
        n=self.c['num_heads']; kv=x if kv is None else kv
        q,k,v=[self.heads(self.linear(z,p+'.'+t),n) for z,t in [(x,'q'),(kv,'k'),(kv,'v')]]
        s=q@k.transpose(0,1,3,2)
        if bias is not None: s=s+bias
        if mask is not None: s=mx.where(mask[:,None,None,:]>0,s,-1e30)
        y=(mx.softmax(s,axis=-1)@v).transpose(0,2,1,3).reshape(x.shape[0],x.shape[1],-1)
        return self.linear(y,p+'.o')
    def t5stack(self,x,root,mask,enc=None):
        length=x.shape[1]; decoder=enc is not None
        r=mx.arange(length)[None,:]-mx.arange(length)[:,None]
        buckets=self.c['relative_attention_num_buckets']; maxd=self.c['relative_attention_max_distance']
        if decoder:
            r=mx.maximum(-r,0); sign=0
        else:
            buckets//=2; sign=(r>0).astype(mx.int32)*buckets; r=mx.abs(r)
        exact=buckets//2
        log=exact+(mx.log(mx.maximum(r,1)/exact)/np.log(maxd/exact)*(buckets-exact)).astype(mx.int32)
        idx=sign+mx.where(r<exact,r,mx.minimum(log,buckets-1))
        b=self.w[root+'.block.0.layer.0.SelfAttention.relative_attention_bias.weight'][idx.astype(mx.int32)].transpose(2,0,1)[None]
        for i in range(self.c['num_decoder_layers'] if decoder else self.c['num_layers']):
            p=root+f'.block.{i}.layer.'
            x=x+self.t5att(self.rms(x,p+'0.layer_norm'),p+'0.SelfAttention',bias=b,mask=None if decoder else mask)
            if decoder: x=x+self.t5att(self.rms(x,p+'1.layer_norm'),p+'1.EncDecAttention',kv=enc,mask=mask)
            f=p+('2' if decoder else '1')
            y=self.rms(x,f+'.layer_norm')
            x=x+self.linear(mx.maximum(self.linear(y,f+'.DenseReluDense.wi'),0),f+'.DenseReluDense.wo')
        return self.rms(x,root+'.final_layer_norm')
    def chronos(self,context):
        c=self.c['chronos_config']; context=context[:,-c['context_length']:]
        mask=~mx.isnan(context); count=mx.maximum(mask.sum(-1,keepdims=True),1)
        loc=mx.where(mask,context,0).sum(-1,keepdims=True)/count
        scale=mx.sqrt(mx.where(mask,(context-loc)**2,0).sum(-1,keepdims=True)/count)
        scale=mx.where(mask.any(-1,keepdims=True),scale,1.0)
        scale=mx.where(scale==0,1e-5,scale)
        x=mx.where(mask,(context-loc)/scale,0); size=c['input_patch_size']
        pad=(-x.shape[1])%size
        x=mx.pad(x,[(0,0),(pad,0)]); mask=mx.pad(mask,[(0,0),(pad,0)])
        x=x.reshape(x.shape[0],-1,size); m=mask.reshape(mask.shape[0],-1,size)
        x=self.residual(mx.concatenate([x,m.astype(x.dtype)],-1),'input_patch_embedding'); am=m.sum(-1)>0
        if c['use_reg_token']:
            x=mx.concatenate([x,mx.broadcast_to(self.w['shared.weight'][1],(x.shape[0],1,x.shape[-1]))],1)
            am=mx.concatenate([am,mx.ones((am.shape[0],1),dtype=mx.bool_)],1)
        enc=self.t5stack(x,'encoder',am)
        dec=mx.broadcast_to(self.w['shared.weight'][0],(x.shape[0],1,x.shape[-1]))
        y=self.residual(self.t5stack(dec,'decoder',am,enc),'output_patch_embedding')
        return y.reshape(x.shape[0],len(c['quantiles']),c['prediction_length'])*scale[...,None]+loc[...,None]

    def deberta(self,ids,mask):
        c=self.c; e='embeddings.'
        x=self.w[e+'word_embeddings.weight'][ids]
        if c.get('position_biased_input',True): x=x+self.w[e+'position_embeddings.weight'][mx.arange(ids.shape[1])]
        if c.get('type_vocab_size',0)>0: x=x+self.w[e+'token_type_embeddings.weight'][0]
        x=self.ln(x,e+'LayerNorm')*mask[...,None]
        rel=self.w['encoder.rel_embeddings.weight']
        if 'encoder.LayerNorm.weight' in self.w: rel=self.ln(rel,'encoder.LayerNorm')
        n=c['num_attention_heads']; span=c.get('position_buckets',c['max_position_embeddings']); mid=span//2
        r=mx.arange(ids.shape[1])[:,None]-mx.arange(ids.shape[1])[None,:]
        ab=mx.where((r<mid)&(r>-mid),mid-1,mx.abs(r))
        lg=mx.ceil(mx.log(ab/mid)/np.log((c['max_position_embeddings']-1)/mid)*(mid-1))+mid
        r=mx.where(ab<=mid,r,lg*mx.sign(r)).astype(mx.int32)
        for i in range(c['num_hidden_layers']):
            p=f'encoder.layer.{i}.'; a=p+'attention.self.'
            q,k,v=[self.heads(self.linear(x,a+t+'_proj'),n) for t in ['query','key','value']]
            pq=self.heads(self.linear(rel[None],a+'query_proj'),n)
            pk=self.heads(self.linear(rel[None],a+'key_proj'),n)
            s=q@k.transpose(0,1,3,2)
            cp=q@pk.transpose(0,1,3,2); pc=k@pq.transpose(0,1,3,2)
            idx=mx.broadcast_to(mx.clip(r+span,0,span*2-1)[None,None],(x.shape[0],n,ids.shape[1],ids.shape[1]))
            jdx=mx.broadcast_to(mx.clip(-r+span,0,span*2-1)[None,None],idx.shape)
            s=(s+mx.take_along_axis(cp,idx,axis=-1)+mx.take_along_axis(pc,jdx,axis=-1).transpose(0,1,3,2))/(q.shape[-1]*3)**.5
            valid=(mask[:,None,:,None]*mask[:,None,None,:])>0
            y=(mx.softmax(mx.where(valid,s,-1e30),axis=-1)@v).transpose(0,2,1,3).reshape(x.shape)
            x=self.ln(x+self.linear(y,p+'attention.output.dense'),p+'attention.output.LayerNorm')
            x=self.ln(x+self.linear(nn.gelu(self.linear(x,p+'intermediate.dense')),p+'output.dense'),p+'output.LayerNorm')
        return x