contriever-mlx / runtime_mlx.py
masahiroid's picture
Publish validated original-checkpoint conversion with provenance and usage
618a741 verified
Raw History Blame Contribute Delete
7.92 kB
"""Native MLX inference; original Hugging Face tensor names are retained."""
import json
from pathlib import Path
import numpy as np
import mlx.core as mx
import mlx.nn as nn
class Model:
def __init__(self, path, gguf=False):
p=Path(path)
self.c=json.loads((p/'config.json').read_text())
if gguf:
from gguf import GGUFReader
self.w={t.name: mx.array(np.array(t.data)) for t in GGUFReader(str(p/'model-f32.gguf')).tensors}
else:
self.w=mx.load(str(p/'model.safetensors'))
def linear(self,x,p):
y=x@self.w[p+'.weight'].T
return y+self.w[p+'.bias'] if p+'.bias' in self.w else y
def ln(self,x,p,eps=None):
return mx.fast.layer_norm(x,self.w[p+'.weight'],self.w.get(p+'.bias'),eps or self.c.get('layer_norm_eps',1e-12))
def heads(self,x,n):
return x.reshape(*x.shape[:-1],n,-1).transpose(0,2,1,3)
def __call__(self,input_ids=None,attention_mask=None,context=None):
if 'chronos_config' in self.c: return self.chronos(mx.array(context))
ids=mx.array(input_ids,dtype=mx.int32); mask=mx.array(attention_mask)
if self.c['model_type']=='deberta-v2': return self.deberta(ids,mask)
roberta=self.c['model_type']=='roberta'; root='roberta.' if roberta else ''
e=root+'embeddings.'
pos=mx.cumsum((ids!=1).astype(mx.int32),axis=1)*(ids!=1)+1 if roberta else mx.arange(ids.shape[1])[None,:]
x=self.w[e+'word_embeddings.weight'][ids]+self.w[e+'position_embeddings.weight'][pos]+self.w[e+'token_type_embeddings.weight'][0]
x=self.ln(x,e+'LayerNorm')
n=self.c['num_attention_heads']
for i in range(self.c['num_hidden_layers']):
p=root+f'encoder.layer.{i}.'
q,k,v=[self.heads(self.linear(x,p+'attention.self.'+t),n) for t in ['query','key','value']]
a=mx.softmax(mx.where(mask[:,None,None,:]>0,(q@k.transpose(0,1,3,2))/(q.shape[-1]**.5),-1e30),axis=-1)
y=(a@v).transpose(0,2,1,3).reshape(x.shape)
x=self.ln(x+self.linear(y,p+'attention.output.dense'),p+'attention.output.LayerNorm')
x=self.ln(x+self.linear(nn.gelu(self.linear(x,p+'intermediate.dense')),p+'output.dense'),p+'output.LayerNorm')
if roberta: return self.linear(mx.tanh(self.linear(x[:,0], 'classifier.dense')),'classifier.out_proj')
return (x*mask[...,None]).sum(1)/mx.maximum(mask.sum(1,keepdims=True),1)
def rms(self,x,p):
return x*mx.rsqrt(mx.mean(x*x,axis=-1,keepdims=True)+self.c['layer_norm_epsilon'])*self.w[p+'.weight']
def residual(self,x,p):
return self.linear(mx.maximum(self.linear(x,p+'.hidden_layer'),0),p+'.output_layer')+self.linear(x,p+'.residual_layer')
def t5att(self,x,p,kv=None,bias=None,mask=None):
n=self.c['num_heads']; kv=x if kv is None else kv
q,k,v=[self.heads(self.linear(z,p+'.'+t),n) for z,t in [(x,'q'),(kv,'k'),(kv,'v')]]
s=q@k.transpose(0,1,3,2)
if bias is not None: s=s+bias
if mask is not None: s=mx.where(mask[:,None,None,:]>0,s,-1e30)
y=(mx.softmax(s,axis=-1)@v).transpose(0,2,1,3).reshape(x.shape[0],x.shape[1],-1)
return self.linear(y,p+'.o')
def t5stack(self,x,root,mask,enc=None):
length=x.shape[1]; decoder=enc is not None
r=mx.arange(length)[None,:]-mx.arange(length)[:,None]
buckets=self.c['relative_attention_num_buckets']; maxd=self.c['relative_attention_max_distance']
if decoder:
r=mx.maximum(-r,0); sign=0
else:
buckets//=2; sign=(r>0).astype(mx.int32)*buckets; r=mx.abs(r)
exact=buckets//2
log=exact+(mx.log(mx.maximum(r,1)/exact)/np.log(maxd/exact)*(buckets-exact)).astype(mx.int32)
idx=sign+mx.where(r<exact,r,mx.minimum(log,buckets-1))
b=self.w[root+'.block.0.layer.0.SelfAttention.relative_attention_bias.weight'][idx.astype(mx.int32)].transpose(2,0,1)[None]
for i in range(self.c['num_decoder_layers'] if decoder else self.c['num_layers']):
p=root+f'.block.{i}.layer.'
x=x+self.t5att(self.rms(x,p+'0.layer_norm'),p+'0.SelfAttention',bias=b,mask=None if decoder else mask)
if decoder: x=x+self.t5att(self.rms(x,p+'1.layer_norm'),p+'1.EncDecAttention',kv=enc,mask=mask)
f=p+('2' if decoder else '1')
y=self.rms(x,f+'.layer_norm')
x=x+self.linear(mx.maximum(self.linear(y,f+'.DenseReluDense.wi'),0),f+'.DenseReluDense.wo')
return self.rms(x,root+'.final_layer_norm')
def chronos(self,context):
c=self.c['chronos_config']; context=context[:,-c['context_length']:]
mask=~mx.isnan(context); count=mx.maximum(mask.sum(-1,keepdims=True),1)
loc=mx.where(mask,context,0).sum(-1,keepdims=True)/count
scale=mx.sqrt(mx.where(mask,(context-loc)**2,0).sum(-1,keepdims=True)/count)
scale=mx.where(mask.any(-1,keepdims=True),scale,1.0)
scale=mx.where(scale==0,1e-5,scale)
x=mx.where(mask,(context-loc)/scale,0); size=c['input_patch_size']
pad=(-x.shape[1])%size
x=mx.pad(x,[(0,0),(pad,0)]); mask=mx.pad(mask,[(0,0),(pad,0)])
x=x.reshape(x.shape[0],-1,size); m=mask.reshape(mask.shape[0],-1,size)
x=self.residual(mx.concatenate([x,m.astype(x.dtype)],-1),'input_patch_embedding'); am=m.sum(-1)>0
if c['use_reg_token']:
x=mx.concatenate([x,mx.broadcast_to(self.w['shared.weight'][1],(x.shape[0],1,x.shape[-1]))],1)
am=mx.concatenate([am,mx.ones((am.shape[0],1),dtype=mx.bool_)],1)
enc=self.t5stack(x,'encoder',am)
dec=mx.broadcast_to(self.w['shared.weight'][0],(x.shape[0],1,x.shape[-1]))
y=self.residual(self.t5stack(dec,'decoder',am,enc),'output_patch_embedding')
return y.reshape(x.shape[0],len(c['quantiles']),c['prediction_length'])*scale[...,None]+loc[...,None]
def deberta(self,ids,mask):
c=self.c; e='embeddings.'
x=self.w[e+'word_embeddings.weight'][ids]
if c.get('position_biased_input',True): x=x+self.w[e+'position_embeddings.weight'][mx.arange(ids.shape[1])]
if c.get('type_vocab_size',0)>0: x=x+self.w[e+'token_type_embeddings.weight'][0]
x=self.ln(x,e+'LayerNorm')*mask[...,None]
rel=self.w['encoder.rel_embeddings.weight']
if 'encoder.LayerNorm.weight' in self.w: rel=self.ln(rel,'encoder.LayerNorm')
n=c['num_attention_heads']; span=c.get('position_buckets',c['max_position_embeddings']); mid=span//2
r=mx.arange(ids.shape[1])[:,None]-mx.arange(ids.shape[1])[None,:]
ab=mx.where((r<mid)&(r>-mid),mid-1,mx.abs(r))
lg=mx.ceil(mx.log(ab/mid)/np.log((c['max_position_embeddings']-1)/mid)*(mid-1))+mid
r=mx.where(ab<=mid,r,lg*mx.sign(r)).astype(mx.int32)
for i in range(c['num_hidden_layers']):
p=f'encoder.layer.{i}.'; a=p+'attention.self.'
q,k,v=[self.heads(self.linear(x,a+t+'_proj'),n) for t in ['query','key','value']]
pq=self.heads(self.linear(rel[None],a+'query_proj'),n)
pk=self.heads(self.linear(rel[None],a+'key_proj'),n)
s=q@k.transpose(0,1,3,2)
cp=q@pk.transpose(0,1,3,2); pc=k@pq.transpose(0,1,3,2)
idx=mx.broadcast_to(mx.clip(r+span,0,span*2-1)[None,None],(x.shape[0],n,ids.shape[1],ids.shape[1]))
jdx=mx.broadcast_to(mx.clip(-r+span,0,span*2-1)[None,None],idx.shape)
s=(s+mx.take_along_axis(cp,idx,axis=-1)+mx.take_along_axis(pc,jdx,axis=-1).transpose(0,1,3,2))/(q.shape[-1]*3)**.5
valid=(mask[:,None,:,None]*mask[:,None,None,:])>0
y=(mx.softmax(mx.where(valid,s,-1e30),axis=-1)@v).transpose(0,2,1,3).reshape(x.shape)
x=self.ln(x+self.linear(y,p+'attention.output.dense'),p+'attention.output.LayerNorm')
x=self.ln(x+self.linear(nn.gelu(self.linear(x,p+'intermediate.dense')),p+'output.dense'),p+'output.LayerNorm')
return x