"""Native MLX inference; original Hugging Face tensor names are retained.""" import json from pathlib import Path import numpy as np import mlx.core as mx import mlx.nn as nn class Model: def __init__(self, path, gguf=False): p=Path(path) self.c=json.loads((p/'config.json').read_text()) if gguf: from gguf import GGUFReader self.w={t.name: mx.array(np.array(t.data)) for t in GGUFReader(str(p/'model-f32.gguf')).tensors} else: self.w=mx.load(str(p/'model.safetensors')) def linear(self,x,p): y=x@self.w[p+'.weight'].T return y+self.w[p+'.bias'] if p+'.bias' in self.w else y def ln(self,x,p,eps=None): return mx.fast.layer_norm(x,self.w[p+'.weight'],self.w.get(p+'.bias'),eps or self.c.get('layer_norm_eps',1e-12)) def heads(self,x,n): return x.reshape(*x.shape[:-1],n,-1).transpose(0,2,1,3) def __call__(self,input_ids=None,attention_mask=None,context=None): if 'chronos_config' in self.c: return self.chronos(mx.array(context)) ids=mx.array(input_ids,dtype=mx.int32); mask=mx.array(attention_mask) if self.c['model_type']=='deberta-v2': return self.deberta(ids,mask) roberta=self.c['model_type']=='roberta'; root='roberta.' if roberta else '' e=root+'embeddings.' pos=mx.cumsum((ids!=1).astype(mx.int32),axis=1)*(ids!=1)+1 if roberta else mx.arange(ids.shape[1])[None,:] x=self.w[e+'word_embeddings.weight'][ids]+self.w[e+'position_embeddings.weight'][pos]+self.w[e+'token_type_embeddings.weight'][0] x=self.ln(x,e+'LayerNorm') n=self.c['num_attention_heads'] for i in range(self.c['num_hidden_layers']): p=root+f'encoder.layer.{i}.' q,k,v=[self.heads(self.linear(x,p+'attention.self.'+t),n) for t in ['query','key','value']] a=mx.softmax(mx.where(mask[:,None,None,:]>0,(q@k.transpose(0,1,3,2))/(q.shape[-1]**.5),-1e30),axis=-1) y=(a@v).transpose(0,2,1,3).reshape(x.shape) x=self.ln(x+self.linear(y,p+'attention.output.dense'),p+'attention.output.LayerNorm') x=self.ln(x+self.linear(nn.gelu(self.linear(x,p+'intermediate.dense')),p+'output.dense'),p+'output.LayerNorm') if roberta: return self.linear(mx.tanh(self.linear(x[:,0], 'classifier.dense')),'classifier.out_proj') return (x*mask[...,None]).sum(1)/mx.maximum(mask.sum(1,keepdims=True),1) def rms(self,x,p): return x*mx.rsqrt(mx.mean(x*x,axis=-1,keepdims=True)+self.c['layer_norm_epsilon'])*self.w[p+'.weight'] def residual(self,x,p): return self.linear(mx.maximum(self.linear(x,p+'.hidden_layer'),0),p+'.output_layer')+self.linear(x,p+'.residual_layer') def t5att(self,x,p,kv=None,bias=None,mask=None): n=self.c['num_heads']; kv=x if kv is None else kv q,k,v=[self.heads(self.linear(z,p+'.'+t),n) for z,t in [(x,'q'),(kv,'k'),(kv,'v')]] s=q@k.transpose(0,1,3,2) if bias is not None: s=s+bias if mask is not None: s=mx.where(mask[:,None,None,:]>0,s,-1e30) y=(mx.softmax(s,axis=-1)@v).transpose(0,2,1,3).reshape(x.shape[0],x.shape[1],-1) return self.linear(y,p+'.o') def t5stack(self,x,root,mask,enc=None): length=x.shape[1]; decoder=enc is not None r=mx.arange(length)[None,:]-mx.arange(length)[:,None] buckets=self.c['relative_attention_num_buckets']; maxd=self.c['relative_attention_max_distance'] if decoder: r=mx.maximum(-r,0); sign=0 else: buckets//=2; sign=(r>0).astype(mx.int32)*buckets; r=mx.abs(r) exact=buckets//2 log=exact+(mx.log(mx.maximum(r,1)/exact)/np.log(maxd/exact)*(buckets-exact)).astype(mx.int32) idx=sign+mx.where(r0 if c['use_reg_token']: x=mx.concatenate([x,mx.broadcast_to(self.w['shared.weight'][1],(x.shape[0],1,x.shape[-1]))],1) am=mx.concatenate([am,mx.ones((am.shape[0],1),dtype=mx.bool_)],1) enc=self.t5stack(x,'encoder',am) dec=mx.broadcast_to(self.w['shared.weight'][0],(x.shape[0],1,x.shape[-1])) y=self.residual(self.t5stack(dec,'decoder',am,enc),'output_patch_embedding') return y.reshape(x.shape[0],len(c['quantiles']),c['prediction_length'])*scale[...,None]+loc[...,None] def deberta(self,ids,mask): c=self.c; e='embeddings.' x=self.w[e+'word_embeddings.weight'][ids] if c.get('position_biased_input',True): x=x+self.w[e+'position_embeddings.weight'][mx.arange(ids.shape[1])] if c.get('type_vocab_size',0)>0: x=x+self.w[e+'token_type_embeddings.weight'][0] x=self.ln(x,e+'LayerNorm')*mask[...,None] rel=self.w['encoder.rel_embeddings.weight'] if 'encoder.LayerNorm.weight' in self.w: rel=self.ln(rel,'encoder.LayerNorm') n=c['num_attention_heads']; span=c.get('position_buckets',c['max_position_embeddings']); mid=span//2 r=mx.arange(ids.shape[1])[:,None]-mx.arange(ids.shape[1])[None,:] ab=mx.where((r-mid),mid-1,mx.abs(r)) lg=mx.ceil(mx.log(ab/mid)/np.log((c['max_position_embeddings']-1)/mid)*(mid-1))+mid r=mx.where(ab<=mid,r,lg*mx.sign(r)).astype(mx.int32) for i in range(c['num_hidden_layers']): p=f'encoder.layer.{i}.'; a=p+'attention.self.' q,k,v=[self.heads(self.linear(x,a+t+'_proj'),n) for t in ['query','key','value']] pq=self.heads(self.linear(rel[None],a+'query_proj'),n) pk=self.heads(self.linear(rel[None],a+'key_proj'),n) s=q@k.transpose(0,1,3,2) cp=q@pk.transpose(0,1,3,2); pc=k@pq.transpose(0,1,3,2) idx=mx.broadcast_to(mx.clip(r+span,0,span*2-1)[None,None],(x.shape[0],n,ids.shape[1],ids.shape[1])) jdx=mx.broadcast_to(mx.clip(-r+span,0,span*2-1)[None,None],idx.shape) s=(s+mx.take_along_axis(cp,idx,axis=-1)+mx.take_along_axis(pc,jdx,axis=-1).transpose(0,1,3,2))/(q.shape[-1]*3)**.5 valid=(mask[:,None,:,None]*mask[:,None,None,:])>0 y=(mx.softmax(mx.where(valid,s,-1e30),axis=-1)@v).transpose(0,2,1,3).reshape(x.shape) x=self.ln(x+self.linear(y,p+'attention.output.dense'),p+'attention.output.LayerNorm') x=self.ln(x+self.linear(nn.gelu(self.linear(x,p+'intermediate.dense')),p+'output.dense'),p+'output.LayerNorm') return x