File size: 6,052 Bytes
a44492d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
--!native
-- SmolLM2-135M-Instruct forward pass. CPU inference is entirely Luau.
local Encoding = game:GetService('EncodingService')
local Http = game:GetService('HttpService')
local Engine = {}
local D,H,KV,HD,FF,LAYERS,VOCAB = 576,9,3,64,1536,30,49152
local EPS,THETA = 1e-5,100000
local rd,wr,ri = buffer.readf32,buffer.writef32,buffer.readi8
local function vec(n) return buffer.create(n*4) end
local lastYield=os.clock()
local cancelCheck=nil
local function checkpoint(force)
 if force or os.clock()-lastYield>0.012 then
  task.wait();lastYield=os.clock()
  if cancelCheck and cancelCheck() then error('CANCELLED',0) end
 end
end
local function matvec(w,x,out)
 local n,m,b=w.rows,w.cols,w.data
 local so=n*m
 for r=0,n-1 do
  local a=r*m;local s=0
  for g=0,m/64-1 do
   local partial=0;local start=g*64
   for c=start,start+63,4 do
    partial+=ri(b,a+c)*rd(x,c*4)+ri(b,a+c+1)*rd(x,(c+1)*4)+ri(b,a+c+2)*rd(x,(c+2)*4)+ri(b,a+c+3)*rd(x,(c+3)*4)
   end
   s+=partial*rd(b,so+(r*m/64+g)*4)
  end
  wr(out,r*4,s)
  if r%128==127 then checkpoint(false) end
 end
end
local function norm(x,w,out)
 local s=0
 for i=0,D-1 do local v=rd(x,i*4);s+=v*v end
 local inv=1/math.sqrt(s/D+EPS)
 for i=0,D-1 do wr(out,i*4,rd(x,i*4)*inv*rd(w.data,i*4)) end
end
local function add(x,b)
 for i=0,D-1 do wr(x,i*4,rd(x,i*4)+rd(b,i*4)) end
end
function Engine.load(folder,onProgress)
 local manifest=Http:JSONDecode(folder.Manifest.Value)
 local weights={};local names={}
 for name in manifest.tensors do table.insert(names,name) end
 table.sort(names)
 for index,name in names do
  local meta=manifest.tensors[name];local node=folder[meta.key]
  local chunks=node:GetChildren();table.sort(chunks,function(a,b) return a.Name<b.Name end)
  local data=buffer.create(meta.bytes);local off=0
  for _,chunk in chunks do
   local decoded=Encoding:Base64Decode(buffer.fromstring(chunk.Value))
   buffer.copy(data,off,decoded);off+=buffer.len(decoded)
  end
  assert(off==meta.bytes,'Corrupt tensor '..name)
  if meta.kind=='q4g64' then
   local count=meta.shape[1]*meta.shape[2]
   local expanded=buffer.create(count+count/64*4)
   for j=0,count/2-1 do
    local packed=buffer.readu8(data,j)
    buffer.writei8(expanded,j*2,bit32.band(packed,15)-8)
    buffer.writei8(expanded,j*2+1,bit32.rshift(packed,4)-8)
    if j%131072==131071 then checkpoint(false) end
   end
   buffer.copy(expanded,count,data,count/2,count/64*4)
   data=expanded
  end
  weights[name]={data=data,rows=meta.shape[1],cols=meta.shape[2]}
  node:Destroy()
  if onProgress then onProgress(index/#names) end
  checkpoint(true)
 end
 Engine.weights=weights;Engine.manifest=manifest
 Engine.scratch={x=vec(D),n=vec(D),q=vec(D),k=vec(KV*HD),v=vec(KV*HD),a=vec(D),o=vec(D),gate=vec(FF),up=vec(FF),logits=vec(VOCAB)}
 return Engine
end
function Engine.newSession(maxContext)
 local s={position=0,maxContext=maxContext or 256,keys={},values={}}
 for l=1,LAYERS do s.keys[l]=vec(s.maxContext*KV*HD);s.values[l]=vec(s.maxContext*KV*HD) end
 return s
end
function Engine.forward(session,token,project,shouldCancel)
 cancelCheck=shouldCancel;checkpoint(false)
 local pos=session.position
 assert(pos<session.maxContext,'Context limit reached')
 local w=Engine.weights;local t=Engine.scratch;local x=t.x
 local emb=w['model.embed_tokens.weight']
 for i=0,D-1 do
  local scale=rd(emb.data,emb.rows*D+(token*D/64+math.floor(i/64))*4)
  wr(x,i*4,ri(emb.data,token*D+i)*scale)
 end
 for layer=0,LAYERS-1 do
  local p='model.layers.'..layer..'.';local att=p..'self_attn.'
  norm(x,w[p..'input_layernorm.weight'],t.n)
  matvec(w[att..'q_proj.weight'],t.n,t.q)
  matvec(w[att..'k_proj.weight'],t.n,t.k)
  matvec(w[att..'v_proj.weight'],t.n,t.v)
  for h=0,H-1 do
   for j=0,HD/2-1 do
    local angle=pos/(THETA^(2*j/HD));local co,si=math.cos(angle),math.sin(angle)
    local a=(h*HD+j)*4;local b=a+HD/2*4;local v1,v2=rd(t.q,a),rd(t.q,b)
    wr(t.q,a,v1*co-v2*si);wr(t.q,b,v2*co+v1*si)
    if h<KV then
     v1,v2=rd(t.k,a),rd(t.k,b);wr(t.k,a,v1*co-v2*si);wr(t.k,b,v2*co+v1*si)
    end
   end
  end
  local kc,vc=session.keys[layer+1],session.values[layer+1]
  buffer.copy(kc,pos*KV*HD*4,t.k);buffer.copy(vc,pos*KV*HD*4,t.v)
  for h=0,H-1 do
   local kh=math.floor(h/(H/KV));local scores=table.create(pos+1);local peak=-math.huge
   for step=0,pos do
    local base=(step*KV*HD+kh*HD)*4;local qbase=h*HD*4;local dot=0
    for j=0,HD-1 do dot+=rd(t.q,qbase+j*4)*rd(kc,base+j*4) end
    dot/=8;scores[step+1]=dot;peak=math.max(peak,dot)
   end
   local total=0
   for step=1,pos+1 do local e=math.exp(scores[step]-peak);scores[step]=e;total+=e end
   for j=0,HD-1 do
    local sum=0
    for step=0,pos do sum+=scores[step+1]/total*rd(vc,(step*KV*HD+kh*HD+j)*4) end
    wr(t.a,(h*HD+j)*4,sum)
   end
  end
  matvec(w[att..'o_proj.weight'],t.a,t.o);add(x,t.o)
  norm(x,w[p..'post_attention_layernorm.weight'],t.n)
  matvec(w[p..'mlp.gate_proj.weight'],t.n,t.gate)
  matvec(w[p..'mlp.up_proj.weight'],t.n,t.up)
  for i=0,FF-1 do local g=rd(t.gate,i*4);wr(t.gate,i*4,g/(1+math.exp(-g))*rd(t.up,i*4)) end
  matvec(w[p..'mlp.down_proj.weight'],t.gate,t.o);add(x,t.o)
  checkpoint(false)
 end
 session.position+=1
 if not project then return nil end
 norm(x,w['model.norm.weight'],t.n);matvec(emb,t.n,t.logits)
 return t.logits
end
function Engine.sample(logits,temperature,rng,previous)
 local best={};local repetition={}
 for _,id in previous or {} do repetition[id]=true end
 for id=0,VOCAB-1 do
  local score=rd(logits,id*4)
  if repetition[id] then score=if score>0 then score/1.08 else score*1.08 end
  if id==0 or id==1 or (id>=3 and id<=16) then score=-math.huge end
  if #best<32 or score>best[#best][2] then
   local k=#best+1
   while k>1 and best[k-1][2]<score do best[k]=best[k-1];k-=1 end
   best[k]={id,score};if #best>32 then best[33]=nil end
  end
 end
 if temperature==0 then return best[1][1] end
 local sum=0
 for _,v in best do v[3]=math.exp((v[2]-best[1][2])/temperature);sum+=v[3] end
 local z=rng:NextNumber()*sum
 for _,v in best do z-=v[3];if z<=0 then return v[1] end end
 return best[1][1]
end
return Engine