Franok / source /web /engine.mjs
yunusemrejr's picture
Publish source, film and evaluations from 699be55df232
2bff5be verified
Raw History Blame Contribute Delete
5.38 kB
// Incremental decoder. The same module runs in a Worker and under Node for parity tests.
export class Engine {
constructor(instance, manifest, bytes) {
const w = (this.w = instance.exports);
const m = (this.c = manifest.config);
this.bits = manifest.format === 'franok-q8-v1' ? 8 : 4;
if (!['franok-q4-v1', 'franok-q8-v1'].includes(manifest.format) || bytes.byteLength !== manifest.bytes)
throw Error('Incomplete model file.');
const dim = m.width,
hd = dim / m.heads,
kv = hd * m.kv_heads;
this.base = Number(w.__heap_base.value);
this.t = manifest.tensors;
let next = this.base + bytes.byteLength;
const alloc = (length) => {
next = Math.ceil(next / 16) * 16;
const start = next;
next += 4 * length;
return start;
};
this.x = alloc(dim);
this.norm = alloc(dim);
this.qkv = alloc(dim + kv * 2);
this.a = alloc(dim);
this.h = alloc(m.hidden * 2);
this.scores = alloc(m.window);
this.logits = alloc(m.vocab);
this.cos = alloc((m.window * hd) / 2);
this.sin = alloc((m.window * hd) / 2);
this.act = alloc(Math.max(dim, m.hidden) / 4);
this.scales = alloc(Math.max(dim, m.hidden) / 32);
this.keys = alloc(m.layers * m.window * kv);
this.values = alloc(m.layers * m.window * kv);
w.memory.grow(Math.max(0, Math.ceil((next - w.memory.buffer.byteLength) / 65536)));
this.f = new Float32Array(w.memory.buffer);
new Uint8Array(w.memory.buffer, this.base, bytes.byteLength).set(new Uint8Array(bytes));
for (let pos = 0; pos < m.window; pos++)
for (let i = 0; i < hd / 2; i++) {
const theta = pos * 10000 ** ((-2 * i) / hd),
j = (pos * hd) / 2 + i;
this.f[this.cos / 4 + j] = Math.cos(theta);
this.f[this.sin / 4 + j] = Math.sin(theta);
}
this.tokens = new Int32Array(m.window);
this.reset();
}
reset() {
this.pos = 0;
this.anchor = 0;
this.last = 0;
}
reusePrefix(ids, limit = ids.length) {
let count = 0;
const end = Math.min(this.pos, ids.length, limit);
while (count < end && ids[count] === this.tokens[count]) count++;
this.pos = count;
this.last = count ? this.tokens[count - 1] : 0;
// The active group reads the final token of the preceding group.
const group = Math.floor((count - 1) / this.c.banks);
this.anchor = group > 0 ? this.tokens[group * this.c.banks - 1] : 0;
return count;
}
hidden() {
if (!this.pos) throw Error('Read a decoder state after processing a token.');
const width = this.c.width;
this.w.rms(this.x, this.ptr('norm.weight'), this.norm, width);
return this.f.slice(this.norm / 4, this.norm / 4 + width);
}
ptr(name) {
return this.base + this.t[name].offset;
}
step(token, needLogits = true) {
const { w, c, f, pos } = this,
d = c.width,
hd = d / c.heads,
kv = c.kv_heads * hd;
if (pos >= c.window || !Number.isInteger(token) || token < 0 || token >= c.vocab)
throw Error('Decoder input outside its context.');
if (pos % c.banks === 0) this.anchor = this.last;
this.last = token;
this.tokens[pos] = token;
const kernel = this.bits === 8 ? w.gemv8 : w.gemv;
const mul = (p, x, y, rows, cols) => kernel(p, x, y, rows, cols, this.act, this.scales);
const tbl = this.ptr('embed.weight'),
rowBytes = (d / 32) * (this.bits === 8 ? 36 : 20);
(this.bits === 8 ? w.unpack8 : w.unpack)(tbl + rowBytes * token, this.x, d);
for (let layer = 0; layer < c.layers; layer++) {
const p = (name) => this.ptr('blocks.' + layer + '.' + name);
w.rms(this.x, p('pre.weight'), this.norm, d);
mul(p('qkv.weight'), this.norm, this.qkv, d + 2 * kv, d);
const k = this.qkv + 4 * d,
v = k + 4 * kv;
const cos = this.cos + 2 * pos * hd,
sin = this.sin + 2 * pos * hd;
w.rotary(this.qkv, cos, sin, c.heads, hd);
w.rotary(k, cos, sin, c.kv_heads, hd);
const cacheOffset = 4 * layer * c.window * kv;
const keys = this.keys + cacheOffset,
vals = this.values + cacheOffset;
f.copyWithin(keys / 4 + pos * kv, k / 4, k / 4 + kv);
f.copyWithin(vals / 4 + pos * kv, v / 4, v / 4 + kv);
w.attend(this.qkv, keys, vals, this.a, this.scores, pos + 1, c.heads, c.kv_heads, hd);
mul(p('out.weight'), this.a, this.norm, d, d);
w.add(this.x, this.norm, d);
w.rms(this.x, p('post.weight'), this.norm, d);
const bank = ((pos % c.banks) + (this.anchor >> (layer % 4)) + layer) % c.banks;
mul(
p('up') + (bank * this.t['blocks.' + layer + '.up'].bytes) / c.banks,
this.norm,
this.h,
2 * c.hidden,
d,
);
w.swiglu(this.h, c.hidden);
mul(
p('down') + (bank * this.t['blocks.' + layer + '.down'].bytes) / c.banks,
this.h,
this.norm,
d,
c.hidden,
);
w.add(this.x, this.norm, d);
}
this.pos++;
if (!needLogits) return;
w.rms(this.x, this.ptr('norm.weight'), this.norm, d);
mul(tbl, this.norm, this.logits, c.vocab, d);
return f.subarray(this.logits / 4, this.logits / 4 + c.vocab);
}
}
export async function createEngine(wasm, manifest, bytes) {
const { instance } = await WebAssembly.instantiate(wasm, { math: { exp: Math.exp } });
return new Engine(instance, manifest, bytes);
}