tiny-scene-three / web /js /nn /model.js
paralaif's picture
Tiny Scene: text to animated three.js scenes with a 4.85M Transformer
dd53693 verified
Raw History Blame Contribute Delete
5.3 kB
// The Transformer from ml/model.py, in plain JavaScript (no dependencies). Greedy decoding with a key/value cache.
function halfToFloat(u16) {
const out = new Float32Array(u16.length);
for (let i = 0; i < u16.length; i++) {
const h = u16[i], e = (h >> 10) & 31, f = h & 1023, s = h & 0x8000 ? -1 : 1;
out[i] = e === 0 ? s * f * 5.960464477539063e-8 : e === 31 ? (f ? NaN : s * Infinity) : s * (1 + f / 1024) * 2 ** (e - 15);
}
return out;
}
export class TinyGPT {
/** meta: parsed model.json, buffer: ArrayBuffer of weights.bin */
constructor(meta, buffer) {
this.meta = meta;
Object.assign(this, meta.config);
this.vocabList = meta.vocab;
this.id = new Map(meta.vocab.map((t, i) => [t, i]));
const all = meta.dtype === 'float16' ? halfToFloat(new Uint16Array(buffer)) : new Float32Array(buffer);
this.w = {};
for (const t of meta.tensors) this.w[t.name] = all.subarray(t.offset, t.offset + t.shape.reduce((a, b) => a * b, 1));
const d = this.d;
this.kc = Array.from({ length: this.layers }, () => new Float32Array(this.ctx * d));
this.vc = Array.from({ length: this.layers }, () => new Float32Array(this.ctx * d));
this.x = new Float32Array(d); this.h = new Float32Array(d); this.qkv = new Float32Array(3 * d);
this.att = new Float32Array(d); this.tmp = new Float32Array(d); this.ff = new Float32Array(4 * d);
this.sc = new Float32Array(this.ctx);
}
static async load(base = './model/') {
const [meta, buf] = await Promise.all([
fetch(base + 'model.json').then((r) => r.json()),
fetch(base + 'weights.bin').then((r) => r.arrayBuffer()),
]);
return new TinyGPT(meta, buf);
}
normalize(text) {
let s = '';
for (const c of text.toLowerCase().replace(/’/g, "'")) if (c.length === 1 && this.id.has(c) && this.id.get(c) < 4 + 64) s += c;
return s.replace(/\s+/g, ' ').trim().slice(0, this.meta.max_prompt);
}
encode(text) {
const { bos, sep } = this.meta.special;
return [bos, ...[...this.normalize(text)].map((c) => this.id.get(c)), sep];
}
_linear(W, b, x, y, nOut, nIn) {
for (let o = 0, k = 0; o < nOut; o++) {
let s = b[o];
for (let i = 0; i < nIn; i++, k++) s += W[k] * x[i];
y[o] = s;
}
}
_norm(x, y, g, b) {
const d = this.d;
let m = 0, v = 0;
for (let i = 0; i < d; i++) m += x[i];
m /= d;
for (let i = 0; i < d; i++) { const c = x[i] - m; v += c * c; }
const inv = 1 / Math.sqrt(v / d + 1e-5);
for (let i = 0; i < d; i++) y[i] = (x[i] - m) * inv * g[i] + b[i];
}
/** Feed one token at position t; returns the hidden state (valid until the next call). */
_step(tok, t) {
const { d, heads, w, x, h, qkv, att, tmp, ff, sc } = this, dh = d / heads, scale = 1 / Math.sqrt(dh);
const te = w['tok.weight'], pe = w['pos.weight'];
for (let i = 0; i < d; i++) x[i] = te[tok * d + i] + pe[t * d + i];
for (let l = 0; l < this.layers; l++) {
const p = `blocks.${l}.`, K = this.kc[l], V = this.vc[l];
this._norm(x, h, w[p + 'ln1.weight'], w[p + 'ln1.bias']);
this._linear(w[p + 'qkv.weight'], w[p + 'qkv.bias'], h, qkv, 3 * d, d);
K.set(qkv.subarray(d, 2 * d), t * d);
V.set(qkv.subarray(2 * d), t * d);
for (let hd = 0; hd < heads; hd++) {
const o = hd * dh;
let mx = -Infinity;
for (let j = 0; j <= t; j++) {
let s = 0;
for (let i = 0, kb = j * d + o; i < dh; i++) s += qkv[o + i] * K[kb + i];
sc[j] = s *= scale;
if (s > mx) mx = s;
}
let z = 0;
for (let j = 0; j <= t; j++) z += sc[j] = Math.exp(sc[j] - mx);
for (let i = 0; i < dh; i++) att[o + i] = 0;
for (let j = 0; j <= t; j++) {
const pj = sc[j] / z;
for (let i = 0, vb = j * d + o; i < dh; i++) att[o + i] += pj * V[vb + i];
}
}
this._linear(w[p + 'proj.weight'], w[p + 'proj.bias'], att, tmp, d, d);
for (let i = 0; i < d; i++) x[i] += tmp[i];
this._norm(x, h, w[p + 'ln2.weight'], w[p + 'ln2.bias']);
this._linear(w[p + 'fc1.weight'], w[p + 'fc1.bias'], h, ff, 4 * d, d);
for (let i = 0; i < 4 * d; i++) { const v = ff[i]; ff[i] = 0.5 * v * (1 + Math.tanh(0.7978845608028654 * (v + 0.044715 * v * v * v))); }
this._linear(w[p + 'fc2.weight'], w[p + 'fc2.bias'], ff, tmp, d, 4 * d);
for (let i = 0; i < d; i++) x[i] += tmp[i];
}
this._norm(x, h, w['ln_f.weight'], w['ln_f.bias']);
return h;
}
_argmax(h) {
const te = this.w['tok.weight'], d = this.d;
let best = -Infinity, arg = 0;
for (let v = 0, k = 0; v < this.vocab; v++) {
let s = 0;
for (let i = 0; i < d; i++, k++) s += te[k] * h[i];
if (s > best) { best = s; arg = v; }
}
return arg;
}
/** text -> { tokens: ['<new>', 'loc=beach', ...], ms } */
generate(text, maxNew = 40) {
const t0 = performance.now(), ids = this.encode(text), out = [];
let h, t = 0;
for (const id of ids) h = this._step(id, t++);
while (out.length < maxNew && t < this.ctx) {
const next = this._argmax(h);
if (next === this.meta.special.eos) break;
out.push(this.vocabList[next]);
h = this._step(next, t++);
}
return { tokens: out, ms: performance.now() - t0 };
}
}