File size: 5,300 Bytes
dd53693
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
// The Transformer from ml/model.py, in plain JavaScript (no dependencies). Greedy decoding with a key/value cache.

function halfToFloat(u16) {
  const out = new Float32Array(u16.length);
  for (let i = 0; i < u16.length; i++) {
    const h = u16[i], e = (h >> 10) & 31, f = h & 1023, s = h & 0x8000 ? -1 : 1;
    out[i] = e === 0 ? s * f * 5.960464477539063e-8 : e === 31 ? (f ? NaN : s * Infinity) : s * (1 + f / 1024) * 2 ** (e - 15);
  }
  return out;
}

export class TinyGPT {
  /** meta: parsed model.json, buffer: ArrayBuffer of weights.bin */
  constructor(meta, buffer) {
    this.meta = meta;
    Object.assign(this, meta.config);
    this.vocabList = meta.vocab;
    this.id = new Map(meta.vocab.map((t, i) => [t, i]));
    const all = meta.dtype === 'float16' ? halfToFloat(new Uint16Array(buffer)) : new Float32Array(buffer);
    this.w = {};
    for (const t of meta.tensors) this.w[t.name] = all.subarray(t.offset, t.offset + t.shape.reduce((a, b) => a * b, 1));
    const d = this.d;
    this.kc = Array.from({ length: this.layers }, () => new Float32Array(this.ctx * d));
    this.vc = Array.from({ length: this.layers }, () => new Float32Array(this.ctx * d));
    this.x = new Float32Array(d); this.h = new Float32Array(d); this.qkv = new Float32Array(3 * d);
    this.att = new Float32Array(d); this.tmp = new Float32Array(d); this.ff = new Float32Array(4 * d);
    this.sc = new Float32Array(this.ctx);
  }

  static async load(base = './model/') {
    const [meta, buf] = await Promise.all([
      fetch(base + 'model.json').then((r) => r.json()),
      fetch(base + 'weights.bin').then((r) => r.arrayBuffer()),
    ]);
    return new TinyGPT(meta, buf);
  }

  normalize(text) {
    let s = '';
    for (const c of text.toLowerCase().replace(/’/g, "'")) if (c.length === 1 && this.id.has(c) && this.id.get(c) < 4 + 64) s += c;
    return s.replace(/\s+/g, ' ').trim().slice(0, this.meta.max_prompt);
  }

  encode(text) {
    const { bos, sep } = this.meta.special;
    return [bos, ...[...this.normalize(text)].map((c) => this.id.get(c)), sep];
  }

  _linear(W, b, x, y, nOut, nIn) {
    for (let o = 0, k = 0; o < nOut; o++) {
      let s = b[o];
      for (let i = 0; i < nIn; i++, k++) s += W[k] * x[i];
      y[o] = s;
    }
  }

  _norm(x, y, g, b) {
    const d = this.d;
    let m = 0, v = 0;
    for (let i = 0; i < d; i++) m += x[i];
    m /= d;
    for (let i = 0; i < d; i++) { const c = x[i] - m; v += c * c; }
    const inv = 1 / Math.sqrt(v / d + 1e-5);
    for (let i = 0; i < d; i++) y[i] = (x[i] - m) * inv * g[i] + b[i];
  }

  /** Feed one token at position t; returns the hidden state (valid until the next call). */
  _step(tok, t) {
    const { d, heads, w, x, h, qkv, att, tmp, ff, sc } = this, dh = d / heads, scale = 1 / Math.sqrt(dh);
    const te = w['tok.weight'], pe = w['pos.weight'];
    for (let i = 0; i < d; i++) x[i] = te[tok * d + i] + pe[t * d + i];
    for (let l = 0; l < this.layers; l++) {
      const p = `blocks.${l}.`, K = this.kc[l], V = this.vc[l];
      this._norm(x, h, w[p + 'ln1.weight'], w[p + 'ln1.bias']);
      this._linear(w[p + 'qkv.weight'], w[p + 'qkv.bias'], h, qkv, 3 * d, d);
      K.set(qkv.subarray(d, 2 * d), t * d);
      V.set(qkv.subarray(2 * d), t * d);
      for (let hd = 0; hd < heads; hd++) {
        const o = hd * dh;
        let mx = -Infinity;
        for (let j = 0; j <= t; j++) {
          let s = 0;
          for (let i = 0, kb = j * d + o; i < dh; i++) s += qkv[o + i] * K[kb + i];
          sc[j] = s *= scale;
          if (s > mx) mx = s;
        }
        let z = 0;
        for (let j = 0; j <= t; j++) z += sc[j] = Math.exp(sc[j] - mx);
        for (let i = 0; i < dh; i++) att[o + i] = 0;
        for (let j = 0; j <= t; j++) {
          const pj = sc[j] / z;
          for (let i = 0, vb = j * d + o; i < dh; i++) att[o + i] += pj * V[vb + i];
        }
      }
      this._linear(w[p + 'proj.weight'], w[p + 'proj.bias'], att, tmp, d, d);
      for (let i = 0; i < d; i++) x[i] += tmp[i];
      this._norm(x, h, w[p + 'ln2.weight'], w[p + 'ln2.bias']);
      this._linear(w[p + 'fc1.weight'], w[p + 'fc1.bias'], h, ff, 4 * d, d);
      for (let i = 0; i < 4 * d; i++) { const v = ff[i]; ff[i] = 0.5 * v * (1 + Math.tanh(0.7978845608028654 * (v + 0.044715 * v * v * v))); }
      this._linear(w[p + 'fc2.weight'], w[p + 'fc2.bias'], ff, tmp, d, 4 * d);
      for (let i = 0; i < d; i++) x[i] += tmp[i];
    }
    this._norm(x, h, w['ln_f.weight'], w['ln_f.bias']);
    return h;
  }

  _argmax(h) {
    const te = this.w['tok.weight'], d = this.d;
    let best = -Infinity, arg = 0;
    for (let v = 0, k = 0; v < this.vocab; v++) {
      let s = 0;
      for (let i = 0; i < d; i++, k++) s += te[k] * h[i];
      if (s > best) { best = s; arg = v; }
    }
    return arg;
  }

  /** text -> { tokens: ['<new>', 'loc=beach', ...], ms } */
  generate(text, maxNew = 40) {
    const t0 = performance.now(), ids = this.encode(text), out = [];
    let h, t = 0;
    for (const id of ids) h = this._step(id, t++);
    while (out.length < maxNew && t < this.ctx) {
      const next = this._argmax(h);
      if (next === this.meta.special.eos) break;
      out.push(this.vocabList[next]);
      h = this._step(next, t++);
    }
    return { tokens: out, ms: performance.now() - t0 };
  }
}