Download data.js from Vibelet/spotlight-demo: direct link, hf CLI and curl.
- Browser
- Download file 13.7 kB
-
https://huggingface.co/Vibelet/spotlight-demo/resolve/main/data.js
- Command line
-
hf download hf://Vibelet/spotlight-demo/data.js
-
curl -L -o data.js https://huggingface.co/Vibelet/spotlight-demo/resolve/main/data.js
13.7 kB
| /* Spotlight Memory demo — synthetic dataset + exact CPU reference. | |
| * | |
| * Mirrors spotlight/memory.py at chunk == 1 (the exact per-token recurrence): | |
| * | |
| * a = 0.5 * G * (1 + tanh(u)) address in (0, G) | |
| * b(t) = cos^2(pi t / 3) for |t| < 3/2 else 0 1D cos^2 bump | |
| * phi_z(x) = b(x - z) / sqrt(sum_j b(x - j)^2) 3 nearest cells | |
| * c = floor(x + 0.5), z in c + {-1, 0, 1} | |
| * w_2d = phi_zx(ax) * phi_zy(ay) 9 cells | |
| * q2, k2 = L2-normalised content, with a constant 1 appended (dim dk + 1) | |
| * beta = sigmoid(.) | |
| * read o_t = sum_j wq_j * (S[cell_j]^T q2_t) | |
| * write S[cell_j] += w_j * beta_t * (v_t - S[cell_j] k2_t) k2_t^T | |
| * | |
| * The cell state buffer is laid out head-outermost, padded lattice N = G + 3: | |
| * cell id = (h * N + py) * N + px px, py in [0, N) | |
| * slot = cell id * (DK + 1) * DV + d * DV + dv | |
| * | |
| * Loaded as a classic <script> in the browser (defines window.Spotlight) and as a | |
| * CommonJS module under node (for offline checks of the reference numbers). | |
| */ | |
| (function (root, factory) { | |
| if (typeof module === 'object' && module.exports) module.exports = factory(); | |
| else root.Spotlight = factory(); | |
| })(typeof self !== 'undefined' ? self : globalThis, function () { | |
| 'use strict'; | |
| var CFG = { | |
| G: 32, // lattice side | |
| H: 4, // heads | |
| DK: 8, // content key/query dim (the +1 constant channel is added) | |
| DV: 8, // value dim | |
| T: 128, // tokens | |
| RADIUS: 1.5, // bump support | |
| N: 35, // padded lattice = G + 1 + 2*K, K = 1 | |
| NOFF: 9, // (2K+1)^2 neighbour cells | |
| DK1: 9, // DK + 1 | |
| S_STRIDE: 72, // DK1 * DV, floats per (cell, head) | |
| IN_STRIDE: 32, // floats per (token, head) of raw inputs | |
| PREP_STRIDE: 48, // floats per (token, head) of prepared routing data | |
| NEIGH_STRIDE: 18, // u32 per (token, head): 9 read + 9 write cell ids | |
| MAX_ADDR: 28.5, // keep addresses off the lattice border | |
| MIN_ADDR: 1.5, | |
| FLOOR_SHARE: 1 / 9, // uniform share used by the dense routing floor | |
| }; | |
| /* ---------------------------------------------------------------- concepts */ | |
| /* Six "concepts": a fixed address prototype (so writes cluster into a few | |
| * cells) plus fixed content vectors. E and F sit close together so their | |
| * 3x3 neighbourhoods overlap — the lattice is shared, not owned. */ | |
| var CONCEPTS = [ | |
| { name: 'A', x: 6.0, y: 6.0, color: '#ff6b6b' }, | |
| { name: 'B', x: 26.0, y: 6.0, color: '#3ddc97' }, | |
| { name: 'C', x: 6.0, y: 26.0, color: '#5b8cff' }, | |
| { name: 'D', x: 26.0, y: 26.0, color: '#ffd166' }, | |
| { name: 'E', x: 15.0, y: 16.0, color: '#c792ea' }, | |
| { name: 'F', x: 17.4, y: 17.6, color: '#ff9f43' }, | |
| ]; | |
| var NOVEL_COLOR = '#8b98a8'; | |
| /* ------------------------------------------------------------------- rand */ | |
| function mulberry32(a) { | |
| return function () { | |
| a |= 0; a = (a + 0x6D2B79F5) | 0; | |
| var t = Math.imul(a ^ (a >>> 15), 1 | a); | |
| t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; | |
| return ((t ^ (t >>> 14)) >>> 0) / 4294967296; | |
| }; | |
| } | |
| function gauss(rnd) { | |
| var u = Math.max(rnd(), 1e-12), v = rnd(); | |
| return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); | |
| } | |
| function randVec(rnd, n, sigma) { | |
| var out = new Float64Array(n); | |
| for (var i = 0; i < n; i++) out[i] = gauss(rnd) * (sigma || 1); | |
| return out; | |
| } | |
| function norm(v) { | |
| var s = 0; | |
| for (var i = 0; i < v.length; i++) s += v[i] * v[i]; | |
| s = Math.sqrt(s) || 1; | |
| var o = new Float64Array(v.length); | |
| for (var j = 0; j < v.length; j++) o[j] = v[j] / s; | |
| return o; | |
| } | |
| function sigmoid(x) { return 1 / (1 + Math.exp(-x)); } | |
| function clamp(x, lo, hi) { return x < lo ? lo : (x > hi ? hi : x); } | |
| /* ------------------------------------------------------------- addressing */ | |
| /* cos^2 bump, support |t| < RADIUS (value and derivative vanish at the edge) */ | |
| function bump(t) { | |
| if (Math.abs(t) >= CFG.RADIUS) return 0; | |
| var c = Math.cos(Math.PI * t / (2 * CFG.RADIUS)); | |
| return c * c; | |
| } | |
| /* 1D window: centre cell and the normalised weights on c + {-1, 0, 1} */ | |
| function window1d(x) { | |
| var c = Math.floor(x + 0.5), f = x - c; | |
| var b = [bump(f + 1), bump(f), bump(f - 1)]; // offsets -1, 0, +1 | |
| var n = Math.sqrt(b[0] * b[0] + b[1] * b[1] + b[2] * b[2] + 1e-12); | |
| return { c: c, w: [b[0] / n, b[1] / n, b[2] / n] }; | |
| } | |
| function addrFromU(u) { return 0.5 * CFG.G * (1 + Math.tanh(u)); } | |
| function uFromAddr(a) { | |
| var t = clamp(2 * a / CFG.G - 1, -0.999999, 0.999999); | |
| return 0.5 * Math.log((1 + t) / (1 - t)); // atanh | |
| } | |
| function cellId(h, px, py) { return (h * CFG.N + py) * CFG.N + px; } | |
| /* ---------------------------------------------------------------- dataset */ | |
| /* Token stream: warm-up writes (clustering), variant flips (delta rule | |
| * overwriting), immediate repeats (the rule saturates), novel addresses | |
| * (fresh cells fill in), then a replay with different values. | |
| * | |
| * Every head gets its own address shift and its own content/value vectors, so | |
| * the four lattices genuinely differ (heads specialise, as in the real model). */ | |
| var HEAD_SHIFT = [[0, 0], [3.0, -2.0], [-3.0, 3.0], [2.5, 2.5]]; | |
| function headShift(h) { return HEAD_SHIFT[h % HEAD_SHIFT.length]; } | |
| function makeDataset(seed, cfg) { | |
| var C = Object.assign({}, CFG, cfg || {}); | |
| var rnd = mulberry32(seed >>> 0); | |
| var T = C.T, H = C.H; | |
| var range = function (n) { var a = []; for (var i = 0; i < n; i++) a.push(i); return a; }; | |
| /* [concept][head] content key, and [concept][head][variant] value */ | |
| var kvec = CONCEPTS.map(function () { return range(H).map(function () { return norm(randVec(rnd, C.DK)); }); }); | |
| var vv = CONCEPTS.map(function () { | |
| return range(H).map(function () { | |
| return [norm(randVec(rnd, C.DV)), norm(randVec(rnd, C.DV))]; | |
| }); | |
| }); | |
| var tokens = []; | |
| function emit(ci, variant, phase) { | |
| var t = tokens.length; | |
| var perHead = [], lab, col; | |
| for (var h = 0; h < H; h++) { | |
| var qa, ka, q2, k2, v; | |
| if (ci < 0) { // novel: unseen address+content | |
| qa = [C.MIN_ADDR + rnd() * (C.MAX_ADDR - C.MIN_ADDR), | |
| C.MIN_ADDR + rnd() * (C.MAX_ADDR - C.MIN_ADDR)]; | |
| ka = [C.MIN_ADDR + rnd() * (C.MAX_ADDR - C.MIN_ADDR), | |
| C.MIN_ADDR + rnd() * (C.MAX_ADDR - C.MIN_ADDR)]; | |
| q2 = norm(randVec(rnd, C.DK)); | |
| k2 = norm(randVec(rnd, C.DK)); | |
| v = norm(randVec(rnd, C.DV)); | |
| } else { | |
| var p = CONCEPTS[ci], sh = headShift(h); | |
| qa = [clamp(p.x + sh[0] + gauss(rnd) * 0.45, C.MIN_ADDR, C.MAX_ADDR), | |
| clamp(p.y + sh[1] + gauss(rnd) * 0.45, C.MIN_ADDR, C.MAX_ADDR)]; | |
| ka = [clamp(p.x + sh[0] + gauss(rnd) * 0.45, C.MIN_ADDR, C.MAX_ADDR), | |
| clamp(p.y + sh[1] + gauss(rnd) * 0.45, C.MIN_ADDR, C.MAX_ADDR)]; | |
| var noise = randVec(rnd, C.DK, 0.08); | |
| var qr = new Float64Array(C.DK); | |
| for (var d = 0; d < C.DK; d++) qr[d] = kvec[ci][h][d] + noise[d]; | |
| q2 = norm(qr); | |
| k2 = kvec[ci][h]; | |
| v = vv[ci][h][variant]; | |
| } | |
| perHead.push({ | |
| qaddr: qa, kaddr: ka, q2: q2, k2: k2, v: v, | |
| beta: sigmoid(0.8 + 1.1 * rnd()), | |
| }); | |
| } | |
| if (ci < 0) { lab = 'novel'; col = NOVEL_COLOR; } | |
| else { lab = CONCEPTS[ci].name + (variant ? '\u00b7v1' : '\u00b7v0'); col = CONCEPTS[ci].color; } | |
| tokens.push({ t: t, ci: ci, variant: variant, phase: phase, label: lab, color: col, perHead: perHead }); | |
| } | |
| var ci, rep; | |
| for (rep = 0; rep < 5; rep++) { for (ci = 0; ci < 6; ci++) emit(ci, rep % 2, 'warm-up'); } | |
| for (rep = 0; rep < 4; rep++) { for (ci = 0; ci < 6; ci++) emit(ci, (rep + 1) % 2, 'overwrite'); } | |
| for (ci = 0; ci < 6; ci++) { emit(ci, 1, 'repeat'); emit(ci, 1, 'repeat'); } | |
| for (rep = 0; rep < 24; rep++) emit(-1, 0, 'novel'); | |
| for (rep = 0; rep < 12; rep++) emit([0, 4, 2][rep % 3], 0, 'revisit'); | |
| while (tokens.length < T) { | |
| var i = tokens.length; | |
| emit((i * 5 + 1) % 6, i % 2, 'recall'); | |
| } | |
| tokens.length = T; | |
| /* pack raw inputs: [uq(2), uk(2), q2raw(DK), k2raw(DK), v(DV), beta_logit] */ | |
| var inp = new Float32Array(T * H * C.IN_STRIDE); | |
| for (var t = 0; t < T; t++) { | |
| var tk = tokens[t]; | |
| for (var h2 = 0; h2 < H; h2++) { | |
| var ph = tk.perHead[h2]; | |
| var b = (t * H + h2) * C.IN_STRIDE; | |
| inp[b + 0] = uFromAddr(ph.qaddr[0]); | |
| inp[b + 1] = uFromAddr(ph.qaddr[1]); | |
| inp[b + 2] = uFromAddr(ph.kaddr[0]); | |
| inp[b + 3] = uFromAddr(ph.kaddr[1]); | |
| for (var d = 0; d < C.DK; d++) { inp[b + 4 + d] = ph.q2[d]; inp[b + 12 + d] = ph.k2[d]; } | |
| for (var e = 0; e < C.DV; e++) inp[b + 20 + e] = ph.v[e]; | |
| inp[b + 28] = logit(ph.beta); | |
| } | |
| } | |
| return { | |
| cfg: C, tokens: tokens, inp: inp, seed: seed, | |
| concepts: CONCEPTS, kvec: kvec, vvec: vv, | |
| }; | |
| } | |
| function logit(p) { return Math.log(p / (1 - p)); } | |
| /* ------------------------------------------------------------------- prep */ | |
| /* CPU mirror of the WGSL prep pass: addresses, 9-cell windows, weights. */ | |
| function cpuPrep(ds) { | |
| var C = ds.cfg, T = C.T, H = C.H, N = C.N; | |
| var qaddr = new Float64Array(T * H * 2), kaddr = new Float64Array(T * H * 2); | |
| var readW = new Float64Array(T * H * 9), writeW = new Float64Array(T * H * 9); | |
| var q2 = new Float64Array(T * H * 9), k2 = new Float64Array(T * H * 9); | |
| var beta = new Float64Array(T * H); | |
| var readIdx = new Uint32Array(T * H * 9), writeIdx = new Uint32Array(T * H * 9); | |
| for (var t = 0; t < T; t++) { | |
| for (var h = 0; h < H; h++) { | |
| var idx = t * H + h, b = idx * C.IN_STRIDE; | |
| var ax = addrFromU(ds.inp[b + 0]), ay = addrFromU(ds.inp[b + 1]); | |
| var bx = addrFromU(ds.inp[b + 2]), by = addrFromU(ds.inp[b + 3]); | |
| var wq = [window1d(ax), window1d(ay)]; | |
| var wk = [window1d(bx), window1d(by)]; | |
| qaddr[idx * 2] = ax; qaddr[idx * 2 + 1] = ay; | |
| kaddr[idx * 2] = bx; kaddr[idx * 2 + 1] = by; | |
| for (var dy = 0; dy < 3; dy++) { | |
| for (var dx = 0; dx < 3; dx++) { | |
| var j = dy * 3 + dx; | |
| readW[idx * 9 + j] = wq[0].w[dx] * wq[1].w[dy]; | |
| writeW[idx * 9 + j] = wk[0].w[dx] * wk[1].w[dy]; | |
| readIdx[idx * 9 + j] = cellId(h, wq[0].c + dx, wq[1].c + dy); | |
| writeIdx[idx * 9 + j] = cellId(h, wk[0].c + dx, wk[1].c + dy); | |
| } | |
| } | |
| var nq = 0, nk = 0, d; | |
| for (d = 0; d < C.DK; d++) { nq += ds.inp[b + 4 + d] ** 2; nk += ds.inp[b + 12 + d] ** 2; } | |
| nq = Math.sqrt(nq + 1e-12); nk = Math.sqrt(nk + 1e-12); | |
| for (d = 0; d < C.DK; d++) { | |
| q2[idx * 9 + d] = ds.inp[b + 4 + d] / nq; | |
| k2[idx * 9 + d] = ds.inp[b + 12 + d] / nk; | |
| } | |
| q2[idx * 9 + C.DK] = 1; // constant channel: dot carries (1 + q2.k2) | |
| k2[idx * 9 + C.DK] = 1; | |
| beta[idx] = sigmoid(ds.inp[b + 28]); | |
| } | |
| } | |
| return { qaddr: qaddr, kaddr: kaddr, readW: readW, writeW: writeW, | |
| q2: q2, k2: k2, beta: beta, readIdx: readIdx, writeIdx: writeIdx }; | |
| } | |
| /* ------------------------------------------------------------------- scan */ | |
| /* Exact token-sequential recurrence. `floor` mixes a uniform share into the | |
| * 9-cell routing weights (the demo's rendering of the dense routing floor). */ | |
| function cpuScan(ds, prep, floor) { | |
| var C = ds.cfg, T = C.T, H = C.H, N = C.N, STRIDE = C.S_STRIDE; | |
| var S = new Float64Array(H * N * N * STRIDE); | |
| var out = new Float32Array(T * H * C.DV); | |
| var cellNorm = new Float64Array(H * N * N); | |
| var f = floor || 0, uni = C.FLOOR_SHARE, keep = 1 - f; | |
| for (var t = 0; t < T; t++) { | |
| for (var h = 0; h < H; h++) { | |
| var idx = t * H + h, j, d, dv; | |
| var q2 = prep.q2, k2 = prep.k2, beta = prep.beta[idx]; | |
| /* read (state before this token's writes) */ | |
| for (dv = 0; dv < C.DV; dv++) { | |
| var acc = 0; | |
| for (j = 0; j < 9; j++) { | |
| var w = keep * prep.readW[idx * 9 + j] + f * uni; | |
| var base = prep.readIdx[idx * 9 + j] * STRIDE; | |
| for (d = 0; d < 9; d++) acc += w * q2[idx * 9 + d] * S[base + d * C.DV + dv]; | |
| } | |
| out[t * H * C.DV + h * C.DV + dv] = acc; | |
| } | |
| /* write */ | |
| for (j = 0; j < 9; j++) { | |
| var ww = (keep * prep.writeW[idx * 9 + j] + f * uni) * beta; | |
| var cb = prep.writeIdx[idx * 9 + j] * STRIDE; | |
| for (dv = 0; dv < C.DV; dv++) { | |
| var dot = 0; | |
| for (d = 0; d < 9; d++) dot += S[cb + d * C.DV + dv] * k2[idx * 9 + d]; | |
| var err = ds.inp[(t * H + h) * C.IN_STRIDE + 20 + dv] - dot; | |
| for (d = 0; d < 9; d++) S[cb + d * C.DV + dv] += ww * err * k2[idx * 9 + d]; | |
| } | |
| } | |
| } | |
| } | |
| for (var c = 0; c < H * N * N; c++) { | |
| var s = 0; | |
| for (var q = 0; q < STRIDE; q++) s += S[c * STRIDE + q] * S[c * STRIDE + q]; | |
| cellNorm[c] = Math.sqrt(s); | |
| } | |
| return { S: S, out: out, cellNorm: cellNorm }; | |
| } | |
| return { | |
| CFG: CFG, CONCEPTS: CONCEPTS, NOVEL_COLOR: NOVEL_COLOR, | |
| makeDataset: makeDataset, cpuPrep: cpuPrep, cpuScan: cpuScan, | |
| bump: bump, window1d: window1d, addrFromU: addrFromU, uFromAddr: uFromAddr, | |
| headShift: headShift, | |
| cellId: cellId, sigmoid: sigmoid, mulberry32: mulberry32, | |
| }; | |
| }); | |