spotlight-demo / data.js
Vibelet's picture
Interactive demo @ step 122070 (fp16 shards)
ff5acd7 verified
Raw History Blame Contribute Delete
13.7 kB
/* Spotlight Memory demo — synthetic dataset + exact CPU reference.
*
* Mirrors spotlight/memory.py at chunk == 1 (the exact per-token recurrence):
*
* a = 0.5 * G * (1 + tanh(u)) address in (0, G)
* b(t) = cos^2(pi t / 3) for |t| < 3/2 else 0 1D cos^2 bump
* phi_z(x) = b(x - z) / sqrt(sum_j b(x - j)^2) 3 nearest cells
* c = floor(x + 0.5), z in c + {-1, 0, 1}
* w_2d = phi_zx(ax) * phi_zy(ay) 9 cells
* q2, k2 = L2-normalised content, with a constant 1 appended (dim dk + 1)
* beta = sigmoid(.)
* read o_t = sum_j wq_j * (S[cell_j]^T q2_t)
* write S[cell_j] += w_j * beta_t * (v_t - S[cell_j] k2_t) k2_t^T
*
* The cell state buffer is laid out head-outermost, padded lattice N = G + 3:
* cell id = (h * N + py) * N + px px, py in [0, N)
* slot = cell id * (DK + 1) * DV + d * DV + dv
*
* Loaded as a classic <script> in the browser (defines window.Spotlight) and as a
* CommonJS module under node (for offline checks of the reference numbers).
*/
(function (root, factory) {
if (typeof module === 'object' && module.exports) module.exports = factory();
else root.Spotlight = factory();
})(typeof self !== 'undefined' ? self : globalThis, function () {
'use strict';
var CFG = {
G: 32, // lattice side
H: 4, // heads
DK: 8, // content key/query dim (the +1 constant channel is added)
DV: 8, // value dim
T: 128, // tokens
RADIUS: 1.5, // bump support
N: 35, // padded lattice = G + 1 + 2*K, K = 1
NOFF: 9, // (2K+1)^2 neighbour cells
DK1: 9, // DK + 1
S_STRIDE: 72, // DK1 * DV, floats per (cell, head)
IN_STRIDE: 32, // floats per (token, head) of raw inputs
PREP_STRIDE: 48, // floats per (token, head) of prepared routing data
NEIGH_STRIDE: 18, // u32 per (token, head): 9 read + 9 write cell ids
MAX_ADDR: 28.5, // keep addresses off the lattice border
MIN_ADDR: 1.5,
FLOOR_SHARE: 1 / 9, // uniform share used by the dense routing floor
};
/* ---------------------------------------------------------------- concepts */
/* Six "concepts": a fixed address prototype (so writes cluster into a few
* cells) plus fixed content vectors. E and F sit close together so their
* 3x3 neighbourhoods overlap — the lattice is shared, not owned. */
var CONCEPTS = [
{ name: 'A', x: 6.0, y: 6.0, color: '#ff6b6b' },
{ name: 'B', x: 26.0, y: 6.0, color: '#3ddc97' },
{ name: 'C', x: 6.0, y: 26.0, color: '#5b8cff' },
{ name: 'D', x: 26.0, y: 26.0, color: '#ffd166' },
{ name: 'E', x: 15.0, y: 16.0, color: '#c792ea' },
{ name: 'F', x: 17.4, y: 17.6, color: '#ff9f43' },
];
var NOVEL_COLOR = '#8b98a8';
/* ------------------------------------------------------------------- rand */
function mulberry32(a) {
return function () {
a |= 0; a = (a + 0x6D2B79F5) | 0;
var t = Math.imul(a ^ (a >>> 15), 1 | a);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
function gauss(rnd) {
var u = Math.max(rnd(), 1e-12), v = rnd();
return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
}
function randVec(rnd, n, sigma) {
var out = new Float64Array(n);
for (var i = 0; i < n; i++) out[i] = gauss(rnd) * (sigma || 1);
return out;
}
function norm(v) {
var s = 0;
for (var i = 0; i < v.length; i++) s += v[i] * v[i];
s = Math.sqrt(s) || 1;
var o = new Float64Array(v.length);
for (var j = 0; j < v.length; j++) o[j] = v[j] / s;
return o;
}
function sigmoid(x) { return 1 / (1 + Math.exp(-x)); }
function clamp(x, lo, hi) { return x < lo ? lo : (x > hi ? hi : x); }
/* ------------------------------------------------------------- addressing */
/* cos^2 bump, support |t| < RADIUS (value and derivative vanish at the edge) */
function bump(t) {
if (Math.abs(t) >= CFG.RADIUS) return 0;
var c = Math.cos(Math.PI * t / (2 * CFG.RADIUS));
return c * c;
}
/* 1D window: centre cell and the normalised weights on c + {-1, 0, 1} */
function window1d(x) {
var c = Math.floor(x + 0.5), f = x - c;
var b = [bump(f + 1), bump(f), bump(f - 1)]; // offsets -1, 0, +1
var n = Math.sqrt(b[0] * b[0] + b[1] * b[1] + b[2] * b[2] + 1e-12);
return { c: c, w: [b[0] / n, b[1] / n, b[2] / n] };
}
function addrFromU(u) { return 0.5 * CFG.G * (1 + Math.tanh(u)); }
function uFromAddr(a) {
var t = clamp(2 * a / CFG.G - 1, -0.999999, 0.999999);
return 0.5 * Math.log((1 + t) / (1 - t)); // atanh
}
function cellId(h, px, py) { return (h * CFG.N + py) * CFG.N + px; }
/* ---------------------------------------------------------------- dataset */
/* Token stream: warm-up writes (clustering), variant flips (delta rule
* overwriting), immediate repeats (the rule saturates), novel addresses
* (fresh cells fill in), then a replay with different values.
*
* Every head gets its own address shift and its own content/value vectors, so
* the four lattices genuinely differ (heads specialise, as in the real model). */
var HEAD_SHIFT = [[0, 0], [3.0, -2.0], [-3.0, 3.0], [2.5, 2.5]];
function headShift(h) { return HEAD_SHIFT[h % HEAD_SHIFT.length]; }
function makeDataset(seed, cfg) {
var C = Object.assign({}, CFG, cfg || {});
var rnd = mulberry32(seed >>> 0);
var T = C.T, H = C.H;
var range = function (n) { var a = []; for (var i = 0; i < n; i++) a.push(i); return a; };
/* [concept][head] content key, and [concept][head][variant] value */
var kvec = CONCEPTS.map(function () { return range(H).map(function () { return norm(randVec(rnd, C.DK)); }); });
var vv = CONCEPTS.map(function () {
return range(H).map(function () {
return [norm(randVec(rnd, C.DV)), norm(randVec(rnd, C.DV))];
});
});
var tokens = [];
function emit(ci, variant, phase) {
var t = tokens.length;
var perHead = [], lab, col;
for (var h = 0; h < H; h++) {
var qa, ka, q2, k2, v;
if (ci < 0) { // novel: unseen address+content
qa = [C.MIN_ADDR + rnd() * (C.MAX_ADDR - C.MIN_ADDR),
C.MIN_ADDR + rnd() * (C.MAX_ADDR - C.MIN_ADDR)];
ka = [C.MIN_ADDR + rnd() * (C.MAX_ADDR - C.MIN_ADDR),
C.MIN_ADDR + rnd() * (C.MAX_ADDR - C.MIN_ADDR)];
q2 = norm(randVec(rnd, C.DK));
k2 = norm(randVec(rnd, C.DK));
v = norm(randVec(rnd, C.DV));
} else {
var p = CONCEPTS[ci], sh = headShift(h);
qa = [clamp(p.x + sh[0] + gauss(rnd) * 0.45, C.MIN_ADDR, C.MAX_ADDR),
clamp(p.y + sh[1] + gauss(rnd) * 0.45, C.MIN_ADDR, C.MAX_ADDR)];
ka = [clamp(p.x + sh[0] + gauss(rnd) * 0.45, C.MIN_ADDR, C.MAX_ADDR),
clamp(p.y + sh[1] + gauss(rnd) * 0.45, C.MIN_ADDR, C.MAX_ADDR)];
var noise = randVec(rnd, C.DK, 0.08);
var qr = new Float64Array(C.DK);
for (var d = 0; d < C.DK; d++) qr[d] = kvec[ci][h][d] + noise[d];
q2 = norm(qr);
k2 = kvec[ci][h];
v = vv[ci][h][variant];
}
perHead.push({
qaddr: qa, kaddr: ka, q2: q2, k2: k2, v: v,
beta: sigmoid(0.8 + 1.1 * rnd()),
});
}
if (ci < 0) { lab = 'novel'; col = NOVEL_COLOR; }
else { lab = CONCEPTS[ci].name + (variant ? '\u00b7v1' : '\u00b7v0'); col = CONCEPTS[ci].color; }
tokens.push({ t: t, ci: ci, variant: variant, phase: phase, label: lab, color: col, perHead: perHead });
}
var ci, rep;
for (rep = 0; rep < 5; rep++) { for (ci = 0; ci < 6; ci++) emit(ci, rep % 2, 'warm-up'); }
for (rep = 0; rep < 4; rep++) { for (ci = 0; ci < 6; ci++) emit(ci, (rep + 1) % 2, 'overwrite'); }
for (ci = 0; ci < 6; ci++) { emit(ci, 1, 'repeat'); emit(ci, 1, 'repeat'); }
for (rep = 0; rep < 24; rep++) emit(-1, 0, 'novel');
for (rep = 0; rep < 12; rep++) emit([0, 4, 2][rep % 3], 0, 'revisit');
while (tokens.length < T) {
var i = tokens.length;
emit((i * 5 + 1) % 6, i % 2, 'recall');
}
tokens.length = T;
/* pack raw inputs: [uq(2), uk(2), q2raw(DK), k2raw(DK), v(DV), beta_logit] */
var inp = new Float32Array(T * H * C.IN_STRIDE);
for (var t = 0; t < T; t++) {
var tk = tokens[t];
for (var h2 = 0; h2 < H; h2++) {
var ph = tk.perHead[h2];
var b = (t * H + h2) * C.IN_STRIDE;
inp[b + 0] = uFromAddr(ph.qaddr[0]);
inp[b + 1] = uFromAddr(ph.qaddr[1]);
inp[b + 2] = uFromAddr(ph.kaddr[0]);
inp[b + 3] = uFromAddr(ph.kaddr[1]);
for (var d = 0; d < C.DK; d++) { inp[b + 4 + d] = ph.q2[d]; inp[b + 12 + d] = ph.k2[d]; }
for (var e = 0; e < C.DV; e++) inp[b + 20 + e] = ph.v[e];
inp[b + 28] = logit(ph.beta);
}
}
return {
cfg: C, tokens: tokens, inp: inp, seed: seed,
concepts: CONCEPTS, kvec: kvec, vvec: vv,
};
}
function logit(p) { return Math.log(p / (1 - p)); }
/* ------------------------------------------------------------------- prep */
/* CPU mirror of the WGSL prep pass: addresses, 9-cell windows, weights. */
function cpuPrep(ds) {
var C = ds.cfg, T = C.T, H = C.H, N = C.N;
var qaddr = new Float64Array(T * H * 2), kaddr = new Float64Array(T * H * 2);
var readW = new Float64Array(T * H * 9), writeW = new Float64Array(T * H * 9);
var q2 = new Float64Array(T * H * 9), k2 = new Float64Array(T * H * 9);
var beta = new Float64Array(T * H);
var readIdx = new Uint32Array(T * H * 9), writeIdx = new Uint32Array(T * H * 9);
for (var t = 0; t < T; t++) {
for (var h = 0; h < H; h++) {
var idx = t * H + h, b = idx * C.IN_STRIDE;
var ax = addrFromU(ds.inp[b + 0]), ay = addrFromU(ds.inp[b + 1]);
var bx = addrFromU(ds.inp[b + 2]), by = addrFromU(ds.inp[b + 3]);
var wq = [window1d(ax), window1d(ay)];
var wk = [window1d(bx), window1d(by)];
qaddr[idx * 2] = ax; qaddr[idx * 2 + 1] = ay;
kaddr[idx * 2] = bx; kaddr[idx * 2 + 1] = by;
for (var dy = 0; dy < 3; dy++) {
for (var dx = 0; dx < 3; dx++) {
var j = dy * 3 + dx;
readW[idx * 9 + j] = wq[0].w[dx] * wq[1].w[dy];
writeW[idx * 9 + j] = wk[0].w[dx] * wk[1].w[dy];
readIdx[idx * 9 + j] = cellId(h, wq[0].c + dx, wq[1].c + dy);
writeIdx[idx * 9 + j] = cellId(h, wk[0].c + dx, wk[1].c + dy);
}
}
var nq = 0, nk = 0, d;
for (d = 0; d < C.DK; d++) { nq += ds.inp[b + 4 + d] ** 2; nk += ds.inp[b + 12 + d] ** 2; }
nq = Math.sqrt(nq + 1e-12); nk = Math.sqrt(nk + 1e-12);
for (d = 0; d < C.DK; d++) {
q2[idx * 9 + d] = ds.inp[b + 4 + d] / nq;
k2[idx * 9 + d] = ds.inp[b + 12 + d] / nk;
}
q2[idx * 9 + C.DK] = 1; // constant channel: dot carries (1 + q2.k2)
k2[idx * 9 + C.DK] = 1;
beta[idx] = sigmoid(ds.inp[b + 28]);
}
}
return { qaddr: qaddr, kaddr: kaddr, readW: readW, writeW: writeW,
q2: q2, k2: k2, beta: beta, readIdx: readIdx, writeIdx: writeIdx };
}
/* ------------------------------------------------------------------- scan */
/* Exact token-sequential recurrence. `floor` mixes a uniform share into the
* 9-cell routing weights (the demo's rendering of the dense routing floor). */
function cpuScan(ds, prep, floor) {
var C = ds.cfg, T = C.T, H = C.H, N = C.N, STRIDE = C.S_STRIDE;
var S = new Float64Array(H * N * N * STRIDE);
var out = new Float32Array(T * H * C.DV);
var cellNorm = new Float64Array(H * N * N);
var f = floor || 0, uni = C.FLOOR_SHARE, keep = 1 - f;
for (var t = 0; t < T; t++) {
for (var h = 0; h < H; h++) {
var idx = t * H + h, j, d, dv;
var q2 = prep.q2, k2 = prep.k2, beta = prep.beta[idx];
/* read (state before this token's writes) */
for (dv = 0; dv < C.DV; dv++) {
var acc = 0;
for (j = 0; j < 9; j++) {
var w = keep * prep.readW[idx * 9 + j] + f * uni;
var base = prep.readIdx[idx * 9 + j] * STRIDE;
for (d = 0; d < 9; d++) acc += w * q2[idx * 9 + d] * S[base + d * C.DV + dv];
}
out[t * H * C.DV + h * C.DV + dv] = acc;
}
/* write */
for (j = 0; j < 9; j++) {
var ww = (keep * prep.writeW[idx * 9 + j] + f * uni) * beta;
var cb = prep.writeIdx[idx * 9 + j] * STRIDE;
for (dv = 0; dv < C.DV; dv++) {
var dot = 0;
for (d = 0; d < 9; d++) dot += S[cb + d * C.DV + dv] * k2[idx * 9 + d];
var err = ds.inp[(t * H + h) * C.IN_STRIDE + 20 + dv] - dot;
for (d = 0; d < 9; d++) S[cb + d * C.DV + dv] += ww * err * k2[idx * 9 + d];
}
}
}
}
for (var c = 0; c < H * N * N; c++) {
var s = 0;
for (var q = 0; q < STRIDE; q++) s += S[c * STRIDE + q] * S[c * STRIDE + q];
cellNorm[c] = Math.sqrt(s);
}
return { S: S, out: out, cellNorm: cellNorm };
}
return {
CFG: CFG, CONCEPTS: CONCEPTS, NOVEL_COLOR: NOVEL_COLOR,
makeDataset: makeDataset, cpuPrep: cpuPrep, cpuScan: cpuScan,
bump: bump, window1d: window1d, addrFromU: addrFromU, uFromAddr: uFromAddr,
headShift: headShift,
cellId: cellId, sigmoid: sigmoid, mulberry32: mulberry32,
};
});