let randomState = 42; function random() { randomState = (1664525 * randomState + 1013904223) % 4294967296; return randomState / 4294967296; } export class Tensor { // When false, operations skip building the graph: static gradEnabled = true; constructor(rows, columns) { this.rows = rows; this.columns = columns; this.data = new Float32Array(rows * columns); // zero-filled this.grad = null; // allocated on demand by backward() this.role = null; // how the trainer updates it: "matrix", "embedding", or null for vectors and filters this._inputs = []; // tensors this one was built from this._backward = () => {}; // leaves do nothing } get(row, column) { // returns a plain number now, not a Value return this.data[row * this.columns + column]; } set(row, column, value) { this.data[row * this.columns + column] = value; } // --- Initialization: these create the model's leaf parameters. --- initToZeroes() { for (let i = 0; i < this.data.length; ++i) { this.data[i] = 0.0; } } initToOnes() { for (let i = 0; i < this.data.length; ++i) { this.data[i] = 1.0; } } initToSmallRandom() { const scale = 1 / Math.sqrt(this.rows); for (let i = 0; i < this.data.length; ++i) { this.data[i] = (random() - 0.5) * 2 * scale; // uniform in [-scale, +scale) } } zeroGrad() { if (this.grad !== null) this.grad.fill(0); } parameters() { return [this]; // a tensor is one parameter block } // --- Element-wise operations (both tensors share the same shape). --- add(other) { const out = new Tensor(this.rows, this.columns); for (let i = 0; i < this.data.length; ++i) { out.data[i] = this.data[i] + other.data[i]; } if (Tensor.gradEnabled) { out._inputs = [this, other]; out._backward = () => { for (let i = 0; i < this.data.length; ++i) { this.grad[i] += out.grad[i]; other.grad[i] += out.grad[i]; } }; } return out; } sub(other) { const out = new Tensor(this.rows, this.columns); for (let i = 0; i < this.data.length; ++i) { out.data[i] = this.data[i] - other.data[i]; } if (Tensor.gradEnabled) { out._inputs = [this, other]; out._backward = () => { for (let i = 0; i < this.data.length; ++i) { this.grad[i] += out.grad[i]; other.grad[i] -= out.grad[i]; } }; } return out; } mul(other) { // element-wise multiply (matching numpy/pytorch); matrix multiply is matmul const out = new Tensor(this.rows, this.columns); for (let i = 0; i < this.data.length; ++i) { out.data[i] = this.data[i] * other.data[i]; } if (Tensor.gradEnabled) { out._inputs = [this, other]; out._backward = () => { for (let i = 0; i < this.data.length; ++i) { this.grad[i] += other.data[i] * out.grad[i]; other.grad[i] += this.data[i] * out.grad[i]; } }; } return out; } scale(scalar) { // multiply every value by one plain number const out = new Tensor(this.rows, this.columns); for (let i = 0; i < this.data.length; ++i) { out.data[i] = this.data[i] * scalar; } if (Tensor.gradEnabled) { out._inputs = [this]; out._backward = () => { for (let i = 0; i < this.data.length; ++i) { this.grad[i] += scalar * out.grad[i]; } }; } return out; } relu() { const out = new Tensor(this.rows, this.columns); for (let i = 0; i < this.data.length; ++i) { out.data[i] = this.data[i] > 0 ? this.data[i] : 0; } if (Tensor.gradEnabled) { out._inputs = [this]; out._backward = () => { for (let i = 0; i < this.data.length; ++i) { this.grad[i] += (this.data[i] > 0 ? 1 : 0) * out.grad[i]; } }; } return out; } silu() { // x * sigmoid(x), the gate in SwiGLU const out = new Tensor(this.rows, this.columns); const sigmoids = new Float32Array(this.data.length); // cached for the backward for (let i = 0; i < this.data.length; ++i) { const x = this.data[i]; const s = 1 / (1 + Math.exp(-x)); sigmoids[i] = s; out.data[i] = x * s; } if (Tensor.gradEnabled) { out._inputs = [this]; out._backward = () => { for (let i = 0; i < this.data.length; ++i) { // d/dx [x * s(x)] = s + x * s * (1 - s) const s = sigmoids[i]; this.grad[i] += (s + this.data[i] * s * (1 - s)) * out.grad[i]; } }; } return out; } // --- Matrix multiplication: the one node that replaces the most scalars. --- matmul(other) { // (this.rows x this.columns) * (other.rows x other.columns) const m = this.rows; const k = this.columns; // shared inner dimension const n = other.columns; const out = new Tensor(m, n); for (let row = 0; row < m; ++row) { for (let col = 0; col < n; ++col) { let sum = 0.0; // accumulate in double, store as float for (let element = 0; element < k; ++element) { sum += this.data[row * k + element] * other.data[element * n + col]; } out.data[row * n + col] = sum; } } if (Tensor.gradEnabled) { out._inputs = [this, other]; out._backward = () => { // dThis = dOut * other^T for (let row = 0; row < m; ++row) { for (let p = 0; p < k; ++p) { let sum = 0.0; for (let col = 0; col < n; ++col) { sum += out.grad[row * n + col] * other.data[p * n + col]; } this.grad[row * k + p] += sum; } } // dOther = this^T * dOut for (let p = 0; p < k; ++p) { for (let col = 0; col < n; ++col) { let sum = 0.0; for (let row = 0; row < m; ++row) { sum += this.data[row * k + p] * out.grad[row * n + col]; } other.grad[p * n + col] += sum; } } }; } return out; } transposed() { const out = new Tensor(this.columns, this.rows); for (let row = 0; row < this.rows; ++row) { for (let col = 0; col < this.columns; ++col) { out.data[col * this.rows + row] = this.data[row * this.columns + col]; } } if (Tensor.gradEnabled) { out._inputs = [this]; out._backward = () => { for (let row = 0; row < this.rows; ++row) { for (let col = 0; col < this.columns; ++col) { this.grad[row * this.columns + col] += out.grad[col * this.rows + row]; } } }; } return out; } // --- Fused operations. Each is one node with a single analytic backward // rule instead of being assembled from many smaller nodes, which is // both faster and far lighter on memory. These are general array // operations, so they live here. RMS normalization and the // cross-entropy loss are also fused, but each belongs to one component // (the norm layer, the trainer), so they are defined there using the // same _inputs/_backward protocol. --- softMaxedRows() { const rows = this.rows; const cols = this.columns; const out = new Tensor(rows, cols); for (let row = 0; row < rows; ++row) { const base = row * cols; let rowMax = this.data[base]; for (let col = 1; col < cols; ++col) { const v = this.data[base + col]; if (v > rowMax) rowMax = v; } let rowSum = 0.0; for (let col = 0; col < cols; ++col) { const e = Math.exp(this.data[base + col] - rowMax); out.data[base + col] = e; rowSum += e; } for (let col = 0; col < cols; ++col) { out.data[base + col] /= rowSum; } } if (Tensor.gradEnabled) { out._inputs = [this]; out._backward = () => { // For each row: dx_j = p_j * (g_j - sum_k g_k p_k) for (let row = 0; row < rows; ++row) { const base = row * cols; let dot = 0.0; for (let col = 0; col < cols; ++col) { dot += out.grad[base + col] * out.data[base + col]; } for (let col = 0; col < cols; ++col) { const p = out.data[base + col]; this.grad[base + col] += p * (out.grad[base + col] - dot); } } }; } return out; } causalMasked() { const rows = this.rows; const cols = this.columns; const out = new Tensor(rows, cols); for (let row = 0; row < rows; ++row) { for (let col = 0; col < cols; ++col) { const index = row * cols + col; out.data[index] = col > row ? -Infinity : this.data[index]; } } if (Tensor.gradEnabled) { out._inputs = [this]; out._backward = () => { // Masked cells are constants, so gradient only flows where col <= row. for (let row = 0; row < rows; ++row) { for (let col = 0; col <= row; ++col) { const index = row * cols + col; this.grad[index] += out.grad[index]; } } }; } return out; } // Embedding lookup: copy the rows named by `indices` into a new tensor. // The gradient scatters back, accumulating when an index repeats. gatherRows(indices) { const out = new Tensor(indices.length, this.columns); for (let r = 0; r < indices.length; ++r) { const src = indices[r] * this.columns; const dst = r * this.columns; for (let col = 0; col < this.columns; ++col) { out.data[dst + col] = this.data[src + col]; } } if (Tensor.gradEnabled) { out._inputs = [this]; out._backward = () => { for (let r = 0; r < indices.length; ++r) { const src = indices[r] * this.columns; const dst = r * this.columns; for (let col = 0; col < this.columns; ++col) { this.grad[src + col] += out.grad[dst + col]; } } }; } return out; } // Stack another tensor's rows below this one's, returning a new, taller // tensor. Both must share the same column count. The KV cache uses this to // grow one row per generated token. The gradient of the top rows flows back // to this, the bottom rows to other. Generation runs with the graph off, so // that rule never fires in practice, but it keeps appendRow a first-class op. appendRow(other) { // A new tensor: this tensor's rows on top, other's rows stacked below. // Both must share the same column count. const out = new Tensor(this.rows + other.rows, this.columns); out.data.set(this.data, 0); // top block out.data.set(other.data, this.data.length); // bottom block if (Tensor.gradEnabled) { out._inputs = [this, other]; out._backward = () => { // Gradient of the top rows flows to this, the bottom rows to other. for (let i = 0; i < this.data.length; ++i) { this.grad[i] += out.grad[i]; } for (let i = 0; i < other.data.length; ++i) { other.grad[i] += out.grad[this.data.length + i]; } }; } return out; } // RoPE: rotate each row's component pairs by an angle set by the row's // position. Query and key rows sit in sequence order, so the row index is // the position — plus positionOffset, which the KV cache uses to rotate a // new token by its true position rather than its row (offset zero, the // default, reproduces the original behavior). The angles are fixed functions // of position and frequency, not learned, so this op adds nothing to // parameters(); the gradient only routes back to the input. The cosine and // sine of each angle are cached, since the backward pass — a rotation by the // negated angle — reuses them. ropeRotated(thetaBase = 10000, positionOffset = 0) { // positionOffset is new const rows = this.rows; const cols = this.columns; // head_dim, must be even const halfCols = cols / 2; const out = new Tensor(rows, cols); // Frequency depends on the pair, not the row // Compute it once and cache const freqs = new Float32Array(halfCols); for (let pair = 0; pair < halfCols; ++pair) { freqs[pair] = 1.0 / Math.pow(thetaBase, (2 * pair) / cols); } // Cached for the backward pass. const cosTable = new Float32Array(rows * halfCols); const sinTable = new Float32Array(rows * halfCols); for (let row = 0; row < rows; ++row) { const base = row * cols; const trig = row * halfCols; for (let pair = 0; pair < halfCols; ++pair) { const angle = (positionOffset + row) * freqs[pair]; // true position = offset + row const cos = Math.cos(angle); const sin = Math.sin(angle); cosTable[trig + pair] = cos; sinTable[trig + pair] = sin; const a = this.data[base + 2 * pair]; const b = this.data[base + 2 * pair + 1]; out.data[base + 2 * pair] = a * cos - b * sin; out.data[base + 2 * pair + 1] = a * sin + b * cos; } } if (Tensor.gradEnabled) { out._inputs = [this]; out._backward = () => { // Rotation is linear, the gradient is the inverse rotation. for (let row = 0; row < rows; ++row) { const base = row * cols; const trig = row * halfCols; for (let pair = 0; pair < halfCols; ++pair) { const cos = cosTable[trig + pair]; const sin = sinTable[trig + pair]; const ga = out.grad[base + 2 * pair]; const gb = out.grad[base + 2 * pair + 1]; this.grad[base + 2 * pair] += ga * cos + gb * sin; this.grad[base + 2 * pair + 1] += -ga * sin + gb * cos; } } }; } return out; } // Short causal convolution, one filter per column (depthwise): out[t] = // sum over j of weight[j] * x[t - j], where weight is [kernel, columns] and // row j weighs the input j rows back. Rows before the first are zeros, or // come from history (the previous kernel - 1 rows, during generation). The // gradient flows to this tensor and the filter; history is a constant. causalConv(weight, history = null) { const rows = this.rows; const cols = this.columns; const taps = weight.rows - 1; const out = new Tensor(rows, cols); const input = (source, col) => { if (source >= 0) return this.data[source * cols + col]; if (history !== null) return history.data[(taps + source) * cols + col]; return 0.0; }; for (let row = 0; row < rows; ++row) { for (let col = 0; col < cols; ++col) { let sum = 0.0; for (let j = 0; j <= taps; ++j) { sum += weight.data[j * cols + col] * input(row - j, col); } out.data[row * cols + col] = sum; } } if (Tensor.gradEnabled) { out._inputs = [this, weight]; out._backward = () => { for (let row = 0; row < rows; ++row) { for (let col = 0; col < cols; ++col) { const g = out.grad[row * cols + col]; for (let j = 0; j <= taps; ++j) { const source = row - j; weight.grad[j * cols + col] += g * input(source, col); if (source >= 0) { this.grad[source * cols + col] += g * weight.data[j * cols + col]; } } } } }; } return out; } // The last count rows, with earlier rows from history (or zeros) when this // tensor is shorter. Carries a convolution's input from one generation step // to the next, so it builds no graph. lastRows(count, history = null) { const out = new Tensor(count, this.columns); for (let r = 0; r < count; ++r) { const source = this.rows - count + r; if (source >= 0) { out.data.set(this.data.subarray(source * this.columns, (source + 1) * this.columns), r * this.columns); } else if (history !== null) { const h = history.rows + source; out.data.set(history.data.subarray(h * this.columns, (h + 1) * this.columns), r * this.columns); } } return out; } backward() { // Topological sort: inputs come before the nodes built from them. const topo = []; const visited = new Set(); const visit = (node) => { if (visited.has(node)) return; visited.add(node); for (let i = 0; i < node._inputs.length; ++i) { visit(node._inputs[i]); } topo.push(node); }; visit(this); // Make sure every node in the graph has a gradient buffer. Freshly // built tensors start at null and get a zeroed buffer here. Parameter // tensors keep the buffer the trainer already zeroed. for (let i = 0; i < topo.length; ++i) { if (topo[i].grad === null) { topo[i].grad = new Float32Array(topo[i].data.length); } } // Seed: derivative of the output with respect to itself is 1. this.grad.fill(1); // Walk in reverse. Each node pushes gradient onto its inputs. for (let i = topo.length - 1; i >= 0; --i) { topo[i]._backward(); } } } export class RMSNorm { constructor(feature_dim) { // Learned per-feature scale, applied after normalization. Starts at one, // so the layer begins as a plain normalize until training moves it. this.gamma = new Tensor(1, feature_dim); this.gamma.initToOnes(); } forward(x) { const rows = x.rows; const n = x.columns; const eps = 1e-5; const gamma = this.gamma; const out = new Tensor(rows, n); const invRmsByRow = new Float32Array(rows); // cached for the backward for (let row = 0; row < rows; ++row) { const base = row * n; let sumSq = 0.0; for (let col = 0; col < n; ++col) { const v = x.data[base + col]; sumSq += v * v; } const invRms = 1 / Math.sqrt(sumSq / n + eps); invRmsByRow[row] = invRms; for (let col = 0; col < n; ++col) { // normalize, then scale by the per-feature gamma out.data[base + col] = x.data[base + col] * invRms * gamma.data[col]; } } if (Tensor.gradEnabled) { out._inputs = [x, gamma]; out._backward = () => { for (let row = 0; row < rows; ++row) { const base = row * n; const invRms = invRmsByRow[row]; const invRms3 = invRms * invRms * invRms; // Upstream gradient passes through gamma before the RMS rule. let dotGX = 0.0; for (let col = 0; col < n; ++col) { const gNorm = out.grad[base + col] * gamma.data[col]; dotGX += gNorm * x.data[base + col]; } for (let col = 0; col < n; ++col) { const g = out.grad[base + col]; const xv = x.data[base + col]; const gNorm = g * gamma.data[col]; x.grad[base + col] += gNorm * invRms - (xv * dotGX * invRms3) / n; // gamma is shared across rows: sum the normalized input times upstream grad gamma.grad[col] += xv * invRms * g; } } }; } return out; } parameters() { return [this.gamma]; } } // Canon layer: a short causal convolution added back onto its input, applied // right after a pre-norm. It lets each position mix in the few positions before // it, which a byte-level model needs constantly to assemble bytes into words. // The filter starts at zero, so the layer starts as a no-op. export class CanonLayer { constructor(feature_dim, kernel) { this.weight = new Tensor(kernel, feature_dim); this.weight.initToZeroes(); } // history: the previous kernel - 1 inputs, kept in the cache's slot. forward(x, cache = null, layerIndex = 0, slot = "convA") { const history = cache === null ? null : cache[slot][layerIndex]; const out = x.add(x.causalConv(this.weight, history)); if (cache !== null) { cache[slot][layerIndex] = x.lastRows(this.weight.rows - 1, history); } return out; } parameters() { return [this.weight]; } } export class MultiHeadAttention { constructor(feature_dim, num_heads, rope_base) { this.num_heads = num_heads; this.head_dim = feature_dim / num_heads; // must divide evenly, and be even for RoPE this.rope_base = rope_base; // Each head owns Q, K, V down to head_dim, and an output projection // back up to feature_dim. The summed outputs equal a concat plus one // big output projection, so no concat operation is needed. this.heads = []; for (let i = 0; i < num_heads; ++i) { const learnedQ = new Tensor(feature_dim, this.head_dim); learnedQ.initToSmallRandom(); const learnedK = new Tensor(feature_dim, this.head_dim); learnedK.initToSmallRandom(); const learnedV = new Tensor(feature_dim, this.head_dim); learnedV.initToSmallRandom(); const learnedO = new Tensor(this.head_dim, feature_dim); learnedO.initToSmallRandom(); for (const matrix of [learnedQ, learnedK, learnedV, learnedO]) { matrix.role = "matrix"; } // QK-norm: each head normalizes its own queries and keys, with a // learned scale, before RoPE. Attention scores stay in a sane range // however large the projections grow. const queryNorm = new RMSNorm(this.head_dim); const keyNorm = new RMSNorm(this.head_dim); this.heads.push({ learnedQ, learnedK, learnedV, learnedO, queryNorm, keyNorm }); } } forward(rms_norm_matrix, cache = null, layerIndex = 0) { // Shape: new_tokens, feature_dim const scale = 1.0 / Math.sqrt(this.head_dim); let output = null; for (let i = 0; i < this.num_heads; ++i) { const head = this.heads[i]; let Q = head.queryNorm.forward(rms_norm_matrix.matmul(head.learnedQ)); // Shape: new_tokens, head_dim let K = head.keyNorm.forward(rms_norm_matrix.matmul(head.learnedK)); // Shape: new_tokens, head_dim let V = rms_norm_matrix.matmul(head.learnedV); // Shape: new_tokens, head_dim // The new tokens sit after whatever this head has already cached. const pastK = cache === null ? null : cache.keys[layerIndex][i]; const pastV = cache === null ? null : cache.values[layerIndex][i]; const positionOffset = pastK === null ? 0 : pastK.rows; // Rotate by true position. Cached keys keep the rotation they were // stored with; the new keys are rotated from the offset onward. Q = Q.ropeRotated(this.rope_base, positionOffset); K = K.ropeRotated(this.rope_base, positionOffset); // Grow the cache with the new keys and values, then attend over the // full history. Without a cache, K and V are just the new tokens. if (cache !== null) { K = pastK === null ? K : pastK.appendRow(K); V = pastV === null ? V : pastV.appendRow(V); cache.keys[layerIndex][i] = K; cache.values[layerIndex][i] = V; } let scores = Q.matmul(K.transposed()); // Shape: new_tokens, total_tokens scores = scores.scale(scale); // The prompt's first pass scores a square block and needs the mask. // Every later token sees only its past, so no mask is needed. if (positionOffset === 0) { scores = scores.causalMasked(); } const probabilities = scores.softMaxedRows(); const mixed = probabilities.matmul(V); // Shape: new_tokens, head_dim // Project this head up to feature_dim and accumulate. const projected = mixed.matmul(head.learnedO); // Shape: new_tokens, feature_dim output = output === null ? projected : output.add(projected); } return output; // Shape: new_tokens, feature_dim } parameters() { const params = []; for (let i = 0; i < this.heads.length; ++i) { const head = this.heads[i]; const matrices = [head.learnedQ, head.learnedK, head.learnedV, head.learnedO, head.queryNorm.gamma, head.keyNorm.gamma]; for (let j = 0; j < matrices.length; ++j) { const matParams = matrices[j].parameters(); for (let k = 0; k < matParams.length; ++k) { params.push(matParams[k]); } } } return params; } } // SwiGLU: a gated MLP. silu(x @ gate) decides how much of each hidden unit // (x @ up) passes, then down projects back. No bias. export class MLP { constructor(feature_dim_size, hidden_dim_size) { this.learnedGate = new Tensor(feature_dim_size, hidden_dim_size); this.learnedGate.initToSmallRandom(); this.learnedUp = new Tensor(feature_dim_size, hidden_dim_size); this.learnedUp.initToSmallRandom(); this.learnedDown = new Tensor(hidden_dim_size, feature_dim_size); this.learnedDown.initToSmallRandom(); for (const matrix of [this.learnedGate, this.learnedUp, this.learnedDown]) { matrix.role = "matrix"; } } forward(x) { // Shape: sequence_length, feature_dim const gate = x.matmul(this.learnedGate).silu(); // Shape: sequence_length, hidden_dim const up = x.matmul(this.learnedUp); // Shape: sequence_length, hidden_dim return gate.mul(up).matmul(this.learnedDown); // Shape: sequence_length, feature_dim } parameters() { return [this.learnedGate, this.learnedUp, this.learnedDown]; } } export class TransformerBlock { constructor(feature_dim, num_heads, rope_base, hidden_dim, conv_kernel) { this.attentionNorm = new RMSNorm(feature_dim); this.canonA = new CanonLayer(feature_dim, conv_kernel); this.attention = new MultiHeadAttention(feature_dim, num_heads, rope_base); this.mlpNorm = new RMSNorm(feature_dim); this.canonC = new CanonLayer(feature_dim, conv_kernel); this.mlp = new MLP(feature_dim, hidden_dim); } forward(x, cache = null, layerIndex = 0) { // Attention sub-layer, with residual. The canon layer mixes in the last // few positions first. The cache holds this layer's keys, values and // both convolution histories. const attentionInput = this.canonA.forward(this.attentionNorm.forward(x), cache, layerIndex, "convA"); const attentionDelta = this.attention.forward(attentionInput, cache, layerIndex); const afterAttention = x.add(attentionDelta); // MLP sub-layer, with residual const mlpInput = this.canonC.forward(this.mlpNorm.forward(afterAttention), cache, layerIndex, "convC"); const mlpDelta = this.mlp.forward(mlpInput); const afterMLP = afterAttention.add(mlpDelta); return afterMLP; } parameters() { const params = []; const components = [this.attentionNorm, this.canonA, this.attention, this.mlpNorm, this.canonC, this.mlp]; for (let i = 0; i < components.length; ++i) { const sub = components[i].parameters(); for (let j = 0; j < sub.length; ++j) { params.push(sub[j]); } } return params; } } export class GabGPT { constructor(vocabSize, featureDim, numHeads, ropeBase, numPrelude, numCore, numCoda, loopCount, mlpHiddenDim, convKernel) { this.vocabSize = vocabSize; this.featureDim = featureDim; this.numHeads = numHeads; this.ropeBase = ropeBase; this.numPrelude = numPrelude; this.numCore = numCore; this.numCoda = numCoda; this.loopCount = loopCount; // how many times the core runs, in training and inference alike this.mlpHiddenDim = mlpHiddenDim; // SwiGLU's hidden width, about 2/3 of 4x to keep the budget this.convKernel = convKernel; // canon layer filter length // Token embedding table (position enters through rotation in attention). this.tokenEmbeddings = new Tensor(vocabSize, featureDim); this.tokenEmbeddings.initToSmallRandom(); this.tokenEmbeddings.role = "embedding"; // Three stacks of transformer blocks: the prelude and coda run once, // the core runs loopCount times with the same weights. const makeBlocks = (count) => { const blocks = []; for (let i = 0; i < count; ++i) { blocks.push(new TransformerBlock(featureDim, numHeads, ropeBase, mlpHiddenDim, convKernel)); } return blocks; }; this.prelude = makeBlocks(numPrelude); this.core = makeBlocks(numCore); this.coda = makeBlocks(numCoda); // Every unique block, in weight-file order. The layout matches an unlooped // model with numPrelude + numCore + numCoda blocks. this.blocks = [...this.prelude, ...this.core, ...this.coda]; // Final norm before the unembedding. this.finalNorm = new RMSNorm(featureDim); } // Layers a token actually passes through. Each one is a separate slot in // the KV cache, so build caches as new KVCache(model.effectiveDepth, numHeads). get effectiveDepth() { return this.numPrelude + this.numCore * this.loopCount + this.numCoda; } embed(ids) { // Token embeddings only; position now enters through rotation in attention return this.tokenEmbeddings.gatherRows(ids); } forward(tokenIdArray, cache = null) { // cache is new // No maxContextLength check: there is no position table to index past. // A sequence beyond the trained length just runs slower and predicts worse. // With a cache, tokenIdArray holds only the new tokens; the block index // threads through so each layer reads and writes its own cached slot. // Every pass through a block is its own layer as far as the cache is // concerned: the same core block sees different inputs on each loop, // so it produces different keys, values and convolution histories. let layerIndex = 0; let x = this.embed(tokenIdArray); // Shape: new_tokens, feature_dim for (let i = 0; i < this.prelude.length; ++i) { x = this.prelude[i].forward(x, cache, layerIndex++); // Shape preserved. } // Input injection: the prelude's output is added back in at the start // of every core pass after the first, so the loop never loses sight // of the input. The first pass already starts from it. const injected = x; for (let loop = 0; loop < this.loopCount; ++loop) { if (loop > 0) { x = x.add(injected); } for (let i = 0; i < this.core.length; ++i) { x = this.core[i].forward(x, cache, layerIndex++); } } for (let i = 0; i < this.coda.length; ++i) { x = this.coda[i].forward(x, cache, layerIndex++); } x = this.finalNorm.forward(x); // Shape: new_tokens, feature_dim // Tied unembedding: (sequence_length, feature_dim) * (feature_dim, vocabSize) const logits = x.matmul(this.tokenEmbeddings.transposed()); // Shape: sequence_length, vocabSize return logits; } // Predict the next token. Temperature reshapes the distribution, top-k and // top-p rule out the unlikely tail, and an optional KV cache skips rerunning // tokens that were processed on an earlier call. predictNextToken(tokenIdArray, temperature = 1.0, topK = 50, topP = 0.9, cache = null) { // With a cache, the stored tokens do not run again. Only the tokens // past the cached length are new. const newTokens = cache === null ? tokenIdArray : tokenIdArray.slice(cache.length); const logits = this.forward(newTokens, cache); // was forward(tokenIdArray) // The last row holds the scores for the token after the sequence, // whether that is the last row of a full prompt or the only row of a // single cached step. const lastRow = logits.rows - 1; // Temperature zero means greedy decoding: the single highest logit. // Softmax is monotonic, so the argmax of the logits is the argmax of // the probabilities, and the softmax can be skipped here. if (temperature === 0) { let bestId = 0; let bestScore = logits.get(lastRow, 0); for (let col = 1; col < this.vocabSize; ++col) { const score = logits.get(lastRow, col); if (score > bestScore) { bestScore = score; bestId = col; } } return bestId; } // Reshape by temperature and turn the last row into probabilities. const scaled = logits.scale(1 / temperature); const probabilities = scaled.softMaxedRows(); // Rank the tokens from most to least likely so the worst can be cut. const ranked = []; for (let col = 0; col < this.vocabSize; ++col) { ranked.push({ id: col, probability: probabilities.get(lastRow, col) }); } ranked.sort((a, b) => b.probability - a.probability); // top-k: keep only the k most likely tokens. const capped = ranked.slice(0, topK); // top-p: from the top, keep tokens until their combined probability // crosses topP, then stop. Always keeps at least the single top token. const kept = []; let keptTotal = 0.0; for (let i = 0; i < capped.length; ++i) { kept.push(capped[i]); keptTotal += capped[i].probability; if (keptTotal >= topP) { break; } } // The survivors cover only part of the number line now, up to keptTotal. // Draw a random number up to that total, then walk them as before. const draw = Math.random() * keptTotal; let cumulative = 0.0; for (let i = 0; i < kept.length; ++i) { cumulative += kept[i].probability; if (draw < cumulative) { return kept[i].id; } } return kept[kept.length - 1].id; // guard against floating-point drift } parameters() { const params = []; // Token embedding table (also used tied as the unembedding). const tokenParams = this.tokenEmbeddings.parameters(); for (let i = 0; i < tokenParams.length; ++i) { params.push(tokenParams[i]); } // Per-block parameters. for (let i = 0; i < this.blocks.length; ++i) { const blockParams = this.blocks[i].parameters(); for (let j = 0; j < blockParams.length; ++j) { params.push(blockParams[j]); } } // Final norm now carries a learned scale (gamma), gathered like the rest. const normParams = this.finalNorm.parameters(); for (let i = 0; i < normParams.length; ++i) { params.push(normParams[i]); } return params; } parameterCount() { let total = 0; for (const t of this.parameters()) { total += t.data.length; } return total; } // Build from a model.json object, or from a weight file's header. static fromConfig(config) { return new GabGPT( config.vocabSize, config.featureDim, config.numHeads, config.ropeBase, config.numPreludeBlocks, config.numCoreBlocks, config.numCodaBlocks, config.loopCount, config.mlpHiddenDim, config.convKernel, ); } // The architecture and shape, using model.json's field names. Written into // every weight file's header, and enough for fromConfig to rebuild the model. toConfig() { return { architecture: "gab10m", vocabSize: this.vocabSize, featureDim: this.featureDim, numHeads: this.numHeads, ropeBase: this.ropeBase, numPreludeBlocks: this.numPrelude, numCoreBlocks: this.numCore, numCodaBlocks: this.numCoda, loopCount: this.loopCount, mlpHiddenDim: this.mlpHiddenDim, convKernel: this.convKernel, parameterCount: this.parameterCount(), }; } // metadata is anything worth keeping with the weights, typically the // model.json the model was built from plus training details. The shape // fields always come from the model itself, so the header cannot lie. serializeToArrayBuffer(metadata = {}) { const weights = new Float32Array(this.parameterCount()); let offset = 0; for (const t of this.parameters()) { weights.set(t.data, offset); offset += t.data.length; } return encodeWeightFile({ ...metadata, ...this.toConfig() }, weights); } // Loads weights with the same layout. // The file's own header is kept in // loadedHeader, so a run can always tell which weights it is running. deserializeFromArrayBuffer(buffer) { const { header, weights } = decodeWeightFile(buffer); checkWeightLayout(header, this.toConfig()); let offset = 0; for (const t of this.parameters()) { t.data.set(weights.subarray(offset, offset + t.data.length)); offset += t.data.length; } this.loadedHeader = header; } // Rebuild the model a weight file describes, and load its weights. static fromArrayBuffer(buffer) { const { header } = decodeWeightFile(buffer); if (header.architecture !== "gab10m") { throw new Error( `this engine runs gab10m, but the file holds ${header.architecture} weights. ` + `To run them here anyway, build the model with fromConfig and call deserializeFromArrayBuffer.` ); } const model = GabGPT.fromConfig(header); model.deserializeFromArrayBuffer(buffer); return model; } } // A key-value cache holds one key tensor and one value tensor for every layer // pass (for a looped model, numBlocks is model.effectiveDepth, not the unique // block count), // for every head. The model reads and writes it but does not own it: the caller // creates one and passes it in, so a single GabGPT can serve many independent // generations at once, each with its own cache. export class KVCache { constructor(numBlocks, numHeads) { // One key tensor and one value tensor per block, per head. Each starts // null and becomes a growing tensor as tokens flow through. this.keys = []; this.values = []; for (let block = 0; block < numBlocks; ++block) { const blockKeys = []; const blockValues = []; for (let head = 0; head < numHeads; ++head) { blockKeys.push(null); blockValues.push(null); } this.keys.push(blockKeys); this.values.push(blockValues); } // Each block's two canon layers keep their last few inputs, so a new // token can be convolved with the ones before it. this.convA = new Array(numBlocks).fill(null); this.convC = new Array(numBlocks).fill(null); } // Tokens cached so far. Every block and head advances together, so the // first head's key tensor speaks for all of them. get length() { const first = this.keys[0][0]; return first === null ? 0 : first.rows; } } // Muon for the matrices, AdamW for everything else, with the full set of // training features: // - Muon (MomentUm Orthogonalized by Newton-Schulz) for every attention and // MLP matrix: Nesterov momentum, then the update is orthogonalized, so it // moves every direction of the matrix by about the same amount instead of // letting a few large directions dominate. It is scaled to the size an // AdamW update would have (0.2 x sqrt(larger side)), so one learning rate // and one schedule serve both optimizers. // - AdamW, with bias correction, for the embedding table (with weight // decay), the norm scales and the canon filters (without) // - decoupled weight decay on the matrices and the embedding table // - linear warmup, then cosine decay to a floor learning rate // - global gradient-norm clipping // - mini-batches: gradients from every example are combined before a step // - loss masking: a 0/1 mask per example marks which targets are graded // - token-weighted batches: the batch loss is the mean over every graded // target in the batch, so a long example counts for more than a short one // Every hyperparameter is an option, so each model can use its own settings. export class MuonAdamWTrainer { constructor(model, { maxLearningRate = 3e-4, minLearningRate = 3e-5, warmupSteps = 100, totalSteps = 10000, beta1 = 0.9, beta2 = 0.95, // 0.95, not 0.999: adapts faster when gradient scale shifts, the usual choice for language models epsilon = 1e-8, weightDecay = 0.1, gradientClip = 1.0, optimizer = "muon", // "muon", or "adamw" to train the matrices with AdamW too muonMomentum = 0.95, muonSteps = 5, // Newton-Schulz iterations } = {}) { this.model = model; this.maxLearningRate = maxLearningRate; this.minLearningRate = minLearningRate; this.warmupSteps = warmupSteps; this.totalSteps = totalSteps; this.beta1 = beta1; this.beta2 = beta2; this.epsilon = epsilon; this.weightDecay = weightDecay; this.gradientClip = gradientClip; this.muonMomentum = muonMomentum; this.muonSteps = muonSteps; // Grab the parameter list once. The tensors inside are stable references, // so the moment buffers below stay matched to them for the whole run. this.params = model.parameters(); // Each tensor's role decides its update. Matrices go to Muon. Weight // decay pulls weights toward zero, which suits matrices and the // embedding table, but not norm scales or filters: each of their values // has one job, and decaying it just fights that job. this.usesMuon = this.params.map((t) => optimizer === "muon" && t.role === "matrix"); this.decays = this.params.map((t) => t.role === "matrix" || t.role === "embedding"); // Adam keeps two running averages per parameter, Muon one momentum // buffer, each the same shape as the parameter. A fresh Float32Array is // already all zeros. this.firstMoments = []; this.secondMoments = []; for (let i = 0; i < this.params.length; ++i) { const length = this.params[i].data.length; this.firstMoments.push(new Float32Array(length)); // m, or Muon's momentum this.secondMoments.push(this.usesMuon[i] ? null : new Float32Array(length)); // v } // Updates applied so far. Drives both bias correction and the schedule. this.step = 0; // Exposed for logging. this.lastLearningRate = 0; this.lastGradientNorm = 0; } // Fused mean cross-entropy over the graded rows. The forward pass uses the // log-sum-exp form for each row of logits, which avoids exponentiating large // values directly: for a row with correct token t, the loss is // log(sum_j e^{x_j}) - x_t. The softmax probabilities are cached, because // the backward rule reuses them. crossEntropyLoss(logits, targetIds, targetMask = null) { const rows = logits.rows; const cols = logits.columns; const probs = new Float32Array(rows * cols); // cached for the backward let total = 0.0; let count = 0; // graded rows, the divisor for the mean for (let row = 0; row < rows; ++row) { // A context-only row contributes no loss and no gradient. if (targetMask !== null && !targetMask[row]) { continue; } const base = row * cols; let rowMax = logits.data[base]; for (let col = 1; col < cols; ++col) { const v = logits.data[base + col]; if (v > rowMax) rowMax = v; } let sum = 0.0; for (let col = 0; col < cols; ++col) { const e = Math.exp(logits.data[base + col] - rowMax); probs[base + col] = e; sum += e; } for (let col = 0; col < cols; ++col) { probs[base + col] /= sum; } const logSumExp = rowMax + Math.log(sum); total += logSumExp - logits.data[base + targetIds[row]]; count += 1; } if (count === 0) { throw new Error("No unmasked targets were provided"); } const loss = new Tensor(1, 1); loss.data[0] = total / count; if (Tensor.gradEnabled) { loss._inputs = [logits]; loss._backward = () => { const seed = loss.grad[0] / count; for (let row = 0; row < rows; ++row) { if (targetMask !== null && !targetMask[row]) { continue; } const base = row * cols; for (let col = 0; col < cols; ++col) { logits.grad[base + col] += seed * probs[base + col]; } logits.grad[base + targetIds[row]] -= seed; } }; } return loss; } learningRate() { const step = this.step; // Warmup: ramp linearly from zero to the peak over the first steps. if (step < this.warmupSteps) { return this.maxLearningRate * (step / this.warmupSteps); } // Past the planned end: hold at the floor. if (step >= this.totalSteps) { return this.minLearningRate; } // Cosine decay: ease from the peak down to the floor. const progress = (step - this.warmupSteps) / (this.totalSteps - this.warmupSteps); const cosine = 0.5 * (1 + Math.cos(Math.PI * progress)); return this.minLearningRate + (this.maxLearningRate - this.minLearningRate) * cosine; } clipGradients() { // Global L2 norm across every gradient in the model. let sumSquares = 0.0; for (let i = 0; i < this.params.length; ++i) { const grad = this.params[i].grad; for (let j = 0; j < grad.length; ++j) { sumSquares += grad[j] * grad[j]; } } const norm = Math.sqrt(sumSquares); // Only scale down when the norm is over the threshold. if (norm > this.gradientClip) { const scale = this.gradientClip / norm; for (let i = 0; i < this.params.length; ++i) { const grad = this.params[i].grad; for (let j = 0; j < grad.length; ++j) { grad[j] *= scale; } } } return norm; // before clipping, for logging } train(tokenIdBatch, tokenMaskBatch = null) { const counts = gradedTargetCounts(tokenIdBatch, tokenMaskBatch); const totalCount = counts.reduce((a, b) => a + b, 0); if (totalCount === 0) { throw new Error("Mini-batch has no graded targets"); } // Clear gradients once, then let every training example add to them. for (let i = 0; i < this.params.length; ++i) { this.params[i].zeroGrad(); } // Each example's loss is a mean over its own graded targets. Scaling it // by that example's share of the batch's graded targets makes the sum // of the scaled losses the mean over every graded target in the batch. let batchLoss = 0.0; for (let batchIndex = 0; batchIndex < tokenIdBatch.length; ++batchIndex) { if (counts[batchIndex] === 0) { continue; // nothing to grade: the example would add nothing } const tokenIds = tokenIdBatch[batchIndex]; const tokenMask = tokenMaskBatch === null ? null : tokenMaskBatch[batchIndex]; const inputIds = tokenIds.slice(0, tokenIds.length - 1); const targetIds = tokenIds.slice(1); // The mask lines up with tokenIds, so shift it like the targets. The // first token is never a target, so its mask is dropped. const targetMask = tokenMask === null ? null : tokenMask.slice(1); const logits = this.model.forward(inputIds); const loss = this.crossEntropyLoss(logits, targetIds, targetMask) .scale(counts[batchIndex] / totalCount); loss.backward(); batchLoss += loss.data[0]; } // Rein in the gradient before it moves anything. this.lastGradientNorm = this.clipGradients(); // Advance the step, then read this step's scheduled learning rate. this.step += 1; const learningRate = this.learningRate(); this.lastLearningRate = learningRate; const beta1 = this.beta1; const beta2 = this.beta2; const epsilon = this.epsilon; // Bias-correction denominators for this step. const correction1 = 1 - Math.pow(beta1, this.step); const correction2 = 1 - Math.pow(beta2, this.step); // Muon or AdamW update, parameter by parameter. for (let i = 0; i < this.params.length; ++i) { const param = this.params[i]; if (this.usesMuon[i]) { this.muonUpdate(param, this.firstMoments[i], learningRate); continue; } const m = this.firstMoments[i]; const v = this.secondMoments[i]; const weightDecay = this.decays[i] ? this.weightDecay : 0; for (let j = 0; j < param.data.length; ++j) { const g = param.grad[j]; const weight = param.data[j]; // Update the running averages of the gradient and its square. m[j] = beta1 * m[j] + (1 - beta1) * g; v[j] = beta2 * v[j] + (1 - beta2) * g * g; // Undo the startup bias toward zero. const mHat = m[j] / correction1; const vHat = v[j] / correction2; // The Adam step: smoothed direction, per-parameter scale. const adamStep = mHat / (Math.sqrt(vHat) + epsilon); // Decoupled weight decay: pull the weight toward zero on its // own, not routed through the adaptive scaling above. param.data[j] = weight - learningRate * (adamStep + weightDecay * weight); } } return batchLoss; } muonUpdate(param, momentum, learningRate) { const mu = this.muonMomentum; const rows = param.rows; const cols = param.columns; // Nesterov momentum: keep a running sum of gradients, then step along // the gradient plus a look-ahead share of that sum. const direction = new Float32Array(param.data.length); for (let j = 0; j < param.data.length; ++j) { momentum[j] = mu * momentum[j] + param.grad[j]; direction[j] = param.grad[j] + mu * momentum[j]; } const orthogonal = newtonSchulz(direction, rows, cols, this.muonSteps); // Scale to an AdamW-sized update, so both share one learning rate. const scale = 0.2 * Math.sqrt(Math.max(rows, cols)); for (let j = 0; j < param.data.length; ++j) { const weight = param.data[j]; param.data[j] = weight - learningRate * (scale * orthogonal[j] + this.weightDecay * weight); } } } // Newton-Schulz iteration: pushes every singular value of a matrix toward one, // keeping its singular vectors. The result is close to the nearest orthogonal // matrix (U V^T from the SVD) at the cost of a few matrix products. The // quintic's coefficients (3.4445, -4.7750, 2.0315) trade exactness for speed: // singular values land roughly between 0.7 and 1.2, which is all Muon needs. // The matrix is first scaled to Frobenius norm one, so every singular value // starts at most one. A tall matrix is worked on transposed, which keeps the // Gram matrix X X^T small. function newtonSchulz(values, rows, cols, steps) { const a = 3.4445, b = -4.7750, c = 2.0315; const transpose = rows > cols; const m = transpose ? cols : rows; // X is m x n with m <= n const n = transpose ? rows : cols; let x = new Float32Array(m * n); let norm = 0.0; for (let r = 0; r < rows; ++r) { for (let k = 0; k < cols; ++k) { const v = values[r * cols + k]; norm += v * v; if (transpose) x[k * n + r] = v; else x[r * n + k] = v; } } norm = Math.sqrt(norm) + 1e-7; for (let i = 0; i < x.length; ++i) x[i] /= norm; const multiply = (left, right, p, q, s) => { // [p, q] x [q, s] const out = new Float32Array(p * s); for (let i = 0; i < p; ++i) { for (let k = 0; k < q; ++k) { const l = left[i * q + k]; for (let j = 0; j < s; ++j) out[i * s + j] += l * right[k * s + j]; } } return out; }; for (let step = 0; step < steps; ++step) { const xT = new Float32Array(n * m); for (let i = 0; i < m; ++i) for (let j = 0; j < n; ++j) xT[j * m + i] = x[i * n + j]; const A = multiply(x, xT, m, n, m); // X X^T, [m, m] const AA = multiply(A, A, m, m, m); const B = new Float32Array(m * m); for (let i = 0; i < B.length; ++i) B[i] = b * A[i] + c * AA[i]; const BX = multiply(B, x, m, m, n); for (let i = 0; i < x.length; ++i) x[i] = a * x[i] + BX[i]; } if (!transpose) return x; const out = new Float32Array(rows * cols); for (let k = 0; k < m; ++k) for (let r = 0; r < n; ++r) out[r * cols + k] = x[k * n + r]; return out; } // Checks a mini-batch and counts each example's graded targets: the positions // after the first whose mask is 1, or every position after the first when // there is no mask. The trainer weights each example's loss by its count. function gradedTargetCounts(tokenIdBatch, tokenMaskBatch) { if (!Array.isArray(tokenIdBatch) || tokenIdBatch.length === 0) { throw new Error("Mini-batch must contain at least one training example"); } if (tokenMaskBatch !== null && tokenMaskBatch.length !== tokenIdBatch.length) { throw new Error( `mask batch length ${tokenMaskBatch.length} does not match ` + `token batch length ${tokenIdBatch.length}` ); } const counts = []; for (let batchIndex = 0; batchIndex < tokenIdBatch.length; ++batchIndex) { const tokenIds = tokenIdBatch[batchIndex]; const tokenMask = tokenMaskBatch === null ? null : tokenMaskBatch[batchIndex]; if (tokenIds.length < 2) { throw new Error(`training example ${batchIndex} needs at least 2 tokens, got ${tokenIds.length}`); } if (tokenMask !== null && tokenMask.length !== tokenIds.length) { throw new Error( `mask length ${tokenMask.length} does not match ` + `token length ${tokenIds.length} for example ${batchIndex}` ); } let count = tokenIds.length - 1; if (tokenMask !== null) { count = 0; for (let i = 1; i < tokenMask.length; ++i) { if (tokenMask[i]) count += 1; } } counts.push(count); } return counts; } // --- Weight files --------------------------------------------------------- // A weight file carries everything needed to rebuild the model that wrote it: // // bytes 0-3 magic "GAB_" // bytes 4-7 format version, uint32 little-endian (currently 1) // bytes 8-11 header length in bytes, uint32 little-endian, a multiple of 4 // header UTF-8 JSON, space-padded: architecture, shape, parameterCount, // plus any metadata the writer adds (chat markers, training info) // weights float32 little-endian, in parameters() order // // Padding the header to a multiple of 4 keeps the weights 4-byte aligned, so // they can be read as a Float32Array in place. const WEIGHT_MAGIC = "GAB_"; const WEIGHT_VERSION = 1; function assertLittleEndian() { if (new Uint8Array(new Uint32Array([1]).buffer)[0] !== 1) { throw new Error("weight files are little-endian, and this platform is not"); } } export function encodeWeightFile(header, weights) { // weights: Float32Array assertLittleEndian(); const json = JSON.stringify(header); let headerBytes = new TextEncoder().encode(json); const padding = (4 - (headerBytes.length % 4)) % 4; if (padding > 0) { headerBytes = new TextEncoder().encode(json + " ".repeat(padding)); } const buffer = new ArrayBuffer(12 + headerBytes.length + weights.length * 4); const bytes = new Uint8Array(buffer); const view = new DataView(buffer); bytes.set(new TextEncoder().encode(WEIGHT_MAGIC), 0); view.setUint32(4, WEIGHT_VERSION, true); view.setUint32(8, headerBytes.length, true); bytes.set(headerBytes, 12); new Float32Array(buffer, 12 + headerBytes.length, weights.length).set(weights); return buffer; } // Accepts an ArrayBuffer, or any typed array over one (such as the Buffer // that Node's readFileSync returns). Returns { header, weights }. export function decodeWeightFile(buffer) { assertLittleEndian(); if (ArrayBuffer.isView(buffer)) { // Copy into a fresh buffer: a view can start at an unaligned offset. buffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength); } if (buffer.byteLength < 12 || new TextDecoder().decode(new Uint8Array(buffer, 0, 4)) !== WEIGHT_MAGIC) { throw new Error("not a GabGPT weight file: missing the GAB_ magic"); } const view = new DataView(buffer); const version = view.getUint32(4, true); if (version !== WEIGHT_VERSION) { throw new Error(`unsupported weight file version ${version}`); } const headerLength = view.getUint32(8, true); const header = JSON.parse(new TextDecoder().decode(new Uint8Array(buffer, 12, headerLength))); const weightCount = (buffer.byteLength - 12 - headerLength) / 4; if (weightCount !== header.parameterCount) { throw new Error(`weight file holds ${weightCount} values, but its header says ${header.parameterCount}`); } return { header, weights: new Float32Array(buffer, 12 + headerLength, weightCount) }; } // Two architectures can share weights when their tensors line up one for one: // the same family, widths and number of unique blocks. A looped model and its // unlooped counterpart qualify. function weightLayout(config) { const looped = config.loopCount != null; return { family: config.architecture.replace("_looped", ""), vocabSize: config.vocabSize, featureDim: config.featureDim, numHeads: config.numHeads ?? null, mlpHiddenDim: config.mlpHiddenDim ?? null, convKernel: config.convKernel ?? null, uniqueBlocks: looped ? config.numPreludeBlocks + config.numCoreBlocks + config.numCodaBlocks : config.numBlocks, parameterCount: config.parameterCount, }; } function checkWeightLayout(fileConfig, modelConfig) { const file = weightLayout(fileConfig); const model = weightLayout(modelConfig); for (const key of Object.keys(model)) { if (file[key] !== model[key]) { throw new Error( `${fileConfig.architecture} weights do not fit a ${modelConfig.architecture} model: ` + `${key} is ${file[key]} in the file and ${model[key]} in the model` ); } } }