Text Generation
Transformers
Safetensors
English
gabgpt
gift-of-gab
small-language-model
trained-from-scratch
byte-level
looped-transformer
browser
conversational
custom_code
Instructions to use gszauer/Gab10M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use gszauer/Gab10M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="gszauer/Gab10M", trust_remote_code=True) messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("gszauer/Gab10M", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use gszauer/Gab10M with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "gszauer/Gab10M" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/Gab10M", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/gszauer/Gab10M
- SGLang
How to use gszauer/Gab10M with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "gszauer/Gab10M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/Gab10M", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "gszauer/Gab10M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/Gab10M", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use gszauer/Gab10M with Docker Model Runner:
docker model run hf.co/gszauer/Gab10M
Download reference.js from gszauer/Gab10M: direct link, hf CLI and curl.
- Browser
- Download file 63.8 kB
-
https://huggingface.co/gszauer/Gab10M/resolve/main/reference.js
- Command line
-
hf download hf://gszauer/Gab10M/reference.js
-
curl -L -o reference.js https://huggingface.co/gszauer/Gab10M/resolve/main/reference.js
63.8 kB
let randomState = 42;
function random() {
randomState =
(1664525 * randomState + 1013904223) % 4294967296;
return randomState / 4294967296;
}
export class Tensor {
// When false, operations skip building the graph:
static gradEnabled = true;
constructor(rows, columns) {
this.rows = rows;
this.columns = columns;
this.data = new Float32Array(rows * columns); // zero-filled
this.grad = null; // allocated on demand by backward()
this.role = null; // how the trainer updates it: "matrix", "embedding", or null for vectors and filters
this._inputs = []; // tensors this one was built from
this._backward = () => {}; // leaves do nothing
}
get(row, column) { // returns a plain number now, not a Value
return this.data[row * this.columns + column];
}
set(row, column, value) {
this.data[row * this.columns + column] = value;
}
// --- Initialization: these create the model's leaf parameters. ---
initToZeroes() {
for (let i = 0; i < this.data.length; ++i) {
this.data[i] = 0.0;
}
}
initToOnes() {
for (let i = 0; i < this.data.length; ++i) {
this.data[i] = 1.0;
}
}
initToSmallRandom() {
const scale = 1 / Math.sqrt(this.rows);
for (let i = 0; i < this.data.length; ++i) {
this.data[i] = (random() - 0.5) * 2 * scale; // uniform in [-scale, +scale)
}
}
zeroGrad() {
if (this.grad !== null) this.grad.fill(0);
}
parameters() {
return [this]; // a tensor is one parameter block
}
// --- Element-wise operations (both tensors share the same shape). ---
add(other) {
const out = new Tensor(this.rows, this.columns);
for (let i = 0; i < this.data.length; ++i) {
out.data[i] = this.data[i] + other.data[i];
}
if (Tensor.gradEnabled) {
out._inputs = [this, other];
out._backward = () => {
for (let i = 0; i < this.data.length; ++i) {
this.grad[i] += out.grad[i];
other.grad[i] += out.grad[i];
}
};
}
return out;
}
sub(other) {
const out = new Tensor(this.rows, this.columns);
for (let i = 0; i < this.data.length; ++i) {
out.data[i] = this.data[i] - other.data[i];
}
if (Tensor.gradEnabled) {
out._inputs = [this, other];
out._backward = () => {
for (let i = 0; i < this.data.length; ++i) {
this.grad[i] += out.grad[i];
other.grad[i] -= out.grad[i];
}
};
}
return out;
}
mul(other) { // element-wise multiply (matching numpy/pytorch); matrix multiply is matmul
const out = new Tensor(this.rows, this.columns);
for (let i = 0; i < this.data.length; ++i) {
out.data[i] = this.data[i] * other.data[i];
}
if (Tensor.gradEnabled) {
out._inputs = [this, other];
out._backward = () => {
for (let i = 0; i < this.data.length; ++i) {
this.grad[i] += other.data[i] * out.grad[i];
other.grad[i] += this.data[i] * out.grad[i];
}
};
}
return out;
}
scale(scalar) { // multiply every value by one plain number
const out = new Tensor(this.rows, this.columns);
for (let i = 0; i < this.data.length; ++i) {
out.data[i] = this.data[i] * scalar;
}
if (Tensor.gradEnabled) {
out._inputs = [this];
out._backward = () => {
for (let i = 0; i < this.data.length; ++i) {
this.grad[i] += scalar * out.grad[i];
}
};
}
return out;
}
relu() {
const out = new Tensor(this.rows, this.columns);
for (let i = 0; i < this.data.length; ++i) {
out.data[i] = this.data[i] > 0 ? this.data[i] : 0;
}
if (Tensor.gradEnabled) {
out._inputs = [this];
out._backward = () => {
for (let i = 0; i < this.data.length; ++i) {
this.grad[i] += (this.data[i] > 0 ? 1 : 0) * out.grad[i];
}
};
}
return out;
}
silu() { // x * sigmoid(x), the gate in SwiGLU
const out = new Tensor(this.rows, this.columns);
const sigmoids = new Float32Array(this.data.length); // cached for the backward
for (let i = 0; i < this.data.length; ++i) {
const x = this.data[i];
const s = 1 / (1 + Math.exp(-x));
sigmoids[i] = s;
out.data[i] = x * s;
}
if (Tensor.gradEnabled) {
out._inputs = [this];
out._backward = () => {
for (let i = 0; i < this.data.length; ++i) {
// d/dx [x * s(x)] = s + x * s * (1 - s)
const s = sigmoids[i];
this.grad[i] += (s + this.data[i] * s * (1 - s)) * out.grad[i];
}
};
}
return out;
}
// --- Matrix multiplication: the one node that replaces the most scalars. ---
matmul(other) { // (this.rows x this.columns) * (other.rows x other.columns)
const m = this.rows;
const k = this.columns; // shared inner dimension
const n = other.columns;
const out = new Tensor(m, n);
for (let row = 0; row < m; ++row) {
for (let col = 0; col < n; ++col) {
let sum = 0.0; // accumulate in double, store as float
for (let element = 0; element < k; ++element) {
sum += this.data[row * k + element] * other.data[element * n + col];
}
out.data[row * n + col] = sum;
}
}
if (Tensor.gradEnabled) {
out._inputs = [this, other];
out._backward = () => {
// dThis = dOut * other^T
for (let row = 0; row < m; ++row) {
for (let p = 0; p < k; ++p) {
let sum = 0.0;
for (let col = 0; col < n; ++col) {
sum += out.grad[row * n + col] * other.data[p * n + col];
}
this.grad[row * k + p] += sum;
}
}
// dOther = this^T * dOut
for (let p = 0; p < k; ++p) {
for (let col = 0; col < n; ++col) {
let sum = 0.0;
for (let row = 0; row < m; ++row) {
sum += this.data[row * k + p] * out.grad[row * n + col];
}
other.grad[p * n + col] += sum;
}
}
};
}
return out;
}
transposed() {
const out = new Tensor(this.columns, this.rows);
for (let row = 0; row < this.rows; ++row) {
for (let col = 0; col < this.columns; ++col) {
out.data[col * this.rows + row] = this.data[row * this.columns + col];
}
}
if (Tensor.gradEnabled) {
out._inputs = [this];
out._backward = () => {
for (let row = 0; row < this.rows; ++row) {
for (let col = 0; col < this.columns; ++col) {
this.grad[row * this.columns + col] += out.grad[col * this.rows + row];
}
}
};
}
return out;
}
// --- Fused operations. Each is one node with a single analytic backward
// rule instead of being assembled from many smaller nodes, which is
// both faster and far lighter on memory. These are general array
// operations, so they live here. RMS normalization and the
// cross-entropy loss are also fused, but each belongs to one component
// (the norm layer, the trainer), so they are defined there using the
// same _inputs/_backward protocol. ---
softMaxedRows() {
const rows = this.rows;
const cols = this.columns;
const out = new Tensor(rows, cols);
for (let row = 0; row < rows; ++row) {
const base = row * cols;
let rowMax = this.data[base];
for (let col = 1; col < cols; ++col) {
const v = this.data[base + col];
if (v > rowMax) rowMax = v;
}
let rowSum = 0.0;
for (let col = 0; col < cols; ++col) {
const e = Math.exp(this.data[base + col] - rowMax);
out.data[base + col] = e;
rowSum += e;
}
for (let col = 0; col < cols; ++col) {
out.data[base + col] /= rowSum;
}
}
if (Tensor.gradEnabled) {
out._inputs = [this];
out._backward = () => {
// For each row: dx_j = p_j * (g_j - sum_k g_k p_k)
for (let row = 0; row < rows; ++row) {
const base = row * cols;
let dot = 0.0;
for (let col = 0; col < cols; ++col) {
dot += out.grad[base + col] * out.data[base + col];
}
for (let col = 0; col < cols; ++col) {
const p = out.data[base + col];
this.grad[base + col] += p * (out.grad[base + col] - dot);
}
}
};
}
return out;
}
causalMasked() {
const rows = this.rows;
const cols = this.columns;
const out = new Tensor(rows, cols);
for (let row = 0; row < rows; ++row) {
for (let col = 0; col < cols; ++col) {
const index = row * cols + col;
out.data[index] = col > row ? -Infinity : this.data[index];
}
}
if (Tensor.gradEnabled) {
out._inputs = [this];
out._backward = () => {
// Masked cells are constants, so gradient only flows where col <= row.
for (let row = 0; row < rows; ++row) {
for (let col = 0; col <= row; ++col) {
const index = row * cols + col;
this.grad[index] += out.grad[index];
}
}
};
}
return out;
}
// Embedding lookup: copy the rows named by `indices` into a new tensor.
// The gradient scatters back, accumulating when an index repeats.
gatherRows(indices) {
const out = new Tensor(indices.length, this.columns);
for (let r = 0; r < indices.length; ++r) {
const src = indices[r] * this.columns;
const dst = r * this.columns;
for (let col = 0; col < this.columns; ++col) {
out.data[dst + col] = this.data[src + col];
}
}
if (Tensor.gradEnabled) {
out._inputs = [this];
out._backward = () => {
for (let r = 0; r < indices.length; ++r) {
const src = indices[r] * this.columns;
const dst = r * this.columns;
for (let col = 0; col < this.columns; ++col) {
this.grad[src + col] += out.grad[dst + col];
}
}
};
}
return out;
}
// Stack another tensor's rows below this one's, returning a new, taller
// tensor. Both must share the same column count. The KV cache uses this to
// grow one row per generated token. The gradient of the top rows flows back
// to this, the bottom rows to other. Generation runs with the graph off, so
// that rule never fires in practice, but it keeps appendRow a first-class op.
appendRow(other) {
// A new tensor: this tensor's rows on top, other's rows stacked below.
// Both must share the same column count.
const out = new Tensor(this.rows + other.rows, this.columns);
out.data.set(this.data, 0); // top block
out.data.set(other.data, this.data.length); // bottom block
if (Tensor.gradEnabled) {
out._inputs = [this, other];
out._backward = () => {
// Gradient of the top rows flows to this, the bottom rows to other.
for (let i = 0; i < this.data.length; ++i) {
this.grad[i] += out.grad[i];
}
for (let i = 0; i < other.data.length; ++i) {
other.grad[i] += out.grad[this.data.length + i];
}
};
}
return out;
}
// RoPE: rotate each row's component pairs by an angle set by the row's
// position. Query and key rows sit in sequence order, so the row index is
// the position — plus positionOffset, which the KV cache uses to rotate a
// new token by its true position rather than its row (offset zero, the
// default, reproduces the original behavior). The angles are fixed functions
// of position and frequency, not learned, so this op adds nothing to
// parameters(); the gradient only routes back to the input. The cosine and
// sine of each angle are cached, since the backward pass — a rotation by the
// negated angle — reuses them.
ropeRotated(thetaBase = 10000, positionOffset = 0) { // positionOffset is new
const rows = this.rows;
const cols = this.columns; // head_dim, must be even
const halfCols = cols / 2;
const out = new Tensor(rows, cols);
// Frequency depends on the pair, not the row
// Compute it once and cache
const freqs = new Float32Array(halfCols);
for (let pair = 0; pair < halfCols; ++pair) {
freqs[pair] = 1.0 / Math.pow(thetaBase, (2 * pair) / cols);
}
// Cached for the backward pass.
const cosTable = new Float32Array(rows * halfCols);
const sinTable = new Float32Array(rows * halfCols);
for (let row = 0; row < rows; ++row) {
const base = row * cols;
const trig = row * halfCols;
for (let pair = 0; pair < halfCols; ++pair) {
const angle = (positionOffset + row) * freqs[pair]; // true position = offset + row
const cos = Math.cos(angle);
const sin = Math.sin(angle);
cosTable[trig + pair] = cos;
sinTable[trig + pair] = sin;
const a = this.data[base + 2 * pair];
const b = this.data[base + 2 * pair + 1];
out.data[base + 2 * pair] = a * cos - b * sin;
out.data[base + 2 * pair + 1] = a * sin + b * cos;
}
}
if (Tensor.gradEnabled) {
out._inputs = [this];
out._backward = () => {
// Rotation is linear, the gradient is the inverse rotation.
for (let row = 0; row < rows; ++row) {
const base = row * cols;
const trig = row * halfCols;
for (let pair = 0; pair < halfCols; ++pair) {
const cos = cosTable[trig + pair];
const sin = sinTable[trig + pair];
const ga = out.grad[base + 2 * pair];
const gb = out.grad[base + 2 * pair + 1];
this.grad[base + 2 * pair] += ga * cos + gb * sin;
this.grad[base + 2 * pair + 1] += -ga * sin + gb * cos;
}
}
};
}
return out;
}
// Short causal convolution, one filter per column (depthwise): out[t] =
// sum over j of weight[j] * x[t - j], where weight is [kernel, columns] and
// row j weighs the input j rows back. Rows before the first are zeros, or
// come from history (the previous kernel - 1 rows, during generation). The
// gradient flows to this tensor and the filter; history is a constant.
causalConv(weight, history = null) {
const rows = this.rows;
const cols = this.columns;
const taps = weight.rows - 1;
const out = new Tensor(rows, cols);
const input = (source, col) => {
if (source >= 0) return this.data[source * cols + col];
if (history !== null) return history.data[(taps + source) * cols + col];
return 0.0;
};
for (let row = 0; row < rows; ++row) {
for (let col = 0; col < cols; ++col) {
let sum = 0.0;
for (let j = 0; j <= taps; ++j) {
sum += weight.data[j * cols + col] * input(row - j, col);
}
out.data[row * cols + col] = sum;
}
}
if (Tensor.gradEnabled) {
out._inputs = [this, weight];
out._backward = () => {
for (let row = 0; row < rows; ++row) {
for (let col = 0; col < cols; ++col) {
const g = out.grad[row * cols + col];
for (let j = 0; j <= taps; ++j) {
const source = row - j;
weight.grad[j * cols + col] += g * input(source, col);
if (source >= 0) {
this.grad[source * cols + col] += g * weight.data[j * cols + col];
}
}
}
}
};
}
return out;
}
// The last count rows, with earlier rows from history (or zeros) when this
// tensor is shorter. Carries a convolution's input from one generation step
// to the next, so it builds no graph.
lastRows(count, history = null) {
const out = new Tensor(count, this.columns);
for (let r = 0; r < count; ++r) {
const source = this.rows - count + r;
if (source >= 0) {
out.data.set(this.data.subarray(source * this.columns, (source + 1) * this.columns), r * this.columns);
} else if (history !== null) {
const h = history.rows + source;
out.data.set(history.data.subarray(h * this.columns, (h + 1) * this.columns), r * this.columns);
}
}
return out;
}
backward() {
// Topological sort: inputs come before the nodes built from them.
const topo = [];
const visited = new Set();
const visit = (node) => {
if (visited.has(node)) return;
visited.add(node);
for (let i = 0; i < node._inputs.length; ++i) {
visit(node._inputs[i]);
}
topo.push(node);
};
visit(this);
// Make sure every node in the graph has a gradient buffer. Freshly
// built tensors start at null and get a zeroed buffer here. Parameter
// tensors keep the buffer the trainer already zeroed.
for (let i = 0; i < topo.length; ++i) {
if (topo[i].grad === null) {
topo[i].grad = new Float32Array(topo[i].data.length);
}
}
// Seed: derivative of the output with respect to itself is 1.
this.grad.fill(1);
// Walk in reverse. Each node pushes gradient onto its inputs.
for (let i = topo.length - 1; i >= 0; --i) {
topo[i]._backward();
}
}
}
export class RMSNorm {
constructor(feature_dim) {
// Learned per-feature scale, applied after normalization. Starts at one,
// so the layer begins as a plain normalize until training moves it.
this.gamma = new Tensor(1, feature_dim);
this.gamma.initToOnes();
}
forward(x) {
const rows = x.rows;
const n = x.columns;
const eps = 1e-5;
const gamma = this.gamma;
const out = new Tensor(rows, n);
const invRmsByRow = new Float32Array(rows); // cached for the backward
for (let row = 0; row < rows; ++row) {
const base = row * n;
let sumSq = 0.0;
for (let col = 0; col < n; ++col) {
const v = x.data[base + col];
sumSq += v * v;
}
const invRms = 1 / Math.sqrt(sumSq / n + eps);
invRmsByRow[row] = invRms;
for (let col = 0; col < n; ++col) {
// normalize, then scale by the per-feature gamma
out.data[base + col] = x.data[base + col] * invRms * gamma.data[col];
}
}
if (Tensor.gradEnabled) {
out._inputs = [x, gamma];
out._backward = () => {
for (let row = 0; row < rows; ++row) {
const base = row * n;
const invRms = invRmsByRow[row];
const invRms3 = invRms * invRms * invRms;
// Upstream gradient passes through gamma before the RMS rule.
let dotGX = 0.0;
for (let col = 0; col < n; ++col) {
const gNorm = out.grad[base + col] * gamma.data[col];
dotGX += gNorm * x.data[base + col];
}
for (let col = 0; col < n; ++col) {
const g = out.grad[base + col];
const xv = x.data[base + col];
const gNorm = g * gamma.data[col];
x.grad[base + col] += gNorm * invRms - (xv * dotGX * invRms3) / n;
// gamma is shared across rows: sum the normalized input times upstream grad
gamma.grad[col] += xv * invRms * g;
}
}
};
}
return out;
}
parameters() {
return [this.gamma];
}
}
// Canon layer: a short causal convolution added back onto its input, applied
// right after a pre-norm. It lets each position mix in the few positions before
// it, which a byte-level model needs constantly to assemble bytes into words.
// The filter starts at zero, so the layer starts as a no-op.
export class CanonLayer {
constructor(feature_dim, kernel) {
this.weight = new Tensor(kernel, feature_dim);
this.weight.initToZeroes();
}
// history: the previous kernel - 1 inputs, kept in the cache's slot.
forward(x, cache = null, layerIndex = 0, slot = "convA") {
const history = cache === null ? null : cache[slot][layerIndex];
const out = x.add(x.causalConv(this.weight, history));
if (cache !== null) {
cache[slot][layerIndex] = x.lastRows(this.weight.rows - 1, history);
}
return out;
}
parameters() {
return [this.weight];
}
}
export class MultiHeadAttention {
constructor(feature_dim, num_heads, rope_base) {
this.num_heads = num_heads;
this.head_dim = feature_dim / num_heads; // must divide evenly, and be even for RoPE
this.rope_base = rope_base;
// Each head owns Q, K, V down to head_dim, and an output projection
// back up to feature_dim. The summed outputs equal a concat plus one
// big output projection, so no concat operation is needed.
this.heads = [];
for (let i = 0; i < num_heads; ++i) {
const learnedQ = new Tensor(feature_dim, this.head_dim);
learnedQ.initToSmallRandom();
const learnedK = new Tensor(feature_dim, this.head_dim);
learnedK.initToSmallRandom();
const learnedV = new Tensor(feature_dim, this.head_dim);
learnedV.initToSmallRandom();
const learnedO = new Tensor(this.head_dim, feature_dim);
learnedO.initToSmallRandom();
for (const matrix of [learnedQ, learnedK, learnedV, learnedO]) {
matrix.role = "matrix";
}
// QK-norm: each head normalizes its own queries and keys, with a
// learned scale, before RoPE. Attention scores stay in a sane range
// however large the projections grow.
const queryNorm = new RMSNorm(this.head_dim);
const keyNorm = new RMSNorm(this.head_dim);
this.heads.push({ learnedQ, learnedK, learnedV, learnedO, queryNorm, keyNorm });
}
}
forward(rms_norm_matrix, cache = null, layerIndex = 0) { // Shape: new_tokens, feature_dim
const scale = 1.0 / Math.sqrt(this.head_dim);
let output = null;
for (let i = 0; i < this.num_heads; ++i) {
const head = this.heads[i];
let Q = head.queryNorm.forward(rms_norm_matrix.matmul(head.learnedQ)); // Shape: new_tokens, head_dim
let K = head.keyNorm.forward(rms_norm_matrix.matmul(head.learnedK)); // Shape: new_tokens, head_dim
let V = rms_norm_matrix.matmul(head.learnedV); // Shape: new_tokens, head_dim
// The new tokens sit after whatever this head has already cached.
const pastK = cache === null ? null : cache.keys[layerIndex][i];
const pastV = cache === null ? null : cache.values[layerIndex][i];
const positionOffset = pastK === null ? 0 : pastK.rows;
// Rotate by true position. Cached keys keep the rotation they were
// stored with; the new keys are rotated from the offset onward.
Q = Q.ropeRotated(this.rope_base, positionOffset);
K = K.ropeRotated(this.rope_base, positionOffset);
// Grow the cache with the new keys and values, then attend over the
// full history. Without a cache, K and V are just the new tokens.
if (cache !== null) {
K = pastK === null ? K : pastK.appendRow(K);
V = pastV === null ? V : pastV.appendRow(V);
cache.keys[layerIndex][i] = K;
cache.values[layerIndex][i] = V;
}
let scores = Q.matmul(K.transposed()); // Shape: new_tokens, total_tokens
scores = scores.scale(scale);
// The prompt's first pass scores a square block and needs the mask.
// Every later token sees only its past, so no mask is needed.
if (positionOffset === 0) {
scores = scores.causalMasked();
}
const probabilities = scores.softMaxedRows();
const mixed = probabilities.matmul(V); // Shape: new_tokens, head_dim
// Project this head up to feature_dim and accumulate.
const projected = mixed.matmul(head.learnedO); // Shape: new_tokens, feature_dim
output = output === null ? projected : output.add(projected);
}
return output; // Shape: new_tokens, feature_dim
}
parameters() {
const params = [];
for (let i = 0; i < this.heads.length; ++i) {
const head = this.heads[i];
const matrices = [head.learnedQ, head.learnedK, head.learnedV, head.learnedO, head.queryNorm.gamma, head.keyNorm.gamma];
for (let j = 0; j < matrices.length; ++j) {
const matParams = matrices[j].parameters();
for (let k = 0; k < matParams.length; ++k) {
params.push(matParams[k]);
}
}
}
return params;
}
}
// SwiGLU: a gated MLP. silu(x @ gate) decides how much of each hidden unit
// (x @ up) passes, then down projects back. No bias.
export class MLP {
constructor(feature_dim_size, hidden_dim_size) {
this.learnedGate = new Tensor(feature_dim_size, hidden_dim_size);
this.learnedGate.initToSmallRandom();
this.learnedUp = new Tensor(feature_dim_size, hidden_dim_size);
this.learnedUp.initToSmallRandom();
this.learnedDown = new Tensor(hidden_dim_size, feature_dim_size);
this.learnedDown.initToSmallRandom();
for (const matrix of [this.learnedGate, this.learnedUp, this.learnedDown]) {
matrix.role = "matrix";
}
}
forward(x) { // Shape: sequence_length, feature_dim
const gate = x.matmul(this.learnedGate).silu(); // Shape: sequence_length, hidden_dim
const up = x.matmul(this.learnedUp); // Shape: sequence_length, hidden_dim
return gate.mul(up).matmul(this.learnedDown); // Shape: sequence_length, feature_dim
}
parameters() {
return [this.learnedGate, this.learnedUp, this.learnedDown];
}
}
export class TransformerBlock {
constructor(feature_dim, num_heads, rope_base, hidden_dim, conv_kernel) {
this.attentionNorm = new RMSNorm(feature_dim);
this.canonA = new CanonLayer(feature_dim, conv_kernel);
this.attention = new MultiHeadAttention(feature_dim, num_heads, rope_base);
this.mlpNorm = new RMSNorm(feature_dim);
this.canonC = new CanonLayer(feature_dim, conv_kernel);
this.mlp = new MLP(feature_dim, hidden_dim);
}
forward(x, cache = null, layerIndex = 0) {
// Attention sub-layer, with residual. The canon layer mixes in the last
// few positions first. The cache holds this layer's keys, values and
// both convolution histories.
const attentionInput = this.canonA.forward(this.attentionNorm.forward(x), cache, layerIndex, "convA");
const attentionDelta = this.attention.forward(attentionInput, cache, layerIndex);
const afterAttention = x.add(attentionDelta);
// MLP sub-layer, with residual
const mlpInput = this.canonC.forward(this.mlpNorm.forward(afterAttention), cache, layerIndex, "convC");
const mlpDelta = this.mlp.forward(mlpInput);
const afterMLP = afterAttention.add(mlpDelta);
return afterMLP;
}
parameters() {
const params = [];
const components = [this.attentionNorm, this.canonA, this.attention, this.mlpNorm, this.canonC, this.mlp];
for (let i = 0; i < components.length; ++i) {
const sub = components[i].parameters();
for (let j = 0; j < sub.length; ++j) {
params.push(sub[j]);
}
}
return params;
}
}
export class GabGPT {
constructor(vocabSize, featureDim, numHeads, ropeBase, numPrelude, numCore, numCoda, loopCount, mlpHiddenDim, convKernel) {
this.vocabSize = vocabSize;
this.featureDim = featureDim;
this.numHeads = numHeads;
this.ropeBase = ropeBase;
this.numPrelude = numPrelude;
this.numCore = numCore;
this.numCoda = numCoda;
this.loopCount = loopCount; // how many times the core runs, in training and inference alike
this.mlpHiddenDim = mlpHiddenDim; // SwiGLU's hidden width, about 2/3 of 4x to keep the budget
this.convKernel = convKernel; // canon layer filter length
// Token embedding table (position enters through rotation in attention).
this.tokenEmbeddings = new Tensor(vocabSize, featureDim);
this.tokenEmbeddings.initToSmallRandom();
this.tokenEmbeddings.role = "embedding";
// Three stacks of transformer blocks: the prelude and coda run once,
// the core runs loopCount times with the same weights.
const makeBlocks = (count) => {
const blocks = [];
for (let i = 0; i < count; ++i) {
blocks.push(new TransformerBlock(featureDim, numHeads, ropeBase, mlpHiddenDim, convKernel));
}
return blocks;
};
this.prelude = makeBlocks(numPrelude);
this.core = makeBlocks(numCore);
this.coda = makeBlocks(numCoda);
// Every unique block, in weight-file order. The layout matches an unlooped
// model with numPrelude + numCore + numCoda blocks.
this.blocks = [...this.prelude, ...this.core, ...this.coda];
// Final norm before the unembedding.
this.finalNorm = new RMSNorm(featureDim);
}
// Layers a token actually passes through. Each one is a separate slot in
// the KV cache, so build caches as new KVCache(model.effectiveDepth, numHeads).
get effectiveDepth() {
return this.numPrelude + this.numCore * this.loopCount + this.numCoda;
}
embed(ids) {
// Token embeddings only; position now enters through rotation in attention
return this.tokenEmbeddings.gatherRows(ids);
}
forward(tokenIdArray, cache = null) { // cache is new
// No maxContextLength check: there is no position table to index past.
// A sequence beyond the trained length just runs slower and predicts worse.
// With a cache, tokenIdArray holds only the new tokens; the block index
// threads through so each layer reads and writes its own cached slot.
// Every pass through a block is its own layer as far as the cache is
// concerned: the same core block sees different inputs on each loop,
// so it produces different keys, values and convolution histories.
let layerIndex = 0;
let x = this.embed(tokenIdArray); // Shape: new_tokens, feature_dim
for (let i = 0; i < this.prelude.length; ++i) {
x = this.prelude[i].forward(x, cache, layerIndex++); // Shape preserved.
}
// Input injection: the prelude's output is added back in at the start
// of every core pass after the first, so the loop never loses sight
// of the input. The first pass already starts from it.
const injected = x;
for (let loop = 0; loop < this.loopCount; ++loop) {
if (loop > 0) {
x = x.add(injected);
}
for (let i = 0; i < this.core.length; ++i) {
x = this.core[i].forward(x, cache, layerIndex++);
}
}
for (let i = 0; i < this.coda.length; ++i) {
x = this.coda[i].forward(x, cache, layerIndex++);
}
x = this.finalNorm.forward(x); // Shape: new_tokens, feature_dim
// Tied unembedding: (sequence_length, feature_dim) * (feature_dim, vocabSize)
const logits = x.matmul(this.tokenEmbeddings.transposed()); // Shape: sequence_length, vocabSize
return logits;
}
// Predict the next token. Temperature reshapes the distribution, top-k and
// top-p rule out the unlikely tail, and an optional KV cache skips rerunning
// tokens that were processed on an earlier call.
predictNextToken(tokenIdArray, temperature = 1.0, topK = 50, topP = 0.9, cache = null) {
// With a cache, the stored tokens do not run again. Only the tokens
// past the cached length are new.
const newTokens = cache === null
? tokenIdArray
: tokenIdArray.slice(cache.length);
const logits = this.forward(newTokens, cache); // was forward(tokenIdArray)
// The last row holds the scores for the token after the sequence,
// whether that is the last row of a full prompt or the only row of a
// single cached step.
const lastRow = logits.rows - 1;
// Temperature zero means greedy decoding: the single highest logit.
// Softmax is monotonic, so the argmax of the logits is the argmax of
// the probabilities, and the softmax can be skipped here.
if (temperature === 0) {
let bestId = 0;
let bestScore = logits.get(lastRow, 0);
for (let col = 1; col < this.vocabSize; ++col) {
const score = logits.get(lastRow, col);
if (score > bestScore) {
bestScore = score;
bestId = col;
}
}
return bestId;
}
// Reshape by temperature and turn the last row into probabilities.
const scaled = logits.scale(1 / temperature);
const probabilities = scaled.softMaxedRows();
// Rank the tokens from most to least likely so the worst can be cut.
const ranked = [];
for (let col = 0; col < this.vocabSize; ++col) {
ranked.push({ id: col, probability: probabilities.get(lastRow, col) });
}
ranked.sort((a, b) => b.probability - a.probability);
// top-k: keep only the k most likely tokens.
const capped = ranked.slice(0, topK);
// top-p: from the top, keep tokens until their combined probability
// crosses topP, then stop. Always keeps at least the single top token.
const kept = [];
let keptTotal = 0.0;
for (let i = 0; i < capped.length; ++i) {
kept.push(capped[i]);
keptTotal += capped[i].probability;
if (keptTotal >= topP) {
break;
}
}
// The survivors cover only part of the number line now, up to keptTotal.
// Draw a random number up to that total, then walk them as before.
const draw = Math.random() * keptTotal;
let cumulative = 0.0;
for (let i = 0; i < kept.length; ++i) {
cumulative += kept[i].probability;
if (draw < cumulative) {
return kept[i].id;
}
}
return kept[kept.length - 1].id; // guard against floating-point drift
}
parameters() {
const params = [];
// Token embedding table (also used tied as the unembedding).
const tokenParams = this.tokenEmbeddings.parameters();
for (let i = 0; i < tokenParams.length; ++i) {
params.push(tokenParams[i]);
}
// Per-block parameters.
for (let i = 0; i < this.blocks.length; ++i) {
const blockParams = this.blocks[i].parameters();
for (let j = 0; j < blockParams.length; ++j) {
params.push(blockParams[j]);
}
}
// Final norm now carries a learned scale (gamma), gathered like the rest.
const normParams = this.finalNorm.parameters();
for (let i = 0; i < normParams.length; ++i) {
params.push(normParams[i]);
}
return params;
}
parameterCount() {
let total = 0;
for (const t of this.parameters()) {
total += t.data.length;
}
return total;
}
// Build from a model.json object, or from a weight file's header.
static fromConfig(config) {
return new GabGPT(
config.vocabSize, config.featureDim, config.numHeads, config.ropeBase,
config.numPreludeBlocks, config.numCoreBlocks, config.numCodaBlocks, config.loopCount,
config.mlpHiddenDim, config.convKernel,
);
}
// The architecture and shape, using model.json's field names. Written into
// every weight file's header, and enough for fromConfig to rebuild the model.
toConfig() {
return {
architecture: "gab10m",
vocabSize: this.vocabSize,
featureDim: this.featureDim,
numHeads: this.numHeads,
ropeBase: this.ropeBase,
numPreludeBlocks: this.numPrelude,
numCoreBlocks: this.numCore,
numCodaBlocks: this.numCoda,
loopCount: this.loopCount,
mlpHiddenDim: this.mlpHiddenDim,
convKernel: this.convKernel,
parameterCount: this.parameterCount(),
};
}
// metadata is anything worth keeping with the weights, typically the
// model.json the model was built from plus training details. The shape
// fields always come from the model itself, so the header cannot lie.
serializeToArrayBuffer(metadata = {}) {
const weights = new Float32Array(this.parameterCount());
let offset = 0;
for (const t of this.parameters()) {
weights.set(t.data, offset);
offset += t.data.length;
}
return encodeWeightFile({ ...metadata, ...this.toConfig() }, weights);
}
// Loads weights with the same layout.
// The file's own header is kept in
// loadedHeader, so a run can always tell which weights it is running.
deserializeFromArrayBuffer(buffer) {
const { header, weights } = decodeWeightFile(buffer);
checkWeightLayout(header, this.toConfig());
let offset = 0;
for (const t of this.parameters()) {
t.data.set(weights.subarray(offset, offset + t.data.length));
offset += t.data.length;
}
this.loadedHeader = header;
}
// Rebuild the model a weight file describes, and load its weights.
static fromArrayBuffer(buffer) {
const { header } = decodeWeightFile(buffer);
if (header.architecture !== "gab10m") {
throw new Error(
`this engine runs gab10m, but the file holds ${header.architecture} weights. ` +
`To run them here anyway, build the model with fromConfig and call deserializeFromArrayBuffer.`
);
}
const model = GabGPT.fromConfig(header);
model.deserializeFromArrayBuffer(buffer);
return model;
}
}
// A key-value cache holds one key tensor and one value tensor for every layer
// pass (for a looped model, numBlocks is model.effectiveDepth, not the unique
// block count),
// for every head. The model reads and writes it but does not own it: the caller
// creates one and passes it in, so a single GabGPT can serve many independent
// generations at once, each with its own cache.
export class KVCache {
constructor(numBlocks, numHeads) {
// One key tensor and one value tensor per block, per head. Each starts
// null and becomes a growing tensor as tokens flow through.
this.keys = [];
this.values = [];
for (let block = 0; block < numBlocks; ++block) {
const blockKeys = [];
const blockValues = [];
for (let head = 0; head < numHeads; ++head) {
blockKeys.push(null);
blockValues.push(null);
}
this.keys.push(blockKeys);
this.values.push(blockValues);
}
// Each block's two canon layers keep their last few inputs, so a new
// token can be convolved with the ones before it.
this.convA = new Array(numBlocks).fill(null);
this.convC = new Array(numBlocks).fill(null);
}
// Tokens cached so far. Every block and head advances together, so the
// first head's key tensor speaks for all of them.
get length() {
const first = this.keys[0][0];
return first === null ? 0 : first.rows;
}
}
// Muon for the matrices, AdamW for everything else, with the full set of
// training features:
// - Muon (MomentUm Orthogonalized by Newton-Schulz) for every attention and
// MLP matrix: Nesterov momentum, then the update is orthogonalized, so it
// moves every direction of the matrix by about the same amount instead of
// letting a few large directions dominate. It is scaled to the size an
// AdamW update would have (0.2 x sqrt(larger side)), so one learning rate
// and one schedule serve both optimizers.
// - AdamW, with bias correction, for the embedding table (with weight
// decay), the norm scales and the canon filters (without)
// - decoupled weight decay on the matrices and the embedding table
// - linear warmup, then cosine decay to a floor learning rate
// - global gradient-norm clipping
// - mini-batches: gradients from every example are combined before a step
// - loss masking: a 0/1 mask per example marks which targets are graded
// - token-weighted batches: the batch loss is the mean over every graded
// target in the batch, so a long example counts for more than a short one
// Every hyperparameter is an option, so each model can use its own settings.
export class MuonAdamWTrainer {
constructor(model, {
maxLearningRate = 3e-4,
minLearningRate = 3e-5,
warmupSteps = 100,
totalSteps = 10000,
beta1 = 0.9,
beta2 = 0.95, // 0.95, not 0.999: adapts faster when gradient scale shifts, the usual choice for language models
epsilon = 1e-8,
weightDecay = 0.1,
gradientClip = 1.0,
optimizer = "muon", // "muon", or "adamw" to train the matrices with AdamW too
muonMomentum = 0.95,
muonSteps = 5, // Newton-Schulz iterations
} = {}) {
this.model = model;
this.maxLearningRate = maxLearningRate;
this.minLearningRate = minLearningRate;
this.warmupSteps = warmupSteps;
this.totalSteps = totalSteps;
this.beta1 = beta1;
this.beta2 = beta2;
this.epsilon = epsilon;
this.weightDecay = weightDecay;
this.gradientClip = gradientClip;
this.muonMomentum = muonMomentum;
this.muonSteps = muonSteps;
// Grab the parameter list once. The tensors inside are stable references,
// so the moment buffers below stay matched to them for the whole run.
this.params = model.parameters();
// Each tensor's role decides its update. Matrices go to Muon. Weight
// decay pulls weights toward zero, which suits matrices and the
// embedding table, but not norm scales or filters: each of their values
// has one job, and decaying it just fights that job.
this.usesMuon = this.params.map((t) => optimizer === "muon" && t.role === "matrix");
this.decays = this.params.map((t) => t.role === "matrix" || t.role === "embedding");
// Adam keeps two running averages per parameter, Muon one momentum
// buffer, each the same shape as the parameter. A fresh Float32Array is
// already all zeros.
this.firstMoments = [];
this.secondMoments = [];
for (let i = 0; i < this.params.length; ++i) {
const length = this.params[i].data.length;
this.firstMoments.push(new Float32Array(length)); // m, or Muon's momentum
this.secondMoments.push(this.usesMuon[i] ? null : new Float32Array(length)); // v
}
// Updates applied so far. Drives both bias correction and the schedule.
this.step = 0;
// Exposed for logging.
this.lastLearningRate = 0;
this.lastGradientNorm = 0;
}
// Fused mean cross-entropy over the graded rows. The forward pass uses the
// log-sum-exp form for each row of logits, which avoids exponentiating large
// values directly: for a row with correct token t, the loss is
// log(sum_j e^{x_j}) - x_t. The softmax probabilities are cached, because
// the backward rule reuses them.
crossEntropyLoss(logits, targetIds, targetMask = null) {
const rows = logits.rows;
const cols = logits.columns;
const probs = new Float32Array(rows * cols); // cached for the backward
let total = 0.0;
let count = 0; // graded rows, the divisor for the mean
for (let row = 0; row < rows; ++row) {
// A context-only row contributes no loss and no gradient.
if (targetMask !== null && !targetMask[row]) {
continue;
}
const base = row * cols;
let rowMax = logits.data[base];
for (let col = 1; col < cols; ++col) {
const v = logits.data[base + col];
if (v > rowMax) rowMax = v;
}
let sum = 0.0;
for (let col = 0; col < cols; ++col) {
const e = Math.exp(logits.data[base + col] - rowMax);
probs[base + col] = e;
sum += e;
}
for (let col = 0; col < cols; ++col) {
probs[base + col] /= sum;
}
const logSumExp = rowMax + Math.log(sum);
total += logSumExp - logits.data[base + targetIds[row]];
count += 1;
}
if (count === 0) {
throw new Error("No unmasked targets were provided");
}
const loss = new Tensor(1, 1);
loss.data[0] = total / count;
if (Tensor.gradEnabled) {
loss._inputs = [logits];
loss._backward = () => {
const seed = loss.grad[0] / count;
for (let row = 0; row < rows; ++row) {
if (targetMask !== null && !targetMask[row]) {
continue;
}
const base = row * cols;
for (let col = 0; col < cols; ++col) {
logits.grad[base + col] += seed * probs[base + col];
}
logits.grad[base + targetIds[row]] -= seed;
}
};
}
return loss;
}
learningRate() {
const step = this.step;
// Warmup: ramp linearly from zero to the peak over the first steps.
if (step < this.warmupSteps) {
return this.maxLearningRate * (step / this.warmupSteps);
}
// Past the planned end: hold at the floor.
if (step >= this.totalSteps) {
return this.minLearningRate;
}
// Cosine decay: ease from the peak down to the floor.
const progress = (step - this.warmupSteps) / (this.totalSteps - this.warmupSteps);
const cosine = 0.5 * (1 + Math.cos(Math.PI * progress));
return this.minLearningRate + (this.maxLearningRate - this.minLearningRate) * cosine;
}
clipGradients() {
// Global L2 norm across every gradient in the model.
let sumSquares = 0.0;
for (let i = 0; i < this.params.length; ++i) {
const grad = this.params[i].grad;
for (let j = 0; j < grad.length; ++j) {
sumSquares += grad[j] * grad[j];
}
}
const norm = Math.sqrt(sumSquares);
// Only scale down when the norm is over the threshold.
if (norm > this.gradientClip) {
const scale = this.gradientClip / norm;
for (let i = 0; i < this.params.length; ++i) {
const grad = this.params[i].grad;
for (let j = 0; j < grad.length; ++j) {
grad[j] *= scale;
}
}
}
return norm; // before clipping, for logging
}
train(tokenIdBatch, tokenMaskBatch = null) {
const counts = gradedTargetCounts(tokenIdBatch, tokenMaskBatch);
const totalCount = counts.reduce((a, b) => a + b, 0);
if (totalCount === 0) {
throw new Error("Mini-batch has no graded targets");
}
// Clear gradients once, then let every training example add to them.
for (let i = 0; i < this.params.length; ++i) {
this.params[i].zeroGrad();
}
// Each example's loss is a mean over its own graded targets. Scaling it
// by that example's share of the batch's graded targets makes the sum
// of the scaled losses the mean over every graded target in the batch.
let batchLoss = 0.0;
for (let batchIndex = 0; batchIndex < tokenIdBatch.length; ++batchIndex) {
if (counts[batchIndex] === 0) {
continue; // nothing to grade: the example would add nothing
}
const tokenIds = tokenIdBatch[batchIndex];
const tokenMask = tokenMaskBatch === null ? null : tokenMaskBatch[batchIndex];
const inputIds = tokenIds.slice(0, tokenIds.length - 1);
const targetIds = tokenIds.slice(1);
// The mask lines up with tokenIds, so shift it like the targets. The
// first token is never a target, so its mask is dropped.
const targetMask = tokenMask === null ? null : tokenMask.slice(1);
const logits = this.model.forward(inputIds);
const loss = this.crossEntropyLoss(logits, targetIds, targetMask)
.scale(counts[batchIndex] / totalCount);
loss.backward();
batchLoss += loss.data[0];
}
// Rein in the gradient before it moves anything.
this.lastGradientNorm = this.clipGradients();
// Advance the step, then read this step's scheduled learning rate.
this.step += 1;
const learningRate = this.learningRate();
this.lastLearningRate = learningRate;
const beta1 = this.beta1;
const beta2 = this.beta2;
const epsilon = this.epsilon;
// Bias-correction denominators for this step.
const correction1 = 1 - Math.pow(beta1, this.step);
const correction2 = 1 - Math.pow(beta2, this.step);
// Muon or AdamW update, parameter by parameter.
for (let i = 0; i < this.params.length; ++i) {
const param = this.params[i];
if (this.usesMuon[i]) {
this.muonUpdate(param, this.firstMoments[i], learningRate);
continue;
}
const m = this.firstMoments[i];
const v = this.secondMoments[i];
const weightDecay = this.decays[i] ? this.weightDecay : 0;
for (let j = 0; j < param.data.length; ++j) {
const g = param.grad[j];
const weight = param.data[j];
// Update the running averages of the gradient and its square.
m[j] = beta1 * m[j] + (1 - beta1) * g;
v[j] = beta2 * v[j] + (1 - beta2) * g * g;
// Undo the startup bias toward zero.
const mHat = m[j] / correction1;
const vHat = v[j] / correction2;
// The Adam step: smoothed direction, per-parameter scale.
const adamStep = mHat / (Math.sqrt(vHat) + epsilon);
// Decoupled weight decay: pull the weight toward zero on its
// own, not routed through the adaptive scaling above.
param.data[j] = weight - learningRate * (adamStep + weightDecay * weight);
}
}
return batchLoss;
}
muonUpdate(param, momentum, learningRate) {
const mu = this.muonMomentum;
const rows = param.rows;
const cols = param.columns;
// Nesterov momentum: keep a running sum of gradients, then step along
// the gradient plus a look-ahead share of that sum.
const direction = new Float32Array(param.data.length);
for (let j = 0; j < param.data.length; ++j) {
momentum[j] = mu * momentum[j] + param.grad[j];
direction[j] = param.grad[j] + mu * momentum[j];
}
const orthogonal = newtonSchulz(direction, rows, cols, this.muonSteps);
// Scale to an AdamW-sized update, so both share one learning rate.
const scale = 0.2 * Math.sqrt(Math.max(rows, cols));
for (let j = 0; j < param.data.length; ++j) {
const weight = param.data[j];
param.data[j] = weight - learningRate * (scale * orthogonal[j] + this.weightDecay * weight);
}
}
}
// Newton-Schulz iteration: pushes every singular value of a matrix toward one,
// keeping its singular vectors. The result is close to the nearest orthogonal
// matrix (U V^T from the SVD) at the cost of a few matrix products. The
// quintic's coefficients (3.4445, -4.7750, 2.0315) trade exactness for speed:
// singular values land roughly between 0.7 and 1.2, which is all Muon needs.
// The matrix is first scaled to Frobenius norm one, so every singular value
// starts at most one. A tall matrix is worked on transposed, which keeps the
// Gram matrix X X^T small.
function newtonSchulz(values, rows, cols, steps) {
const a = 3.4445, b = -4.7750, c = 2.0315;
const transpose = rows > cols;
const m = transpose ? cols : rows; // X is m x n with m <= n
const n = transpose ? rows : cols;
let x = new Float32Array(m * n);
let norm = 0.0;
for (let r = 0; r < rows; ++r) {
for (let k = 0; k < cols; ++k) {
const v = values[r * cols + k];
norm += v * v;
if (transpose) x[k * n + r] = v; else x[r * n + k] = v;
}
}
norm = Math.sqrt(norm) + 1e-7;
for (let i = 0; i < x.length; ++i) x[i] /= norm;
const multiply = (left, right, p, q, s) => { // [p, q] x [q, s]
const out = new Float32Array(p * s);
for (let i = 0; i < p; ++i) {
for (let k = 0; k < q; ++k) {
const l = left[i * q + k];
for (let j = 0; j < s; ++j) out[i * s + j] += l * right[k * s + j];
}
}
return out;
};
for (let step = 0; step < steps; ++step) {
const xT = new Float32Array(n * m);
for (let i = 0; i < m; ++i) for (let j = 0; j < n; ++j) xT[j * m + i] = x[i * n + j];
const A = multiply(x, xT, m, n, m); // X X^T, [m, m]
const AA = multiply(A, A, m, m, m);
const B = new Float32Array(m * m);
for (let i = 0; i < B.length; ++i) B[i] = b * A[i] + c * AA[i];
const BX = multiply(B, x, m, m, n);
for (let i = 0; i < x.length; ++i) x[i] = a * x[i] + BX[i];
}
if (!transpose) return x;
const out = new Float32Array(rows * cols);
for (let k = 0; k < m; ++k) for (let r = 0; r < n; ++r) out[r * cols + k] = x[k * n + r];
return out;
}
// Checks a mini-batch and counts each example's graded targets: the positions
// after the first whose mask is 1, or every position after the first when
// there is no mask. The trainer weights each example's loss by its count.
function gradedTargetCounts(tokenIdBatch, tokenMaskBatch) {
if (!Array.isArray(tokenIdBatch) || tokenIdBatch.length === 0) {
throw new Error("Mini-batch must contain at least one training example");
}
if (tokenMaskBatch !== null && tokenMaskBatch.length !== tokenIdBatch.length) {
throw new Error(
`mask batch length ${tokenMaskBatch.length} does not match ` +
`token batch length ${tokenIdBatch.length}`
);
}
const counts = [];
for (let batchIndex = 0; batchIndex < tokenIdBatch.length; ++batchIndex) {
const tokenIds = tokenIdBatch[batchIndex];
const tokenMask = tokenMaskBatch === null ? null : tokenMaskBatch[batchIndex];
if (tokenIds.length < 2) {
throw new Error(`training example ${batchIndex} needs at least 2 tokens, got ${tokenIds.length}`);
}
if (tokenMask !== null && tokenMask.length !== tokenIds.length) {
throw new Error(
`mask length ${tokenMask.length} does not match ` +
`token length ${tokenIds.length} for example ${batchIndex}`
);
}
let count = tokenIds.length - 1;
if (tokenMask !== null) {
count = 0;
for (let i = 1; i < tokenMask.length; ++i) {
if (tokenMask[i]) count += 1;
}
}
counts.push(count);
}
return counts;
}
// --- Weight files ---------------------------------------------------------
// A weight file carries everything needed to rebuild the model that wrote it:
//
// bytes 0-3 magic "GAB_"
// bytes 4-7 format version, uint32 little-endian (currently 1)
// bytes 8-11 header length in bytes, uint32 little-endian, a multiple of 4
// header UTF-8 JSON, space-padded: architecture, shape, parameterCount,
// plus any metadata the writer adds (chat markers, training info)
// weights float32 little-endian, in parameters() order
//
// Padding the header to a multiple of 4 keeps the weights 4-byte aligned, so
// they can be read as a Float32Array in place.
const WEIGHT_MAGIC = "GAB_";
const WEIGHT_VERSION = 1;
function assertLittleEndian() {
if (new Uint8Array(new Uint32Array([1]).buffer)[0] !== 1) {
throw new Error("weight files are little-endian, and this platform is not");
}
}
export function encodeWeightFile(header, weights) { // weights: Float32Array
assertLittleEndian();
const json = JSON.stringify(header);
let headerBytes = new TextEncoder().encode(json);
const padding = (4 - (headerBytes.length % 4)) % 4;
if (padding > 0) {
headerBytes = new TextEncoder().encode(json + " ".repeat(padding));
}
const buffer = new ArrayBuffer(12 + headerBytes.length + weights.length * 4);
const bytes = new Uint8Array(buffer);
const view = new DataView(buffer);
bytes.set(new TextEncoder().encode(WEIGHT_MAGIC), 0);
view.setUint32(4, WEIGHT_VERSION, true);
view.setUint32(8, headerBytes.length, true);
bytes.set(headerBytes, 12);
new Float32Array(buffer, 12 + headerBytes.length, weights.length).set(weights);
return buffer;
}
// Accepts an ArrayBuffer, or any typed array over one (such as the Buffer
// that Node's readFileSync returns). Returns { header, weights }.
export function decodeWeightFile(buffer) {
assertLittleEndian();
if (ArrayBuffer.isView(buffer)) {
// Copy into a fresh buffer: a view can start at an unaligned offset.
buffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
}
if (buffer.byteLength < 12 || new TextDecoder().decode(new Uint8Array(buffer, 0, 4)) !== WEIGHT_MAGIC) {
throw new Error("not a GabGPT weight file: missing the GAB_ magic");
}
const view = new DataView(buffer);
const version = view.getUint32(4, true);
if (version !== WEIGHT_VERSION) {
throw new Error(`unsupported weight file version ${version}`);
}
const headerLength = view.getUint32(8, true);
const header = JSON.parse(new TextDecoder().decode(new Uint8Array(buffer, 12, headerLength)));
const weightCount = (buffer.byteLength - 12 - headerLength) / 4;
if (weightCount !== header.parameterCount) {
throw new Error(`weight file holds ${weightCount} values, but its header says ${header.parameterCount}`);
}
return { header, weights: new Float32Array(buffer, 12 + headerLength, weightCount) };
}
// Two architectures can share weights when their tensors line up one for one:
// the same family, widths and number of unique blocks. A looped model and its
// unlooped counterpart qualify.
function weightLayout(config) {
const looped = config.loopCount != null;
return {
family: config.architecture.replace("_looped", ""),
vocabSize: config.vocabSize,
featureDim: config.featureDim,
numHeads: config.numHeads ?? null,
mlpHiddenDim: config.mlpHiddenDim ?? null,
convKernel: config.convKernel ?? null,
uniqueBlocks: looped
? config.numPreludeBlocks + config.numCoreBlocks + config.numCodaBlocks
: config.numBlocks,
parameterCount: config.parameterCount,
};
}
function checkWeightLayout(fileConfig, modelConfig) {
const file = weightLayout(fileConfig);
const model = weightLayout(modelConfig);
for (const key of Object.keys(model)) {
if (file[key] !== model[key]) {
throw new Error(
`${fileConfig.architecture} weights do not fit a ${modelConfig.architecture} model: ` +
`${key} is ${file[key]} in the file and ${model[key]} in the model`
);
}
}
}
let randomState = 42;
function random() {
randomState =
(1664525 * randomState + 1013904223) % 4294967296;
return randomState / 4294967296;
}
export class Tensor {
// When false, operations skip building the graph:
static gradEnabled = true;
constructor(rows, columns) {
this.rows = rows;
this.columns = columns;
this.data = new Float32Array(rows * columns); // zero-filled
this.grad = null; // allocated on demand by backward()
this.role = null; // how the trainer updates it: "matrix", "embedding", or null for vectors and filters
this._inputs = []; // tensors this one was built from
this._backward = () => {}; // leaves do nothing
}
get(row, column) { // returns a plain number now, not a Value
return this.data[row * this.columns + column];
}
set(row, column, value) {
this.data[row * this.columns + column] = value;
}
// --- Initialization: these create the model's leaf parameters. ---
initToZeroes() {
for (let i = 0; i < this.data.length; ++i) {
this.data[i] = 0.0;
}
}
initToOnes() {
for (let i = 0; i < this.data.length; ++i) {
this.data[i] = 1.0;
}
}
initToSmallRandom() {
const scale = 1 / Math.sqrt(this.rows);
for (let i = 0; i < this.data.length; ++i) {
this.data[i] = (random() - 0.5) * 2 * scale; // uniform in [-scale, +scale)
}
}
zeroGrad() {
if (this.grad !== null) this.grad.fill(0);
}
parameters() {
return [this]; // a tensor is one parameter block
}
// --- Element-wise operations (both tensors share the same shape). ---
add(other) {
const out = new Tensor(this.rows, this.columns);
for (let i = 0; i < this.data.length; ++i) {
out.data[i] = this.data[i] + other.data[i];
}
if (Tensor.gradEnabled) {
out._inputs = [this, other];
out._backward = () => {
for (let i = 0; i < this.data.length; ++i) {
this.grad[i] += out.grad[i];
other.grad[i] += out.grad[i];
}
};
}
return out;
}
sub(other) {
const out = new Tensor(this.rows, this.columns);
for (let i = 0; i < this.data.length; ++i) {
out.data[i] = this.data[i] - other.data[i];
}
if (Tensor.gradEnabled) {
out._inputs = [this, other];
out._backward = () => {
for (let i = 0; i < this.data.length; ++i) {
this.grad[i] += out.grad[i];
other.grad[i] -= out.grad[i];
}
};
}
return out;
}
mul(other) { // element-wise multiply (matching numpy/pytorch); matrix multiply is matmul
const out = new Tensor(this.rows, this.columns);
for (let i = 0; i < this.data.length; ++i) {
out.data[i] = this.data[i] * other.data[i];
}