slack-moji / src /lib /engine.js
cfahlgren1's picture
cfahlgren1 HF Staff
switch to int8 weights that stay int8 in memory, add a pixel loader with load stages, and stop the reload loop after an out-of-memory crash
8e8aecc
Raw History Blame Contribute Delete
10.8 kB
// Tessera-small in the browser: CLIP text encoder + denoiser via onnxruntime-web (WebGPU, or CPU),
// with the masked-diffusion sampler ported from tessera/sampler.py (16 steps, CFG 3,
// confidence-ordered unmasking with Gumbel choice noise and spatial NMS).
import { ClipTokenizer } from "./tokenizer.js";
const ORT_VERSION = "1.22.0";
const ORT_DIR = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ORT_VERSION}/dist/`;
export const GRID = 16;
export const N = GRID * GRID;
export const K = 512;
export const MASK = 512;
const TEXT_LEN = 24;
const TEXT_DIM = 512;
/** Error message for a run whose logits came back NaN or infinite. The client matches on it. */
export const BAD_OUTPUT = "bad-model-output";
function mulberry32(seed) {
let a = seed >>> 0;
return () => {
a = (a + 0x6d2b79f5) >>> 0;
let t = a;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
async function webgpuAvailable() {
const gpu = globalThis.navigator?.gpu;
if (!gpu) return false;
try {
return !!(await gpu.requestAdapter());
} catch {
return false;
}
}
/**
* Check the model's output for one fixed prompt against a reference computed on the CPU.
* Some GPUs and drivers run the model without an error but return NaN or garbage, so a
* session that loads isn't proof that it works. Returns why it failed, or null if it passed.
*/
async function selfTest(ort, den, clip, ref) {
const text = await clip.run({
input_ids: new ort.Tensor("int64", BigInt64Array.from(ref.ids, BigInt), [1, TEXT_LEN]),
});
const out = await den.run({
x: new ort.Tensor("int64", new BigInt64Array(N).fill(BigInt(MASK)), [1, N]),
text_states: text.last_hidden_state,
text_pooled: text.pooler_output,
mask_ratio: new ort.Tensor("float32", new Float32Array([1]), [1]),
sc_probs: new ort.Tensor("float32", new Float32Array(N * K), [1, N, K]),
drop_text: new ort.Tensor("bool", new Uint8Array(1), [1]),
});
const logits = out.logits.data;
let agree = 0;
let topDiff = 0;
for (let i = 0; i < N; i++) {
let best = 0;
for (let j = 0; j < K; j++) {
const v = logits[i * K + j];
if (!Number.isFinite(v)) return `logit ${i * K + j} is ${v}`;
if (v > logits[i * K + best]) best = j;
}
if (best === ref.argmax[i]) agree++;
topDiff += Math.abs(logits[i * K + best] - ref.top[i]);
}
const agreement = agree / N;
const meanTopDiff = topDiff / N;
if (agreement < 0.8 || meanTopDiff > 0.5) {
return `output doesn't match the reference (agreement ${agreement.toFixed(2)}, top-logit error ${meanTopDiff.toFixed(2)})`;
}
return null;
}
/**
* Build the engine from already-downloaded model bytes. Runs on WebGPU when it passes the
* self-test, otherwise on the CPU (WebAssembly). Meant to run inside a Web Worker.
*/
export async function createEngine(bytes, tokJson, palette, selfTestRef, { forceCpu = false, onStage } = {}) {
const ort = await import(/* @vite-ignore */ `${ORT_DIR}ort.webgpu.min.mjs`);
ort.env.wasm.wasmPaths = ORT_DIR;
const tokenizer = new ClipTokenizer(tokJson);
// One session at a time, so the peak memory holds one model's copy, not both.
// Constant folding is off because it would expand every int8 weight into an fp32 copy at load
// time, about 4x the memory, which is more than phones give a tab. The weights are dequantized
// on the fly instead.
const create = async (ep) => {
onStage?.("load");
const opts = {
executionProviders: [ep],
graphOptimizationLevel: "all",
extra: { optimization: { disable_specified_optimizers: "ConstantFolding" } },
};
const den = await ort.InferenceSession.create(bytes.den, opts);
const clip = await ort.InferenceSession.create(bytes.clip, opts);
return [den, clip];
};
const onCpu = async (fallbackReason) => {
const [den, clip] = await create("wasm");
return new Engine(ort, den, clip, tokenizer, palette, "cpu", fallbackReason);
};
if (forceCpu) return onCpu("forced");
if (!(await webgpuAvailable())) return onCpu("no-webgpu");
let den, clip;
try {
[den, clip] = await create("webgpu");
} catch (e) {
console.warn("WebGPU session failed, falling back to the CPU", e);
return onCpu("gpu-error");
}
onStage?.("check");
let failure;
try {
failure = await selfTest(ort, den, clip, selfTestRef);
} catch (e) {
failure = String(e?.message ?? e);
}
if (!failure) return new Engine(ort, den, clip, tokenizer, palette, "webgpu", null);
console.warn(`WebGPU failed the self-test (${failure}), falling back to the CPU`);
await Promise.all([den.release(), clip.release()]);
return onCpu("gpu-bad-output");
}
export class Engine {
constructor(ort, den, clip, tokenizer, palette, device, fallbackReason) {
this.device = device;
this.fallbackReason = fallbackReason;
this.ort = ort;
this.den = den;
this.clip = clip;
this.tokenizer = tokenizer;
this.palette = palette;
}
async encodeText(prompt) {
const ids = this.tokenizer.encodeFixed(prompt, TEXT_LEN);
const out = await this.clip.run({
input_ids: new this.ort.Tensor("int64", BigInt64Array.from(ids, BigInt), [1, TEXT_LEN]),
});
return { states: out.last_hidden_state.data, pooled: out.pooler_output.data };
}
/**
* Generate `n` sprites for one prompt. onStep(step, tokens, masked) is awaited after every step
* so the caller can paint the canvas filling in.
*/
async generate(prompt, { n = 4, seed = 0, steps = 16, cfgScale = 3, choiceTemperature = 2, nmsThreshold = 0.9, onStep } = {}) {
const { Tensor } = this.ort;
const rand = mulberry32(seed);
const { states, pooled } = await this.encodeText(prompt);
const B2 = n * 2;
const ts = new Float32Array(B2 * TEXT_LEN * TEXT_DIM);
const tp = new Float32Array(B2 * TEXT_DIM);
for (let i = 0; i < B2; i++) {
ts.set(states, i * TEXT_LEN * TEXT_DIM);
tp.set(pooled, i * TEXT_DIM);
}
const drop = Uint8Array.from({ length: B2 }, (_, i) => (i >= n ? 1 : 0));
const tsT = new Tensor("float32", ts, [B2, TEXT_LEN, TEXT_DIM]);
const tpT = new Tensor("float32", tp, [B2, TEXT_DIM]);
const dropT = new Tensor("bool", drop, [B2]);
const x = new Int32Array(n * N).fill(MASK);
const masked = new Uint8Array(n * N).fill(1);
const sc = new Float32Array(B2 * N * K);
const guided = new Float32Array(N * K);
const probs = new Float32Array(N * K);
const sampled = new Int32Array(N);
const pSel = new Float32Array(N);
const score = new Float32Array(N);
const nStart = N;
for (let k = 0; k < steps; k++) {
const frac = (k + 1) / steps;
const x2 = new BigInt64Array(B2 * N);
const ratio = new Float32Array(B2);
for (let b = 0; b < n; b++) {
let m = 0;
for (let i = 0; i < N; i++) {
const v = BigInt(x[b * N + i]);
x2[b * N + i] = v;
x2[(b + n) * N + i] = v;
m += masked[b * N + i];
}
ratio[b] = ratio[b + n] = m / N;
}
const out = await this.den.run({
x: new Tensor("int64", x2, [B2, N]),
text_states: tsT,
text_pooled: tpT,
mask_ratio: new Tensor("float32", ratio, [B2]),
sc_probs: new Tensor("float32", sc, [B2, N, K]),
drop_text: dropT,
});
const logits = out.logits.data;
for (let i = 0; i < logits.length; i++) {
if (!Number.isFinite(logits[i])) throw new Error(BAD_OUTPUT);
}
for (let b = 0; b < n; b++) {
const cOff = b * N * K;
const uOff = (b + n) * N * K;
// Classifier-free guidance, softmax, and one categorical sample per pixel.
for (let i = 0; i < N; i++) {
const row = i * K;
let max = -Infinity;
for (let j = 0; j < K; j++) {
const lu = logits[uOff + row + j];
const g = lu + cfgScale * (logits[cOff + row + j] - lu);
guided[row + j] = g;
if (g > max) max = g;
}
let sum = 0;
for (let j = 0; j < K; j++) {
const e = Math.exp(guided[row + j] - max);
probs[row + j] = e;
sum += e;
}
let u = rand() * sum;
let pick = K - 1;
for (let j = 0; j < K; j++) {
u -= probs[row + j];
if (u <= 0) {
pick = j;
break;
}
}
for (let j = 0; j < K; j++) probs[row + j] /= sum;
sampled[i] = pick;
pSel[i] = probs[row + pick];
}
// How many pixels to commit this step (cosine schedule).
let nMasked = 0;
for (let i = 0; i < N; i++) nMasked += masked[b * N + i];
let remaining = Math.floor(nStart * Math.cos((Math.PI / 2) * frac));
if (k === steps - 1) remaining = 0;
const nCommit = Math.min(Math.max(nMasked - remaining, 1), nMasked);
// Confidence score with annealed Gumbel noise.
for (let i = 0; i < N; i++) {
let u = rand();
u = Math.min(Math.max(u, 1e-10), 1 - 1e-10);
const gumbel = -Math.log(-Math.log(u));
score[i] = Math.log(Math.max(pSel[i], 1e-12)) + choiceTemperature * (1 - frac) * gumbel;
}
// Spatial NMS: prefer certain pixels and local maxima among uncertain masked ones.
const order = [];
for (let i = 0; i < N; i++) {
if (!masked[b * N + i]) continue;
const certain = pSel[i] >= nmsThreshold;
let elig = certain;
if (!certain) {
const r = (i / GRID) | 0;
const c = i % GRID;
let localMax = -Infinity;
for (let dr = -1; dr <= 1; dr++) {
for (let dc = -1; dc <= 1; dc++) {
const rr = r + dr;
const cc = c + dc;
if (rr < 0 || rr >= GRID || cc < 0 || cc >= GRID) continue;
const j = rr * GRID + cc;
if (masked[b * N + j] && pSel[j] < nmsThreshold && score[j] > localMax) localMax = score[j];
}
}
elig = score[i] >= localMax;
}
order.push([score[i] + (elig ? 1e4 : 0), i]);
}
order.sort((a, c) => c[0] - a[0]);
for (let q = 0; q < nCommit && q < order.length; q++) {
const i = order[q][1];
x[b * N + i] = sampled[i];
masked[b * N + i] = 0;
}
// Self-conditioning: feed this step's guided distribution to both CFG branches.
sc.set(probs, b * N * K);
sc.set(probs, (b + n) * N * K);
}
if (onStep) await onStep(k, x, masked);
}
return x;
}
}