Spaces:
Running
Running
cfahlgren1 HF Staff
switch to int8 weights that stay int8 in memory, add a pixel loader with load stages, and stop the reload loop after an out-of-memory crash
8e8aecc Download src/lib/engine.js from cfahlgren1/slack-moji: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/spaces/cfahlgren1/slack-moji/resolve/main/src/lib/engine.js
- Command line
-
hf download hf://spaces/cfahlgren1/slack-moji/src/lib/engine.js
-
curl -L -o engine.js https://huggingface.co/spaces/cfahlgren1/slack-moji/resolve/main/src/lib/engine.js
10.8 kB
| // Tessera-small in the browser: CLIP text encoder + denoiser via onnxruntime-web (WebGPU, or CPU), | |
| // with the masked-diffusion sampler ported from tessera/sampler.py (16 steps, CFG 3, | |
| // confidence-ordered unmasking with Gumbel choice noise and spatial NMS). | |
| import { ClipTokenizer } from "./tokenizer.js"; | |
| const ORT_VERSION = "1.22.0"; | |
| const ORT_DIR = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ORT_VERSION}/dist/`; | |
| export const GRID = 16; | |
| export const N = GRID * GRID; | |
| export const K = 512; | |
| export const MASK = 512; | |
| const TEXT_LEN = 24; | |
| const TEXT_DIM = 512; | |
| /** Error message for a run whose logits came back NaN or infinite. The client matches on it. */ | |
| export const BAD_OUTPUT = "bad-model-output"; | |
| function mulberry32(seed) { | |
| let a = seed >>> 0; | |
| return () => { | |
| a = (a + 0x6d2b79f5) >>> 0; | |
| let t = a; | |
| t = Math.imul(t ^ (t >>> 15), t | 1); | |
| t ^= t + Math.imul(t ^ (t >>> 7), t | 61); | |
| return ((t ^ (t >>> 14)) >>> 0) / 4294967296; | |
| }; | |
| } | |
| async function webgpuAvailable() { | |
| const gpu = globalThis.navigator?.gpu; | |
| if (!gpu) return false; | |
| try { | |
| return !!(await gpu.requestAdapter()); | |
| } catch { | |
| return false; | |
| } | |
| } | |
| /** | |
| * Check the model's output for one fixed prompt against a reference computed on the CPU. | |
| * Some GPUs and drivers run the model without an error but return NaN or garbage, so a | |
| * session that loads isn't proof that it works. Returns why it failed, or null if it passed. | |
| */ | |
| async function selfTest(ort, den, clip, ref) { | |
| const text = await clip.run({ | |
| input_ids: new ort.Tensor("int64", BigInt64Array.from(ref.ids, BigInt), [1, TEXT_LEN]), | |
| }); | |
| const out = await den.run({ | |
| x: new ort.Tensor("int64", new BigInt64Array(N).fill(BigInt(MASK)), [1, N]), | |
| text_states: text.last_hidden_state, | |
| text_pooled: text.pooler_output, | |
| mask_ratio: new ort.Tensor("float32", new Float32Array([1]), [1]), | |
| sc_probs: new ort.Tensor("float32", new Float32Array(N * K), [1, N, K]), | |
| drop_text: new ort.Tensor("bool", new Uint8Array(1), [1]), | |
| }); | |
| const logits = out.logits.data; | |
| let agree = 0; | |
| let topDiff = 0; | |
| for (let i = 0; i < N; i++) { | |
| let best = 0; | |
| for (let j = 0; j < K; j++) { | |
| const v = logits[i * K + j]; | |
| if (!Number.isFinite(v)) return `logit ${i * K + j} is ${v}`; | |
| if (v > logits[i * K + best]) best = j; | |
| } | |
| if (best === ref.argmax[i]) agree++; | |
| topDiff += Math.abs(logits[i * K + best] - ref.top[i]); | |
| } | |
| const agreement = agree / N; | |
| const meanTopDiff = topDiff / N; | |
| if (agreement < 0.8 || meanTopDiff > 0.5) { | |
| return `output doesn't match the reference (agreement ${agreement.toFixed(2)}, top-logit error ${meanTopDiff.toFixed(2)})`; | |
| } | |
| return null; | |
| } | |
| /** | |
| * Build the engine from already-downloaded model bytes. Runs on WebGPU when it passes the | |
| * self-test, otherwise on the CPU (WebAssembly). Meant to run inside a Web Worker. | |
| */ | |
| export async function createEngine(bytes, tokJson, palette, selfTestRef, { forceCpu = false, onStage } = {}) { | |
| const ort = await import(/* @vite-ignore */ `${ORT_DIR}ort.webgpu.min.mjs`); | |
| ort.env.wasm.wasmPaths = ORT_DIR; | |
| const tokenizer = new ClipTokenizer(tokJson); | |
| // One session at a time, so the peak memory holds one model's copy, not both. | |
| // Constant folding is off because it would expand every int8 weight into an fp32 copy at load | |
| // time, about 4x the memory, which is more than phones give a tab. The weights are dequantized | |
| // on the fly instead. | |
| const create = async (ep) => { | |
| onStage?.("load"); | |
| const opts = { | |
| executionProviders: [ep], | |
| graphOptimizationLevel: "all", | |
| extra: { optimization: { disable_specified_optimizers: "ConstantFolding" } }, | |
| }; | |
| const den = await ort.InferenceSession.create(bytes.den, opts); | |
| const clip = await ort.InferenceSession.create(bytes.clip, opts); | |
| return [den, clip]; | |
| }; | |
| const onCpu = async (fallbackReason) => { | |
| const [den, clip] = await create("wasm"); | |
| return new Engine(ort, den, clip, tokenizer, palette, "cpu", fallbackReason); | |
| }; | |
| if (forceCpu) return onCpu("forced"); | |
| if (!(await webgpuAvailable())) return onCpu("no-webgpu"); | |
| let den, clip; | |
| try { | |
| [den, clip] = await create("webgpu"); | |
| } catch (e) { | |
| console.warn("WebGPU session failed, falling back to the CPU", e); | |
| return onCpu("gpu-error"); | |
| } | |
| onStage?.("check"); | |
| let failure; | |
| try { | |
| failure = await selfTest(ort, den, clip, selfTestRef); | |
| } catch (e) { | |
| failure = String(e?.message ?? e); | |
| } | |
| if (!failure) return new Engine(ort, den, clip, tokenizer, palette, "webgpu", null); | |
| console.warn(`WebGPU failed the self-test (${failure}), falling back to the CPU`); | |
| await Promise.all([den.release(), clip.release()]); | |
| return onCpu("gpu-bad-output"); | |
| } | |
| export class Engine { | |
| constructor(ort, den, clip, tokenizer, palette, device, fallbackReason) { | |
| this.device = device; | |
| this.fallbackReason = fallbackReason; | |
| this.ort = ort; | |
| this.den = den; | |
| this.clip = clip; | |
| this.tokenizer = tokenizer; | |
| this.palette = palette; | |
| } | |
| async encodeText(prompt) { | |
| const ids = this.tokenizer.encodeFixed(prompt, TEXT_LEN); | |
| const out = await this.clip.run({ | |
| input_ids: new this.ort.Tensor("int64", BigInt64Array.from(ids, BigInt), [1, TEXT_LEN]), | |
| }); | |
| return { states: out.last_hidden_state.data, pooled: out.pooler_output.data }; | |
| } | |
| /** | |
| * Generate `n` sprites for one prompt. onStep(step, tokens, masked) is awaited after every step | |
| * so the caller can paint the canvas filling in. | |
| */ | |
| async generate(prompt, { n = 4, seed = 0, steps = 16, cfgScale = 3, choiceTemperature = 2, nmsThreshold = 0.9, onStep } = {}) { | |
| const { Tensor } = this.ort; | |
| const rand = mulberry32(seed); | |
| const { states, pooled } = await this.encodeText(prompt); | |
| const B2 = n * 2; | |
| const ts = new Float32Array(B2 * TEXT_LEN * TEXT_DIM); | |
| const tp = new Float32Array(B2 * TEXT_DIM); | |
| for (let i = 0; i < B2; i++) { | |
| ts.set(states, i * TEXT_LEN * TEXT_DIM); | |
| tp.set(pooled, i * TEXT_DIM); | |
| } | |
| const drop = Uint8Array.from({ length: B2 }, (_, i) => (i >= n ? 1 : 0)); | |
| const tsT = new Tensor("float32", ts, [B2, TEXT_LEN, TEXT_DIM]); | |
| const tpT = new Tensor("float32", tp, [B2, TEXT_DIM]); | |
| const dropT = new Tensor("bool", drop, [B2]); | |
| const x = new Int32Array(n * N).fill(MASK); | |
| const masked = new Uint8Array(n * N).fill(1); | |
| const sc = new Float32Array(B2 * N * K); | |
| const guided = new Float32Array(N * K); | |
| const probs = new Float32Array(N * K); | |
| const sampled = new Int32Array(N); | |
| const pSel = new Float32Array(N); | |
| const score = new Float32Array(N); | |
| const nStart = N; | |
| for (let k = 0; k < steps; k++) { | |
| const frac = (k + 1) / steps; | |
| const x2 = new BigInt64Array(B2 * N); | |
| const ratio = new Float32Array(B2); | |
| for (let b = 0; b < n; b++) { | |
| let m = 0; | |
| for (let i = 0; i < N; i++) { | |
| const v = BigInt(x[b * N + i]); | |
| x2[b * N + i] = v; | |
| x2[(b + n) * N + i] = v; | |
| m += masked[b * N + i]; | |
| } | |
| ratio[b] = ratio[b + n] = m / N; | |
| } | |
| const out = await this.den.run({ | |
| x: new Tensor("int64", x2, [B2, N]), | |
| text_states: tsT, | |
| text_pooled: tpT, | |
| mask_ratio: new Tensor("float32", ratio, [B2]), | |
| sc_probs: new Tensor("float32", sc, [B2, N, K]), | |
| drop_text: dropT, | |
| }); | |
| const logits = out.logits.data; | |
| for (let i = 0; i < logits.length; i++) { | |
| if (!Number.isFinite(logits[i])) throw new Error(BAD_OUTPUT); | |
| } | |
| for (let b = 0; b < n; b++) { | |
| const cOff = b * N * K; | |
| const uOff = (b + n) * N * K; | |
| // Classifier-free guidance, softmax, and one categorical sample per pixel. | |
| for (let i = 0; i < N; i++) { | |
| const row = i * K; | |
| let max = -Infinity; | |
| for (let j = 0; j < K; j++) { | |
| const lu = logits[uOff + row + j]; | |
| const g = lu + cfgScale * (logits[cOff + row + j] - lu); | |
| guided[row + j] = g; | |
| if (g > max) max = g; | |
| } | |
| let sum = 0; | |
| for (let j = 0; j < K; j++) { | |
| const e = Math.exp(guided[row + j] - max); | |
| probs[row + j] = e; | |
| sum += e; | |
| } | |
| let u = rand() * sum; | |
| let pick = K - 1; | |
| for (let j = 0; j < K; j++) { | |
| u -= probs[row + j]; | |
| if (u <= 0) { | |
| pick = j; | |
| break; | |
| } | |
| } | |
| for (let j = 0; j < K; j++) probs[row + j] /= sum; | |
| sampled[i] = pick; | |
| pSel[i] = probs[row + pick]; | |
| } | |
| // How many pixels to commit this step (cosine schedule). | |
| let nMasked = 0; | |
| for (let i = 0; i < N; i++) nMasked += masked[b * N + i]; | |
| let remaining = Math.floor(nStart * Math.cos((Math.PI / 2) * frac)); | |
| if (k === steps - 1) remaining = 0; | |
| const nCommit = Math.min(Math.max(nMasked - remaining, 1), nMasked); | |
| // Confidence score with annealed Gumbel noise. | |
| for (let i = 0; i < N; i++) { | |
| let u = rand(); | |
| u = Math.min(Math.max(u, 1e-10), 1 - 1e-10); | |
| const gumbel = -Math.log(-Math.log(u)); | |
| score[i] = Math.log(Math.max(pSel[i], 1e-12)) + choiceTemperature * (1 - frac) * gumbel; | |
| } | |
| // Spatial NMS: prefer certain pixels and local maxima among uncertain masked ones. | |
| const order = []; | |
| for (let i = 0; i < N; i++) { | |
| if (!masked[b * N + i]) continue; | |
| const certain = pSel[i] >= nmsThreshold; | |
| let elig = certain; | |
| if (!certain) { | |
| const r = (i / GRID) | 0; | |
| const c = i % GRID; | |
| let localMax = -Infinity; | |
| for (let dr = -1; dr <= 1; dr++) { | |
| for (let dc = -1; dc <= 1; dc++) { | |
| const rr = r + dr; | |
| const cc = c + dc; | |
| if (rr < 0 || rr >= GRID || cc < 0 || cc >= GRID) continue; | |
| const j = rr * GRID + cc; | |
| if (masked[b * N + j] && pSel[j] < nmsThreshold && score[j] > localMax) localMax = score[j]; | |
| } | |
| } | |
| elig = score[i] >= localMax; | |
| } | |
| order.push([score[i] + (elig ? 1e4 : 0), i]); | |
| } | |
| order.sort((a, c) => c[0] - a[0]); | |
| for (let q = 0; q < nCommit && q < order.length; q++) { | |
| const i = order[q][1]; | |
| x[b * N + i] = sampled[i]; | |
| masked[b * N + i] = 0; | |
| } | |
| // Self-conditioning: feed this step's guided distribution to both CFG branches. | |
| sc.set(probs, b * N * K); | |
| sc.set(probs, (b + n) * N * K); | |
| } | |
| if (onStep) await onStep(k, x, masked); | |
| } | |
| return x; | |
| } | |
| } | |