crescent-moon / web /model.js
jfrick's picture
Update the demo from the repository
afc8274 verified
Raw History Blame Contribute Delete
25.3 kB
// The slot detector's forward pass on WebGPU: loading the export of
// `scripts/export_web.py` and running one 378 x 378 crop through the vision encoder, the
// projection, the decoder (image prefix and slots together, block by block) and the
// read-out heads. `docs/webgpu.md` describes the pass.
import { CHOICES, Kernels, TUNING, encode, storage, upload } from "./kernels.js";
const PATCH = 14, GRID = 27, CROP = PATCH * GRID;
const IMAGE_ROWS = GRID * GRID; // 729 image positions
const VISION = { dim: 1152, hidden: 4304, heads: 16, blocks: 27 };
const TEXT = { dim: 2048, hidden: 8192, heads: 32, blocks: 24, vocabulary: 51200 };
const BINS = 1024;
const pad32 = (n) => Math.ceil(n / 32) * 32;
function halfToFloat(bytes, count) {
const halves = new Uint16Array(bytes.buffer, bytes.byteOffset, count);
const out = new Float32Array(count);
for (let i = 0; i < count; i++) {
const h = halves[i], exponent = (h >> 10) & 31, fraction = h & 1023;
const magnitude = exponent === 0 ? fraction * 2 ** -24
: exponent === 31 ? (fraction ? NaN : Infinity)
: (fraction + 1024) * 2 ** (exponent - 25);
out[i] = h & 0x8000 ? -magnitude : magnitude;
}
return out;
}
// Fetches a shard, from the browser's cache when it is there. `version` identifies the
// shard's content: a cached copy of the same file with another version is replaced.
// The shard is read into one array of its `size`, so memory holds it once.
async function fetchShard(url, version, size, onBytes) {
const cache = "caches" in globalThis ? await caches.open("crescent-moon-weights") : null;
const key = `${url}?v=${version}`;
let response = cache && await cache.match(key);
if (!response) {
if (cache) {
for (const request of await cache.keys()) {
if (request.url.split("?")[0] === url) await cache.delete(request);
}
}
response = await fetch(key);
if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`);
if (cache) {
try { await cache.put(key, response.clone()); } catch { /* over quota: keep going uncached */ }
}
}
const reader = response.body.getReader();
const bytes = new Uint8Array(size);
let at = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
if (at + value.byteLength > size) throw new Error(`${url}: longer than its ${size} bytes`);
bytes.set(value, at);
at += value.byteLength;
onBytes(value.byteLength);
}
if (at < size) throw new Error(`${url}: ${at} of ${size} bytes`);
return bytes;
}
// Reads every tensor of an export into GPU buffers. Quantised layers become
// { weight, scale, bias, rows, columns, groups, scaleStride, bits }; dense FP16 tensors
// named in `packedHalf` stay packed (the half-linear kernel decodes them); the others
// become f32 buffers. `progress(loadedBytes, totalBytes)` is called as shards arrive.
export async function loadWeights(device, base, manifest, progress = () => {}) {
const places = [];
for (const [name, entry] of Object.entries(manifest.tensors)) {
const parts = entry.format === "quantised" ? ["weight", "scale", "bias"] : ["data"];
for (const part of parts) places.push({ name, part, entry, place: entry[part] });
}
const total = places.reduce((sum, p) => sum + p.place.bytes, 0);
let loaded = 0;
const tensors = {};
for (let shard = 0; shard < manifest.shards.length; shard++) {
const inShard = places.filter((p) => p.place.shard === shard);
const version = inShard[0].place.sha256.slice(0, 16);
const size = Math.max(...inShard.map((p) => p.place.offset + p.place.bytes));
const bytes = await fetchShard(new URL(manifest.shards[shard], base).href, version, size, (n) => {
loaded += n;
progress(loaded, total);
});
for (const { name, part, entry, place } of places) {
if (place.shard !== shard) continue;
const blob = bytes.subarray(place.offset, place.offset + place.bytes);
const tensor = (tensors[name] ??= {});
if (entry.format === "quantised") {
const groups = Math.ceil(entry.columns / manifest.group);
Object.assign(tensor, {
rows: entry.rows, columns: entry.columns, bits: entry.bits, groups,
scaleStride: groups + (groups % 2),
});
tensor[part] = part === "bias" ? upload(device, halfToFloat(blob, entry.rows)) : upload(device, blob);
} else {
const columns = entry.shape[entry.shape.length - 1];
const count = place.bytes / 2;
tensor.shape = entry.shape;
const dense = /^slots\.(adapters\..*|objectness\.weight)$/.test(name);
if (dense && columns % 2) throw new Error(`${name}: odd row length`);
tensor.buffer = dense ? upload(device, blob) : upload(device, halfToFloat(blob, count));
if (name === "slots.objectness.bias") tensor.value = halfToFloat(blob, 1)[0];
}
}
}
return tensors;
}
// Rotary angles as Moondream's precompute_freqs_cis(32, positions): (cos, sin) per
// position and each of 16 frequencies.
function rotaryTable(positions) {
const table = new Float32Array(positions * 32);
for (let position = 0; position < positions; position++) {
for (let j = 0; j < 16; j++) {
const frequency = Math.fround(1 / Math.fround(Math.pow(10000, Math.fround(2 * j) / 32)));
const angle = Math.fround(position * frequency);
table[position * 32 + 2 * j] = Math.cos(angle);
table[position * 32 + 2 * j + 1] = Math.sin(angle);
}
}
return table;
}
// Patch rows [729, 608] of a 378 x 378 RGBA or RGB crop: per patch, channel by channel,
// row by row, pixels scaled to [-1, 1]; the last 20 columns of each row stay zero.
export function patches(pixels, channels) {
const stride = pad32(3 * PATCH * PATCH);
const out = new Float32Array(IMAGE_ROWS * stride);
for (let py = 0; py < GRID; py++) {
for (let px = 0; px < GRID; px++) {
const row = (py * GRID + px) * stride;
for (let c = 0; c < 3; c++) {
for (let y = 0; y < PATCH; y++) {
const source = ((py * PATCH + y) * CROP + px * PATCH) * channels + c;
const target = row + (c * PATCH + y) * PATCH;
for (let x = 0; x < PATCH; x++) out[target + x] = pixels[source + x * channels] / 127.5 - 1;
}
}
}
}
return out;
}
export class SlotModel {
// `tensors` from loadWeights; `manifest` of the same export; `tuning` from `tune`.
constructor(device, manifest, tensors, tuning = TUNING) {
this.device = device;
this.manifest = manifest;
this.slots = manifest.head.slots;
this.width = manifest.selection.coordinates + manifest.head.label_tokens;
this.slotRows = this.slots * this.width;
// The image prefix: BOS, then the image positions, pooled when the head was trained so.
this.pool = manifest.head.prefix_pool;
this.prefix = 1 + Math.ceil(GRID / this.pool) ** 2;
this.rows = this.prefix + this.slotRows;
this.objectnessBias = tensors["slots.objectness.bias"].value;
this.tuning = tuning;
this.kernels = new Kernels(device, tuning);
this.build(tensors);
}
build(t) {
const k = this.kernels, device = this.device;
const f32 = (count) => storage(device, count * 4);
const V = VISION, T = TEXT, rows = this.rows, slotRows = this.slotRows, PREFIX = this.prefix;
const labelTokens = this.manifest.head.label_tokens, coordinates = this.manifest.selection.coordinates;
const hiddenStride = pad32(V.hidden);
const b = this.buffers = {
patches: f32(IMAGE_ROWS * pad32(3 * PATCH * PATCH)),
vx: f32(IMAGE_ROWS * V.dim), vn: f32(IMAGE_ROWS * V.dim), vqkv: f32(IMAGE_ROWS * 3 * V.dim),
vmix: f32(IMAGE_ROWS * V.dim), vh: f32(IMAGE_ROWS * hiddenStride),
pin: f32(IMAGE_ROWS * 2 * V.dim), ph: f32(IMAGE_ROWS * T.hidden),
projected: f32(IMAGE_ROWS * T.dim),
x: f32(rows * T.dim), normed: f32(rows * T.dim), qkv: f32(rows * 3 * T.dim),
mixed: f32(rows * T.dim), hidden: f32(rows * T.hidden), down: f32(slotRows * this.manifest.head.rank),
coordIn: f32(this.slots * 2 * T.dim), sizeIn: f32(this.slots * T.dim),
labelIn: f32(this.slots * labelTokens * T.dim), objectIn: f32(this.slots * T.dim),
readHidden: f32(this.slots * labelTokens * T.hidden),
coordOut: f32(this.slots * 2 * BINS), sizeOut: f32(this.slots * 2 * BINS),
labelOut: f32(this.slots * labelTokens * T.vocabulary),
// objectness [slots], then (index, probability) pairs: x and y bins [slots, 2],
// width and height bins [slots, 2], label tokens [slots, label tokens]
results: f32(this.slots * (1 + 2 * (4 + labelTokens))),
};
// The decoder's input: BOS, room for the image embeddings, then every slot position.
b.initial = f32(rows * T.dim);
b.rotary = upload(device, rotaryTable(rows));
this.setup = [
{ copy: [t["text.bos"].buffer, 0, b.initial, 0, T.dim * 4] },
{ copy: [t["slots.inputs"].buffer, 0, b.initial, PREFIX * T.dim * 4, slotRows * T.dim * 4] },
];
device.queue.submit([encode(device, this.setup)]);
const stages = this.stages = { vision: [], projection: [], decoder: [], readout: [] };
const q = (x, name, y, v) => k.quantisedLinear(t[name].bits, x, t[name], y, v);
const norm = (x, name, y, v) => k.layerNorm(x, t[`${name}.weight`].buffer, t[`${name}.bias`].buffer, y, v);
// Vision encoder: patch embedding added to the position embedding, 27 pre-norm blocks.
let s = stages.vision;
s.push({ copy: [t["vision.pos_emb"].buffer, 0, b.vx, 0, IMAGE_ROWS * V.dim * 4] });
s.push(q(b.patches, "vision.patch_emb", b.vx,
{ rows: IMAGE_ROWS, xStride: pad32(3 * PATCH * PATCH), yStride: V.dim, add: true }));
for (let i = 0; i < V.blocks; i++) {
const name = `vision.blocks.${i}`, all = { rows: IMAGE_ROWS, xStride: V.dim, yStride: V.dim };
s.push(norm(b.vx, `${name}.ln1`, b.vn, { ...all, dim: V.dim }));
s.push(q(b.vn, `${name}.attn.qkv`, b.vqkv, { ...all, yStride: 3 * V.dim }));
s.push(...k.attention(b.vqkv, b.vmix,
{ rows: IMAGE_ROWS, stride: 3 * V.dim, dim: V.dim, heads: V.heads, yStride: V.dim }));
s.push(q(b.vmix, `${name}.attn.proj`, b.vx, { ...all, add: true }));
s.push(norm(b.vx, `${name}.ln2`, b.vn, { ...all, dim: V.dim }));
s.push(q(b.vn, `${name}.mlp.fc1`, b.vh, { ...all, yStride: hiddenStride, gelu: true }));
s.push(q(b.vh, `${name}.mlp.fc2`, b.vx, { ...all, xStride: hiddenStride, add: true }));
}
// Projection: with one crop the local features are the global ones, side by side.
s = stages.projection;
for (const yCol0 of [0, V.dim]) {
s.push(norm(b.vx, "vision.post_ln", b.pin,
{ rows: IMAGE_ROWS, xStride: V.dim, dim: V.dim, yStride: 2 * V.dim, yCol0 }));
}
s.push(q(b.pin, "vision.proj_mlp.fc1", b.ph,
{ rows: IMAGE_ROWS, xStride: 2 * V.dim, yStride: T.hidden, gelu: true }));
s.push({ copy: [b.initial, 0, b.x, 0, rows * T.dim * 4] });
s.push(q(b.ph, "vision.proj_mlp.fc2", b.projected,
{ rows: IMAGE_ROWS, xStride: T.hidden, yStride: T.dim }));
s.push(k.poolGrid(b.projected, b.x,
{ side: GRID, pool: this.pool, xStride: T.dim, dim: T.dim, yRow0: 1, yStride: T.dim }));
// Decoder: prefix rows attend to the prefix, slot rows to everything; adapters add
// to slot rows only.
s = stages.decoder;
const rank = this.manifest.head.rank;
const adapt = (block, part, x, xStride, ins, y, yStride, outs) => {
const name = `slots.adapters.${block}.${part}`;
s.push(k.halfLinear(x, t[`${name}.down`].buffer, b.down,
{ rows: slotRows, xRow0: PREFIX, xStride, ins, outs: rank, yStride: rank }));
s.push(k.halfLinear(b.down, t[`${name}.up`].buffer, y,
{ rows: slotRows, xStride: rank, ins: rank, outs, yRow0: PREFIX, yStride, add: true }));
};
this.blockEnds = [];
for (let i = 0; i < T.blocks; i++) {
const name = `text.blocks.${i}`, all = { rows, xStride: T.dim, yStride: T.dim };
s.push(norm(b.x, `${name}.ln`, b.normed, { ...all, dim: T.dim }));
s.push(q(b.normed, `${name}.attn.qkv`, b.qkv, { ...all, yStride: 3 * T.dim }));
adapt(i, "qkv", b.normed, T.dim, T.dim, b.qkv, 3 * T.dim, 3 * T.dim);
s.push(k.rotary(b.rotary, b.qkv, { rows, stride: 3 * T.dim, dim: T.dim, heads: T.heads }));
s.push(...k.attention(b.qkv, b.mixed,
{ rows, stride: 3 * T.dim, dim: T.dim, heads: T.heads, prefix: PREFIX, yStride: T.dim }));
s.push(q(b.mixed, `${name}.attn.proj`, b.x, { ...all, add: true }));
adapt(i, "proj", b.mixed, T.dim, T.dim, b.x, T.dim, T.dim);
s.push(q(b.normed, `${name}.mlp.fc1`, b.hidden, { ...all, yStride: T.hidden }));
adapt(i, "fc1", b.normed, T.dim, T.dim, b.hidden, T.hidden, T.hidden);
s.push(k.gelu(b.hidden, 0, rows * T.hidden));
s.push(q(b.hidden, `${name}.mlp.fc2`, b.x, { ...all, xStride: T.hidden, add: true }));
adapt(i, "fc2", b.hidden, T.hidden, T.hidden, b.x, T.dim, T.dim);
this.blockEnds.push(s.length);
}
// Read-out. Positions 0 and 1 of a slot go to the coordinate decoder and position 2
// to the size decoder, from the raw hidden state; the label positions and the
// objectness layer read the normalised one.
s = stages.readout;
const row = T.dim * 4;
s.push(norm(b.x, "text.post_ln", b.normed,
{ rows: slotRows, xRow0: PREFIX, xStride: T.dim, dim: T.dim, yRow0: PREFIX, yStride: T.dim }));
for (let slot = 0; slot < this.slots; slot++) {
const from = (PREFIX + slot * this.width) * row;
s.push({ copy: [b.x, from, b.coordIn, slot * 2 * row, 2 * row] });
s.push({ copy: [b.x, from + 2 * row, b.sizeIn, slot * row, row] });
s.push({ copy: [b.normed, from, b.objectIn, slot * row, row] });
s.push({ copy: [b.normed, from + coordinates * row, b.labelIn, slot * labelTokens * row, labelTokens * row] });
}
const pairs = this.slots * 2; // f32 offset of (index, probability) pairs after objectness
s.push(k.halfLinear(b.objectIn, t["slots.objectness.weight"].buffer, b.results,
{ rows: this.slots, xStride: T.dim, ins: T.dim, outs: 1, yStride: 1 }));
s.push(q(b.coordIn, "region.coord_decoder.fc1", b.readHidden,
{ rows: this.slots * 2, xStride: T.dim, yStride: T.hidden, gelu: true }));
s.push(q(b.readHidden, "region.coord_decoder.fc2", b.coordOut,
{ rows: this.slots * 2, xStride: T.hidden, yStride: BINS }));
s.push(k.argmax(b.coordOut, b.results,
{ rows: this.slots * 2, xStride: BINS, segments: 1, width: BINS, y0: this.slots / 2 }));
s.push(q(b.sizeIn, "region.size_decoder.fc1", b.readHidden,
{ rows: this.slots, xStride: T.dim, yStride: T.hidden, gelu: true }));
s.push(q(b.readHidden, "region.size_decoder.fc2", b.sizeOut,
{ rows: this.slots, xStride: T.hidden, yStride: 2 * BINS }));
s.push(k.argmax(b.sizeOut, b.results,
{ rows: this.slots, xStride: 2 * BINS, segments: 2, width: BINS, y0: this.slots / 2 + pairs }));
s.push(q(b.labelIn, "text.lm_head", b.labelOut,
{ rows: this.slots * labelTokens, xStride: T.dim, yStride: T.vocabulary }));
s.push(k.argmax(b.labelOut, b.results, {
rows: this.slots * labelTokens, xStride: T.vocabulary, segments: 1, width: T.vocabulary,
y0: this.slots / 2 + 2 * pairs,
}));
this.resultBytes = b.results.size;
this.staging = device.createBuffer({
size: this.resultBytes, usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
});
}
// Runs one crop. `patchRows` from patches(). Returns per slot the objectness logit,
// bins [x, y, w, h] with their probabilities and label tokens with theirs, and the
// seconds each stage took (the GPU queue is drained after each stage when `timed`).
// `probe(stage, index, total)` may be async and is awaited after every step when given;
// tests use it to read intermediate buffers.
async run(patchRows, { timed = false, probe = null } = {}) {
const device = this.device, b = this.buffers;
device.queue.writeBuffer(b.patches, 0, patchRows);
const seconds = {};
for (const [stage, steps] of Object.entries(this.stages)) {
const start = performance.now();
if (probe) {
for (let i = 0; i < steps.length; i++) {
device.queue.submit([encode(device, [steps[i]])]);
await probe(stage, i + 1, steps.length);
}
} else {
// One submission per decoder block keeps command buffers short.
const cuts = stage === "decoder" ? this.blockEnds : [steps.length];
let from = 0;
for (const to of cuts) {
device.queue.submit([encode(device, steps.slice(from, to))]);
from = to;
}
}
if (timed) {
await device.queue.onSubmittedWorkDone();
seconds[stage] = (performance.now() - start) / 1000;
}
}
const encoder = device.createCommandEncoder();
encoder.copyBufferToBuffer(b.results, 0, this.staging, 0, this.resultBytes);
device.queue.submit([encoder.finish()]);
await this.staging.mapAsync(GPUMapMode.READ);
const out = new Float32Array(this.staging.getMappedRange().slice(0));
this.staging.unmap();
const n = this.slots, labelTokens = this.manifest.head.label_tokens;
const slots = [];
for (let slot = 0; slot < n; slot++) {
const pair = (base, i) => [out[n + 2 * (base + i)], out[n + 2 * (base + i) + 1]];
const picks = [pair(0, slot * 2), pair(0, slot * 2 + 1), pair(2 * n, slot * 2), pair(2 * n, slot * 2 + 1)];
const tokens = Array.from({ length: labelTokens }, (_, i) => pair(4 * n, slot * labelTokens + i));
slots.push({
objectness: out[slot] + this.objectnessBias,
bins: picks.map((p) => p[0]), binProbabilities: picks.map((p) => p[1]),
tokens: tokens.map((p) => p[0]), tokenProbabilities: tokens.map((p) => p[1]),
});
}
return { slots, seconds };
}
}
// Requests a device for the model: timestamp-free, with the storage limits it needs.
// `features` are requested where the adapter has them.
export async function requestDevice(features = []) {
if (!navigator.gpu) throw new Error("This browser has no WebGPU");
const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" });
if (!adapter) throw new Error("No WebGPU adapter");
const need = 256 * 2 ** 20;
const requiredLimits = {
maxStorageBufferBindingSize: Math.min(need, adapter.limits.maxStorageBufferBindingSize),
maxBufferSize: Math.min(need, adapter.limits.maxBufferSize),
};
const requiredFeatures = features.filter((feature) => adapter.features.has(feature));
return adapter.requestDevice({ requiredLimits, requiredFeatures });
}
// The fastest of the kernels' CHOICES on this device, timed on the work of one decoder
// block (its MLP's first product and its attention, over as many rows as a frame has, on
// zeros). A GPU speeds up over its first seconds of work, so the choices are timed in
// rounds, one after the other, and a choice's time is its fastest round. Choices are
// compiled during the first half of `budget` seconds (the default first; those not
// reached are left out) and rounds run for the second half, three at least; the default
// stays unless another choice is over 5% faster. A GPU that
// needs over `slow` seconds for one product keeps the default untimed. After three rounds a
// choice over twice its kind's best is left out of later rounds. `half` adds the products in f16
// to the choices where the device has shader-f16.
// Returns { outs, rows, queries, rounds, seconds: { choice: seconds } }.
export async function tune(device, manifest, tensors, { budget = 10, slow = 3, half = false } = {}) {
const T = TEXT;
const rows = 1 + Math.ceil(GRID / manifest.head.prefix_pool) ** 2
+ manifest.head.slots * (manifest.selection.coordinates + manifest.head.label_tokens);
const x = storage(device, rows * T.dim * 4), hidden = storage(device, rows * T.hidden * 4);
const qkv = storage(device, rows * 3 * T.dim * 4);
const layer = tensors["text.blocks.0.mlp.fc1"];
const now = () => performance.now() / 1000;
const time = async (make, repeats) => {
const start = now();
device.queue.submit([encode(device, Array.from({ length: repeats }, make).flat())]);
await device.queue.onSubmittedWorkDone();
return (now() - start) / repeats;
};
const product = (tuning) => {
const kernels = new Kernels(device, tuning);
return () => kernels.quantisedLinear(layer.bits, x, layer, hidden,
{ rows, xStride: T.dim, yStride: T.hidden });
};
const attention = (tuning) => {
const kernels = new Kernels(device, tuning);
return () => kernels.attention(qkv, x,
{ rows, stride: 3 * T.dim, dim: T.dim, heads: T.heads, yStride: T.dim });
};
const candidates = [
...CHOICES.tiles.map(([outs, r]) => ({
kind: "product", name: `product ${outs} x ${r}`, set: { outs, rows: r, half: false },
make: product({ ...TUNING, outs, rows: r }),
})),
...(half && device.features.has("shader-f16") ? CHOICES.tiles : []).map(([outs, r]) => ({
kind: "product", name: `product ${outs} x ${r} f16`, set: { outs, rows: r, half: true },
make: product({ ...TUNING, outs, rows: r, half: true }),
})),
...CHOICES.queries.map((queries) => ({
kind: "attention", name: `attention ${queries}`, set: { queries, split: false },
make: attention({ ...TUNING, queries }),
})),
];
// The split attention is compiled right after the default.
candidates.splice(candidates.findIndex((c) => c.kind === "attention") + 1, 0, {
kind: "attention", name: "attention split", set: { split: true },
make: attention({ ...TUNING, split: true }),
});
const seconds = {}, started = now();
let chosen = { ...TUNING }, rounds = 0;
// The weights' first use is slow on some devices; it stays out of the times.
seconds["first use"] = await time(product(TUNING), 1);
if (await time(product(TUNING), 1) <= slow) {
// Compile: the defaults lead CHOICES, and each kind gets half of the compile time.
const ready = [];
for (const kind of ["product", "attention"]) {
const end = now() + budget / 4;
for (const candidate of candidates.filter((c) => c.kind === kind)) {
if (ready.some((c) => c.kind === kind) && now() > end) break;
await time(candidate.make, 1);
ready.push(candidate);
}
}
for (const candidate of ready) candidate.best = Infinity;
// One round that is not counted: the GPU's first seconds of work are slow.
for (const candidate of ready) await time(candidate.make, 2);
while (rounds < 3 || now() - started < budget) {
for (const candidate of ready) {
if (candidate.out) continue;
candidate.best = Math.min(candidate.best, await time(candidate.make, 2));
}
rounds++;
// After three rounds a choice far behind its kind's best is not timed again.
for (const kind of rounds < 3 ? [] : ["product", "attention"]) {
const same = ready.filter((c) => c.kind === kind);
const best = Math.min(...same.map((c) => c.best));
for (const candidate of same) candidate.out = candidate.best > 2 * best;
}
}
for (const kind of ["product", "attention"]) {
// The default (first) stays unless another choice is over 5% faster.
const [first, ...others] = ready.filter((c) => c.kind === kind);
const best = others.reduce((a, c) => (c.best < a.best ? c : a), first);
if (best.best < 0.95 * first.best) chosen = { ...chosen, ...best.set };
}
for (const candidate of ready) seconds[candidate.name] = candidate.best;
}
for (const buffer of [x, hidden, qkv]) buffer.destroy();
return { ...chosen, rounds, seconds };
}
// `tune`, remembered in this browser (the choice belongs to the device's GPU). A device
// on which the timing fails runs the default kernels.
async function tuned(device, manifest, tensors) {
const key = `crescent-moon-tuning-6-${manifest.head.prefix_pool}-${JSON.stringify(CHOICES)}`;
try {
const kept = JSON.parse(localStorage.getItem(key));
if (kept) return kept;
} catch { /* no storage, or nothing usable in it: measure */ }
let tuning;
try {
tuning = await tune(device, manifest, tensors);
} catch (error) {
return { ...TUNING, seconds: {}, error: String(error) };
}
try { localStorage.setItem(key, JSON.stringify(tuning)); } catch { /* not kept */ }
return tuning;
}
// `progress(done, total)` follows the download in bytes; `stage(text)` names what follows.
export async function loadModel(base, progress, stage = () => {}) {
const device = await requestDevice(["shader-f16"]);
const manifest = await (await fetch(new URL("manifest.json", base))).json();
const tensors = await loadWeights(device, base, manifest, progress);
stage("Choosing the fastest kernels for this GPU");
const tuning = await tuned(device, manifest, tensors);
stage("Preparing the detector on the GPU");
const model = new SlotModel(device, manifest, tensors, tuning);
const problems = await model.kernels.problems();
if (problems.length) throw new Error(`This browser rejected a kernel: ${problems[0]}`);
return model;
}
export { CROP };