Spaces:
Sleeping
Sleeping
Upgrade to codehygiene v0.6.1, Bonsai 1.7B WebGPU + ZeroGPU refiner, and Data Retention page
Browse files- web/src/bonsai-webgpu.js +36 -5
- web/src/bonsai-worker.js +6 -5
- web/src/browser-refiner.js +2 -0
web/src/bonsai-webgpu.js
CHANGED
|
@@ -151,12 +151,21 @@ function sharedPrefixLength(left, right) {
|
|
| 151 |
return idx;
|
| 152 |
}
|
| 153 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
export async function probeWebGpuSupport() {
|
| 155 |
if (!globalThis.navigator?.gpu) return false;
|
| 156 |
try {
|
| 157 |
const adapter = await navigator.gpu.requestAdapter();
|
| 158 |
-
|
| 159 |
-
return true;
|
| 160 |
} catch {
|
| 161 |
return false;
|
| 162 |
}
|
|
@@ -172,12 +181,17 @@ export class BonsaiWebGpuEngine {
|
|
| 172 |
this.cachedTokens = [];
|
| 173 |
}
|
| 174 |
|
| 175 |
-
static async load({
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
if (!globalThis.navigator?.gpu) {
|
| 177 |
throw new Error("WebGPU is unavailable in this browser.");
|
| 178 |
}
|
| 179 |
const adapter = await navigator.gpu.requestAdapter();
|
| 180 |
-
if (!adapter
|
| 181 |
throw new Error("Hardware WebGPU adapter is unavailable.");
|
| 182 |
}
|
| 183 |
const device = await adapter.requestDevice();
|
|
@@ -274,7 +288,24 @@ export class BonsaiWebGpuEngine {
|
|
| 274 |
const pipelines = { embed, rmsNorm, gemv, gemm, qkRopeKv, gqaAttn, siluMul, copyLast, topk };
|
| 275 |
const ops = BonsaiWebGpuEngine.buildLayerOps(device, buffers, pipelines);
|
| 276 |
await device.queue.onSubmittedWorkDone();
|
| 277 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 278 |
}
|
| 279 |
|
| 280 |
static buildLayerOps(device, b, p) {
|
|
|
|
| 151 |
return idx;
|
| 152 |
}
|
| 153 |
|
| 154 |
+
const SOFTWARE_ADAPTER_RE = /swiftshader|basic render|llvmpipe|lavapipe|warp|software/i;
|
| 155 |
+
const MAX_STEP_MS = 95;
|
| 156 |
+
|
| 157 |
+
function isUsableHardwareAdapter(adapter) {
|
| 158 |
+
if (!adapter || adapter.info?.isFallbackAdapter) return false;
|
| 159 |
+
const info = adapter.info ?? {};
|
| 160 |
+
const label = `${info.vendor ?? ""} ${info.architecture ?? ""} ${info.device ?? ""} ${info.description ?? ""}`;
|
| 161 |
+
return !SOFTWARE_ADAPTER_RE.test(label);
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
export async function probeWebGpuSupport() {
|
| 165 |
if (!globalThis.navigator?.gpu) return false;
|
| 166 |
try {
|
| 167 |
const adapter = await navigator.gpu.requestAdapter();
|
| 168 |
+
return isUsableHardwareAdapter(adapter);
|
|
|
|
| 169 |
} catch {
|
| 170 |
return false;
|
| 171 |
}
|
|
|
|
| 181 |
this.cachedTokens = [];
|
| 182 |
}
|
| 183 |
|
| 184 |
+
static async load({
|
| 185 |
+
baseUrl = DEFAULT_MODEL_BASE,
|
| 186 |
+
skipSpeedGate = false,
|
| 187 |
+
signal,
|
| 188 |
+
onProgress = () => {},
|
| 189 |
+
} = {}) {
|
| 190 |
if (!globalThis.navigator?.gpu) {
|
| 191 |
throw new Error("WebGPU is unavailable in this browser.");
|
| 192 |
}
|
| 193 |
const adapter = await navigator.gpu.requestAdapter();
|
| 194 |
+
if (!isUsableHardwareAdapter(adapter)) {
|
| 195 |
throw new Error("Hardware WebGPU adapter is unavailable.");
|
| 196 |
}
|
| 197 |
const device = await adapter.requestDevice();
|
|
|
|
| 288 |
const pipelines = { embed, rmsNorm, gemv, gemm, qkRopeKv, gqaAttn, siluMul, copyLast, topk };
|
| 289 |
const ops = BonsaiWebGpuEngine.buildLayerOps(device, buffers, pipelines);
|
| 290 |
await device.queue.onSubmittedWorkDone();
|
| 291 |
+
const engine = new BonsaiWebGpuEngine(device, tokenizer, buffers, pipelines, ops);
|
| 292 |
+
if (!skipSpeedGate) {
|
| 293 |
+
await engine.verifyRealtimeThroughput();
|
| 294 |
+
}
|
| 295 |
+
return engine;
|
| 296 |
+
}
|
| 297 |
+
|
| 298 |
+
async verifyRealtimeThroughput() {
|
| 299 |
+
this.encodeForwardStep([151644], 0, { computeLogits: true });
|
| 300 |
+
await this.readNextToken(0.0, 1.0);
|
| 301 |
+
const t0 = performance.now();
|
| 302 |
+
this.encodeForwardStep([8948], 1, { computeLogits: true });
|
| 303 |
+
await this.readNextToken(0.0, 1.0);
|
| 304 |
+
const stepMs = performance.now() - t0;
|
| 305 |
+
this.cachedTokens = [];
|
| 306 |
+
if (stepMs > MAX_STEP_MS) {
|
| 307 |
+
throw new Error(`WebGPU decode step too slow (${Math.round(stepMs)} ms); using ZeroGPU.`);
|
| 308 |
+
}
|
| 309 |
}
|
| 310 |
|
| 311 |
static buildLayerOps(device, b, p) {
|
web/src/bonsai-worker.js
CHANGED
|
@@ -13,12 +13,13 @@ let enginePromise = null;
|
|
| 13 |
let loadedBaseUrl = null;
|
| 14 |
const cancelledIds = new Set();
|
| 15 |
|
| 16 |
-
function getEngine(baseUrl, onLoadProgress) {
|
| 17 |
const resolvedBase = baseUrl || DEFAULT_MODEL_BASE;
|
| 18 |
if (!enginePromise || loadedBaseUrl !== resolvedBase) {
|
| 19 |
loadedBaseUrl = resolvedBase;
|
| 20 |
enginePromise = BonsaiWebGpuEngine.load({
|
| 21 |
baseUrl: resolvedBase,
|
|
|
|
| 22 |
onProgress: onLoadProgress,
|
| 23 |
}).catch((err) => {
|
| 24 |
enginePromise = null;
|
|
@@ -102,7 +103,7 @@ async function runSingleAttempt({
|
|
| 102 |
return { cancelled: false, verdict, tokens };
|
| 103 |
}
|
| 104 |
|
| 105 |
-
async function runFixJob(id, body, modelBaseUrl) {
|
| 106 |
const isCancelled = () => cancelledIds.has(id);
|
| 107 |
const emit = (name, data) => {
|
| 108 |
if (!isCancelled()) {
|
|
@@ -137,7 +138,7 @@ async function runFixJob(id, body, modelBaseUrl) {
|
|
| 137 |
}
|
| 138 |
|
| 139 |
let lastLoadEmit = 0;
|
| 140 |
-
const engine = await getEngine(modelBaseUrl, (done, total) => {
|
| 141 |
const now = performance.now();
|
| 142 |
if (done < total && now - lastLoadEmit < PROGRESS_INTERVAL_MS) return;
|
| 143 |
lastLoadEmit = now;
|
|
@@ -235,7 +236,7 @@ async function runFixJob(id, body, modelBaseUrl) {
|
|
| 235 |
let queueChain = Promise.resolve();
|
| 236 |
|
| 237 |
globalThis.addEventListener("message", (messageEvent) => {
|
| 238 |
-
const { type, id, body, modelBaseUrl } = messageEvent.data;
|
| 239 |
if (type === "cancel") {
|
| 240 |
cancelledIds.add(id);
|
| 241 |
return;
|
|
@@ -248,7 +249,7 @@ globalThis.addEventListener("message", (messageEvent) => {
|
|
| 248 |
return;
|
| 249 |
}
|
| 250 |
try {
|
| 251 |
-
await runFixJob(id, body, modelBaseUrl);
|
| 252 |
globalThis.postMessage({ id, done: true });
|
| 253 |
} catch (err) {
|
| 254 |
const message = err && err.message ? err.message : String(err);
|
|
|
|
| 13 |
let loadedBaseUrl = null;
|
| 14 |
const cancelledIds = new Set();
|
| 15 |
|
| 16 |
+
function getEngine(baseUrl, forceBrowser, onLoadProgress) {
|
| 17 |
const resolvedBase = baseUrl || DEFAULT_MODEL_BASE;
|
| 18 |
if (!enginePromise || loadedBaseUrl !== resolvedBase) {
|
| 19 |
loadedBaseUrl = resolvedBase;
|
| 20 |
enginePromise = BonsaiWebGpuEngine.load({
|
| 21 |
baseUrl: resolvedBase,
|
| 22 |
+
skipSpeedGate: Boolean(forceBrowser),
|
| 23 |
onProgress: onLoadProgress,
|
| 24 |
}).catch((err) => {
|
| 25 |
enginePromise = null;
|
|
|
|
| 103 |
return { cancelled: false, verdict, tokens };
|
| 104 |
}
|
| 105 |
|
| 106 |
+
async function runFixJob(id, body, modelBaseUrl, forceBrowser) {
|
| 107 |
const isCancelled = () => cancelledIds.has(id);
|
| 108 |
const emit = (name, data) => {
|
| 109 |
if (!isCancelled()) {
|
|
|
|
| 138 |
}
|
| 139 |
|
| 140 |
let lastLoadEmit = 0;
|
| 141 |
+
const engine = await getEngine(modelBaseUrl, forceBrowser, (done, total) => {
|
| 142 |
const now = performance.now();
|
| 143 |
if (done < total && now - lastLoadEmit < PROGRESS_INTERVAL_MS) return;
|
| 144 |
lastLoadEmit = now;
|
|
|
|
| 236 |
let queueChain = Promise.resolve();
|
| 237 |
|
| 238 |
globalThis.addEventListener("message", (messageEvent) => {
|
| 239 |
+
const { type, id, body, modelBaseUrl, forceBrowser } = messageEvent.data;
|
| 240 |
if (type === "cancel") {
|
| 241 |
cancelledIds.add(id);
|
| 242 |
return;
|
|
|
|
| 249 |
return;
|
| 250 |
}
|
| 251 |
try {
|
| 252 |
+
await runFixJob(id, body, modelBaseUrl, forceBrowser);
|
| 253 |
globalThis.postMessage({ id, done: true });
|
| 254 |
} catch (err) {
|
| 255 |
const message = err && err.message ? err.message : String(err);
|
web/src/browser-refiner.js
CHANGED
|
@@ -100,6 +100,7 @@ function runWorkerSession(body, { signal, onEvent }) {
|
|
| 100 |
id,
|
| 101 |
body,
|
| 102 |
modelBaseUrl: configuredModelBaseUrl(),
|
|
|
|
| 103 |
});
|
| 104 |
});
|
| 105 |
}
|
|
@@ -110,6 +111,7 @@ export async function streamCleanup(body, { signal, onEvent }) {
|
|
| 110 |
if (!outcome.fallback || signal?.aborted) {
|
| 111 |
return;
|
| 112 |
}
|
|
|
|
| 113 |
if (configuredMode() === "browser") {
|
| 114 |
onEvent("error", { code: "webgpu_failed", message: outcome.error || "WebGPU failed" });
|
| 115 |
return;
|
|
|
|
| 100 |
id,
|
| 101 |
body,
|
| 102 |
modelBaseUrl: configuredModelBaseUrl(),
|
| 103 |
+
forceBrowser: configuredMode() === "browser",
|
| 104 |
});
|
| 105 |
});
|
| 106 |
}
|
|
|
|
| 111 |
if (!outcome.fallback || signal?.aborted) {
|
| 112 |
return;
|
| 113 |
}
|
| 114 |
+
webGpuAvailablePromise = Promise.resolve(false);
|
| 115 |
if (configuredMode() === "browser") {
|
| 116 |
onEvent("error", { code: "webgpu_failed", message: outcome.error || "WebGPU failed" });
|
| 117 |
return;
|