import * as ort from "onnxruntime-web/webgpu"; import { SPECTROGRAM_BINS, SPECTROGRAM_FRAMES } from "./dsp"; import { readFloat16 } from "./float16"; export interface LocalCapability { supported: boolean; adapter: string; shaderF16: boolean; backend: "webgpu" | "wasm"; reason?: string; } export type LocalBackend = "webgpu" | "wasm"; interface Manifest { precision: "fp16" | "fp32"; runtime_contract: { sample_rate: number; input_frames: number; segment_seconds: number; channels: number; max_tokens: number; }; } type Progress = (message: string) => void; export interface SegmentDiagnostics { precision: "fp16" | "fp32"; backend: LocalBackend; cacheLocation: "gpu" | "cpu"; logitsArray: string; steps: number; endedChannels: number; finiteLogits: number; totalLogits: number; firstTokens: number[]; channelPreviews: number[][]; encoderMs: number; crossCacheMs: number; decoderMs: number; } export interface SegmentResult { channels: number[][]; diagnostics: SegmentDiagnostics; } function adapterLabel(adapter: GPUAdapter): string { const info = adapter.info; return [info.vendor, info.architecture, info.device, info.description] .filter(Boolean) .join(" · ") || "WebGPU adapter"; } function wasmThreads(): number { // The HF page may embed us without cross-origin isolation. ORT can still // run there, but SharedArrayBuffer and multi-threaded WASM need isolation. if (!globalThis.crossOriginIsolated) return 1; const logicalCores = navigator.hardwareConcurrency || 1; return Math.min(4, Math.ceil(logicalCores / 2)); } function wasmLabel(): string { const threads = wasmThreads(); return `WebAssembly CPU · ${threads} ${threads === 1 ? "thread" : "threads"}`; } export async function probeLocalCapability(preferred?: LocalBackend): Promise { if (preferred === "wasm") { return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" }; } if (!globalThis.isSecureContext || !navigator.gpu) { return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" }; } const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" }); if (!adapter) { return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" }; } return { supported: true, adapter: adapterLabel(adapter), shaderF16: adapter.features.has("shader-f16"), backend: "webgpu", }; } const floatView = new Float32Array(1); const integerView = new Uint32Array(floatView.buffer); function floatToHalf(value: number): number { floatView[0] = value; const bits = integerView[0]; const sign = (bits >> 16) & 0x8000; let exponent = ((bits >> 23) & 0xff) - 127 + 15; let mantissa = bits & 0x7fffff; if (exponent <= 0) { if (exponent < -10) return sign; mantissa = (mantissa | 0x800000) >> (1 - exponent); return sign | ((mantissa + 0x1000) >> 13); } if (exponent >= 31) return sign | (mantissa ? 0x7e00 : 0x7c00); mantissa += 0x1000; if (mantissa & 0x800000) { mantissa = 0; exponent += 1; if (exponent >= 31) return sign | 0x7c00; } return sign | (exponent << 10) | (mantissa >> 13); } function floatData(values: Float32Array, precision: "fp16" | "fp32") { if (precision === "fp32") return values; const result = new Uint16Array(values.length); for (let i = 0; i < values.length; i++) result[i] = floatToHalf(values[i]); return result; } function int64(values: number[], dims: readonly number[]): ort.Tensor { return new ort.Tensor("int64", BigInt64Array.from(values, BigInt), dims); } function argmaxChannels(logits: ort.Tensor, precision: "fp16" | "fp32") { const data = logits.data as ArrayLike; const vocabulary = 596; const tokens: number[] = []; let finiteValues = 0; for (let channel = 0; channel < 13; channel++) { let bestIndex = 0; let bestValue = Number.NEGATIVE_INFINITY; const start = channel * vocabulary; for (let token = 0; token < vocabulary; token++) { const value = precision === "fp16" ? readFloat16(data, start + token) : Number(data[start + token]); if (Number.isFinite(value)) finiteValues += 1; if (value > bestValue) { bestValue = value; bestIndex = token; } } tokens.push(bestIndex); } const constructor = (data as { constructor?: { name?: string } }).constructor; return { tokens, arrayType: constructor?.name || "unknown", finiteValues, totalValues: data.length, }; } export class LocalHendrixModel { readonly manifest: Manifest; readonly precision: "fp16" | "fp32"; readonly adapter: string; readonly backend: LocalBackend; private constructor( manifest: Manifest, adapter: string, backend: LocalBackend, private readonly cacheLocation: "gpu" | "cpu", private readonly encoder: ort.InferenceSession, private readonly crossCache: ort.InferenceSession, private readonly decoder: ort.InferenceSession, ) { this.manifest = manifest; this.precision = manifest.precision; this.adapter = adapter; this.backend = backend; } static async create( modelBase = "/models", progress: Progress = () => {}, precisionOverride?: "fp16" | "fp32", gpuResidentOutputs = true, backend: LocalBackend = "webgpu", ): Promise { ort.env.logLevel = "error"; ort.env.wasm.wasmPaths = "/ort/"; let adapterName: string; if (backend === "webgpu") { if (!globalThis.isSecureContext || !navigator.gpu) { throw new Error("WebGPU needs Safari or Chrome over HTTPS (localhost also works). "); } progress("Checking WebGPU…"); const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" }); if (!adapter) throw new Error("No WebGPU adapter is available."); ort.env.webgpu.adapter = adapter; adapterName = adapterLabel(adapter); } else { progress("Starting the local CPU runtime…"); ort.env.wasm.numThreads = wasmThreads(); adapterName = wasmLabel(); gpuResidentOutputs = false; } // Safari's WebGPU FP16 path currently returns a finite but collapsed token-0 // stream for this graph. Keep FP16 available for explicit diagnostics, but // use the verified FP32 graph by default on the two Apple demo targets. const precision = precisionOverride ?? "fp32"; if (backend === "wasm" && precision === "fp16") { throw new Error("The CPU browser path uses the verified FP32 model."); } if (backend === "webgpu" && precision === "fp16") { const adapter = ort.env.webgpu.adapter; if (!adapter?.features.has("shader-f16")) { throw new Error("This WebGPU adapter cannot run the FP16 model."); } } const root = `${modelBase}/${precision}`; progress(`Loading the ${precision.toUpperCase()} model…`); const response = await fetch(`${root}/manifest.json`, { cache: "no-cache" }); if (!response.ok) throw new Error(`${precision.toUpperCase()} model manifest: HTTP ${response.status}`); const manifest = (await response.json()) as Manifest; if (manifest.precision !== precision) throw new Error("Model precision does not match this GPU."); const baseOptions: ort.InferenceSession.SessionOptions = { executionProviders: [backend], graphOptimizationLevel: "all", logSeverityLevel: 3, }; progress("Compiling the encoder…"); const encoder = await ort.InferenceSession.create(`${root}/encoder.onnx`, { ...baseOptions, // The encoder state immediately feeds the cache graph. Leaving it on the // GPU avoids a download followed by an upload between the two sessions. ...(gpuResidentOutputs ? { preferredOutputLocation: "gpu-buffer" as const } : {}), }); progress("Compiling the attention cache…"); const crossCache = await ort.InferenceSession.create(`${root}/cross_cache.onnx`, { ...baseOptions, // Cross-attention K/V tensors are reused at every token step. ...(gpuResidentOutputs ? { preferredOutputLocation: "gpu-buffer" as const } : {}), }); progress("Compiling the note decoder…"); const decoderOutputs: Record = { logits: "cpu" }; for (let layer = 0; layer < 8; layer++) { decoderOutputs[`layer_${layer}_present_self_key`] = "gpu-buffer"; decoderOutputs[`layer_${layer}_present_self_value`] = "gpu-buffer"; } const decoder = await ort.InferenceSession.create(`${root}/decoder_step.onnx`, { ...baseOptions, // Only logits need to reach JavaScript for argmax. Growing self-attention // caches remain GPU-resident and feed the next decoder step directly. ...(gpuResidentOutputs ? { preferredOutputLocation: decoderOutputs } : {}), }); return new LocalHendrixModel( manifest, adapterName, backend, gpuResidentOutputs ? "gpu" : "cpu", encoder, crossCache, decoder, ); } async transcribeSegment(input: Float32Array, signal?: AbortSignal): Promise { if (signal?.aborted) throw new DOMException("Cancelled", "AbortError"); const type = this.precision === "fp16" ? "float16" : "float32"; const encoderInput = new ort.Tensor( type, floatData(input, this.precision), [1, SPECTROGRAM_FRAMES, SPECTROGRAM_BINS], ); const encoderStarted = performance.now(); const encoderOutput = await this.encoder.run({ spectrogram: encoderInput }); const encoderMs = performance.now() - encoderStarted; encoderInput.dispose(); const encoded = encoderOutput.encoder_hidden_states; const crossStarted = performance.now(); const crossOutput = await this.crossCache.run({ encoder_hidden_states: encoded }); const crossCacheMs = performance.now() - crossStarted; encoded.dispose(); const cross = this.crossCache.outputNames.map((name) => crossOutput[name]); let selfCache: ort.Tensor[] = Array.from({ length: 16 }, () => new ort.Tensor( type, this.precision === "fp16" ? new Uint16Array(0) : new Float32Array(0), [13, 6, 0, 64], )); let next = Array(13).fill(0) as number[]; const unfinished = Array(13).fill(true) as boolean[]; const channels = Array.from({ length: 13 }, () => [] as number[]); let logitsArray = "unknown"; let finiteLogits = 0; let totalLogits = 0; let firstTokens: number[] = []; let steps = 0; let decoderMs = 0; try { for (let position = 0; position < this.manifest.runtime_contract.max_tokens; position++) { if (signal?.aborted) throw new DOMException("Cancelled", "AbortError"); const tokenIds = int64(next, [1, 13, 1]); const positionIds = int64([position], [1]); const feeds: Record = { token_ids: tokenIds, position_ids: positionIds }; for (let layer = 0; layer < 8; layer++) { feeds[`layer_${layer}_self_key`] = selfCache[layer * 2]; feeds[`layer_${layer}_self_value`] = selfCache[layer * 2 + 1]; feeds[`layer_${layer}_cross_key`] = cross[layer * 2]; feeds[`layer_${layer}_cross_value`] = cross[layer * 2 + 1]; } const decoderStarted = performance.now(); const output = await this.decoder.run(feeds); decoderMs += performance.now() - decoderStarted; tokenIds.dispose(); positionIds.dispose(); const argmax = argmaxChannels(output.logits, this.precision); const raw = argmax.tokens; if (position === 0) { logitsArray = argmax.arrayType; finiteLogits = argmax.finiteValues; totalLogits = argmax.totalValues; firstTokens = raw.slice(); } steps = position + 1; output.logits.dispose(); const nextCache = this.decoder.outputNames.slice(1).map((name) => output[name]); for (const tensor of selfCache) tensor.dispose(); selfCache = nextCache; next = raw; for (let channel = 0; channel < 13; channel++) { const token = unfinished[channel] ? raw[channel] : 0; channels[channel].push(token); if (token === 1) unfinished[channel] = false; } if (unfinished.every((value) => !value)) break; // Let Safari paint progress and handle a Cancel tap between token steps. if (position % 8 === 7) { await new Promise((resolve) => requestAnimationFrame(() => resolve())); } } return { channels, diagnostics: { precision: this.precision, backend: this.backend, cacheLocation: this.cacheLocation, logitsArray, steps, endedChannels: unfinished.filter((value) => !value).length, finiteLogits, totalLogits, firstTokens, channelPreviews: channels.slice(0, 3).map((tokens) => tokens.slice(0, 12)), encoderMs, crossCacheMs, decoderMs, }, }; } finally { for (const tensor of selfCache) tensor.dispose(); for (const tensor of cross) tensor.dispose(); } } async dispose() { await Promise.all([this.encoder.release(), this.crossCache.release(), this.decoder.release()]); } }