Download source/demo/web/src/local/model.ts from jackboyla/hendrix: direct link, hf CLI and curl.
- Browser
- Download file 13.4 kB
-
https://huggingface.co/spaces/jackboyla/hendrix/resolve/main/source/demo/web/src/local/model.ts
- Command line
-
hf download hf://spaces/jackboyla/hendrix/source/demo/web/src/local/model.ts
-
curl -L -o model.ts https://huggingface.co/spaces/jackboyla/hendrix/resolve/main/source/demo/web/src/local/model.ts
13.4 kB
| import * as ort from "onnxruntime-web/webgpu"; | |
| import { SPECTROGRAM_BINS, SPECTROGRAM_FRAMES } from "./dsp"; | |
| import { readFloat16 } from "./float16"; | |
| export interface LocalCapability { | |
| supported: boolean; | |
| adapter: string; | |
| shaderF16: boolean; | |
| backend: "webgpu" | "wasm"; | |
| reason?: string; | |
| } | |
| export type LocalBackend = "webgpu" | "wasm"; | |
| interface Manifest { | |
| precision: "fp16" | "fp32"; | |
| runtime_contract: { | |
| sample_rate: number; | |
| input_frames: number; | |
| segment_seconds: number; | |
| channels: number; | |
| max_tokens: number; | |
| }; | |
| } | |
| type Progress = (message: string) => void; | |
| export interface SegmentDiagnostics { | |
| precision: "fp16" | "fp32"; | |
| backend: LocalBackend; | |
| cacheLocation: "gpu" | "cpu"; | |
| logitsArray: string; | |
| steps: number; | |
| endedChannels: number; | |
| finiteLogits: number; | |
| totalLogits: number; | |
| firstTokens: number[]; | |
| channelPreviews: number[][]; | |
| encoderMs: number; | |
| crossCacheMs: number; | |
| decoderMs: number; | |
| } | |
| export interface SegmentResult { | |
| channels: number[][]; | |
| diagnostics: SegmentDiagnostics; | |
| } | |
| function adapterLabel(adapter: GPUAdapter): string { | |
| const info = adapter.info; | |
| return [info.vendor, info.architecture, info.device, info.description] | |
| .filter(Boolean) | |
| .join(" · ") || "WebGPU adapter"; | |
| } | |
| function wasmThreads(): number { | |
| // The HF page may embed us without cross-origin isolation. ORT can still | |
| // run there, but SharedArrayBuffer and multi-threaded WASM need isolation. | |
| if (!globalThis.crossOriginIsolated) return 1; | |
| const logicalCores = navigator.hardwareConcurrency || 1; | |
| return Math.min(4, Math.ceil(logicalCores / 2)); | |
| } | |
| function wasmLabel(): string { | |
| const threads = wasmThreads(); | |
| return `WebAssembly CPU · ${threads} ${threads === 1 ? "thread" : "threads"}`; | |
| } | |
| export async function probeLocalCapability(preferred?: LocalBackend): Promise<LocalCapability> { | |
| if (preferred === "wasm") { | |
| return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" }; | |
| } | |
| if (!globalThis.isSecureContext || !navigator.gpu) { | |
| return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" }; | |
| } | |
| const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" }); | |
| if (!adapter) { | |
| return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" }; | |
| } | |
| return { | |
| supported: true, | |
| adapter: adapterLabel(adapter), | |
| shaderF16: adapter.features.has("shader-f16"), | |
| backend: "webgpu", | |
| }; | |
| } | |
| const floatView = new Float32Array(1); | |
| const integerView = new Uint32Array(floatView.buffer); | |
| function floatToHalf(value: number): number { | |
| floatView[0] = value; | |
| const bits = integerView[0]; | |
| const sign = (bits >> 16) & 0x8000; | |
| let exponent = ((bits >> 23) & 0xff) - 127 + 15; | |
| let mantissa = bits & 0x7fffff; | |
| if (exponent <= 0) { | |
| if (exponent < -10) return sign; | |
| mantissa = (mantissa | 0x800000) >> (1 - exponent); | |
| return sign | ((mantissa + 0x1000) >> 13); | |
| } | |
| if (exponent >= 31) return sign | (mantissa ? 0x7e00 : 0x7c00); | |
| mantissa += 0x1000; | |
| if (mantissa & 0x800000) { | |
| mantissa = 0; | |
| exponent += 1; | |
| if (exponent >= 31) return sign | 0x7c00; | |
| } | |
| return sign | (exponent << 10) | (mantissa >> 13); | |
| } | |
| function floatData(values: Float32Array, precision: "fp16" | "fp32") { | |
| if (precision === "fp32") return values; | |
| const result = new Uint16Array(values.length); | |
| for (let i = 0; i < values.length; i++) result[i] = floatToHalf(values[i]); | |
| return result; | |
| } | |
| function int64(values: number[], dims: readonly number[]): ort.Tensor { | |
| return new ort.Tensor("int64", BigInt64Array.from(values, BigInt), dims); | |
| } | |
| function argmaxChannels(logits: ort.Tensor, precision: "fp16" | "fp32") { | |
| const data = logits.data as ArrayLike<number>; | |
| const vocabulary = 596; | |
| const tokens: number[] = []; | |
| let finiteValues = 0; | |
| for (let channel = 0; channel < 13; channel++) { | |
| let bestIndex = 0; | |
| let bestValue = Number.NEGATIVE_INFINITY; | |
| const start = channel * vocabulary; | |
| for (let token = 0; token < vocabulary; token++) { | |
| const value = precision === "fp16" ? readFloat16(data, start + token) : Number(data[start + token]); | |
| if (Number.isFinite(value)) finiteValues += 1; | |
| if (value > bestValue) { | |
| bestValue = value; | |
| bestIndex = token; | |
| } | |
| } | |
| tokens.push(bestIndex); | |
| } | |
| const constructor = (data as { constructor?: { name?: string } }).constructor; | |
| return { | |
| tokens, | |
| arrayType: constructor?.name || "unknown", | |
| finiteValues, | |
| totalValues: data.length, | |
| }; | |
| } | |
| export class LocalHendrixModel { | |
| readonly manifest: Manifest; | |
| readonly precision: "fp16" | "fp32"; | |
| readonly adapter: string; | |
| readonly backend: LocalBackend; | |
| private constructor( | |
| manifest: Manifest, | |
| adapter: string, | |
| backend: LocalBackend, | |
| private readonly cacheLocation: "gpu" | "cpu", | |
| private readonly encoder: ort.InferenceSession, | |
| private readonly crossCache: ort.InferenceSession, | |
| private readonly decoder: ort.InferenceSession, | |
| ) { | |
| this.manifest = manifest; | |
| this.precision = manifest.precision; | |
| this.adapter = adapter; | |
| this.backend = backend; | |
| } | |
| static async create( | |
| modelBase = "/models", | |
| progress: Progress = () => {}, | |
| precisionOverride?: "fp16" | "fp32", | |
| gpuResidentOutputs = true, | |
| backend: LocalBackend = "webgpu", | |
| ): Promise<LocalHendrixModel> { | |
| ort.env.logLevel = "error"; | |
| ort.env.wasm.wasmPaths = "/ort/"; | |
| let adapterName: string; | |
| if (backend === "webgpu") { | |
| if (!globalThis.isSecureContext || !navigator.gpu) { | |
| throw new Error("WebGPU needs Safari or Chrome over HTTPS (localhost also works). "); | |
| } | |
| progress("Checking WebGPU…"); | |
| const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" }); | |
| if (!adapter) throw new Error("No WebGPU adapter is available."); | |
| ort.env.webgpu.adapter = adapter; | |
| adapterName = adapterLabel(adapter); | |
| } else { | |
| progress("Starting the local CPU runtime…"); | |
| ort.env.wasm.numThreads = wasmThreads(); | |
| adapterName = wasmLabel(); | |
| gpuResidentOutputs = false; | |
| } | |
| // Safari's WebGPU FP16 path currently returns a finite but collapsed token-0 | |
| // stream for this graph. Keep FP16 available for explicit diagnostics, but | |
| // use the verified FP32 graph by default on the two Apple demo targets. | |
| const precision = precisionOverride ?? "fp32"; | |
| if (backend === "wasm" && precision === "fp16") { | |
| throw new Error("The CPU browser path uses the verified FP32 model."); | |
| } | |
| if (backend === "webgpu" && precision === "fp16") { | |
| const adapter = ort.env.webgpu.adapter; | |
| if (!adapter?.features.has("shader-f16")) { | |
| throw new Error("This WebGPU adapter cannot run the FP16 model."); | |
| } | |
| } | |
| const root = `${modelBase}/${precision}`; | |
| progress(`Loading the ${precision.toUpperCase()} model…`); | |
| const response = await fetch(`${root}/manifest.json`, { cache: "no-cache" }); | |
| if (!response.ok) throw new Error(`${precision.toUpperCase()} model manifest: HTTP ${response.status}`); | |
| const manifest = (await response.json()) as Manifest; | |
| if (manifest.precision !== precision) throw new Error("Model precision does not match this GPU."); | |
| const baseOptions: ort.InferenceSession.SessionOptions = { | |
| executionProviders: [backend], | |
| graphOptimizationLevel: "all", | |
| logSeverityLevel: 3, | |
| }; | |
| progress("Compiling the encoder…"); | |
| const encoder = await ort.InferenceSession.create(`${root}/encoder.onnx`, { | |
| ...baseOptions, | |
| // The encoder state immediately feeds the cache graph. Leaving it on the | |
| // GPU avoids a download followed by an upload between the two sessions. | |
| ...(gpuResidentOutputs ? { preferredOutputLocation: "gpu-buffer" as const } : {}), | |
| }); | |
| progress("Compiling the attention cache…"); | |
| const crossCache = await ort.InferenceSession.create(`${root}/cross_cache.onnx`, { | |
| ...baseOptions, | |
| // Cross-attention K/V tensors are reused at every token step. | |
| ...(gpuResidentOutputs ? { preferredOutputLocation: "gpu-buffer" as const } : {}), | |
| }); | |
| progress("Compiling the note decoder…"); | |
| const decoderOutputs: Record<string, ort.Tensor.DataLocation> = { logits: "cpu" }; | |
| for (let layer = 0; layer < 8; layer++) { | |
| decoderOutputs[`layer_${layer}_present_self_key`] = "gpu-buffer"; | |
| decoderOutputs[`layer_${layer}_present_self_value`] = "gpu-buffer"; | |
| } | |
| const decoder = await ort.InferenceSession.create(`${root}/decoder_step.onnx`, { | |
| ...baseOptions, | |
| // Only logits need to reach JavaScript for argmax. Growing self-attention | |
| // caches remain GPU-resident and feed the next decoder step directly. | |
| ...(gpuResidentOutputs ? { preferredOutputLocation: decoderOutputs } : {}), | |
| }); | |
| return new LocalHendrixModel( | |
| manifest, | |
| adapterName, | |
| backend, | |
| gpuResidentOutputs ? "gpu" : "cpu", | |
| encoder, | |
| crossCache, | |
| decoder, | |
| ); | |
| } | |
| async transcribeSegment(input: Float32Array, signal?: AbortSignal): Promise<SegmentResult> { | |
| if (signal?.aborted) throw new DOMException("Cancelled", "AbortError"); | |
| const type = this.precision === "fp16" ? "float16" : "float32"; | |
| const encoderInput = new ort.Tensor( | |
| type, | |
| floatData(input, this.precision), | |
| [1, SPECTROGRAM_FRAMES, SPECTROGRAM_BINS], | |
| ); | |
| const encoderStarted = performance.now(); | |
| const encoderOutput = await this.encoder.run({ spectrogram: encoderInput }); | |
| const encoderMs = performance.now() - encoderStarted; | |
| encoderInput.dispose(); | |
| const encoded = encoderOutput.encoder_hidden_states; | |
| const crossStarted = performance.now(); | |
| const crossOutput = await this.crossCache.run({ encoder_hidden_states: encoded }); | |
| const crossCacheMs = performance.now() - crossStarted; | |
| encoded.dispose(); | |
| const cross = this.crossCache.outputNames.map((name) => crossOutput[name]); | |
| let selfCache: ort.Tensor[] = Array.from({ length: 16 }, () => new ort.Tensor( | |
| type, | |
| this.precision === "fp16" ? new Uint16Array(0) : new Float32Array(0), | |
| [13, 6, 0, 64], | |
| )); | |
| let next = Array(13).fill(0) as number[]; | |
| const unfinished = Array(13).fill(true) as boolean[]; | |
| const channels = Array.from({ length: 13 }, () => [] as number[]); | |
| let logitsArray = "unknown"; | |
| let finiteLogits = 0; | |
| let totalLogits = 0; | |
| let firstTokens: number[] = []; | |
| let steps = 0; | |
| let decoderMs = 0; | |
| try { | |
| for (let position = 0; position < this.manifest.runtime_contract.max_tokens; position++) { | |
| if (signal?.aborted) throw new DOMException("Cancelled", "AbortError"); | |
| const tokenIds = int64(next, [1, 13, 1]); | |
| const positionIds = int64([position], [1]); | |
| const feeds: Record<string, ort.Tensor> = { token_ids: tokenIds, position_ids: positionIds }; | |
| for (let layer = 0; layer < 8; layer++) { | |
| feeds[`layer_${layer}_self_key`] = selfCache[layer * 2]; | |
| feeds[`layer_${layer}_self_value`] = selfCache[layer * 2 + 1]; | |
| feeds[`layer_${layer}_cross_key`] = cross[layer * 2]; | |
| feeds[`layer_${layer}_cross_value`] = cross[layer * 2 + 1]; | |
| } | |
| const decoderStarted = performance.now(); | |
| const output = await this.decoder.run(feeds); | |
| decoderMs += performance.now() - decoderStarted; | |
| tokenIds.dispose(); | |
| positionIds.dispose(); | |
| const argmax = argmaxChannels(output.logits, this.precision); | |
| const raw = argmax.tokens; | |
| if (position === 0) { | |
| logitsArray = argmax.arrayType; | |
| finiteLogits = argmax.finiteValues; | |
| totalLogits = argmax.totalValues; | |
| firstTokens = raw.slice(); | |
| } | |
| steps = position + 1; | |
| output.logits.dispose(); | |
| const nextCache = this.decoder.outputNames.slice(1).map((name) => output[name]); | |
| for (const tensor of selfCache) tensor.dispose(); | |
| selfCache = nextCache; | |
| next = raw; | |
| for (let channel = 0; channel < 13; channel++) { | |
| const token = unfinished[channel] ? raw[channel] : 0; | |
| channels[channel].push(token); | |
| if (token === 1) unfinished[channel] = false; | |
| } | |
| if (unfinished.every((value) => !value)) break; | |
| // Let Safari paint progress and handle a Cancel tap between token steps. | |
| if (position % 8 === 7) { | |
| await new Promise<void>((resolve) => requestAnimationFrame(() => resolve())); | |
| } | |
| } | |
| return { | |
| channels, | |
| diagnostics: { | |
| precision: this.precision, | |
| backend: this.backend, | |
| cacheLocation: this.cacheLocation, | |
| logitsArray, | |
| steps, | |
| endedChannels: unfinished.filter((value) => !value).length, | |
| finiteLogits, | |
| totalLogits, | |
| firstTokens, | |
| channelPreviews: channels.slice(0, 3).map((tokens) => tokens.slice(0, 12)), | |
| encoderMs, | |
| crossCacheMs, | |
| decoderMs, | |
| }, | |
| }; | |
| } finally { | |
| for (const tensor of selfCache) tensor.dispose(); | |
| for (const tensor of cross) tensor.dispose(); | |
| } | |
| } | |
| async dispose() { | |
| await Promise.all([this.encoder.release(), this.crossCache.release(), this.decoder.release()]); | |
| } | |
| } | |