hendrix / source /demo /web /src /local /model.ts
jackboyla's picture
Deploy checked Hendrix D26 browser demo
b2f2f57 verified
Raw History Blame Contribute Delete
13.4 kB
import * as ort from "onnxruntime-web/webgpu";
import { SPECTROGRAM_BINS, SPECTROGRAM_FRAMES } from "./dsp";
import { readFloat16 } from "./float16";
export interface LocalCapability {
supported: boolean;
adapter: string;
shaderF16: boolean;
backend: "webgpu" | "wasm";
reason?: string;
}
export type LocalBackend = "webgpu" | "wasm";
interface Manifest {
precision: "fp16" | "fp32";
runtime_contract: {
sample_rate: number;
input_frames: number;
segment_seconds: number;
channels: number;
max_tokens: number;
};
}
type Progress = (message: string) => void;
export interface SegmentDiagnostics {
precision: "fp16" | "fp32";
backend: LocalBackend;
cacheLocation: "gpu" | "cpu";
logitsArray: string;
steps: number;
endedChannels: number;
finiteLogits: number;
totalLogits: number;
firstTokens: number[];
channelPreviews: number[][];
encoderMs: number;
crossCacheMs: number;
decoderMs: number;
}
export interface SegmentResult {
channels: number[][];
diagnostics: SegmentDiagnostics;
}
function adapterLabel(adapter: GPUAdapter): string {
const info = adapter.info;
return [info.vendor, info.architecture, info.device, info.description]
.filter(Boolean)
.join(" · ") || "WebGPU adapter";
}
function wasmThreads(): number {
// The HF page may embed us without cross-origin isolation. ORT can still
// run there, but SharedArrayBuffer and multi-threaded WASM need isolation.
if (!globalThis.crossOriginIsolated) return 1;
const logicalCores = navigator.hardwareConcurrency || 1;
return Math.min(4, Math.ceil(logicalCores / 2));
}
function wasmLabel(): string {
const threads = wasmThreads();
return `WebAssembly CPU · ${threads} ${threads === 1 ? "thread" : "threads"}`;
}
export async function probeLocalCapability(preferred?: LocalBackend): Promise<LocalCapability> {
if (preferred === "wasm") {
return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" };
}
if (!globalThis.isSecureContext || !navigator.gpu) {
return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" };
}
const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" });
if (!adapter) {
return { supported: true, adapter: wasmLabel(), shaderF16: false, backend: "wasm" };
}
return {
supported: true,
adapter: adapterLabel(adapter),
shaderF16: adapter.features.has("shader-f16"),
backend: "webgpu",
};
}
const floatView = new Float32Array(1);
const integerView = new Uint32Array(floatView.buffer);
function floatToHalf(value: number): number {
floatView[0] = value;
const bits = integerView[0];
const sign = (bits >> 16) & 0x8000;
let exponent = ((bits >> 23) & 0xff) - 127 + 15;
let mantissa = bits & 0x7fffff;
if (exponent <= 0) {
if (exponent < -10) return sign;
mantissa = (mantissa | 0x800000) >> (1 - exponent);
return sign | ((mantissa + 0x1000) >> 13);
}
if (exponent >= 31) return sign | (mantissa ? 0x7e00 : 0x7c00);
mantissa += 0x1000;
if (mantissa & 0x800000) {
mantissa = 0;
exponent += 1;
if (exponent >= 31) return sign | 0x7c00;
}
return sign | (exponent << 10) | (mantissa >> 13);
}
function floatData(values: Float32Array, precision: "fp16" | "fp32") {
if (precision === "fp32") return values;
const result = new Uint16Array(values.length);
for (let i = 0; i < values.length; i++) result[i] = floatToHalf(values[i]);
return result;
}
function int64(values: number[], dims: readonly number[]): ort.Tensor {
return new ort.Tensor("int64", BigInt64Array.from(values, BigInt), dims);
}
function argmaxChannels(logits: ort.Tensor, precision: "fp16" | "fp32") {
const data = logits.data as ArrayLike<number>;
const vocabulary = 596;
const tokens: number[] = [];
let finiteValues = 0;
for (let channel = 0; channel < 13; channel++) {
let bestIndex = 0;
let bestValue = Number.NEGATIVE_INFINITY;
const start = channel * vocabulary;
for (let token = 0; token < vocabulary; token++) {
const value = precision === "fp16" ? readFloat16(data, start + token) : Number(data[start + token]);
if (Number.isFinite(value)) finiteValues += 1;
if (value > bestValue) {
bestValue = value;
bestIndex = token;
}
}
tokens.push(bestIndex);
}
const constructor = (data as { constructor?: { name?: string } }).constructor;
return {
tokens,
arrayType: constructor?.name || "unknown",
finiteValues,
totalValues: data.length,
};
}
export class LocalHendrixModel {
readonly manifest: Manifest;
readonly precision: "fp16" | "fp32";
readonly adapter: string;
readonly backend: LocalBackend;
private constructor(
manifest: Manifest,
adapter: string,
backend: LocalBackend,
private readonly cacheLocation: "gpu" | "cpu",
private readonly encoder: ort.InferenceSession,
private readonly crossCache: ort.InferenceSession,
private readonly decoder: ort.InferenceSession,
) {
this.manifest = manifest;
this.precision = manifest.precision;
this.adapter = adapter;
this.backend = backend;
}
static async create(
modelBase = "/models",
progress: Progress = () => {},
precisionOverride?: "fp16" | "fp32",
gpuResidentOutputs = true,
backend: LocalBackend = "webgpu",
): Promise<LocalHendrixModel> {
ort.env.logLevel = "error";
ort.env.wasm.wasmPaths = "/ort/";
let adapterName: string;
if (backend === "webgpu") {
if (!globalThis.isSecureContext || !navigator.gpu) {
throw new Error("WebGPU needs Safari or Chrome over HTTPS (localhost also works). ");
}
progress("Checking WebGPU…");
const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" });
if (!adapter) throw new Error("No WebGPU adapter is available.");
ort.env.webgpu.adapter = adapter;
adapterName = adapterLabel(adapter);
} else {
progress("Starting the local CPU runtime…");
ort.env.wasm.numThreads = wasmThreads();
adapterName = wasmLabel();
gpuResidentOutputs = false;
}
// Safari's WebGPU FP16 path currently returns a finite but collapsed token-0
// stream for this graph. Keep FP16 available for explicit diagnostics, but
// use the verified FP32 graph by default on the two Apple demo targets.
const precision = precisionOverride ?? "fp32";
if (backend === "wasm" && precision === "fp16") {
throw new Error("The CPU browser path uses the verified FP32 model.");
}
if (backend === "webgpu" && precision === "fp16") {
const adapter = ort.env.webgpu.adapter;
if (!adapter?.features.has("shader-f16")) {
throw new Error("This WebGPU adapter cannot run the FP16 model.");
}
}
const root = `${modelBase}/${precision}`;
progress(`Loading the ${precision.toUpperCase()} model…`);
const response = await fetch(`${root}/manifest.json`, { cache: "no-cache" });
if (!response.ok) throw new Error(`${precision.toUpperCase()} model manifest: HTTP ${response.status}`);
const manifest = (await response.json()) as Manifest;
if (manifest.precision !== precision) throw new Error("Model precision does not match this GPU.");
const baseOptions: ort.InferenceSession.SessionOptions = {
executionProviders: [backend],
graphOptimizationLevel: "all",
logSeverityLevel: 3,
};
progress("Compiling the encoder…");
const encoder = await ort.InferenceSession.create(`${root}/encoder.onnx`, {
...baseOptions,
// The encoder state immediately feeds the cache graph. Leaving it on the
// GPU avoids a download followed by an upload between the two sessions.
...(gpuResidentOutputs ? { preferredOutputLocation: "gpu-buffer" as const } : {}),
});
progress("Compiling the attention cache…");
const crossCache = await ort.InferenceSession.create(`${root}/cross_cache.onnx`, {
...baseOptions,
// Cross-attention K/V tensors are reused at every token step.
...(gpuResidentOutputs ? { preferredOutputLocation: "gpu-buffer" as const } : {}),
});
progress("Compiling the note decoder…");
const decoderOutputs: Record<string, ort.Tensor.DataLocation> = { logits: "cpu" };
for (let layer = 0; layer < 8; layer++) {
decoderOutputs[`layer_${layer}_present_self_key`] = "gpu-buffer";
decoderOutputs[`layer_${layer}_present_self_value`] = "gpu-buffer";
}
const decoder = await ort.InferenceSession.create(`${root}/decoder_step.onnx`, {
...baseOptions,
// Only logits need to reach JavaScript for argmax. Growing self-attention
// caches remain GPU-resident and feed the next decoder step directly.
...(gpuResidentOutputs ? { preferredOutputLocation: decoderOutputs } : {}),
});
return new LocalHendrixModel(
manifest,
adapterName,
backend,
gpuResidentOutputs ? "gpu" : "cpu",
encoder,
crossCache,
decoder,
);
}
async transcribeSegment(input: Float32Array, signal?: AbortSignal): Promise<SegmentResult> {
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
const type = this.precision === "fp16" ? "float16" : "float32";
const encoderInput = new ort.Tensor(
type,
floatData(input, this.precision),
[1, SPECTROGRAM_FRAMES, SPECTROGRAM_BINS],
);
const encoderStarted = performance.now();
const encoderOutput = await this.encoder.run({ spectrogram: encoderInput });
const encoderMs = performance.now() - encoderStarted;
encoderInput.dispose();
const encoded = encoderOutput.encoder_hidden_states;
const crossStarted = performance.now();
const crossOutput = await this.crossCache.run({ encoder_hidden_states: encoded });
const crossCacheMs = performance.now() - crossStarted;
encoded.dispose();
const cross = this.crossCache.outputNames.map((name) => crossOutput[name]);
let selfCache: ort.Tensor[] = Array.from({ length: 16 }, () => new ort.Tensor(
type,
this.precision === "fp16" ? new Uint16Array(0) : new Float32Array(0),
[13, 6, 0, 64],
));
let next = Array(13).fill(0) as number[];
const unfinished = Array(13).fill(true) as boolean[];
const channels = Array.from({ length: 13 }, () => [] as number[]);
let logitsArray = "unknown";
let finiteLogits = 0;
let totalLogits = 0;
let firstTokens: number[] = [];
let steps = 0;
let decoderMs = 0;
try {
for (let position = 0; position < this.manifest.runtime_contract.max_tokens; position++) {
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
const tokenIds = int64(next, [1, 13, 1]);
const positionIds = int64([position], [1]);
const feeds: Record<string, ort.Tensor> = { token_ids: tokenIds, position_ids: positionIds };
for (let layer = 0; layer < 8; layer++) {
feeds[`layer_${layer}_self_key`] = selfCache[layer * 2];
feeds[`layer_${layer}_self_value`] = selfCache[layer * 2 + 1];
feeds[`layer_${layer}_cross_key`] = cross[layer * 2];
feeds[`layer_${layer}_cross_value`] = cross[layer * 2 + 1];
}
const decoderStarted = performance.now();
const output = await this.decoder.run(feeds);
decoderMs += performance.now() - decoderStarted;
tokenIds.dispose();
positionIds.dispose();
const argmax = argmaxChannels(output.logits, this.precision);
const raw = argmax.tokens;
if (position === 0) {
logitsArray = argmax.arrayType;
finiteLogits = argmax.finiteValues;
totalLogits = argmax.totalValues;
firstTokens = raw.slice();
}
steps = position + 1;
output.logits.dispose();
const nextCache = this.decoder.outputNames.slice(1).map((name) => output[name]);
for (const tensor of selfCache) tensor.dispose();
selfCache = nextCache;
next = raw;
for (let channel = 0; channel < 13; channel++) {
const token = unfinished[channel] ? raw[channel] : 0;
channels[channel].push(token);
if (token === 1) unfinished[channel] = false;
}
if (unfinished.every((value) => !value)) break;
// Let Safari paint progress and handle a Cancel tap between token steps.
if (position % 8 === 7) {
await new Promise<void>((resolve) => requestAnimationFrame(() => resolve()));
}
}
return {
channels,
diagnostics: {
precision: this.precision,
backend: this.backend,
cacheLocation: this.cacheLocation,
logitsArray,
steps,
endedChannels: unfinished.filter((value) => !value).length,
finiteLogits,
totalLogits,
firstTokens,
channelPreviews: channels.slice(0, 3).map((tokens) => tokens.slice(0, 12)),
encoderMs,
crossCacheMs,
decoderMs,
},
};
} finally {
for (const tensor of selfCache) tensor.dispose();
for (const tensor of cross) tensor.dispose();
}
}
async dispose() {
await Promise.all([this.encoder.release(), this.crossCache.release(), this.decoder.release()]);
}
}