fuze-eletrik commited on
Commit
a98ddfa
·
verified ·
1 Parent(s): e45ca08

Delete client.js

Browse files
Files changed (1) hide show
  1. client.js +0 -762
client.js DELETED
@@ -1,762 +0,0 @@
1
- async () => {
2
- const root = document.getElementById("sam-browser-app");
3
- if (!root || root.dataset.initialized === "yes") return;
4
- root.dataset.initialized = "yes";
5
- const $ = (id) => root.querySelector("#" + id);
6
- const MODEL_BASE = "https://huggingface.co/matbee/sam-audio-small-onnx/resolve/main";
7
- const CACHE_NAME = "sam-audio-webgpu-models-v1";
8
- const MODEL_FILES = {
9
- encoder: "dacvae_encoder.onnx",
10
- decoder: "dacvae_decoder.onnx",
11
- t5: "t5_encoder.onnx",
12
- dit: "dit_single_step.onnx"
13
- };
14
- let ort = null;
15
- let tokenizer = null;
16
- let transformersModule = null;
17
- let spanTokenizer = null;
18
- let spanConfig = null;
19
- let sessions = null;
20
- let sessionsProvider = null;
21
- let runtimeProvider = null;
22
- let lastAttemptProvider = null;
23
- let previousUrls = [];
24
- function setLog(message) {
25
- // Compact user-facing progress message; technical diagnostics are intentionally hidden.
26
- if (message) setStatus(String(message).split("\n")[0]);
27
- }
28
- function setStatus(message, state = "busy") {
29
- $("sam-status").textContent = message;
30
- $("sam-status-box").dataset.state = state;
31
- }
32
- const waveforms = {};
33
- const colors = {input: "#8b879b", target: "#8b7cf6", residual: "#40b995"};
34
- function formatDuration(seconds) {
35
- if (!Number.isFinite(seconds) || seconds < 0) return "—";
36
- const mins = Math.floor(seconds / 60);
37
- const secs = Math.floor(seconds % 60);
38
- return mins ? `${mins}:${String(secs).padStart(2, "0")}` : `0:${String(secs).padStart(2, "0")}`;
39
- }
40
- function drawWaveform(canvasId, samples, kind) {
41
- const canvas = $(canvasId);
42
- if (!canvas) return;
43
- const rect = canvas.getBoundingClientRect();
44
- if (!rect.width || !rect.height) return;
45
- const dpr = Math.max(1, Math.min(window.devicePixelRatio || 1, 2));
46
- canvas.width = Math.round(rect.width * dpr);
47
- canvas.height = Math.round(rect.height * dpr);
48
- const ctx = canvas.getContext("2d");
49
- ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
50
- const width = rect.width, height = rect.height, mid = height / 2;
51
- ctx.clearRect(0, 0, width, height);
52
- ctx.strokeStyle = "rgba(128,128,150,.13)"; ctx.lineWidth = 1;
53
- ctx.beginPath(); ctx.moveTo(0, mid); ctx.lineTo(width, mid); ctx.stroke();
54
- if (!samples || !samples.length) return;
55
- const bars = Math.max(1, Math.floor(width / 3));
56
- const stride = Math.max(1, Math.floor(samples.length / bars));
57
- const color = colors[kind] || colors.input;
58
- ctx.strokeStyle = color; ctx.lineWidth = Math.max(1, Math.min(2, width / bars * .7)); ctx.lineCap = "round";
59
- ctx.beginPath();
60
- for (let x = 0; x < bars; x++) {
61
- const start = x * stride;
62
- const end = Math.min(samples.length, start + stride);
63
- let peak = 0;
64
- for (let i = start; i < end; i++) peak = Math.max(peak, Math.abs(samples[i] || 0));
65
- const amplitude = Math.max(1.25, peak * (height * .43));
66
- const px = (x + .5) * width / bars;
67
- ctx.moveTo(px, mid - amplitude); ctx.lineTo(px, mid + amplitude);
68
- }
69
- ctx.stroke();
70
- waveforms[canvasId] = {samples, kind};
71
- updateWaveProgress(canvasId);
72
- }
73
- function updateWaveProgress(canvasId) {
74
- const canvas = $(canvasId);
75
- const player = $(canvasId.replace("-wave", "-player"));
76
- const entry = waveforms[canvasId];
77
- if (!canvas || !entry || !player || !player.duration) return;
78
- const rect = canvas.getBoundingClientRect();
79
- const dpr = Math.max(1, Math.min(window.devicePixelRatio || 1, 2));
80
- const ctx = canvas.getContext("2d");
81
- const x = Math.max(0, Math.min(rect.width, player.currentTime / player.duration * rect.width));
82
- ctx.save(); ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
83
- ctx.fillStyle = "rgba(255,255,255,.13)"; ctx.fillRect(0, 0, x, rect.height);
84
- ctx.restore();
85
- }
86
- function attachWavePlayer(canvasId, playerId) {
87
- const canvas = $(canvasId), player = $(playerId);
88
- if (!canvas || !player) return;
89
- player.addEventListener("timeupdate", () => drawWaveform(canvasId, waveforms[canvasId]?.samples, waveforms[canvasId]?.kind || "input"));
90
- canvas.addEventListener("click", event => {
91
- if (!Number.isFinite(player.duration) || !player.duration) return;
92
- const rect = canvas.getBoundingClientRect();
93
- player.currentTime = Math.max(0, Math.min(1, (event.clientX - rect.left) / rect.width)) * player.duration;
94
- });
95
- }
96
- function setWave(id, samples, kind, durationId, emptyId) {
97
- const empty = $(emptyId);
98
- if (samples && samples.length) {
99
- if (empty) empty.hidden = true;
100
- drawWaveform(id, samples, kind);
101
- const duration = $(durationId);
102
- if (duration) duration.textContent = formatDuration(samples.length / 48000);
103
- } else {
104
- if (empty) empty.hidden = false;
105
- const canvas = $(id), ctx = canvas.getContext("2d");
106
- ctx.clearRect(0, 0, canvas.width, canvas.height);
107
- delete waveforms[id];
108
- }
109
- }
110
-
111
- function isMobileDevice() {
112
- // Prefer CPU on phones/tablets: mobile WebGPU drivers may expose the API
113
- // but fail during large model inference. Desktop users can still override manually.
114
- return /Android|iPhone|iPad|iPod|Mobile|Tablet/i.test(navigator.userAgent) ||
115
- (navigator.userAgentData?.mobile === true);
116
- }
117
- async function chooseProvider() {
118
- const requested = $("sam-provider").value;
119
- if (requested === "webgpu" || requested === "wasm") return requested;
120
- if (isMobileDevice()) {
121
- setLog("Automatic device selection: mobile device detected. Using CPU (WASM) for compatibility.");
122
- return "wasm";
123
- }
124
- try {
125
- if (navigator.gpu) {
126
- const adapter = await navigator.gpu.requestAdapter();
127
- if (adapter) {
128
- setLog("Automatic device selection: desktop WebGPU adapter found. Using GPU (WebGPU).");
129
- return "webgpu";
130
- }
131
- }
132
- } catch (error) {
133
- setLog("WebGPU adapter check failed. Using CPU (WASM).\n" + String(error));
134
- }
135
- setLog("Automatic device selection: WebGPU is unavailable. Using CPU (WASM).");
136
- return "wasm";
137
- }
138
- async function loadOrt() {
139
- if (!ort) {
140
- ort = await import("https://cdn.jsdelivr.net/npm/onnxruntime-web@1.23.2/+esm");
141
- if (ort.env?.wasm) {
142
- // Let ONNX Runtime choose its supported thread count. Multi-threading
143
- // is enabled only when the browser provides the required isolation.
144
- ort.env.wasm.numThreads = 0;
145
- ort.env.wasm.proxy = false;
146
- }
147
- if (ort.env) ort.env.logLevel = "warning";
148
- }
149
- return ort;
150
- }
151
- async function loadTokenizer() {
152
- if (!tokenizer) {
153
- transformersModule = await import("https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.2");
154
- // Let Transformers.js persist tokenizer assets in its browser cache when supported.
155
- transformersModule.env.useBrowserCache = true;
156
- // Use the Transformers.js-compatible T5 tokenizer repository.
157
- tokenizer = await transformersModule.AutoTokenizer.from_pretrained("Xenova/t5-base");
158
- }
159
- return tokenizer;
160
- }
161
- async function loadSpanTokenizer() {
162
- if (!transformersModule) {
163
- transformersModule = await import("https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.2");
164
- transformersModule.env.useBrowserCache = true;
165
- }
166
- if (!spanTokenizer) {
167
- // Load the exact tokenizer files from the model repo through our shared
168
- // Cache Storage, then instantiate the tokenizer directly from its JSON.
169
- const tokenizerBase = MODEL_BASE + "/peaframe_tokenizer";
170
- const tokenizerJsonBytes = await fetchCachedBytes(tokenizerBase + "/tokenizer.json", "PEAFrame tokenizer · tokenizer.json");
171
- const tokenizerConfigBytes = await fetchCachedBytes(tokenizerBase + "/tokenizer_config.json", "PEAFrame tokenizer · tokenizer_config.json");
172
- await fetchCachedBytes(tokenizerBase + "/special_tokens_map.json", "PEAFrame tokenizer · special_tokens_map.json");
173
- const tokenizerJson = JSON.parse(new TextDecoder().decode(tokenizerJsonBytes));
174
- const tokenizerConfig = JSON.parse(new TextDecoder().decode(tokenizerConfigBytes));
175
- if (typeof transformersModule.PreTrainedTokenizer !== "function") {
176
- throw new Error("This Transformers.js build does not expose PreTrainedTokenizer.");
177
- }
178
- spanTokenizer = new transformersModule.PreTrainedTokenizer(tokenizerJson, tokenizerConfig);
179
- }
180
- return spanTokenizer;
181
- }
182
- async function loadSpanPredictor(provider) {
183
- if (!sessions) throw new Error("Load the base audio models before span prediction.");
184
- if (sessions.peaframe) return sessions.peaframe;
185
- setStatus("Loading optional span-prediction model…");
186
- setLog("Automatic Span Prediction requires the optional PEAFrame ONNX model and external weights. First download can be several GB; assets are cached in this browser.");
187
- const configBytes = await fetchCachedBytes(MODEL_BASE + "/peaframe_config.json", "PEAFrame configuration");
188
- spanConfig = JSON.parse(new TextDecoder().decode(configBytes));
189
- const session = await createSession("peaframe.onnx", provider);
190
- try {
191
- await loadSpanTokenizer();
192
- } catch (error) {
193
- try { await session.release(); } catch (_) {}
194
- throw new Error("Could not load the PEAFrame tokenizer in this browser. Manual Anchors remain available. Details: " + String(error?.message || error));
195
- }
196
- sessions.peaframe = session;
197
- return session;
198
- }
199
- function sigmoid(value) {
200
- if (value >= 0) { const z = Math.exp(-value); return 1 / (1 + z); }
201
- const z = Math.exp(value); return z / (1 + z);
202
- }
203
- async function predictSpans(samples, prompt, threshold, provider) {
204
- const session = await loadSpanPredictor(provider);
205
- const tok = await loadSpanTokenizer();
206
- const encoded = await tok(prompt, {padding: true, truncation: true, max_length: 512});
207
- const rawIds = encoded.input_ids?.data ?? encoded.input_ids;
208
- const rawMask = encoded.attention_mask?.data ?? encoded.attention_mask;
209
- const ids = Array.from(rawIds, Number);
210
- const attention = Array.from(rawMask, Number);
211
- const expectedSamples = 160000; // fixed PEAFrame ONNX input: ~3.33 s at 48 kHz
212
- const hopLength = Number(spanConfig?.hop_length || 1920);
213
- const sampleRate = Number(spanConfig?.sampling_rate || 48000);
214
- const stride = expectedSamples >> 1;
215
- const probabilityChunks = [];
216
- const starts = [];
217
- const startsToProcess = samples.length <= expectedSamples ? [0] : Array.from({length: Math.ceil((samples.length - expectedSamples) / stride) + 1}, (_, i) => i * stride);
218
- for (let ci = 0; ci < startsToProcess.length; ci++) {
219
- const start = startsToProcess[ci];
220
- const end = Math.min(start + expectedSamples, samples.length);
221
- const chunk = new Float32Array(expectedSamples);
222
- chunk.set(samples.subarray(start, end));
223
- const output = await session.run({
224
- input_ids: int64Tensor(ids, [1, ids.length]),
225
- input_values: floatTensor(chunk, [1, 1, expectedSamples]),
226
- attention_mask: int64Tensor(attention, [1, attention.length])
227
- });
228
- const audioEmbeds = output.audio_embeds || output[session.outputNames[0]];
229
- const textEmbeds = output.text_embeds || output[session.outputNames[1]];
230
- if (!audioEmbeds || !textEmbeds) throw new Error("PEAFrame returned unexpected outputs; expected audio_embeds and text_embeds.");
231
- const frames = audioEmbeds.dims[1];
232
- const dim = audioEmbeds.dims[2];
233
- const ae = audioEmbeds.data, te = textEmbeds.data;
234
- const probabilities = new Float32Array(frames);
235
- for (let frame = 0; frame < frames; frame++) {
236
- let dot = 0;
237
- const offset = frame * dim;
238
- for (let d = 0; d < dim; d++) dot += ae[offset + d] * te[d];
239
- probabilities[frame] = sigmoid(dot * Number(spanConfig?.logit_scale ?? 2.298705816268921) + Number(spanConfig?.logit_bias ?? -10.002328872680664));
240
- }
241
- const usableFrames = Math.min(frames, Math.ceil((end - start) / hopLength));
242
- probabilityChunks.push(probabilities.slice(0, usableFrames));
243
- starts.push(start);
244
- setStatus("Predicting sound spans " + (ci + 1) + "/" + startsToProcess.length + "…");
245
- await new Promise(resolve => setTimeout(resolve, 0));
246
- }
247
- const totalFrames = Math.ceil(samples.length / hopLength);
248
- const merged = new Float32Array(totalFrames);
249
- const counts = new Uint16Array(totalFrames);
250
- for (let ci = 0; ci < probabilityChunks.length; ci++) {
251
- const chunkStart = Math.floor(starts[ci] / hopLength);
252
- const probs = probabilityChunks[ci];
253
- for (let j = 0; j < probs.length && chunkStart + j < totalFrames; j++) {
254
- merged[chunkStart + j] += probs[j];
255
- counts[chunkStart + j]++;
256
- }
257
- }
258
- for (let i = 0; i < merged.length; i++) merged[i] = counts[i] ? merged[i] / counts[i] : 0;
259
- const spans = [];
260
- let active = false, startFrame = 0;
261
- for (let i = 0; i < merged.length; i++) {
262
- const detected = merged[i] > threshold;
263
- if (detected && !active) { startFrame = i; active = true; }
264
- else if (!detected && active) {
265
- spans.push([startFrame * hopLength / sampleRate, i * hopLength / sampleRate]);
266
- active = false;
267
- }
268
- }
269
- if (active) spans.push([startFrame * hopLength / sampleRate, samples.length / sampleRate]);
270
- return spans;
271
- }
272
- function parseManualAnchors(raw, duration) {
273
- const lines = raw.split(/\r?\n/).map(line => line.trim()).filter(Boolean);
274
- const spans = [];
275
- for (let index = 0; index < lines.length; index++) {
276
- const parts = lines[index].split(/[\s,;]+/);
277
- if (parts.length !== 3 || !["+", "-"].includes(parts[0])) {
278
- throw new Error("Invalid manual anchor on line " + (index + 1) + '. Use: + 4.5 7.0 or - 0 3.0.');
279
- }
280
- const start = Number(parts[1]), end = Number(parts[2]);
281
- if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end <= start || end > duration) {
282
- throw new Error("Invalid time range on manual anchor line " + (index + 1) + ". Times must be within the audio duration and end must be greater than start.");
283
- }
284
- spans.push({sign: parts[0], start, end});
285
- }
286
- if (!spans.length) throw new Error("Manual Anchors is enabled, but no valid anchor ranges were entered.");
287
- return spans;
288
- }
289
- function buildAnchorTensors(spans, sequenceLength) {
290
- const ids = [0, 3]; // <null>=0, <pad>=3, as in the Python reference
291
- const alignment = new BigInt64Array(sequenceLength);
292
- alignment.fill(1); // unanchored frames map to <pad>
293
- for (const span of spans) {
294
- const startFrame = Math.max(0, Math.min(sequenceLength, Math.floor(span.start * 48000 / 1920)));
295
- const endFrame = Math.max(0, Math.min(sequenceLength, Math.floor(span.end * 48000 / 1920)));
296
- if (startFrame >= endFrame) continue;
297
- const anchorIndex = ids.length;
298
- alignment.fill(BigInt(anchorIndex), startFrame, endFrame);
299
- ids.push(span.sign === "-" ? 2 : 1);
300
- }
301
- return {
302
- ids: new ort.Tensor("int64", BigInt64Array.from(ids, value => BigInt(value)), [1, ids.length]),
303
- alignment: new ort.Tensor("int64", alignment, [1, sequenceLength])
304
- };
305
- }
306
-
307
- async function fetchCachedBytes(url, label) {
308
- // Cache Storage persists across page reloads. If unavailable or quota is exceeded,
309
- // continue with a normal network download rather than failing inference.
310
- let cache = null;
311
- if ("caches" in window) {
312
- try { cache = await caches.open(CACHE_NAME); } catch (_) {}
313
- }
314
- if (cache) {
315
- try {
316
- const cached = await cache.match(url);
317
- if (cached) {
318
- setLog("Using browser cache: " + label + "\nNo network download needed for this file.");
319
- return new Uint8Array(await cached.arrayBuffer());
320
- }
321
- } catch (_) {}
322
- }
323
- setLog("Downloading " + label + "…\nThe file will be cached for future runs if browser storage permits.");
324
- const response = await fetch(url);
325
- if (!response.ok) throw new Error("Download failed (" + response.status + ") for " + url);
326
- if (cache) {
327
- try {
328
- // Cache a clone so the original response remains available for the ONNX runtime.
329
- await cache.put(url, response.clone());
330
- } catch (error) {
331
- setLog("Downloaded " + label + ", but the browser could not persist it in cache (quota/private mode). Continuing without persistent cache.\n" + String(error));
332
- }
333
- }
334
- return new Uint8Array(await response.arrayBuffer());
335
- }
336
- async function configureRuntime(provider) {
337
- const runtime = await loadOrt();
338
- if (runtimeProvider !== null && runtimeProvider !== provider) {
339
- throw new Error(
340
- "The CPU worker and GPU modes require different ONNX Runtime initialization. " +
341
- "Reload this page before switching between CPU and GPU, then select the desired mode before the first inference."
342
- );
343
- }
344
- if (runtimeProvider === null) {
345
- if (runtime.env?.wasm) {
346
- // In CPU mode, run WASM inference in a proxy Web Worker so heavy
347
- // inference does not block the browser's UI thread.
348
- // WebGPU and the WASM proxy worker cannot be used together.
349
- runtime.env.wasm.proxy = provider === "wasm";
350
- runtime.env.wasm.numThreads = 0; // automatic, subject to browser support
351
- }
352
- runtimeProvider = provider;
353
- }
354
- return runtime;
355
- }
356
- async function createSession(filename, provider) {
357
- const url = MODEL_BASE + "/" + filename;
358
- const runtime = await configureRuntime(provider);
359
- const modelBytes = await fetchCachedBytes(url, filename);
360
- const dataUrl = url + ".data";
361
- const externalBytes = await fetchCachedBytes(dataUrl, filename + ".data (external weights)");
362
- return runtime.InferenceSession.create(modelBytes, {
363
- executionProviders: [provider],
364
- graphOptimizationLevel: "all",
365
- externalData: [{ path: filename + ".data", data: externalBytes.buffer }]
366
- });
367
- }
368
- async function clearModelCache() {
369
- if (sessions) {
370
- for (const session of Object.values(sessions)) {
371
- try { await session.release(); } catch (_) {}
372
- }
373
- sessions = null;
374
- sessionsProvider = null;
375
- }
376
- spanTokenizer = null;
377
- spanConfig = null;
378
- tokenizer = null;
379
- transformersModule = null;
380
- if ("caches" in window) {
381
- try { await caches.delete(CACHE_NAME); } catch (_) {}
382
- }
383
- setStatus("Model cache cleared · reload before next run", "success");
384
- }
385
- async function loadModels(provider) {
386
- if (sessions && sessionsProvider === provider) return sessions;
387
- if (sessions) {
388
- for (const session of Object.values(sessions)) {
389
- try { await session.release(); } catch (_) {}
390
- }
391
- sessions = null;
392
- sessionsProvider = null;
393
- }
394
- const loaded = {};
395
- const order = [
396
- ["encoder", "dacvae_encoder.onnx"],
397
- ["t5", "t5_encoder.onnx"],
398
- ["dit", "dit_single_step.onnx"],
399
- ["decoder", "dacvae_decoder.onnx"]
400
- ];
401
- try {
402
- for (const [key, filename] of order) {
403
- setStatus("Loading " + filename + "…");
404
- setLog("Loading model component: " + filename + "\nThe browser may download a large external .onnx.data file. Keep this tab open.");
405
- loaded[key] = await createSession(filename, provider);
406
- setLog("Loaded " + filename + "\nInputs: " + loaded[key].inputNames.join(", ") + "\nOutputs: " + loaded[key].outputNames.join(", "));
407
- }
408
- await loadTokenizer();
409
- sessions = loaded;
410
- sessionsProvider = provider;
411
- return sessions;
412
- } catch (error) {
413
- for (const session of Object.values(loaded)) {
414
- try { await session.release(); } catch (_) {}
415
- }
416
- throw error;
417
- }
418
- }
419
- function getTensorMetadata(session, name) {
420
- try { return session.inputMetadata?.[name] || {}; } catch (_) { return {}; }
421
- }
422
- function floatTensor(data, dims) {
423
- // IMPORTANT: with ort.env.wasm.proxy=true, ONNX Runtime transfers input
424
- // ArrayBuffers to its worker. Passing the original typed array detaches it,
425
- // leaving length 0 for later ODE steps. Always pass a fresh copy.
426
- const copiedData = data instanceof Float32Array ? new Float32Array(data) : Float32Array.from(data);
427
- return new ort.Tensor("float32", copiedData, dims);
428
- }
429
- function int64Tensor(data, dims) {
430
- const values = Array.from(data, value => BigInt(value));
431
- return new ort.Tensor("int64", BigInt64Array.from(values), dims);
432
- }
433
- function boolTensor(data, dims) {
434
- return new ort.Tensor("bool", Uint8Array.from(data, value => value ? 1 : 0), dims);
435
- }
436
- async function decodeAudio(file) {
437
- const bytes = await file.arrayBuffer();
438
- const AudioContextClass = window.AudioContext || window.webkitAudioContext;
439
- const context = new AudioContextClass();
440
- try {
441
- const decoded = await context.decodeAudioData(bytes.slice(0));
442
- const offline = new OfflineAudioContext(1, Math.ceil(decoded.duration * 48000), 48000);
443
- const source = offline.createBufferSource();
444
- const mono = offline.createBuffer(1, decoded.length, decoded.sampleRate);
445
- const channels = [];
446
- for (let i = 0; i < decoded.numberOfChannels; i++) channels.push(decoded.getChannelData(i));
447
- const monoData = mono.getChannelData(0);
448
- for (let i = 0; i < monoData.length; i++) {
449
- let sum = 0;
450
- for (const channel of channels) sum += channel[i];
451
- monoData[i] = sum / channels.length;
452
- }
453
- source.buffer = mono;
454
- source.connect(offline.destination);
455
- source.start();
456
- const resampled = await offline.startRendering();
457
- return {samples: new Float32Array(resampled.getChannelData(0)), sampleRate: 48000};
458
- } finally {
459
- await context.close().catch(() => {});
460
- }
461
- }
462
- async function runSession(session, outputName, feeds) {
463
- const outputs = await session.run(feeds);
464
- const tensor = outputs[outputName] || outputs[session.outputNames[0]];
465
- if (!tensor) throw new Error("Expected output not found: " + outputName);
466
- return tensor.data;
467
- }
468
- async function encodeAudio(samples) {
469
- const session = sessions.encoder;
470
- // In WASM proxy mode, ONNX Runtime may transfer/detach the tensor's
471
- // ArrayBuffer when sending it to the worker. Never give it audio.samples
472
- // directly: keep the decoded source intact for duration checks and output trimming.
473
- const inputSamples = new Float32Array(samples);
474
- const input = new ort.Tensor("float32", inputSamples, [1, 1, inputSamples.length]);
475
- const out = await session.run({audio: input});
476
- const tensor = out.latent_features || out[session.outputNames[0]];
477
- return {data: new Float32Array(tensor.data), dims: tensor.dims};
478
- }
479
- async function encodeText(text) {
480
- const tok = await loadTokenizer();
481
- const encoded = await tok(text, {
482
- padding: "max_length",
483
- truncation: true,
484
- max_length: 77
485
- });
486
- const idsRaw = encoded.input_ids?.data ?? encoded.input_ids;
487
- const maskRaw = encoded.attention_mask?.data ?? encoded.attention_mask;
488
- const ids = Array.from(idsRaw);
489
- const mask = Array.from(maskRaw);
490
- const idsTensor = int64Tensor(ids, [1, ids.length]);
491
- const maskTensor = int64Tensor(mask, [1, mask.length]);
492
- const out = await sessions.t5.run({input_ids: idsTensor, attention_mask: maskTensor});
493
- const tensor = out.hidden_states || out[sessions.t5.outputNames[0]];
494
- return {
495
- features: new Float32Array(tensor.data),
496
- featureDims: tensor.dims,
497
- mask: Uint8Array.from(mask, value => value ? 1 : 0),
498
- maskDims: [1, mask.length]
499
- };
500
- }
501
- async function runDit(noisy, timeValue, audioFeatures, text, sequenceLength, anchors = null) {
502
- const batch = 1;
503
- const anchorIds = anchors?.ids || new ort.Tensor("int64", BigInt64Array.from([0n, 3n]), [1, 2]);
504
- const anchorAlignment = anchors?.alignment || new ort.Tensor("int64", new BigInt64Array(sequenceLength), [1, sequenceLength]);
505
- const mask = new Uint8Array(sequenceLength).fill(1);
506
- const video = new Float32Array(1024 * sequenceLength);
507
- const feeds = {
508
- noisy_audio: floatTensor(noisy, [batch, sequenceLength, 256]),
509
- time: floatTensor(new Float32Array([timeValue]), [1]),
510
- audio_features: floatTensor(audioFeatures, [batch, sequenceLength, 256]),
511
- text_features: floatTensor(text.features, text.featureDims),
512
- text_mask: boolTensor(text.mask, text.maskDims),
513
- masked_video_features: floatTensor(video, [batch, 1024, sequenceLength]),
514
- anchor_ids: anchorIds,
515
- anchor_alignment: anchorAlignment,
516
- audio_pad_mask: boolTensor(mask, [batch, sequenceLength])
517
- };
518
- const outputs = await sessions.dit.run(feeds);
519
- const tensor = outputs.velocity || outputs[sessions.dit.outputNames[0]];
520
- return new Float32Array(tensor.data);
521
- }
522
- async function runOde(initial, audioFeatures, text, steps, sequenceLength, anchors = null) {
523
- let x = new Float32Array(initial);
524
- const dt = 1 / steps;
525
- for (let i = 0; i < steps; i++) {
526
- setStatus("Denoising step " + (i + 1) + "/" + steps + " (1/2)…");
527
- const t = i * dt;
528
- const k1 = await runDit(x, t, audioFeatures, text, sequenceLength, anchors);
529
- const midpoint = new Float32Array(x.length);
530
- for (let j = 0; j < x.length; j++) midpoint[j] = x[j] + (dt / 2) * k1[j];
531
- setStatus("Denoising step " + (i + 1) + "/" + steps + " (2/2)…");
532
- const k2 = await runDit(midpoint, t + dt / 2, audioFeatures, text, sequenceLength, anchors);
533
- for (let j = 0; j < x.length; j++) x[j] += dt * k2[j];
534
- // Yield so status/progress updates can paint between long CPU steps.
535
- await new Promise(resolve => setTimeout(resolve, 0));
536
- }
537
- return x;
538
- }
539
- async function decodeLatent(latent, latentChannels, timeSteps) {
540
- const session = sessions.decoder;
541
- const chunkSize = 25;
542
- const hopLength = 1920;
543
- const chunks = [];
544
- for (let start = 0; start < timeSteps; start += chunkSize) {
545
- const actual = Math.min(chunkSize, timeSteps - start);
546
- const chunk = new Float32Array(latentChannels * chunkSize);
547
- for (let c = 0; c < latentChannels; c++) {
548
- for (let t = 0; t < actual; t++) {
549
- chunk[c * chunkSize + t] = latent[c * timeSteps + start + t];
550
- }
551
- }
552
- const out = await session.run({
553
- latent_features: floatTensor(chunk, [1, latentChannels, chunkSize])
554
- });
555
- const tensor = out.waveform || out[session.outputNames[0]];
556
- const data = tensor.data;
557
- const count = actual * hopLength;
558
- chunks.push(new Float32Array(data.slice(0, count)));
559
- setStatus("Decoding audio " + Math.min(start + chunkSize, timeSteps) + "/" + timeSteps + " latent frames…");
560
- }
561
- const totalLength = chunks.reduce((sum, part) => sum + part.length, 0);
562
- const output = new Float32Array(totalLength);
563
- let offset = 0;
564
- for (const chunk of chunks) { output.set(chunk, offset); offset += chunk.length; }
565
- return output;
566
- }
567
- function wavBlob(samples, sampleRate) {
568
- const buffer = new ArrayBuffer(44 + samples.length * 2);
569
- const view = new DataView(buffer);
570
- function writeString(offset, value) { for (let i = 0; i < value.length; i++) view.setUint8(offset + i, value.charCodeAt(i)); }
571
- writeString(0, "RIFF"); view.setUint32(4, 36 + samples.length * 2, true);
572
- writeString(8, "WAVE"); writeString(12, "fmt "); view.setUint32(16, 16, true);
573
- view.setUint16(20, 1, true); view.setUint16(22, 1, true); view.setUint32(24, sampleRate, true);
574
- view.setUint32(28, sampleRate * 2, true); view.setUint16(32, 2, true); view.setUint16(34, 16, true);
575
- writeString(36, "data"); view.setUint32(40, samples.length * 2, true);
576
- for (let i = 0; i < samples.length; i++) {
577
- const sample = Math.max(-1, Math.min(1, samples[i]));
578
- view.setInt16(44 + i * 2, sample < 0 ? sample * 32768 : sample * 32767, true);
579
- }
580
- return new Blob([buffer], {type:"audio/wav"});
581
- }
582
- function showOutput(playerId, linkId, samples, sampleRate, filename, waveId, kind, durationId, emptyId) {
583
- const url = URL.createObjectURL(wavBlob(samples, sampleRate));
584
- previousUrls.push(url);
585
- const player = $(playerId);
586
- player.src = url;
587
- player.load();
588
- const link = $(linkId); link.href = url; link.download = filename; link.hidden = false;
589
- setWave(waveId, samples, kind, durationId, emptyId);
590
- }
591
- function showInput(file, samples) {
592
- const url = URL.createObjectURL(file);
593
- previousUrls.push(url);
594
- const player = $("sam-input-player"); player.src = url; player.load();
595
- const link = $("sam-input-download"); link.href = url; link.download = file.name; link.hidden = false; link.textContent = "↓ Download original";
596
- $("sam-input-format").textContent = file.name.length > 24 ? file.name.slice(0, 21) + "…" : file.name;
597
- setWave("sam-input-wave", samples, "input", "sam-input-duration", "sam-input-empty");
598
- }
599
- async function separate(providerOverride = null) {
600
- const file = $("sam-audio-file").files?.[0];
601
- const prompt = $("sam-prompt").value.trim();
602
- const steps = Number.parseInt($("sam-steps").value, 10);
603
- const provider = providerOverride || await chooseProvider();
604
- lastAttemptProvider = provider;
605
- const deviceLabel = provider === "wasm" ? "CPU (WASM)" : "GPU (WebGPU)";
606
- if (!file) throw new Error("Choose an audio file first.");
607
- if (!prompt) throw new Error("Enter a text prompt.");
608
- if (!Number.isInteger(steps) || steps < 1 || steps > 16) throw new Error("ODE steps must be between 1 and 16.");
609
- const audio = await decodeAudio(file);
610
- const expectedSamples = audio.samples.length;
611
- if (expectedSamples > 48000 * 60) throw new Error("Audio must be 60 seconds or shorter.");
612
- const predictSpansEnabled = $("sam-predict-spans").checked;
613
- const manualAnchorsEnabled = $("sam-manual-anchors-enabled").checked;
614
- const manualSpans = manualAnchorsEnabled ? parseManualAnchors($("sam-manual-anchors").value, expectedSamples / 48000) : [];
615
- const spanThreshold = Number($("sam-span-threshold").value);
616
- setStatus("Loading models for " + deviceLabel + "…");
617
- if (provider === "wasm") {
618
- setLog("CPU mode selected. ONNX Runtime will use WebAssembly in a background worker where supported, keeping the page responsive. This may take longer than GPU inference.");
619
- }
620
- await loadModels(provider);
621
- setStatus("Encoding audio on " + deviceLabel + "…");
622
- const encoded = await encodeAudio(audio.samples);
623
- if (encoded.dims.length !== 3) throw new Error("Unexpected encoder output dimensions: " + JSON.stringify(encoded.dims));
624
- const [batch, latentChannels, timeSteps] = encoded.dims;
625
- if (batch !== 1 || latentChannels !== 128) {
626
- throw new Error("Unexpected audio latent shape " + JSON.stringify(encoded.dims) + "; expected [1, 128, time].");
627
- }
628
- setStatus("Encoding text…");
629
- const text = await encodeText(prompt);
630
- const mixture = new Float32Array(timeSteps * 256);
631
- for (let t = 0; t < timeSteps; t++) {
632
- for (let c = 0; c < latentChannels; c++) {
633
- const value = encoded.data[c * timeSteps + t];
634
- mixture[t * 256 + c] = value;
635
- mixture[t * 256 + latentChannels + c] = value;
636
- }
637
- }
638
- const initial = new Float32Array(timeSteps * 256);
639
- // Standard normal noise, matching the reference implementation's randn initialization.
640
- for (let i = 0; i < initial.length; i += 2) {
641
- const u1 = Math.max(Number.EPSILON, Math.random());
642
- const u2 = Math.random();
643
- const radius = Math.sqrt(-2 * Math.log(u1));
644
- initial[i] = radius * Math.cos(2 * Math.PI * u2);
645
- if (i + 1 < initial.length) initial[i + 1] = radius * Math.sin(2 * Math.PI * u2);
646
- }
647
- let anchorSpans = [];
648
- if (manualAnchorsEnabled) {
649
- anchorSpans = manualSpans;
650
- setStatus("Applying " + anchorSpans.length + " manual anchor range(s)…");
651
- } else if (predictSpansEnabled) {
652
- setStatus("Detecting target sound spans…");
653
- const detected = await predictSpans(audio.samples, prompt, spanThreshold, provider);
654
- anchorSpans = detected.map(([start, end]) => ({sign: "+", start, end}));
655
- if (anchorSpans.length) {
656
- setStatus("Detected " + anchorSpans.length + " sound span(s). Preparing separation…");
657
- } else {
658
- setStatus("No matching spans detected · continuing without span guidance…");
659
- }
660
- }
661
- const anchors = anchorSpans.length ? buildAnchorTensors(anchorSpans, timeSteps) : null;
662
- const result = await runOde(initial, mixture, text, steps, timeSteps, anchors);
663
- const targetLatent = new Float32Array(latentChannels * timeSteps);
664
- const residualLatent = new Float32Array(latentChannels * timeSteps);
665
- for (let t = 0; t < timeSteps; t++) {
666
- for (let c = 0; c < latentChannels; c++) {
667
- targetLatent[c * timeSteps + t] = result[t * 256 + c];
668
- residualLatent[c * timeSteps + t] = result[t * 256 + latentChannels + c];
669
- }
670
- }
671
- setStatus("Decoding target…");
672
- const target = await decodeLatent(targetLatent, latentChannels, timeSteps);
673
- setStatus("Decoding residual…");
674
- const residual = await decodeLatent(residualLatent, latentChannels, timeSteps);
675
- if (audio.samples.length !== expectedSamples) {
676
- throw new Error("Internal audio buffer was unexpectedly detached or resized: expected " + expectedSamples + " samples, got " + audio.samples.length + ".");
677
- }
678
- showOutput("sam-target-player", "sam-target-download", target.slice(0, expectedSamples), 48000, "target.wav", "sam-target-wave", "target", "sam-target-duration", "sam-target-empty");
679
- showOutput("sam-residual-player", "sam-residual-download", residual.slice(0, expectedSamples), 48000, "residual.wav", "sam-residual-wave", "residual", "sam-residual-duration", "sam-residual-empty");
680
- setStatus("Separation complete · " + deviceLabel, "success");
681
- }
682
- attachWavePlayer("sam-input-wave", "sam-input-player");
683
- attachWavePlayer("sam-target-wave", "sam-target-player");
684
- attachWavePlayer("sam-residual-wave", "sam-residual-player");
685
- $("sam-audio-file").addEventListener("change", async () => {
686
- const file = $("sam-audio-file").files?.[0];
687
- if (!file) return;
688
- for (const url of previousUrls) URL.revokeObjectURL(url);
689
- previousUrls = [];
690
- for (const [playerId, linkId, waveId, durationId, emptyId, waitingText] of [
691
- ["sam-input-player", "sam-input-download", "sam-input-wave", "sam-input-duration", "sam-input-empty", "No audio loaded"],
692
- ["sam-target-player", "sam-target-download", "sam-target-wave", "sam-target-duration", "sam-target-empty", "Waiting for result"],
693
- ["sam-residual-player", "sam-residual-download", "sam-residual-wave", "sam-residual-duration", "sam-residual-empty", "Waiting for result"]
694
- ]) {
695
- const player = $(playerId); player.pause(); player.removeAttribute("src"); player.load();
696
- const link = $(linkId); link.removeAttribute("href"); link.hidden = true;
697
- $(durationId).textContent = waitingText;
698
- $(emptyId).hidden = false;
699
- const canvas = $(waveId); const ctx = canvas.getContext("2d"); ctx.clearRect(0, 0, canvas.width, canvas.height);
700
- delete waveforms[waveId];
701
- }
702
- try {
703
- setStatus("Preparing audio preview…", "busy");
704
- const preview = await decodeAudio(file);
705
- showInput(file, preview.samples);
706
- setStatus("Source ready · " + formatDuration(preview.samples.length / preview.sampleRate), "idle");
707
- } catch (error) {
708
- setStatus("Could not preview this audio: " + String(error?.message || error), "error");
709
- }
710
- });
711
- window.addEventListener("resize", () => {
712
- for (const [id, entry] of Object.entries(waveforms)) drawWaveform(id, entry.samples, entry.kind);
713
- });
714
- $("sam-predict-spans").addEventListener("change", () => {
715
- $("sam-span-threshold-wrap").hidden = !$("sam-predict-spans").checked;
716
- });
717
- $("sam-manual-anchors-enabled").addEventListener("change", () => {
718
- $("sam-manual-anchors-wrap").hidden = !$("sam-manual-anchors-enabled").checked;
719
- });
720
- $("sam-span-threshold").addEventListener("input", () => {
721
- $("sam-span-threshold-value").textContent = Number($("sam-span-threshold").value).toFixed(2);
722
- });
723
- $("sam-clear-cache").addEventListener("click", async () => {
724
- const button = $("sam-clear-cache");
725
- button.disabled = true;
726
- try { await clearModelCache(); }
727
- finally { button.disabled = false; }
728
- });
729
- $("sam-run").addEventListener("click", async () => {
730
- const button = $("sam-run");
731
- button.disabled = true;
732
- try {
733
- await separate();
734
- } catch (error) {
735
- const message = String(error?.message || error);
736
- const isWebGpuBufferFailure = lastAttemptProvider === "webgpu" &&
737
- /createBuffer failed.*mappedAtCreation|mappedAtCreation.*too large/i.test(message);
738
- if (isWebGpuBufferFailure) {
739
- const retryOnCpu = window.confirm(
740
- "WebGPU failed to allocate a GPU buffer on this device. This can be a browser/driver or GPU-memory limitation.\n\nRetry the same audio using CPU (WASM)? This may be slower."
741
- );
742
- if (retryOnCpu) {
743
- $("sam-provider").value = "wasm";
744
- setStatus("Retrying on CPU…");
745
- setLog("WebGPU buffer allocation failed. Retrying locally with CPU (WASM).\nThe audio is not uploaded to the server.");
746
- try {
747
- await separate("wasm");
748
- setStatus("Separation complete · CPU fallback used", "success");
749
- } catch (cpuError) {
750
- setStatus("CPU retry failed: " + String(cpuError?.message || cpuError), "error");
751
- }
752
- } else {
753
- setStatus("GPU allocation failed · try CPU (WASM)", "error");
754
- }
755
- } else {
756
- setStatus("Separation failed: " + message, "error");
757
- }
758
- } finally {
759
- button.disabled = false;
760
- }
761
- });
762
- }